Remove some Rust module allow directives

- clean coverage directory before running specs to avoid simplecov
  slowdowns
- add valgrind task to run valgrind; normal spec does not run it
- delay combining and reporting simplecov results until end of task to
  speed up task execution

CHANGELOG.md

@@ -1,3 +1,39 @@
+## v5.0.0
+
+### New Features
+
+- Add Rust target language output.
+
+### API Changes
+
+- The matched text argument passed to lexer user code blocks is now named
+  `match_text` for every target language. It was previously named `match` for
+  C, C++, and D, and `match_` for Rust. Any lexer user code block which
+  references the matched text must be updated to use the new name. The
+  `match_length` argument (C and C++) is unchanged.
+- Tree generation mode now stores all tree nodes in a compact arena owned by
+  the parser context (a flat node array plus a shared child-link array).
+  This replaces the previous design of one heap allocation per node with
+  layout-punned typed structs.
+- Tree nodes are now referenced by lightweight handles rather than pointers.
+  `p_result()` and the field accessors return handle values.
+- The whole tree is freed together with the context by `p_context_delete()`.
+  The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
+  tree node handles are only valid while the context is alive.
+- Tree node field access changed per target language:
+  - C: per-field accessor functions (e.g. `p_Start_pItems(node)`) plus tree
+    walk macros (e.g. `p_tree_walk_Start(node, pItems, pItem, pToken1, token)`),
+    and generic accessors `p_node_valid()`, `p_node_position()`,
+    `p_node_end_position()`, `p_node_n_fields()`, `p_node_data()`, `p_node_id()`.
+  - C++: handle methods called with `()` (e.g. `node.pItems().pToken1().token()`),
+    plus the same C-style functions/macros for convenience.
+  - D: `@property` accessors preserving the previous field-access syntax
+    (e.g. `node.pItems.pToken1.token`); null checks use `.valid` instead of
+    `is null`.
+- Tree-mode parser rule user code: `$$` and `$1` etc. now yield node handles.
+  Reference child fields through the target-language accessors described above
+  rather than through struct pointer members.
+
 ## v4.8.1
 
 ### Fixes

README.md

@@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C, C++, or D language outputs
+  * targets C, C++, D, or Rust language outputs
   * optionally supports automatic full parse tree generation
   * supports starting parsing from multiple start rules
   * tracks input text start and end positions for all matched tokens/rules
@@ -69,7 +69,7 @@ token times /\*/;
 token power /\*\*/;
 token integer /\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');

Rakefile.rb

@@ -1,3 +1,4 @@
+require "fileutils"
 require "rake/clean"
 require "rspec/core/rake_task"
 require "simplecov"
@@ -11,7 +12,10 @@ end
 
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
+    ENV["partial_specs"] = "1"
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  else
+    FileUtils.rm_rf("coverage")
   end
 end
 task :spec do |task, args|
@@ -19,7 +23,7 @@ task :spec do |task, args|
     original_stdout = $stdout
     sio = StringIO.new
     $stdout = sio
-    SimpleCov.collate Dir["coverage/.resultset.json"]
+    SimpleCov.collate Dir["coverage/parts/*/.resultset.json"]
     $stdout = original_stdout
     sio.string.lines.each do |line|
       $stdout.write(line) unless line =~ /Coverage report generated for/
@@ -27,6 +31,15 @@ task :spec do |task, args|
   end
 end
 
+task :valgrind do
+  begin
+    ENV["spec-valgrind"] = "1"
+    Rake::Task[:spec].execute
+  ensure
+    ENV.delete("spec-valgrind")
+  end
+end
+
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.
 desc "Dist Specs"
@@ -43,4 +56,4 @@ task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
 
-task :all => [:spec, :dspec, :user_guide]
+task :all => [:valgrind, :dspec, :user_guide]

UPGRADING.md

@@ -1,3 +1,56 @@
+## v5.0.0
+
+The generated API for tree generation mode (`tree;`) has been changed
+significantly for this version.
+Aside from the lexer user code block matched text rename described below, the
+lexer/parser value APIs for non-tree grammars are unchanged.
+
+### Lexer user code block matched text
+
+The matched text argument passed to lexer user code blocks has been renamed
+from `match` to `match_text` for all target languages.
+
+- C, C++, and D: rename references to `match` in lexer user code blocks to
+  `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
+
+The `match_length` argument (C, C++) is unchanged.
+This rename only affects lexer user code blocks; parser rule user code blocks
+never had a matched text argument.
+
+### Tree memory management
+
+- Remove all calls to `p_tree_delete()` / `p_tree_delete_XXX()`. Tree nodes now
+  live in the parser context and are freed by `p_context_delete()`.
+- Tree node handles (returned by `p_result()` and the field accessors) are only
+  valid while the context is alive. Do not use them after `p_context_delete()`.
+
+### Tree node field access
+
+Tree nodes are now referenced by handle values instead of pointers, and field
+access differs per target language:
+
+- C: replace `node->field` with the accessor function `p_TYPE_field(node)`, or
+  use the tree walk macro `p_tree_walk_TYPE(node, field1, field2, ...)`. Replace
+  `x != NULL` / `x == NULL` node checks with `p_node_valid(x)` /
+  `!p_node_valid(x)`. Read positions with `p_node_position(node)` /
+  `p_node_end_position(node)`, token payload with `p_TYPE_token(node)` /
+  `p_TYPE_pvalue(node)` or `p_node_data(node)->field`, and compare node identity
+  with `p_node_id(a) == p_node_id(b)`.
+- C++: replace `node->field` with the handle method `node.field()`. Use
+  `node.valid()`, `node.position()`, `node.token()`, `node.pvalue()`, and
+  `node.data()->field` for user token fields. (The C-style functions and macros
+  above are also available.)
+- D: replace pointer declarations (`Start * s`) with value handles (`Start s`)
+  and replace `x !is null` / `x is null` with `x.valid` / `!x.valid`. Field
+  access syntax (`node.field.field`) is otherwise unchanged.
+
+### Tree-mode parser rule user code
+
+In tree generation mode `$$` and `$1`, `$2`, ... now expand to node handles.
+Reference child fields through the target-language accessors above (for example
+`$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
+in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
+
 ## v4.0.0
 
 ### API Changes

assets/parser.c.erb

@@ -68,6 +68,18 @@ const char * <%= @grammar.prefix %>token_names[] = {
     context->text_position.row = 1u;
     context->text_position.col = 1u;
     context->mode = <%= @lexer.mode_id("default") %>;
+<% if @grammar.tree %>
+
+    /* Reserve node ID 0 as the null tree node. */
+<%   if @cpp %>
+    context-><%= @grammar.prefix %>tree_nodes.resize(1);
+<%   else %>
+    context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
+    context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
+    memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
+    context-><%= @grammar.prefix %>tree_nodes_length = 1u;
+<%   end %>
+<% end %>
 
     return context;
 }
@@ -84,9 +96,27 @@ const char * <%= @grammar.prefix %>token_names[] = {
  */
 void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
 {
+<% if @grammar.tree && @grammar.free_token_node != "" %>
+<%   if @cpp %>
+    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
+<%   else %>
+    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
+<%   end %>
+    {
+        if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
+        {
+            <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+        }
+    }
+<% end %>
 <% if @cpp %>
     delete context;
 <% else %>
+<%   if @grammar.tree %>
+    free(context-><%= @grammar.prefix %>tree_nodes);
+    free(context-><%= @grammar.prefix %>tree_children);
+<%   end %>
     free(context);
 <% end %>
 }
@@ -289,7 +319,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param match_length
  *   Matched text length.
@@ -300,7 +330,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   not explicitly return a token.
  */
 static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match,
+    lexer_user_code_id_t code_id, uint8_t const * match_text,
     size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -504,9 +534,9 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match = &context->input[context->input_index];
+            uint8_t const * match_text = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -713,8 +743,8 @@ typedef struct
     size_t state_id;
 
 <% if @grammar.tree %>
-    /** tree node. */
-    void * tree_node;
+    /** Tree node ID. */
+    <%= @grammar.prefix %>node_id_t node_id;
 <% else %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
@@ -723,18 +753,6 @@ typedef struct
 <% end %>
 } state_value_t;
 
-<% if @grammar.tree %>
-/** Common tree node structure. */
-typedef struct TreeNode_s
-{
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-    struct TreeNode_s * fields[];
-} TreeNode;
-<% end %>
-
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
@@ -872,6 +890,77 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
+<% if @grammar.tree %>
+/* Tree arena helpers. */
+
+/**
+ * Allocate a new (zeroed) tree node in the context arena.
+ *
+ * @return The new node ID.
+ */
+static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
+{
+<% if @cpp %>
+    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
+    context-><%= @grammar.prefix %>tree_nodes.emplace_back();
+    return id;
+<% else %>
+    if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
+    {
+        size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
+        <%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
+        memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
+        free(context-><%= @grammar.prefix %>tree_nodes);
+        context-><%= @grammar.prefix %>tree_nodes = new_nodes;
+        context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
+    }
+    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
+    memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
+    context-><%= @grammar.prefix %>tree_nodes_length += 1u;
+    return id;
+<% end %>
+}
+
+/**
+ * Reserve n contiguous (zeroed) child slots in the shared children array.
+ *
+ * @return The offset of the first reserved slot.
+ */
+static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
+{
+<% if @cpp %>
+    <%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
+    context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
+    return offset;
+<% else %>
+    size_t offset = context-><%= @grammar.prefix %>tree_children_length;
+    size_t needed = offset + n;
+    if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
+    {
+        size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
+        while (new_capacity < needed)
+        {
+            new_capacity *= 2u;
+        }
+        <%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
+        if (context-><%= @grammar.prefix %>tree_children != NULL)
+        {
+            memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
+            free(context-><%= @grammar.prefix %>tree_children);
+        }
+        context-><%= @grammar.prefix %>tree_children = new_children;
+        context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
+    }
+    memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
+    context-><%= @grammar.prefix %>tree_children_length = needed;
+    return (<%= @grammar.prefix %>node_id_t)offset;
+<% end %>
+}
+
+/* Tree node field accessor functions. */
+<%= c_tree_accessor_defs %>
+<% end %>
+
 <% unless @grammar.tree %>
 /**
  * Get the rule position (start or end) for the currently matched rule.
@@ -936,7 +1025,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -986,7 +1075,7 @@ static size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1041,7 +1130,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     size_t reduced_rule_set = INVALID_ID;
     size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    void * reduced_parser_node;
+    <%= @grammar.prefix %>node_id_t reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>position_t reduced_position;
     <%= @grammar.prefix %>position_t reduced_end_position;
@@ -1087,7 +1176,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -1114,7 +1203,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                     context->input_index -= token_info.length;
                     context->text_position = token_info.position;
 <% if @grammar.tree %>
-                    context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
 <% else %>
                     context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -1137,11 +1226,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-<%   if @cpp %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
-<%   else %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
-<%   end %>
+                <%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
+                <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
                 token_tree_node->position = token_info.position;
                 token_tree_node->end_position = token_info.end_position;
                 token_tree_node->n_fields = 0u;
@@ -1149,7 +1235,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                new_state_info->tree_node = token_tree_node;
+                new_state_info->node_id = token_node_id;
 <% else %>
                 new_state_info->position = token_info.position;
                 new_state_info->end_position = token_info.end_position;
@@ -1161,7 +1247,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                new_state_info->tree_node = reduced_parser_node;
+                new_state_info->node_id = reduced_parser_node;
 <% else %>
                 new_state_info->pvalue = reduced_parser_value;
                 new_state_info->position = reduced_position;
@@ -1191,50 +1277,54 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
-                TreeNode * node = (TreeNode *)malloc(bytes);
-                memset(node, 0, bytes);
-                node->position = INVALID_POSITION;
-                node->end_position = INVALID_POSITION;
-                node->n_fields = n_fields;
+                uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                /* Reserve child slots. New slots are zero-initialized
+                 * (null node ID) so absent optional children remain null. */
+                <%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                        context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                        context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
                     }
                 }
+                <%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
+                <%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
+                node->position = INVALID_POSITION;
+                node->end_position = INVALID_POSITION;
+                node->child_offset = child_offset;
+                node->n_fields = n_fields;
+                node->is_token = 0u;
                 bool position_found = false;
-                for (size_t i = 0; i < n_fields; i++)
+                for (uint16_t i = 0; i < n_fields; i++)
                 {
-                    TreeNode * child = node->fields[i];
-                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
+                    <%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
+                    if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
                     {
                         if (!position_found)
                         {
-                            node->position = child->position;
+                            node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
                             position_found = true;
                         }
-                        node->end_position = child->end_position;
+                        node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
                     }
                 }
-                reduced_parser_node = node;
+                reduced_parser_node = node_id;
             }
             else
             {
-                reduced_parser_node = NULL;
+                reduced_parser_node = 0u;
             }
 <%   if @grammar.parser_user_code_used? %>
             if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1310,14 +1400,14 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
+    return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
+    return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
 }
 <%   end %>
 <% else %>
@@ -1420,48 +1510,3 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
-<% if @grammar.tree %>
-
-static void tree_delete(TreeNode * node)
-{
-    if (node->is_token)
-    {
-        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
-<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
-<%   if @cpp %>
-        delete token_tree_node;
-<%   else %>
-        free(token_tree_node);
-<%   end %>
-    }
-    else if (node->n_fields > 0u)
-    {
-        for (size_t i = 0u; i < node->n_fields; i++)
-        {
-            if (node->fields[i] != NULL)
-            {
-                tree_delete(node->fields[i]);
-            }
-        }
-        free(node);
-    }
-}
-
-/**
- * Free all tree node memory.
- */
-void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete((TreeNode *)tree);
-}
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-
-/**
- * Free all tree node memory.
- */
-void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete((TreeNode *)tree);
-}
-<%   end %>
-<% end %>

assets/parser.d.erb

@@ -105,22 +105,23 @@ public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ?
 <% end %>
 
 <% if @grammar.tree %>
-/** Common tree node structure. */
-private struct TreeNode
-{
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    ushort n_fields;
-    bool is_token;
-    void *[0] fields;
-}
+/** Tree node ID type (index into the context node arena). ID 0 is null. */
+public alias <%= @grammar.prefix %>node_id_t = uint;
 
-/** Tree node types. @{ */
-public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+/**
+ * Tree node record.
+ *
+ * All tree nodes are stored contiguously in the context node arena. Child
+ * links are stored in a shared children array: a node's children
+ * occupy children[child_offset .. child_offset + n_fields]. Token payload
+ * fields (token, pvalue, and any user fields) are only meaningful when
+ * is_token is true.
+ */
+private struct <%= @grammar.prefix %>node_data_t
 {
-    /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    <%= @grammar.prefix %>node_id_t child_offset;
     ushort n_fields;
     bool is_token;
     <%= @grammar.prefix %>token_t token;
@@ -128,22 +129,84 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 <%= @grammar.token_user_fields %>
 }
 
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
+/** Tree node handle types. @{ */
+
+/** Token tree node handle. */
+public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 {
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    ushort n_fields;
-    bool is_token;
-<%     rule_set.tree_fields.each do |fields| %>
-    union
+    private <%= @grammar.prefix %>context_t * __context;
+    private <%= @grammar.prefix %>node_id_t __id;
+
+    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
     {
-<%       fields.each do |field_name, type| %>
-        <%= type %> * <%= field_name %>;
-<%       end %>
+        this.__context = context;
+        this.__id = id;
     }
+
+    /** Return whether this handle refers to a valid (non-null) node. */
+    @property bool valid()
+    {
+        return __id != 0u;
+    }
+
+    /** Access the underlying node record (token, pvalue, and user fields). */
+    @property ref <%= @grammar.prefix %>node_data_t __node()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id];
+    }
+
+    alias __node this;
+}
+
+<%   tree_node_rule_sets.each do |rule_set| %>
+/** <%= rule_set.name %> tree node handle. */
+public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_suffix %>
+{
+    private <%= @grammar.prefix %>context_t * __context;
+    private <%= @grammar.prefix %>node_id_t __id;
+
+    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
+    {
+        this.__context = context;
+        this.__id = id;
+    }
+
+    /** Return whether this handle refers to a valid (non-null) node. */
+    @property bool valid()
+    {
+        return __id != 0u;
+    }
+
+    /** Text position of the first code point spanned by this node. */
+    @property <%= @grammar.prefix %>position_t position()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id].position;
+    }
+
+    /** Text position of the last code point spanned by this node. */
+    @property <%= @grammar.prefix %>position_t end_position()
+    {
+        return __context.<%= @grammar.prefix %>tree_nodes[__id].end_position;
+    }
+
+    /** Number of child fields in this node. */
+    @property ushort n_fields()
+    {
+        return __id ? __context.<%= @grammar.prefix %>tree_nodes[__id].n_fields : cast(ushort)0u;
+    }
+<%     rule_set.tree_fields.each_with_index do |fields, i| %>
+<%       fields.each do |field_name, type| %>
+
+    /** Access the <%= field_name %> child node. */
+    @property <%= type %> <%= field_name %>()
+    {
+        if (__id == 0u)
+        {
+            return <%= type %>(__context, 0u);
+        }
+        return <%= type %>(__context, __context.<%= @grammar.prefix %>tree_children[__context.<%= @grammar.prefix %>tree_nodes[__id].child_offset + <%= i %>u]);
+    }
+<%       end %>
 <%     end %>
 }
 
@@ -196,7 +259,13 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    void * parse_result;
+    <%= @grammar.prefix %>node_id_t parse_result;
+
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    <%= @grammar.prefix %>node_data_t[] <%= @grammar.prefix %>tree_nodes;
+
+    /** Shared tree child links. */
+    <%= @grammar.prefix %>node_id_t[] <%= @grammar.prefix %>tree_children;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -253,8 +322,6 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *
  * @param input
  *   Text input.
- * @param input_length
- *   Text input length.
  *
  * @return Context structure for lexer/parser.
  */
@@ -268,6 +335,11 @@ private enum size_t INVALID_ID = cast(size_t)-1;
     context.text_position.row = 1u;
     context.text_position.col = 1u;
     context.mode = <%= @lexer.mode_id("default") %>;
+<% if @grammar.tree %>
+
+    /* Reserve node ID 0 as the null tree node. */
+    context.<%= @grammar.prefix %>tree_nodes = new <%= @grammar.prefix %>node_data_t[](1);
+<% end %>
 
     return context;
 }
@@ -280,6 +352,16 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  */
 void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
 {
+<% if @grammar.tree && @grammar.free_token_node != "" %>
+    foreach (ref node; context.<%= @grammar.prefix %>tree_nodes)
+    {
+        if (node.is_token)
+        {
+            <%= @grammar.prefix %>node_data_t * token_tree_node = &node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+        }
+    }
+<% end %>
 }
 
 /**************************************************************************
@@ -478,7 +560,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param out_token_info
  *   Lexer token info in progress.
@@ -487,7 +569,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   not explicitly return a token.
  */
 private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, string match,
+    lexer_user_code_id_t code_id, string match_text,
     <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -683,9 +765,9 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
         }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
-            string match = context.input[context.input_index..(context.input_index + match_info.length)];
+            string match_text = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state.code_id, match, &token_info);
+                match_info.accepting_state.code_id, match_text, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -888,8 +970,8 @@ private struct state_value_t
     size_t state_id;
 
 <% if @grammar.tree %>
-    /** Tree node. */
-    void * tree_node;
+    /** Tree node ID. */
+    <%= @grammar.prefix %>node_id_t node_id;
 <% else %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
@@ -1007,7 +1089,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1057,7 +1139,7 @@ private size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1112,7 +1194,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
     size_t reduced_rule_set = INVALID_ID;
     size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    void * reduced_parser_node;
+    <%= @grammar.prefix %>node_id_t reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>position_t reduced_position;
     <%= @grammar.prefix %>position_t reduced_end_position;
@@ -1153,7 +1235,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context.parse_result = statevalues[$-1].tree_node;
+                context.parse_result = statevalues[$-1].node_id;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -1179,7 +1261,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                     context.input_index -= token_info.length;
                     context.text_position = token_info.position;
 <% if @grammar.tree %>
-                    context.parse_result = statevalues[$-1].tree_node;
+                    context.parse_result = statevalues[$-1].node_id;
 <% else %>
                     context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -1199,9 +1281,17 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
+                <%= @grammar.prefix %>node_id_t token_node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
+                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
+                <%= @grammar.prefix %>node_data_t * token_tree_node = &context.<%= @grammar.prefix %>tree_nodes[token_node_id];
+                token_tree_node.position = token_info.position;
+                token_tree_node.end_position = token_info.end_position;
+                token_tree_node.n_fields = 0u;
+                token_tree_node.is_token = true;
+                token_tree_node.token = token;
+                token_tree_node.pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                statevalues[$-1].tree_node = token_tree_node;
+                statevalues[$-1].node_id = token_node_id;
 <% else %>
                 statevalues[$-1].position = token_info.position;
                 statevalues[$-1].end_position = token_info.end_position;
@@ -1213,7 +1303,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                statevalues[$-1].tree_node = reduced_parser_node;
+                statevalues[$-1].node_id = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
                 statevalues[$-1].position = reduced_position;
@@ -1242,55 +1332,56 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = statevalues[$ - 1].tree_node;
+                reduced_parser_node = statevalues[$ - 1].node_id;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
-                TreeNode * node = cast(TreeNode *)malloc(node_size);
-                GC.addRange(node, node_size);
-                node.position = <%= @grammar.prefix %>position_t.INVALID;
-                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
-                node.n_fields = cast(ushort)n_fields;
-                node.is_token = false;
-                foreach (i; 0..n_fields)
-                {
-                    node.fields[i] = null;
-                }
+                ushort n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                /* Reserve child slots. New slots are zero-initialized
+                 * (null node ID) so absent optional children remain null. */
+                <%= @grammar.prefix %>node_id_t child_offset = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_children.length;
+                context.<%= @grammar.prefix %>tree_children.length += n_fields;
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                        context.<%= @grammar.prefix %>tree_children[child_offset + i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
                     }
                 }
                 else
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                        context.<%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
                     }
                 }
+                <%= @grammar.prefix %>node_id_t node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
+                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
+                <%= @grammar.prefix %>node_data_t * node = &context.<%= @grammar.prefix %>tree_nodes[node_id];
+                node.position = <%= @grammar.prefix %>position_t.INVALID;
+                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.child_offset = child_offset;
+                node.n_fields = n_fields;
+                node.is_token = false;
                 bool position_found = false;
                 foreach (i; 0..n_fields)
                 {
-                    TreeNode * child = cast(TreeNode *)node.fields[i];
-                    if (child && child.position.valid)
+                    <%= @grammar.prefix %>node_id_t child_id = context.<%= @grammar.prefix %>tree_children[child_offset + i];
+                    if (child_id != 0u && context.<%= @grammar.prefix %>tree_nodes[child_id].position.valid)
                     {
                         if (!position_found)
                         {
-                            node.position = child.position;
+                            node.position = context.<%= @grammar.prefix %>tree_nodes[child_id].position;
                             position_found = true;
                         }
-                        node.end_position = child.end_position;
+                        node.end_position = context.<%= @grammar.prefix %>tree_nodes[child_id].end_position;
                     }
                 }
-                reduced_parser_node = node;
+                reduced_parser_node = node_id;
             }
             else
             {
-                reduced_parser_node = null;
+                reduced_parser_node = 0u;
             }
 <%   if @grammar.parser_user_code_used? %>
             if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1360,14 +1451,14 @@ public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.p
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
+    return <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %>(context, context.parse_result);
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
+    return <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %>(context, context.parse_result);
 }
 <%   end %>
 <% else %>
@@ -1383,35 +1474,6 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
-<% if @grammar.tree %>
-private void tree_delete(TreeNode * node)
-{
-    if (!node.is_token)
-    {
-        for (size_t i = 0u; i < node.n_fields; i++)
-        {
-            if (node.fields[i])
-            {
-                tree_delete(cast(TreeNode *)node.fields[i]);
-            }
-        }
-        GC.removeRange(node);
-        free(node);
-    }
-}
-
-void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete(cast(TreeNode *)tree);
-}
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
-{
-    tree_delete(cast(TreeNode *)tree);
-}
-<%   end %>
-<% end %>
-
 /**
  * Get the current text input position.
  *

assets/parser.h.erb

@@ -8,6 +8,9 @@
 
 #include <stdint.h>
 #include <stddef.h>
+<% if @cpp %>
+#include <vector>
+<% end %>
 
 /**************************************************************************
  * Public types
@@ -88,47 +91,29 @@ static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "defa
 <% end %>
 
 <% if @grammar.tree %>
-/** Tree node types. @{ */
-typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+/** Tree node ID type (index into the context node arena). ID 0 is null. */
+typedef uint32_t <%= @grammar.prefix %>node_id_t;
+
+/**
+ * Tree node record.
+ *
+ * All tree nodes are stored contiguously in the context node arena. Child
+ * links are stored in a shared children array: a node's children
+ * occupy children[child_offset .. child_offset + n_fields]. Token payload
+ * fields (token, pvalue, and any user fields) are only meaningful when
+ * is_token is nonzero.
+ */
+typedef struct
 {
-    <% # TreeNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    <%= @grammar.prefix %>node_id_t child_offset;
     uint16_t n_fields;
     uint8_t is_token;
-<%= @grammar.token_user_fields %>
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
-<%   end %>
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
-{
-    <% # TreeNode fields must be present in the same order here. # %>
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-<%     rule_set.tree_fields.each do |fields| %>
-    union
-    {
-<%       fields.each do |field_name, type| %>
-        struct <%= type %> * <%= field_name %>;
-<%       end %>
-    };
-<%     end %>
-} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
-
-<%   end %>
-/** @} */
+<%= @grammar.token_user_fields %>
+} <%= @grammar.prefix %>node_data_t;
 <% end %>
 
 /** Lexed token information. */
@@ -179,7 +164,25 @@ typedef struct
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    void * parse_result;
+    <%= @grammar.prefix %>node_id_t parse_result;
+
+<%   if @cpp %>
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    std::vector<<%= @grammar.prefix %>node_data_t> <%= @grammar.prefix %>tree_nodes;
+
+    /** Shared tree child links. */
+    std::vector<<%= @grammar.prefix %>node_id_t> <%= @grammar.prefix %>tree_children;
+<%   else %>
+    /** Tree node arena. Node ID 0 is reserved as the null node. */
+    <%= @grammar.prefix %>node_data_t * <%= @grammar.prefix %>tree_nodes;
+    size_t <%= @grammar.prefix %>tree_nodes_length;
+    size_t <%= @grammar.prefix %>tree_nodes_capacity;
+
+    /** Shared tree child links. */
+    <%= @grammar.prefix %>node_id_t * <%= @grammar.prefix %>tree_children;
+    size_t <%= @grammar.prefix %>tree_children_length;
+    size_t <%= @grammar.prefix %>tree_children_capacity;
+<%   end %>
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -193,6 +196,10 @@ typedef struct
 <%= @grammar.context_user_fields %>
 } <%= @grammar.prefix %>context_t;
 
+<% if @grammar.tree %>
+<%= c_tree_types_header %>
+<% end %>
+
 /**************************************************************************
  * Public data
  *************************************************************************/
@@ -217,9 +224,9 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
 <%   end %>
 
 <% if @grammar.tree %>
-<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
 <%   end %>
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
@@ -228,13 +235,6 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
 <%   end %>
 <% end %>
 
-<% if @grammar.tree %>
-void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
-<%   end %>
-<% end %>
-
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
 void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);

lib/propane/generator.rb

@@ -18,6 +18,8 @@ class Propane
         elsif output_file =~ %r{\.(cc|cpp|cxx)$}
           @cpp = true
           "c"
+        elsif output_file.end_with?(".rs")
+          "rust"
         else
           raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
@@ -31,7 +33,8 @@ class Propane
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension || @language}.erb")
+        template_language = @language == "rust" ? "rs" : @language
+        template = Assets.get("parser.#{extension || template_language}.erb")
         if extension
           output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         else
@@ -39,7 +42,12 @@ class Propane
         end
         erb = ERB.new(template, trim_mode: "<>")
         result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
-          if line == "#linereset\n"
+          if @language == "rust"
+            # Rust has no #line directive support, so strip the directives that
+            # the grammar embeds in user code blocks.
+            line = line.sub(/^#line \d+ "[^"]*"/, "")
+            line == "#linereset\n" ? "" : line
+          elsif line == "#linereset\n"
             %[#line #{i + 2} "#{output_file}"\n]
           else
             line
@@ -275,6 +283,8 @@ class Propane
           "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
         when "d"
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "rust"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
       code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
@@ -284,6 +294,8 @@ class Propane
           "context->#{fieldname}"
         when "d"
           "context.#{fieldname}"
+        when "rust"
+          "context.#{fieldname}"
         end
       end
       code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
@@ -293,16 +305,21 @@ class Propane
           "token_tree_node->#{fieldname}"
         when "d"
           "token_tree_node.#{fieldname}"
+        when "rust"
+          "token_tree_node.#{fieldname}"
         end
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
           if @grammar.tree
+            typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
             case @language
             when "c"
-              "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+              tree_handle(typename, "_node_id")
             when "d"
-              "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+              tree_handle(typename, "_node_id")
+            when "rust"
+              tree_handle(typename, "_node_id")
             end
           else
             case @language
@@ -310,6 +327,8 @@ class Propane
               "_pvalue->v_#{rule.ptypename}"
             when "d"
               "_pvalue.v_#{rule.ptypename}"
+            when "rust"
+              "(*_pvalue.v_#{rule.ptypename}_mut())"
             end
           end
         end
@@ -344,6 +363,8 @@ class Propane
               "out_token_info->pvalue"
             when "d"
               "out_token_info.pvalue"
+            when "rust"
+              "out_token_info.pvalue"
             end
           else
             case @language
@@ -351,6 +372,8 @@ class Propane
               "out_token_info->pvalue.v_#{pattern.ptypename}"
             when "d"
               "out_token_info.pvalue.v_#{pattern.ptypename}"
+            when "rust"
+              "(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())"
             end
           end
         end
@@ -360,6 +383,8 @@ class Propane
             "out_token_info->position"
           when "d"
             "out_token_info.position"
+          when "rust"
+            "out_token_info.position"
           end
         end
         code = code.gsub(/\$\{end_position\}/) do |match|
@@ -368,6 +393,8 @@ class Propane
             "out_token_info->end_position"
           when "d"
             "out_token_info.end_position"
+          when "rust"
+            "out_token_info.end_position"
           end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
@@ -381,6 +408,8 @@ class Propane
             "context->mode = #{mode_id}u"
           when "d"
             "context.mode = #{mode_id}u"
+          when "rust"
+            "context.mode = #{mode_id}"
           end
         end
       end
@@ -402,7 +431,7 @@ class Propane
     def parser_component_reference(rule, index)
       component = rule.components[index - 1]
       if @grammar.tree
-        # In tree mode a component reference yields a pointer to that
+        # In tree mode a component reference yields a handle to that
         # component's tree node. An optional component propagates its target
         # node (or null), so use the optional target's node type.
         if component.is_a?(RuleSet) && component.optional?
@@ -412,9 +441,11 @@ class Propane
         typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
         case @language
         when "c"
-          "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
+          tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
         when "d"
-          "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
+          tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
+        when "rust"
+          tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id")
         end
       else
         case @language
@@ -422,10 +453,308 @@ class Propane
           "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
         when "d"
           "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
+        when "rust"
+          "statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()"
         end
       end
     end
 
+    # Construct a tree node handle expression for the target language.
+    #
+    # A handle is a small value pairing the parser context with a node ID
+    # (an index into the context's node arena). All handle types share this
+    # layout; the distinct types exist for documentation and, in C, to drive
+    # the tree walk macro's type threading.
+    #
+    # @param typename [String]
+    #   Handle type name.
+    # @param id_expr [String]
+    #   Expression yielding the node ID.
+    #
+    # @return [String]
+    #   Handle constructor expression.
+    def tree_handle(typename, id_expr)
+      if @cpp
+        "(#{typename}{context, #{id_expr}})"
+      elsif @language == "c"
+        "((#{typename}){context, #{id_expr}})"
+      elsif @language == "rust"
+        "(#{typename} { context, id: #{id_expr} })"
+      else
+        "#{typename}(context, #{id_expr})"
+      end
+    end
+
+    # Get the list of non-optional, non-internal rule sets that get a tree node
+    # handle type generated for them.
+    #
+    # @return [Array<Propane::RuleSet>]
+    #   Rule sets with generated tree node handle types.
+    def tree_node_rule_sets
+      @parser.rule_sets.reject do |name, rule_set|
+        name.start_with?("$") || rule_set.optional?
+      end.map {|name, rule_set| rule_set}
+    end
+
+    # Maximum number of chained fields supported by a single C tree walk macro
+    # invocation. Deeper navigation can be expressed by nesting walk calls.
+    C_TREE_WALK_MAX = 16
+
+    # Get the tree node handle type name for a node name.
+    #
+    # @param name [String]
+    #   Rule set name, or "Token".
+    #
+    # @return [String]
+    #   Handle type name.
+    def h_type(name)
+      "#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
+    end
+
+    # Get the list of all tree node handle type names (Token plus rule sets).
+    #
+    # @return [Array<String>]
+    #   Handle type names.
+    def tree_handle_types
+      [h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
+    end
+
+    # Enumerate the navigation fields of a rule set's tree node.
+    #
+    # @yield [rtype, field_name, child_type, slot]
+    #   Handle type name, field accessor name, child handle type, and child
+    #   slot index.
+    def each_tree_field(rule_set)
+      rtype = h_type(rule_set.name)
+      rule_set.tree_fields.each_with_index do |fields, slot|
+        fields.each do |field_name, child_type|
+          yield rtype, field_name, child_type, slot
+        end
+      end
+    end
+
+    # Generate the C/C++ tree node handle type section for the header.
+    #
+    # @return [String]
+    #   Header handle section.
+    def c_tree_types_header
+      @cpp ? cpp_tree_types_header : c_only_tree_types_header
+    end
+
+    # Generate the C (non-C++) tree node handle type section for the header.
+    def c_only_tree_types_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Tree node handle types. @{ */"
+      tree_handle_types.each do |t|
+        out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
+      end
+      out << ""
+      out << c_common_accessors_header
+      out << "/** @} */"
+      out.join("\n")
+    end
+
+    # Generate the C-style (function + macro) tree node accessors shared by the
+    # C and C++ headers. In C++ these are provided in addition to the handle
+    # methods so that C-style code (and the tree walk macros) also works.
+    def c_common_accessors_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Generic tree node accessors (usable on any handle type). */"
+      out << "#define #{p}node_valid(h) ((h).__id != 0u)"
+      out << "#define #{p}node_id(h) ((h).__id)"
+      out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
+      out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
+      out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
+      out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
+      out << ""
+      out << "/** Tree node field accessor functions. */"
+      out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
+      out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
+        end
+      end
+      out << ""
+      out << c_tree_walk_macros
+      out.join("\n")
+    end
+
+    # Generate the C tree walk macro machinery.
+    def c_tree_walk_macros
+      p = @grammar.prefix
+      max = C_TREE_WALK_MAX
+      out = []
+      out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
+      out << "#define #{p}CAT_(a, b) a##b"
+      out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
+      out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
+      out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
+      argn = (1..max).map {|i| "_#{i}"}.join(", ")
+      rseq = (0..max).to_a.reverse.join(", ")
+      out << "#define #{p}ARG_N(#{argn}, N, ...) N"
+      out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
+      (1..max).each do |n|
+        fparams = (1..n).map {|k| "f#{k}"}.join(", ")
+        call = "h"
+        (1..n).each do |k|
+          texpr = "R"
+          (1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
+          call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
+        end
+        out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
+      end
+      out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
+      # Type transition map (navigation fields only).
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
+        end
+      end
+      # Per-handle-type walk entry points.
+      tree_handle_types.each do |t|
+        out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
+      end
+      out.join("\n")
+    end
+
+    # Generate the C tree node accessor function definitions for the source.
+    #
+    # @return [String]
+    #   Accessor function definitions.
+    def c_tree_accessor_defs
+      p = @grammar.prefix
+      tt = h_type("Token")
+      out = []
+      out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
+      out << "{"
+      out << "    return node.__context->#{p}tree_nodes[node.__id].token;"
+      out << "}"
+      out << ""
+      out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
+      out << "{"
+      out << "    return node.__context->#{p}tree_nodes[node.__id].pvalue;"
+      out << "}"
+      tree_node_rule_sets.each do |rule_set|
+        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
+          out << ""
+          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
+          out << "{"
+          out << "    #{child_type} result;"
+          out << "    result.__context = node.__context;"
+          out << "    if (node.__id == 0u)"
+          out << "    {"
+          out << "        result.__id = 0u;"
+          out << "        return result;"
+          out << "    }"
+          out << "    result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
+          out << "    return result;"
+          out << "}"
+        end
+      end
+      out.join("\n")
+    end
+
+    # Generate the C++ tree node handle type section for the header.
+    def cpp_tree_types_header
+      p = @grammar.prefix
+      out = []
+      out << "/** Tree node handle types. @{ */"
+      tree_handle_types.each {|t| out << "struct #{t};"}
+      out << ""
+      # Token handle (all methods inline; no handle-typed returns).
+      tt = h_type("Token")
+      out << "struct #{tt}"
+      out << "{"
+      out << "    #{p}context_t * __context;"
+      out << "    #{p}node_id_t __id;"
+      out << "    bool valid() const { return __id != 0u; }"
+      out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+      out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+      out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+      out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      out << "    #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "    #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << "};"
+      out << ""
+      # Rule set handles: navigation methods declared, defined out-of-line below.
+      tree_node_rule_sets.each do |rule_set|
+        rtype = h_type(rule_set.name)
+        out << "struct #{rtype}"
+        out << "{"
+        out << "    #{p}context_t * __context;"
+        out << "    #{p}node_id_t __id;"
+        out << "    bool valid() const { return __id != 0u; }"
+        out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
+          out << "    #{child_type} #{field_name}() const;"
+        end
+        out << "};"
+        out << ""
+      end
+      # Out-of-line navigation method bodies (all handle types now complete).
+      tree_node_rule_sets.each do |rule_set|
+        rtype = h_type(rule_set.name)
+        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
+          out << "inline #{child_type} #{rtype}::#{field_name}() const"
+          out << "{"
+          out << "    if (__id == 0u)"
+          out << "    {"
+          out << "        return #{child_type}{__context, 0u};"
+          out << "    }"
+          out << "    return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
+          out << "}"
+        end
+      end
+      out << ""
+      out << "/*"
+      out << " * C-style function and macro accessors, provided in addition to the handle"
+      out << " * methods above so that C-style code and the tree walk macros also work."
+      out << " */"
+      out << c_common_accessors_header
+      out << "/** @} */"
+      out.join("\n")
+    end
+
+    # Rust keywords that must be escaped as raw identifiers when used as a
+    # generated identifier (e.g. a field alias named `type`).
+    RUST_KEYWORDS = %w[
+      as break const continue dyn else enum extern false fn for if impl in let
+      loop match mod move mut pub ref return static struct trait true type
+      unsafe use where while async await abstract become box do final macro
+      override priv typeof unsized virtual yield try gen
+    ]
+
+    # Escape a name as a Rust raw identifier if it is a reserved keyword.
+    #
+    # @param name [String]
+    #   Identifier name.
+    #
+    # @return [String]
+    #   Name, escaped as a raw identifier if necessary.
+    def rust_ident(name)
+      RUST_KEYWORDS.include?(name) ? "r##{name}" : name
+    end
+
+    # Map a ptype type string to a valid Rust type.
+    #
+    # The default ptype is a C "void *"; for Rust with no declared ptype we use
+    # the unit type instead.
+    #
+    # @param typestring [String]
+    #   ptype type string.
+    #
+    # @return [String]
+    #   Rust type string.
+    def rust_ptype(typestring)
+      typestring == "void *" ? "()" : typestring
+    end
+
     # Get the lex function to use.
     #
     # @return [String]
@@ -459,6 +788,8 @@ class Propane
           "uint8_t"
         when "d"
           "ubyte"
+        when "rust"
+          "u8"
         end
       elsif max <= 0xFFFF
         case @language
@@ -466,11 +797,15 @@ class Propane
           "uint16_t"
         when "d"
           "ushort"
+        when "rust"
+          "u16"
         end
       else
         case @language
         when "c"
           "uint32_t"
+        when "rust"
+          "u32"
         else
           "uint"
         end

spec/json_parser.c.propane

@@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n = 0.0;
     bool negative = false;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative = false;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
+  char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {(char)match[0], 0};
+  char s[] = {(char)match_text[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_parser.d.propane

@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n;
     bool negative;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
+  string_value ~= "{" ~ match_text[2..6] ~ "}";
 >>
 string: /[^\\]/ <<
-  string_value ~= match;
+  string_value ~= match_text;
 >>
 Start -> Value <<
   $$ = $1;

spec/json_parser.rust.propane

@@ -0,0 +1,176 @@
+<<
+pub const JSON_OBJECT: usize = 0;
+pub const JSON_ARRAY: usize = 1;
+pub const JSON_NUMBER: usize = 2;
+pub const JSON_STRING: usize = 3;
+pub const JSON_TRUE: usize = 4;
+pub const JSON_FALSE: usize = 5;
+pub const JSON_NULL: usize = 6;
+
+#[derive(Clone, Default)]
+pub enum JSONValue {
+    #[default]
+    Null,
+    Object(Vec<(String, JSONValue)>),
+    Array(Vec<JSONValue>),
+    Number(f64),
+    StringVal(String),
+    True,
+    False,
+}
+
+impl JSONValue {
+    pub fn id(&self) -> usize {
+        match self {
+            JSONValue::Object(_) => JSON_OBJECT,
+            JSONValue::Array(_) => JSON_ARRAY,
+            JSONValue::Number(_) => JSON_NUMBER,
+            JSONValue::StringVal(_) => JSON_STRING,
+            JSONValue::True => JSON_TRUE,
+            JSONValue::False => JSON_FALSE,
+            JSONValue::Null => JSON_NULL,
+        }
+    }
+    pub fn number(&self) -> f64 {
+        if let JSONValue::Number(n) = self { *n } else { 0.0 }
+    }
+    pub fn string(&self) -> &str {
+        if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
+    }
+    pub fn object_len(&self) -> usize {
+        if let JSONValue::Object(e) = self { e.len() } else { 0 }
+    }
+    pub fn array_len(&self) -> usize {
+        if let JSONValue::Array(e) = self { e.len() } else { 0 }
+    }
+}
+>>
+
+context_user_fields <<
+    pub string_value: String,
+>>
+
+ptype JSONValue;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
+    $$ = JSONValue::Number(n);
+>>
+token true <<
+  $$ = JSONValue::True;
+>>
+token false <<
+  $$ = JSONValue::False;
+>>
+token null <<
+  $$ = JSONValue::Null;
+>>
+/"/ <<
+  $mode(string);
+  ${context.string_value} = String::new();
+>>
+string: token string /"/ <<
+  $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
+  $mode(default);
+>>
+string: /\\"/ <<
+  ${context.string_value}.push('"');
+>>
+string: /\\\\/ <<
+  ${context.string_value}.push('\\');
+>>
+string: /\\\// <<
+  ${context.string_value}.push('/');
+>>
+string: /\\b/ <<
+  ${context.string_value}.push('\u{0008}');
+>>
+string: /\\f/ <<
+  ${context.string_value}.push('\u{000C}');
+>>
+string: /\\n/ <<
+  ${context.string_value}.push('\n');
+>>
+string: /\\r/ <<
+  ${context.string_value}.push('\r');
+>>
+string: /\\t/ <<
+  ${context.string_value}.push('\t');
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
+  ${context.string_value}.push_str(&s);
+>>
+string: /[^\\]/ <<
+  ${context.string_value}.push(match_text[0] as char);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONValue::Object(Vec::new());
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  let mut obj = $1;
+  if let JSONValue::Object(kve) = $3 {
+      if let JSONValue::Object(entries) = &mut obj {
+          entries.extend(kve);
+      }
+  }
+  $$ = obj;
+>>
+KeyValue -> string colon Value <<
+  let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
+  $$ = JSONValue::Object(vec![(name, $3)]);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONValue::Array(Vec::new());
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  let mut arr = $1;
+  if let JSONValue::Array(elems) = &mut arr {
+      elems.push($3);
+  }
+  $$ = arr;
+>>

spec/macros.c.propane

@@ -14,7 +14,7 @@ token rbrace /\}/;
 token plus /\+/;
 token macro;
 token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/macros.d.propane

@@ -14,7 +14,7 @@ token macro;
 token macroname /@[a-zA-Z_]\w*/;
 token num /\d+/ <<
     int n = 0;
-    foreach (c; match)
+    foreach (c; match_text)
     {
         n *= 10;
         n += (c - '0');

spec/macros.rust.propane

@@ -0,0 +1,80 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        if context.expanding {
+            let ei = context.expand_i;
+            context.expand_i += 1;
+            if context.expand_i >= context.token_infos.len() {
+                context.expanding = false;
+            }
+            *out_token_info = context.token_infos[ei].clone();
+            return P_SUCCESS;
+        }
+
+        let lex_result = p_lex(context, out_token_info);
+        if lex_result != P_SUCCESS {
+            return lex_result;
+        }
+
+        if out_token_info.token == TOKEN_macro {
+            context.defining = true;
+        } else if out_token_info.token == TOKEN_macroname {
+            if !context.defining {
+                context.expanding = true;
+                context.expand_i = 0;
+                continue;
+            }
+        } else if out_token_info.token == TOKEN_lbrace {
+            if context.defining {
+                /* Capture the macro body tokens (up to the closing '}'). */
+                let mut infos: Vec<p_token_info_t> = Vec::new();
+                loop {
+                    let mut ti = p_token_info_t::default();
+                    assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
+                    if ti.token == TOKEN_rbrace {
+                        break;
+                    }
+                    infos.push(ti);
+                }
+                context.token_infos = infos;
+                context.defining = false;
+            }
+        } else {
+            context.defining = false;
+        }
+        return lex_result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub defining: bool,
+    pub expanding: bool,
+    pub expand_i: usize,
+    pub token_infos: Vec<p_token_info_t>,
+    pub nums: Vec<i64>,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -12,7 +12,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.d.propane

@@ -10,7 +10,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.rust.propane

@@ -0,0 +1,41 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        let value = p_result_Start(context);
+        /* p_parse_inner rewound the input so ')' was not consumed; consume it. */
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    P_SUCCESS
+}
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.rust.propane

@@ -0,0 +1,44 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        let start_position = out_token_info.position;
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        /* Read the inner subtree's span before re-borrowing context to lex. */
+        let inner = p_result_Start(context);
+        assert!(inner.valid());
+        let inner_start_col = inner.position().col;
+        let inner_end_col = inner.end_position().col;
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        assert_eq!(start_position.col + 1, inner_start_col);
+        assert_eq!(rparen_info.position.col - 1, inner_end_col);
+        /* Synthesize a num token spanning the whole "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    P_SUCCESS
+}
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane_spec.rb

@@ -39,6 +39,11 @@ class MyFormatter
 end
 SimpleCov.start do
   command_name(#{command_name.inspect})
+  # Write this process's results to its own directory. Sharing one resultset
+  # file means every propane invocation must read, merge, and rewrite the
+  # results of all previous invocations, which grows quadratically over the
+  # suite. The parts are merged once at the end by the spec Rake task.
+  coverage_dir(#{"coverage/parts/#{command_name}".inspect})
   filters.clear
   add_filter do |src|
     !(src.filename[SimpleCov.root])
@@ -53,7 +58,7 @@ EOF
     if options[:args]
       command += options[:args]
     else
-      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{lang_ext(options[:language])} --log spec/run/testparser#{options[:name]}.log]
     end
     command += (options[:extra_args] || [])
     if (options[:capture])
@@ -65,9 +70,16 @@ EOF
     end
   end
 
+  # Map a spec language name to the generated parser source file extension.
+  def lang_ext(language)
+    language == "rust" ? "rs" : language
+  end
+
   def compile(test_files, options = {})
     test_files = Array(test_files).map do |test_file|
-      if !File.exist?(test_file) && test_file.end_with?(".cpp")
+      if test_file.end_with?(".rust")
+        test_file.sub(%r{\.rust$}, ".rs")
+      elsif !File.exist?(test_file) && test_file.end_with?(".cpp")
         test_file.sub(%r{\.cpp$}, ".c")
       else
         test_file
@@ -75,15 +87,33 @@ EOF
     end
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
-      "spec/run/testparser#{name}.#{options[:language]}"
+      "spec/run/testparser#{name}.#{lang_ext(options[:language])}"
     end
     case options[:language]
     when "c"
-      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[gcc -g -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "cpp"
-      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[g++ -g -x c++ -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
+      command = [*%w[ldc2 -g -w -de --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
+    when "rust"
+      # Compile each generated parser to an rlib, then compile the test crate
+      # against them. Safe Rust needs no valgrind, but it is run anyway.
+      # Both compiles deny warnings so that a warning introduced into the
+      # generated parser (or into a spec test file) fails the spec rather than
+      # scrolling past unnoticed.
+      externs = []
+      options[:parsers].each do |name|
+        crate = "testparser#{name}"
+        rlib = "spec/run/lib#{crate}.rlib"
+        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -D warnings],
+                 "--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"]
+        expect(system(*rustc)).to be_truthy
+        externs += ["--extern", "#{crate}=#{rlib}"]
+      end
+      # rustc takes a single crate root; any additional Rust test files are
+      # expected to be pulled in as modules or provided by the parser crate.
+      command = [*%w[rustc --edition 2021 -D warnings -o spec/run/testparser], *externs, test_files.first]
     end
     result = system(*command)
     expect(result).to be_truthy
@@ -97,7 +127,7 @@ EOF
     results = Results.new(stdout, stderr, status)
     # Valgrind is only reliably available on Linux, so limit the leak check to
     # Linux platforms.
-    if RUBY_PLATFORM =~ /linux/
+    if RUBY_PLATFORM =~ /linux/ && ENV["spec-valgrind"]
       stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
       vgout = stdout + stderr
       File.binwrite("spec/run/.vgout", vgout)
@@ -285,9 +315,9 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c cpp].each do |language|
+  {"d" => "D", "c" => "C", "cpp" => "C++", "rust" => "Rust"}.each do |language, language_name|
 
-    context "#{language.upcase} language" do
+    context "#{language_name} language" do
 
       it "generates a lexer" do
         write_grammar <<EOF
@@ -316,7 +346,7 @@ token int /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -327,7 +357,7 @@ EOF
 ptype int;
 token int /\\d+/ <<
   int v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -335,6 +365,16 @@ token int /\\d+/ <<
   $$ = v;
 >>
 Start -> int << $$ = $1; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype i64;
+token int /\\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+Start -> int << $$ = $1; >>
 EOF
         end
         run_propane(language: language)
@@ -378,7 +418,7 @@ token integer /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -409,7 +449,7 @@ token times /\\*/;
 token power /\\*\\*/;
 token integer /\\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -429,6 +469,31 @@ E3 -> E4 << $$ = $1; >>
 E3 -> E3 power E4 << $$ = pow($1, $3); >>
 E4 -> integer << $$ = $1; >>
 E4 -> lparen E1 rparen << $$ = $2; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u64;
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  let mut v: u64 = 0;
+  for c in match_text { v = v * 10 + (*c - b'0') as u64; }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = $1.pow($3 as u32); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
         end
         run_propane(language: language)
@@ -505,6 +570,16 @@ token def;
 Start -> Abcs def;
 Abcs -> ;
 Abcs -> abc Abcs;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc <<
+  println!("abc!");
+>>
+token def;
+Start -> Abcs def;
+Abcs -> ;
+Abcs -> abc Abcs;
 EOF
         end
         run_propane(language: language)
@@ -541,6 +616,12 @@ import std.stdio;
 token abc;
 /def/ << writeln("def!"); >>
 Start -> abc;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+/def/ << println!("def!"); >>
+Start -> abc;
 EOF
         end
         run_propane(language: language)
@@ -583,6 +664,16 @@ token abc;
   return $token(abc);
 >>
 Start -> abc;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+/def/ << println!("def!"); >>
+/ghi/ <<
+  println!("ghi!");
+  return $token(abc);
+>>
+Start -> abc;
 EOF
         end
         run_propane(language: language)
@@ -641,6 +732,25 @@ string: /"/ <<
   return $token(string);
 >>
 Start -> abc string def;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  println!("begin string mode");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  println!("captured string");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
 EOF
         end
         run_propane(language: language)
@@ -668,7 +778,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
   return $token(ident);
 >>
@@ -689,7 +799,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
 >>
 token dot /\\./ <<
@@ -699,6 +809,24 @@ default, identonly: drop /\\s+/;
 Start -> abc dot ident <<
   writeln("ident: ", $3);
 >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u8;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match_text[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  println!("ident: {}", $3 as char);
+>>
 EOF
         end
         run_propane(language: language)
@@ -736,6 +864,14 @@ token b;
 Start -> A B << writeln("Start!"); >>
 A -> a << writeln("A!"); >>
 B -> b << writeln("B!"); >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+token a;
+token b;
+Start -> A B << println!("Start!"); >>
+A -> a << println!("A!"); >>
+B -> b << println!("B!"); >>
 EOF
         end
         run_propane(language: language)
@@ -751,7 +887,7 @@ EOF
 
       it "executes user code associated with a parser rule in tree mode" do
         case language
-        when "c", "cpp"
+        when "c"
           write_grammar <<EOF
 tree;
 context_user_fields <<
@@ -769,18 +905,49 @@ ptype int;
 token a << $$ = 11; >>
 token b << $$ = 22; >>
 Start -> A:ay B:bee C <<
-  ${context.start_n_fields} = $$->n_fields;
-  ${context.start_a_value} = $$->pA->pToken1->pvalue;
-  ${context.a_value} = $1->pToken1->pvalue;
-  ${context.b_value} = $2->pToken1->pvalue;
-  ${context.b_token} = $2->pToken1->token;
-  ${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
-  ${context.alias_a_value} = ${ay}->pToken1->pvalue;
-  ${context.alias_b_value} = ${bee}->pToken1->pvalue;
+  ${context.start_n_fields} = p_node_n_fields($$);
+  ${context.start_a_value} = p_tree_walk_Start($$, pA, pToken1, pvalue);
+  ${context.a_value} = p_tree_walk_A($1, pToken1, pvalue);
+  ${context.b_value} = p_tree_walk_B($2, pToken1, pvalue);
+  ${context.b_token} = p_tree_walk_B($2, pToken1, token);
+  ${context.c_field_is_null} = p_node_valid(p_Start_pC($$)) ? 0 : 1;
+  ${context.alias_a_value} = p_tree_walk_A(${ay}, pToken1, pvalue);
+  ${context.alias_b_value} = p_tree_walk_B(${bee}, pToken1, pvalue);
 >>
 A -> a;
 B -> b;
-C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
+C -> << ${context.c_is_null} = p_node_valid($$) ? 0 : 1; >>
+EOF
+        when "cpp"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields();
+  ${context.start_a_value} = $$.pA().pToken1().pvalue();
+  ${context.a_value} = $1.pToken1().pvalue();
+  ${context.b_value} = $2.pToken1().pvalue();
+  ${context.b_token} = $2.pToken1().token();
+  ${context.c_field_is_null} = $$.pC().valid() ? 0 : 1;
+  ${context.alias_a_value} = ${ay}.pToken1().pvalue();
+  ${context.alias_b_value} = ${bee}.pToken1().pvalue();
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = $$.valid() ? 0 : 1; >>
 EOF
         when "d"
           write_grammar <<EOF
@@ -805,13 +972,44 @@ Start -> A:ay B:bee C <<
   ${context.a_value} = $1.pToken1.pvalue;
   ${context.b_value} = $2.pToken1.pvalue;
   ${context.b_token} = $2.pToken1.token;
-  ${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
+  ${context.c_field_is_null} = ($$.pC.valid) ? 0 : 1;
   ${context.alias_a_value} = ${ay}.pToken1.pvalue;
   ${context.alias_b_value} = ${bee}.pToken1.pvalue;
 >>
 A -> a;
 B -> b;
-C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
+C -> << ${context.c_is_null} = ($$.valid) ? 0 : 1; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    pub start_n_fields: i64,
+    pub start_a_value: i64,
+    pub a_value: i64,
+    pub b_value: i64,
+    pub b_token: p_token_t,
+    pub c_is_null: i64,
+    pub c_field_is_null: i64,
+    pub alias_a_value: i64,
+    pub alias_b_value: i64,
+>>
+ptype i64;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields() as i64;
+  ${context.start_a_value} = $$.pA().pToken1().pvalue();
+  ${context.a_value} = $1.pToken1().pvalue();
+  ${context.b_value} = $2.pToken1().pvalue();
+  ${context.b_token} = $2.pToken1().token();
+  ${context.c_field_is_null} = if $$.pC().valid() { 0 } else { 1 };
+  ${context.alias_a_value} = ${ay}.pToken1().pvalue();
+  ${context.alias_b_value} = ${bee}.pToken1().pvalue();
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = if $$.valid() { 0 } else { 1 }; >>
 EOF
         end
         run_propane(language: language)
@@ -822,12 +1020,15 @@ EOF
       end
 
       it "parses lists" do
+        ptype = language == "d" ? "uint" : (language == "rust" ? "u32" : "uint32_t")
+        zero = language == "rust" ? "0" : "0u"
+        one = language == "rust" ? "1" : "1u"
         write_grammar <<EOF
-ptype #{language == "d" ? "uint" : "uint32_t"};
+ptype #{ptype};
 token a;
 Start -> As << $$ = $1; >>
-As -> << $$ = 0u; >>
-As -> As a << $$ = $1 + 1u; >>
+As -> << $$ = #{zero}; >>
+As -> As a << $$ = $1 + #{one}; >>
 EOF
         run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
@@ -866,7 +1067,7 @@ EOF
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * t = (char *)malloc(match_length + 1);
-  strncpy(t, (char *)match, match_length);
+  strncpy(t, (char *)match_text, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
 >>
@@ -878,7 +1079,14 @@ EOF
 import std.stdio;
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  writeln("Matched token is ", match);
+  writeln("Matched token is ", match_text);
+>>
+Start -> id;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  println!("Matched token is {}", std::str::from_utf8(match_text).unwrap());
 >>
 Start -> id;
 EOF
@@ -909,7 +1117,17 @@ EOF
           write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
-  $$ = match.length;
+  $$ = match_text.length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u64;
+token word /[a-z]+/ <<
+  $$ = match_text.len() as u64;
 >>
 Start -> word <<
   $$ = $1;
@@ -1066,6 +1284,25 @@ Start -> Words;
 Words -> ;
 Words -> word Words;
 Words -> stop Words;
+EOF
+        when "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub last_start: p_position_t,
+    pub last_end: p_position_t,
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
 EOF
         end
         run_propane(language: language)
@@ -1128,6 +1365,21 @@ tokenid t;
   return $token(t);
 >>
 Start -> t;
+EOF
+        when "rust"
+          write_grammar <<EOF
+tokenid t;
+/\\a/ << println!("A"); >>
+/\\b/ << println!("B"); >>
+/\\t/ << println!("T"); >>
+/\\n/ << println!("N"); >>
+/\\v/ << println!("V"); >>
+/\\f/ << println!("F"); >>
+/\\r/ << println!("R"); >>
+/t/ <<
+  return $token(t);
+>>
+Start -> t;
 EOF
         end
         run_propane(language: language)
@@ -1164,7 +1416,7 @@ EOF
         write_grammar <<EOF
 tree;
 
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 
 token a << $$ = 11; >>
 token b << $$ = 22; >>
@@ -1208,7 +1460,7 @@ tree;
 tree_prefix P ;
 tree_suffix  S;
 
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 
 token a << $$ = 11; >>
 token b << $$ = 22; >>
@@ -1289,6 +1541,24 @@ Start -> a? b R? <<
 >>
 R -> c d << $$ = "cd"; >>
 R (string) -> d c << $$ = "dc"; >>
+EOF
+        elsif language == "rust"
+          write_grammar <<EOF
+ptype i64;
+ptype float = f64;
+ptype string = String;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  println!("a: {}", $1);
+  println!("b: {}", $2);
+  println!("R: {}", $3);
+>>
+R -> c d << $$ = "cd".to_string(); >>
+R (string) -> d c << $$ = "dc".to_string(); >>
 EOF
         else
           write_grammar <<EOF
@@ -1319,7 +1589,7 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
-          "a: 0#{language == "d" ? "" : ".0"}",
+          "a: 0#{["d", "rust"].include?(language) ? "" : ".0"}",
           "b: 2",
           "R: ",
           "a: 1.5",
@@ -1356,6 +1626,18 @@ tree;
 #include <stdio.h>
 >>
 
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        end
+        if language == "rust"
+          write_grammar <<EOF
+tree;
 token a;
 token b;
 token c;
@@ -1367,6 +1649,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component_tree.#{language}", language: language)
+        # (rust grammar written above overrides the C/D form for the Rust run)
         results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
@@ -1397,6 +1680,18 @@ tree;
 #include <stdio.h>
 >>
 
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        end
+        if language == "rust"
+          write_grammar <<EOF
+tree;
 token a;
 token b;
 token c;
@@ -1497,14 +1792,26 @@ EOF
       end
 
       it "allows specifying field aliases when tree mode is not enabled" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype String;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
+>>
+drop /\\s+/;
+Start -> id:first id:second <<
+  println!("first is {}", ${first});
+  println!("second is {}", ${second});
+>>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id:first id:second <<
@@ -1521,7 +1828,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -1543,14 +1850,29 @@ EOF
       end
 
       it "aliases the correct field when multiple rules are in a rule set when tree mode is not enabled" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype String;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  println!("first is {}", ${first});
+  println!("second is {}", ${second});
+>>
+Foo -> ;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id;
@@ -1570,7 +1892,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -1609,7 +1931,7 @@ EOF
 
       it "allows multiple starting rules" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1629,7 +1951,7 @@ EOF
 
       it "supports parse_inner APIs that treat provided tokens as follow tokens" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 Start -> Y << $$ = $1; >>
@@ -1644,7 +1966,7 @@ EOF
 
       it "parse_inner APIs block success when the outer rule is unfinished" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1660,7 +1982,7 @@ EOF
 
       it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a;
 token b;
 start Start;
@@ -1719,7 +2041,7 @@ EOF
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1764,14 +2086,23 @@ EOF
       end
 
       it "executes code blocks associated with drop statements" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  eprint!("comment: {}", std::str::from_utf8(match_text).unwrap());
+>>
+token a;
+Start -> a;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  stderr.write("comment: ", match);
+  stderr.write("comment: ", match_text);
 >>
 token a;
 Start -> a;
@@ -1784,7 +2115,7 @@ EOF
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+  fprintf(stderr, "comment: %.*s", (int)match_length, match_text);
 >>
 token a;
 Start -> a;
@@ -1798,7 +2129,24 @@ EOF
       end
 
       it "allows user-defined context fields" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub comments: String,
+    pub acount: u32,
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
+>>
+token a <<
+    ${context.acount} += 1;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 context_user_fields <<
     string comments;
@@ -1806,7 +2154,7 @@ context_user_fields <<
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token a <<
     ${context.acount}++;
@@ -1833,7 +2181,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -1858,7 +2206,28 @@ EOF
       end
 
       it "allows custom token user fields" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub comments: String,
+>>
+token_user_fields <<
+    pub comments: String,
+>>
+on_token_node <<
+    ${token.comments} = std::mem::take(&mut ${context.comments});
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 context_user_fields <<
     string comments;
@@ -1873,7 +2242,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -1909,7 +2278,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -1940,7 +2309,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::string((const char *)match, match_length);
+    ${context.comments} += std::string((const char *)match_text, match_length);
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -1955,7 +2324,41 @@ EOF
       end
 
       it "allows a custom lex function" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let mut result = P_SUCCESS;
+    if context.count > 0 {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(context.count);
+        context.count -= 1;
+    } else {
+        result = p_lex(context, out_token_info);
+        if out_token_info.token == TOKEN_c {
+            context.count = 3;
+        }
+    }
+    result
+}
+>>
+
+context_user_fields <<
+    pub count: usize,
+>>
+ptype usize;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
@@ -2037,7 +2440,19 @@ EOF
       end
 
       it "provides accessor functions to extract user values from a parser value" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype i64;
+ptype float = f64;
+ptype string = String;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello".to_string(); >>
+Start -> num flt str;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 ptype int;
 ptype float = float;
@@ -2070,7 +2485,49 @@ EOF
       end
 
       it "allows accessing rule and component text positions" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    println!("ident start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("ident end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("Num start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("Num end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("Start start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("Start end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    println!("Empty start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("Empty end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("tok2 start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("tok2 end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("R start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("R end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    println!("tok1 start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("tok1 end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("Empty start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("Empty end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("R2 start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("R2 end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    println!("StartR start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("StartR end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    println!("StartEmpty start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("StartEmpty end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;

spec/rewind.c.propane

@@ -14,7 +14,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.d.propane

@@ -11,7 +11,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.rust.propane

@@ -0,0 +1,67 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        let result = p_lex(context, out_token_info);
+        if result != P_SUCCESS {
+            return result;
+        }
+        if out_token_info.token == TOKEN_repeat {
+            let mut count_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
+            assert_eq!(TOKEN_num, count_info.token);
+            let mut brace_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
+            assert_eq!(TOKEN_lbrace, brace_info.token);
+            context.remaining = p_value_get(&count_info.pvalue);
+            context.body_index = p_input_index(context);
+            context.body_position = p_position(context);
+            continue;
+        }
+        if out_token_info.token == TOKEN_rbrace {
+            if context.remaining > 1 {
+                context.remaining -= 1;
+                let bi = context.body_index;
+                let bp = context.body_position;
+                p_set_input_index(context, bi);
+                p_set_position(context, bp);
+                continue;
+            }
+            context.remaining = 0;
+            continue;
+        }
+        if out_token_info.token == TOKEN_num {
+            context.num_cols.push(out_token_info.position.col);
+        }
+        return result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub nums: Vec<i64>,
+    pub num_cols: Vec<u32>,
+    pub remaining: i64,
+    pub body_index: usize,
+    pub body_position: p_position_t,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << ${context.nums}.push($1 + $3); >>

spec/spec_helper.rb

@@ -15,6 +15,10 @@ unless ENV["dist_specs"]
       command_name "RSpec"
     end
     project_name "Propane"
+    # Keep this process's results separate from the propane subprocess results
+    # so that nothing has to merge on the fly; the spec Rake task collates all
+    # of the parts once the suite is done.
+    coverage_dir "coverage/parts/rspec"
     merge_timeout 3600
     formatter(MyFormatter)
   end

spec/test_basic_math_grammar.rs

@@ -0,0 +1,16 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u64); 4] = [
+        (b"1 + 2 * 3 + 4", 11),
+        (b"1 * 2 ** 4 * 3", 48),
+        (b"(1 + 2) * 3 + 4", 13),
+        (b"(2 * 2) ** 3 + 4 + 5", 73),
+    ];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_custom_lex_fn.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"cbacba");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(0x932187932187, p_result(&c));
+    p_context_delete(c);
+}

spec/test_drop_code_block.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    # comment 1\n#    comment 2\na\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_error_positions.rs

@@ -0,0 +1,35 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a 42");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a\n123\na  a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(4, p_position(&c).col);
+    assert_eq!(TOKEN_a, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"12");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(1, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+    assert_eq!(TOKEN_num, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\nab");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(2, p_position(&c).col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\na\n\n77\na   \xAA");
+    assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
+    assert_eq!(6, p_position(&c).row);
+    assert_eq!(5, p_position(&c).col);
+    assert_eq!("a", p_token_names[TOKEN_a as usize]);
+    assert_eq!("num", p_token_names[TOKEN_num as usize]);
+    p_context_delete(c);
+}

spec/test_field_aliases.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"foo1\nbar2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_input_index.rs

@@ -0,0 +1,28 @@
+use testparser::*;
+
+fn main() {
+    let c = p_context_new(b"ab");
+    assert_eq!(0, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a b");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    let follow = [TOKEN_b];
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+}

spec/test_lexer.rs

@@ -0,0 +1,49 @@
+use testparser::*;
+
+fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
+    assert_eq!(row, ti.position.row);
+    assert_eq!(col, ti.position.col);
+    assert_eq!(erow, ti.end_position.row);
+    assert_eq!(ecol, ti.end_position.col);
+    assert_eq!(len, ti.length);
+    assert_eq!(token, ti.token);
+}
+
+fn main() {
+    let mut cp: p_code_point_t = 0;
+    let mut cpl: u8 = 0;
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
+    assert_eq!('5' as u32, cp);
+    assert_eq!(1, cpl);
+
+    assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
+    assert_eq!(0xA9, cp);
+    assert_eq!(2, cpl);
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
+    assert_eq!(0x1F9E1, cp);
+    assert_eq!(4, cpl);
+
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
+
+    let mut context = p_context_new(b"5 + 4 * \n677 + 567");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
+    p_context_delete(context);
+}

spec/test_lexer_match_text.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"identifier_123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+}

spec/test_lexer_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc \"a string\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc \"abc def\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_multiple_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc.def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc .  abc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_positions.rs

@@ -0,0 +1,38 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"abc\n  defg hi\n!");
+    let mut ti = p_token_info_t::default();
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(1, c.last_start.row);
+    assert_eq!(1, c.last_start.col);
+    assert_eq!(1, c.last_end.row);
+    assert_eq!(3, c.last_end.col);
+    assert_eq!(c.last_start.row, ti.position.row);
+    assert_eq!(c.last_start.col, ti.position.col);
+    assert_eq!(c.last_end.row, ti.end_position.row);
+    assert_eq!(c.last_end.col, ti.end_position.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(3, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(6, c.last_end.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(8, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(9, c.last_end.col);
+
+    assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
+    assert_eq!(42, p_user_terminate_code(&c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+
+    p_context_delete(c);
+}

spec/test_lexer_result_value.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(1, p_result(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"fabulous");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(8, p_result(&context));
+    p_context_delete(context);
+}

spec/test_lexer_unknown_character.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(123, p_result(&context));
+    p_context_delete(context);
+}

spec/test_macros.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n";
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(vec![166, 223, 88, 223], c.nums);
+    p_context_delete(c);
+}

spec/test_match_backslashes.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_multiple_parsers.rs

@@ -0,0 +1,12 @@
+use testparsermyp1 as m1;
+use testparsermyp2 as m2;
+
+fn main() {
+    let mut context1 = m1::myp1_context_new(b"a\n1");
+    assert_eq!(m1::MYP1_SUCCESS, m1::myp1_parse(&mut context1));
+    m1::myp1_context_delete(context1);
+
+    let mut context2 = m2::myp2_context_new(b"bcb");
+    assert_eq!(m2::MYP2_SUCCESS, m2::myp2_parse(&mut context2));
+    m2::myp2_context_delete(context2);
+}

spec/test_named_optional_rule_component_tree.c

@@ -9,46 +9,42 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
-    assert(start->r == NULL);
+    Start start = p_result(context);
+    assert(!p_node_valid(p_Start_a(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
+    assert(!p_node_valid(p_Start_pR3(start)));
+    assert(!p_node_valid(p_Start_pR(start)));
+    assert(!p_node_valid(p_Start_r(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->a != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->r != NULL);
-    assert(start->pR == start->pR3);
-    assert(start->pR == start->r);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
+    assert(p_node_valid(p_Start_a(start)));
+    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR3(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert(p_node_valid(p_Start_r(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_r(start)));
+    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->r != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
+    assert(!p_node_valid(p_Start_a(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_r(start)));
+    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
-

spec/test_named_optional_rule_component_tree.d

@@ -12,40 +12,40 @@ unittest
     string input = "b";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
+    Start start = p_result(context);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
-    assert(start.r is null);
+    assert(!start.pR3.valid);
+    assert(!start.pR.valid);
+    assert(!start.r.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 != null);
+    assert(start.pToken1.valid);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
-    assert(start.r != null);
+    assert(start.pToken2.valid);
+    assert(start.pR3.valid);
+    assert(start.pR.valid);
+    assert(start.r.valid);
     assert(start.pR == start.pR3);
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
+    assert(start.pR.valid);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_named_optional_rule_component_tree.rs

@@ -0,0 +1,43 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+        assert!(!start.r().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.a().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert!(start.r().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(start.pR().node_id(), start.r().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.r().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_optional_rule_component.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_optional_rule_component_tree.c

@@ -9,43 +9,39 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
+    Start start = p_result(context);
+    assert(!p_node_valid(p_Start_pToken1(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
+    assert(!p_node_valid(p_Start_pR3(start)));
+    assert(!p_node_valid(p_Start_pR(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->pToken1 != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->pR == start->pR3);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
+    assert(p_node_valid(p_Start_pToken1(start)));
+    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR3(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
+    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->pR != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
+    assert(!p_node_valid(p_Start_pToken1(start)));
+    assert(p_node_valid(p_Start_pToken2(start)));
+    assert(p_node_valid(p_Start_pR(start)));
+    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
-

spec/test_optional_rule_component_tree.d

@@ -12,37 +12,37 @@ unittest
     string input = "b";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
+    Start start = p_result(context);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
+    assert(!start.pR3.valid);
+    assert(!start.pR.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 != null);
+    assert(start.pToken1.valid);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
+    assert(start.pToken2.valid);
+    assert(start.pR3.valid);
+    assert(start.pR.valid);
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
+    assert(!start.pToken1.valid);
+    assert(start.pToken2.valid);
+    assert(start.pR.valid);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_optional_rule_component_tree.rs

@@ -0,0 +1,40 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pToken1().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.pR().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_parse_inner.rs

@@ -0,0 +1,30 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF]));
+    p_context_delete(c);
+}

spec/test_parse_inner_nested.rs

@@ -0,0 +1,17 @@
+use testparser::*;
+
+fn eval(input: &[u8]) -> i64 {
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    p_context_delete(c);
+    v
+}
+
+fn main() {
+    assert_eq!(5, eval(b"2 + 3"));
+    assert_eq!(3, eval(b"(1 + 2)"));
+    assert_eq!(14, eval(b"2 + (3 + 4) + 5"));
+    assert_eq!(37, eval(b"2 + (10 + (20 + 5))"));
+    assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.c

@@ -3,21 +3,8 @@
 #include <string.h>
 #include "testutils.h"
 
-/* Grammar (tree generation mode; parentheses handled by the lex function):
- *   tree;
- *   lex_fn mylexfn;
- *   token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/;
- *   Start -> Expr;
- *   Expr  -> num;
- *   Expr  -> Expr plus num;
- *
- * The same lexer-driven nested parse as test_parse_inner_nested, but in tree
- * generation mode. Each "( ... )" group is parsed by a reentrant
- * p_parse_inner_Start() call from the lex function; the resulting subtree is
- * discarded and a single synthesized num token is handed to the outer parse.
- * The synthesized token's position is set to span the whole group ('(' start
- * through ')' end), so this verifies that positions survive the nested-parse
- * boundary and land correctly in the outer tree. */
+/* Grammar: see the D variant / spec. Tree generation mode; parentheses handled
+ * by the lex function. Tree nodes live in the context arena. */
 
 size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
 {
@@ -36,8 +23,8 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         {
             return inner_result;
         }
-        Start * inner = p_result_Start(context);
-        assert_not_null(inner);
+        Start inner = p_result_Start(context);
+        assert(p_node_valid(inner));
         /* p_parse_inner rewound the input so that ')' was not consumed; consume
          * it now. */
         p_token_info_t rparen_info;
@@ -45,10 +32,11 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         assert(rparen_result == P_SUCCESS);
         assert(rparen_info.token == TOKEN_rparen);
         /* The subtree covers the region strictly between the parentheses. */
-        assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col);
-        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col);
-        p_tree_delete_Start(inner);
-        /* Synthesize a num token spanning the entire "( ... )" group. */
+        assert_eq((size_t)(start_position.col + 1u), (size_t)p_node_position(inner).col);
+        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)p_node_end_position(inner).col);
+        /* The inner subtree is discarded (the lexer synthesizes a num token in
+         * its place), but its nodes remain in the shared context arena and are
+         * freed with the context. */
         out_token_info->token = TOKEN_num;
         out_token_info->position = start_position;
         out_token_info->end_position = rparen_info.end_position;
@@ -64,40 +52,39 @@ int main()
     p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
 
-    Start * tree = p_result(context);
-    assert_not_null(tree);
+    Start tree = p_result(context);
+    assert(p_node_valid(tree));
 
     /* Start -> Expr, where the top Expr is "Expr plus num". */
-    Expr * top = tree->pExpr;
-    assert_not_null(top);
-    assert_not_null(top->pExpr);
-    assert_not_null(top->pToken2);
-    assert_not_null(top->pToken3);
+    Expr top = p_Start_pExpr(tree);
+    assert(p_node_valid(top));
+    assert(p_node_valid(p_Expr_pExpr(top)));
+    assert(p_node_valid(p_Expr_pToken2(top)));
+    assert(p_node_valid(p_Expr_pToken3(top)));
 
     /* The '+' joining the two groups is at column 9. */
-    assert_eq(1u, (size_t)top->pToken2->position.row);
-    assert_eq(9u, (size_t)top->pToken2->position.col);
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken2(top)).row);
+    assert_eq(9u, (size_t)p_node_position(p_Expr_pToken2(top)).col);
 
     /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
-    assert_eq(1u, (size_t)top->pToken3->position.row);
-    assert_eq(11u, (size_t)top->pToken3->position.col);
-    assert_eq(1u, (size_t)top->pToken3->end_position.row);
-    assert_eq(17u, (size_t)top->pToken3->end_position.col);
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken3(top)).row);
+    assert_eq(11u, (size_t)p_node_position(p_Expr_pToken3(top)).col);
+    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken3(top)).row);
+    assert_eq(17u, (size_t)p_node_end_position(p_Expr_pToken3(top)).col);
 
     /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
      * columns 1..7. */
-    Expr * left = top->pExpr;
-    assert_not_null(left->pToken1);
-    assert_eq(1u, (size_t)left->pToken1->position.row);
-    assert_eq(1u, (size_t)left->pToken1->position.col);
-    assert_eq(1u, (size_t)left->pToken1->end_position.row);
-    assert_eq(7u, (size_t)left->pToken1->end_position.col);
+    Expr left = p_Expr_pExpr(top);
+    assert(p_node_valid(p_Expr_pToken1(left)));
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).row);
+    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).col);
+    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken1(left)).row);
+    assert_eq(7u, (size_t)p_node_end_position(p_Expr_pToken1(left)).col);
 
     /* The whole tree spans columns 1..17. */
-    assert_eq(1u, (size_t)tree->position.col);
-    assert_eq(17u, (size_t)tree->end_position.col);
+    assert_eq(1u, (size_t)p_node_position(tree).col);
+    assert_eq(17u, (size_t)p_node_end_position(tree).col);
 
-    p_tree_delete(tree);
     p_context_delete(context);
 
     return 0;

spec/test_parse_inner_nested_tree.d

@@ -20,8 +20,8 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         {
             return inner_result;
         }
-        Start * inner = p_result_Start(context);
-        assert(inner !is null);
+        Start inner = p_result_Start(context);
+        assert(inner.valid);
         /* p_parse_inner rewound the input so that ')' was not consumed; consume
          * it now. */
         p_token_info_t rparen_info;
@@ -31,7 +31,9 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         /* The subtree covers the region strictly between the parentheses. */
         assert_eq(start_position.col + 1u, inner.position.col);
         assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
-        p_tree_delete_Start(inner);
+        /* The inner subtree is discarded (the lexer synthesizes a num token in
+         * its place), but its nodes remain in the shared context arena and are
+         * freed with the context. */
         /* Synthesize a num token spanning the entire "( ... )" group. */
         out_token_info.token = TOKEN_num;
         out_token_info.position = start_position;
@@ -53,15 +55,15 @@ unittest
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
 
-    Start * tree = p_result(context);
-    assert(tree !is null);
+    Start tree = p_result(context);
+    assert(tree.valid);
 
     /* Start -> Expr, where the top Expr is "Expr plus num". */
-    Expr * top = tree.pExpr;
-    assert(top !is null);
-    assert(top.pExpr !is null);
-    assert(top.pToken2 !is null);
-    assert(top.pToken3 !is null);
+    Expr top = tree.pExpr;
+    assert(top.valid);
+    assert(top.pExpr.valid);
+    assert(top.pToken2.valid);
+    assert(top.pToken3.valid);
 
     /* The '+' joining the two groups is at column 9. */
     assert_eq(1u, top.pToken2.position.row);
@@ -75,8 +77,8 @@ unittest
 
     /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
      * columns 1..7. */
-    Expr * left = top.pExpr;
-    assert(left.pToken1 !is null);
+    Expr left = top.pExpr;
+    assert(left.pToken1.valid);
     assert_eq(1u, left.pToken1.position.row);
     assert_eq(1u, left.pToken1.position.col);
     assert_eq(1u, left.pToken1.end_position.row);
@@ -86,6 +88,5 @@ unittest
     assert_eq(1u, tree.position.col);
     assert_eq(17u, tree.end_position.col);
 
-    p_tree_delete(tree);
     p_context_delete(context);
 }

spec/test_parse_inner_nested_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"(3 + 4) + (5 + 6)");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let tree = p_result(&c);
+        assert!(tree.valid());
+
+        /* Start -> Expr, where the top Expr is "Expr plus num". */
+        let top = tree.pExpr();
+        assert!(top.valid());
+        assert!(top.pExpr().valid());
+        assert!(top.pToken2().valid());
+        assert!(top.pToken3().valid());
+
+        /* The '+' joining the two groups is at column 9. */
+        assert_eq!(1, top.pToken2().position().row);
+        assert_eq!(9, top.pToken2().position().col);
+
+        /* Right operand: synthesized num for "(5 + 6)", columns 11..17. */
+        assert_eq!(11, top.pToken3().position().col);
+        assert_eq!(17, top.pToken3().end_position().col);
+
+        /* Left operand: synthesized num for "(3 + 4)", columns 1..7. */
+        let left = top.pExpr();
+        assert!(left.pToken1().valid());
+        assert_eq!(1, left.pToken1().position().col);
+        assert_eq!(7, left.pToken1().end_position().col);
+
+        /* The whole tree spans columns 1..17. */
+        assert_eq!(1, tree.position().col);
+        assert_eq!(17, tree.end_position().col);
+    }
+    p_context_delete(c);
+}

spec/test_parse_inner_recursive.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_shared.rs

@@ -0,0 +1,47 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[]));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_tree.c

@@ -5,24 +5,8 @@
 
 int main()
 {
-    /* Grammar (tree generation mode, shared reduce state):
-     *   tree;
-     *   token a; token b;
-     *   start Start;
-     *   start R1;
-     *   Start -> R1 a;
-     *   Start -> R2 b;
-     *   R1 -> a b;
-     *   R2 -> a b;
-     *
-     * Exercises p_parse_inner_R1() with a non-EOF follow token in tree
-     * generation mode. Verifies:
-     *   * The reduced tree for R1 is well-formed after a follow-token
-     *     completion.
-     *   * The follow token is not consumed and remains available for a
-     *     subsequent p_lex() call.
-     *   * p_tree_delete_R1() cleans up the returned tree without leaks
-     *     (verified in CI via valgrind). */
+    /* See the D variant / grammar comments for details. In tree mode the tree
+     * nodes live in the context arena and are freed with p_context_delete(). */
 
     /* Baseline: p_parse_R1 works on "ab" and the returned tree is
      * well-formed. */
@@ -30,13 +14,12 @@ int main()
         char const * input = "ab";
         p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
         assert(p_parse_R1(context) == P_SUCCESS);
-        R1 * tree = p_result_R1(context);
-        assert_not_null(tree);
-        assert_not_null(tree->pToken1);
-        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
-        assert_not_null(tree->pToken2);
-        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
-        p_tree_delete_R1(tree);
+        R1 tree = p_result_R1(context);
+        assert(p_node_valid(tree));
+        assert(p_node_valid(p_R1_pToken1(tree)));
+        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
+        assert(p_node_valid(p_R1_pToken2(tree)));
+        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
         p_context_delete(context);
     }
 
@@ -50,23 +33,23 @@ int main()
         assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
 
         /* Tree is well-formed. */
-        R1 * tree = p_result_R1(context);
-        assert_not_null(tree);
-        assert_not_null(tree->pToken1);
-        assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
-        assert_eq(1u, (size_t)tree->pToken1->position.row);
-        assert_eq(1u, (size_t)tree->pToken1->position.col);
-        assert_not_null(tree->pToken2);
-        assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
-        assert_eq(1u, (size_t)tree->pToken2->position.row);
-        assert_eq(2u, (size_t)tree->pToken2->position.col);
+        R1 tree = p_result_R1(context);
+        assert(p_node_valid(tree));
+        assert(p_node_valid(p_R1_pToken1(tree)));
+        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).row);
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).col);
+        assert(p_node_valid(p_R1_pToken2(tree)));
+        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
+        assert_eq(1u, (size_t)p_node_position(p_R1_pToken2(tree)).row);
+        assert_eq(2u, (size_t)p_node_position(p_R1_pToken2(tree)).col);
 
-        /* The R1 tree covers positions 1..2 — the third `b` at column 3 is
+        /* The R1 tree covers positions 1..2 - the third `b` at column 3 is
          * the follow token and is not part of the tree. */
-        assert_eq(1u, (size_t)tree->position.row);
-        assert_eq(1u, (size_t)tree->position.col);
-        assert_eq(1u, (size_t)tree->end_position.row);
-        assert_eq(2u, (size_t)tree->end_position.col);
+        assert_eq(1u, (size_t)p_node_position(tree).row);
+        assert_eq(1u, (size_t)p_node_position(tree).col);
+        assert_eq(1u, (size_t)p_node_end_position(tree).row);
+        assert_eq(2u, (size_t)p_node_end_position(tree).col);
 
         /* Follow token remains in the input. */
         p_position_t pos = p_position(context);
@@ -78,10 +61,6 @@ int main()
         assert_eq(1u, (size_t)token_info.position.row);
         assert_eq(3u, (size_t)token_info.position.col);
 
-        /* p_tree_delete_R1 must free every node reachable from the tree
-         * without leaking anything. valgrind (invoked by the spec runner on
-         * Linux) will detect any missed frees. */
-        p_tree_delete_R1(tree);
         p_context_delete(context);
     }
 

spec/test_parse_inner_tree.d

@@ -16,13 +16,13 @@ unittest
         string input = "ab";
         p_context_t * context = p_context_new(input);
         assert(p_parse_R1(context) == P_SUCCESS);
-        R1 * tree = p_result_R1(context);
-        assert(tree !is null);
-        assert(tree.pToken1 !is null);
+        R1 tree = p_result_R1(context);
+        assert(tree.valid);
+        assert(tree.pToken1.valid);
         assert(tree.pToken1.token == TOKEN_a);
-        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.valid);
         assert(tree.pToken2.token == TOKEN_b);
-        p_tree_delete_R1(tree);
+        p_context_delete(context);
     }
 
     /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
@@ -35,13 +35,13 @@ unittest
         assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
 
         /* Tree is well-formed. */
-        R1 * tree = p_result_R1(context);
-        assert(tree !is null);
-        assert(tree.pToken1 !is null);
+        R1 tree = p_result_R1(context);
+        assert(tree.valid);
+        assert(tree.pToken1.valid);
         assert(tree.pToken1.token == TOKEN_a);
         assert(tree.pToken1.position.row == 1);
         assert(tree.pToken1.position.col == 1);
-        assert(tree.pToken2 !is null);
+        assert(tree.pToken2.valid);
         assert(tree.pToken2.token == TOKEN_b);
         assert(tree.pToken2.position.row == 1);
         assert(tree.pToken2.position.col == 2);
@@ -63,6 +63,6 @@ unittest
         assert(token_info.position.row == 1);
         assert(token_info.position.col == 3);
 
-        p_tree_delete_R1(tree);
+        p_context_delete(context);
     }
 }

spec/test_parse_inner_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert!(tree.pToken1().valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert!(tree.pToken2().valid());
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert_eq!(1, tree.pToken1().position().row);
+        assert_eq!(1, tree.pToken1().position().col);
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+        assert_eq!(2, tree.pToken2().position().col);
+        assert_eq!(1, tree.position().col);
+        assert_eq!(2, tree.end_position().col);
+    }
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+}

spec/test_parser_identical_rules_lookahead.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"aba"[..], &b"abb"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_parser_rule_from_multiple_states.rs

@@ -0,0 +1,18 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context));
+    assert_eq!(1, p_position(&context).row);
+    assert_eq!(2, p_position(&context).col);
+    assert_eq!(TOKEN___EOF, p_token(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bb");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_rule_user_code.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_user_code_tree.c

@@ -28,11 +28,10 @@ int main()
     assert_eq(11, context->alias_a_value);
     assert_eq(22, context->alias_b_value);
 
-    Start * start = p_result(context);
-    assert(start->pA != NULL);
-    assert(start->pB != NULL);
-    assert(start->pC == NULL);
-    p_tree_delete(start);
+    Start start = p_result(context);
+    assert(p_node_valid(p_Start_pA(start)));
+    assert(p_node_valid(p_Start_pB(start)));
+    assert(!p_node_valid(p_Start_pC(start)));
     p_context_delete(context);
 
     return 0;

spec/test_parser_user_code_tree.d

@@ -31,9 +31,9 @@ unittest
     assert_eq(11, context.alias_a_value);
     assert_eq(22, context.alias_b_value);
 
-    Start * start = p_result(context);
-    assert(start.pA !is null);
-    assert(start.pB !is null);
-    assert(start.pC is null);
-    p_tree_delete(start);
+    Start start = p_result(context);
+    assert(start.pA.valid);
+    assert(start.pB.valid);
+    assert(!start.pC.valid);
+    p_context_delete(context);
 }

spec/test_parser_user_code_tree.rs

@@ -0,0 +1,24 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+
+    assert_eq!(3, context.start_n_fields);
+    assert_eq!(11, context.start_a_value);
+    assert_eq!(11, context.a_value);
+    assert_eq!(22, context.b_value);
+    assert_eq!(TOKEN_b, context.b_token);
+    assert_eq!(1, context.c_is_null);
+    assert_eq!(1, context.c_field_is_null);
+    assert_eq!(11, context.alias_a_value);
+    assert_eq!(22, context.alias_b_value);
+
+    {
+        let start = p_result(&context);
+        assert!(start.pA().valid());
+        assert!(start.pB().valid());
+        assert!(!start.pC().valid());
+    }
+    p_context_delete(context);
+}

spec/test_parsing_json.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"{}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_ARRAY, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"-45.6");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_NUMBER, p_result(&c).id());
+    assert_eq!(-45.6, p_result(&c).number());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"{\"hi\":true}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    assert_eq!(JSON_ARRAY, v.id());
+    assert_eq!(5, v.array_len());
+    p_context_delete(c);
+}

spec/test_parsing_lists.rs

@@ -0,0 +1,11 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_pattern.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"defabcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_positions.rs

@@ -0,0 +1,19 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    Hello\n\n        4200\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"\n tok2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"  tok1");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_return_token_from_pattern.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"defghidef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_rewind.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+
+    assert_eq!(vec![30, 30, 30, 10], c.nums);
+    assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols);
+
+    p_context_delete(c);
+}

spec/test_set_position.rs

@@ -0,0 +1,66 @@
+use testparser::*;
+
+fn main() {
+    let mut ti = p_token_info_t::default();
+
+    /* Baseline: default (1, 1). */
+    let mut c = p_context_new(b"ab");
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(1, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+
+    /* p_set_position overrides the initial position. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 5, col: 20 });
+    let pos = p_position(&c);
+    assert_eq!(5, pos.row);
+    assert_eq!(20, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(20, ti.position.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(21, ti.position.col);
+    p_context_delete(c);
+
+    /* Set position before a full parse. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 3, col: 7 });
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    /* Set position before a failing parse: error position is relative. */
+    let mut c = p_context_new(b"aa");
+    p_set_position(&mut c, p_position_t { row: 10, col: 2 });
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    let ep = p_position(&c);
+    assert_eq!(10, ep.row);
+    assert_eq!(3, ep.col);
+    p_context_delete(c);
+
+    /* p_set_input_index rewinds the byte cursor to re-read a section. */
+    let mut c = p_context_new(b"ab");
+    let start_index = p_input_index(&c);
+    let start_position = p_position(&c);
+    assert_eq!(0, start_index);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(2, p_input_index(&c));
+    p_set_input_index(&mut c, start_index);
+    p_set_position(&mut c, start_position);
+    assert_eq!(0, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+}

spec/test_start_rule.rs

@@ -0,0 +1,2 @@
+fn main() {
+}

spec/test_start_rule_tree.c

@@ -9,11 +9,10 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
-    Top * top = p_result(context);
-    assert(top->pToken != NULL);
-    assert_eq(TOKEN_hi, top->pToken->token);
+    Top top = p_result(context);
+    assert(p_node_valid(p_Top_pToken(top)));
+    assert_eq(TOKEN_hi, p_tree_walk_Top(top, pToken, token));
 
-    p_tree_delete(top);
     p_context_delete(context);
 
     return 0;

spec/test_start_rule_tree.d

@@ -13,7 +13,7 @@ unittest
     p_context_t * context;
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
-    Top * top = p_result(context);
-    assert(top.pToken !is null);
+    Top top = p_result(context);
+    assert(top.pToken.valid);
     assert_eq(TOKEN_hi, top.pToken.token);
 }

spec/test_start_rule_tree.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"hi");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let top = p_result(&context);
+        assert!(top.pToken().valid());
+        assert_eq!(TOKEN_hi, top.pToken().token());
+    }
+    p_context_delete(context);
+}

spec/test_starting_rules.rs

@@ -0,0 +1,18 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(8, p_result(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
+    assert_eq!(8, p_result_Bs(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_R(&mut c));
+    assert_eq!(3, p_result_R(&c));
+    p_context_delete(c);
+}

spec/test_starting_rules_tree.c

@@ -9,31 +9,28 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert_not_null(start->bs);
-    assert_not_null(start->bs->b);
-    assert_not_null(start->bs->bs->b);
-    assert_not_null(start->bs->bs->bs->b);
-    assert_not_null(start->bs->bs->bs->bs->b);
-    p_tree_delete(start);
+    Start start = p_result(context);
+    assert(p_node_valid(p_Start_bs(start)));
+    assert(p_node_valid(p_tree_walk_Start(start, bs, b)));
+    assert(p_node_valid(p_tree_walk_Start(start, bs, bs, b)));
+    assert(p_node_valid(p_tree_walk_Start(start, bs, bs, bs, b)));
+    assert(p_node_valid(p_tree_walk_Start(start, bs, bs, bs, bs, b)));
     p_context_delete(context);
 
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse_Bs(context) == P_SUCCESS);
-    Bs * bs = p_result_Bs(context);
-    assert_not_null(bs->b);
-    assert_not_null(bs->bs->b);
-    assert_not_null(bs->bs->bs->b);
-    assert_not_null(bs->bs->bs->bs->b);
-    p_tree_delete_Bs(bs);
+    Bs bs = p_result_Bs(context);
+    assert(p_node_valid(p_Bs_b(bs)));
+    assert(p_node_valid(p_tree_walk_Bs(bs, bs, b)));
+    assert(p_node_valid(p_tree_walk_Bs(bs, bs, bs, b)));
+    assert(p_node_valid(p_tree_walk_Bs(bs, bs, bs, bs, b)));
     p_context_delete(context);
 
     input = "c";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse_R(context) == P_SUCCESS);
-    R * r = p_result_R(context);
-    assert_not_null(r->c);
-    p_tree_delete_R(r);
+    R r = p_result_R(context);
+    assert(p_node_valid(p_R_c(r)));
     p_context_delete(context);
 
     return 0;

spec/test_starting_rules_tree.d

@@ -12,30 +12,30 @@ unittest
     string input = "bbbb";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.bs);
-    assert(start.bs.b);
-    assert(start.bs.bs.b);
-    assert(start.bs.bs.bs.b);
-    assert(start.bs.bs.bs.bs.b);
+    Start start = p_result(context);
+    assert(start.bs.valid);
+    assert(start.bs.b.valid);
+    assert(start.bs.bs.b.valid);
+    assert(start.bs.bs.bs.b.valid);
+    assert(start.bs.bs.bs.bs.b.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     context = p_context_new(input);
     assert(p_parse_Bs(context) == P_SUCCESS);
-    Bs * bs = p_result_Bs(context);
-    assert(bs.b);
-    assert(bs.bs.b);
-    assert(bs.bs.bs.b);
-    assert(bs.bs.bs.bs.b);
+    Bs bs = p_result_Bs(context);
+    assert(bs.b.valid);
+    assert(bs.bs.b.valid);
+    assert(bs.bs.bs.b.valid);
+    assert(bs.bs.bs.bs.b.valid);
 
-    p_tree_delete_Bs(bs);
+    p_context_delete(context);
 
     input = "c";
     context = p_context_new(input);
     assert(p_parse_R(context) == P_SUCCESS);
-    R * r = p_result_R(context);
-    assert(r.c);
+    R r = p_result_R(context);
+    assert(r.c.valid);
 
-    p_tree_delete_R(r);
+    p_context_delete(context);
 }

spec/test_starting_rules_tree.rs

@@ -0,0 +1,34 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let start = p_result(&c);
+        assert!(start.bs().valid());
+        assert!(start.bs().b().valid());
+        assert!(start.bs().bs().b().valid());
+        assert!(start.bs().bs().bs().b().valid());
+        assert!(start.bs().bs().bs().bs().b().valid());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
+    {
+        let bs = p_result_Bs(&c);
+        assert!(bs.b().valid());
+        assert!(bs.bs().b().valid());
+        assert!(bs.bs().bs().b().valid());
+        assert!(bs.bs().bs().bs().b().valid());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_R(&mut c));
+    {
+        let r = p_result_R(&c);
+        assert!(r.c().valid());
+    }
+    p_context_delete(c);
+}

spec/test_token_user_fields.c

@@ -18,29 +18,22 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start->pIDs);
-    assert(start->pIDs->id);
-#ifdef __cplusplus
-    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
-#else
-    assert(start->pIDs->id->comments);
-    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
-#endif
-    assert(start->pIDs->pIDs);
-    assert(start->pIDs->pIDs->id);
-#ifdef __cplusplus
-    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
-#else
-    assert(start->pIDs->pIDs->id->comments);
-    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
-#endif
+    Start start = p_result(context);
+    IDs ids = p_Start_pIDs(start);
+    assert(p_node_valid(ids));
+    Token id0 = p_IDs_id(ids);
+    assert(p_node_valid(id0));
+    assert(p_node_data(id0)->comments);
+    assert(strcmp(p_node_data(id0)->comments, "# c1\n#  c2\n") == 0);
+    IDs ids2 = p_IDs_pIDs(ids);
+    assert(p_node_valid(ids2));
+    Token id1 = p_IDs_id(ids2);
+    assert(p_node_valid(id1));
+    assert(p_node_data(id1)->comments);
+    assert(strcmp(p_node_data(id1)->comments, "# s1\n#   s2\n") == 0);
 
-#ifndef __cplusplus
     free(context->comments);
-#endif
     p_context_delete(context);
-    p_tree_delete(start);
 
     return 0;
 }

spec/test_token_user_fields.cpp

@@ -0,0 +1,31 @@
+#include "testparser.h"
+#include <cassert>
+#include <cstring>
+#include <string>
+
+int main()
+{
+    char const * input =
+        "# c1\n"
+        "#  c2\n"
+        "\n"
+        "first\n"
+        "\n   \n  \n"
+        "  # s1\n"
+        "   #   s2\n"
+        "second\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start start = p_result(context);
+    assert(start.pIDs().valid());
+    assert(start.pIDs().id().valid());
+    assert(start.pIDs().id().data()->comments == "# c1\n#  c2\n");
+    assert(start.pIDs().pIDs().valid());
+    assert(start.pIDs().pIDs().id().valid());
+    assert(start.pIDs().pIDs().id().data()->comments == "# s1\n#   s2\n");
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_token_user_fields.d

@@ -19,13 +19,13 @@ unittest
         "second\n";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
-    assert(start.pIDs);
-    assert(start.pIDs.id);
+    Start start = p_result(context);
+    assert(start.pIDs.valid);
+    assert(start.pIDs.id.valid);
     assert(start.pIDs.id.comments == "# c1\n#  c2\n");
-    assert(start.pIDs.pIDs);
-    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.valid);
+    assert(start.pIDs.pIDs.id.valid);
     assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_token_user_fields.rs

@@ -0,0 +1,17 @@
+use testparser::*;
+
+fn main() {
+    let input = b"# c1\n#  c2\n\nfirst\n\n   \n  \n  # s1\n   #   s2\nsecond\n";
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let start = p_result(&c);
+        assert!(start.pIDs().valid());
+        assert!(start.pIDs().id().valid());
+        assert_eq!("# c1\n#  c2\n", start.pIDs().id().data().comments.as_str());
+        assert!(start.pIDs().pIDs().valid());
+        assert!(start.pIDs().pIDs().id().valid());
+        assert_eq!("# s1\n#   s2\n", start.pIDs().pIDs().id().data().comments.as_str());
+    }
+    p_context_delete(c);
+}

spec/test_tree.c

@@ -9,55 +9,52 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
-    Start * start = p_result(context);
-    assert(start->pItems1 != NULL);
-    assert(start->pItems != NULL);
-    Items * items = start->pItems;
-    assert(items->pItem != NULL);
-    assert(items->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_a, items->pItem->pToken1->token);
-    assert_eq(11, items->pItem->pToken1->pvalue);
-    assert(items->pItemsMore != NULL);
-    ItemsMore * itemsmore = items->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore != NULL);
-    itemsmore = itemsmore->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore == NULL);
+    Start start = p_result(context);
+    assert(p_node_valid(p_Start_pItems1(start)));
+    assert(p_node_valid(p_Start_pItems(start)));
+    Items items = p_Start_pItems(start);
+    assert(p_node_valid(p_Items_pItem(items)));
+    assert(p_node_valid(p_tree_walk_Items(items, pItem, pToken1)));
+    assert_eq(TOKEN_a, p_tree_walk_Items(items, pItem, pToken1, token));
+    assert_eq(11, p_tree_walk_Items(items, pItem, pToken1, pvalue));
+    assert(p_node_valid(p_Items_pItemsMore(items)));
+    ItemsMore itemsmore = p_Items_pItemsMore(items);
+    assert(p_node_valid(p_ItemsMore_pItem(itemsmore)));
+    assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem)));
+    assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem)));
+    assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1)));
+    assert_eq(TOKEN_b, p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1, token));
+    assert_eq(22, p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1, pvalue));
+    assert(p_node_valid(p_ItemsMore_pItemsMore(itemsmore)));
+    itemsmore = p_ItemsMore_pItemsMore(itemsmore);
+    assert(p_node_valid(p_ItemsMore_pItem(itemsmore)));
+    assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pToken1)));
+    assert_eq(TOKEN_b, p_tree_walk_ItemsMore(itemsmore, pItem, pToken1, token));
+    assert_eq(22, p_tree_walk_ItemsMore(itemsmore, pItem, pToken1, pvalue));
+    assert(!p_node_valid(p_ItemsMore_pItemsMore(itemsmore)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start->pItems == NULL);
+    assert(!p_node_valid(p_Start_pItems(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "2 1";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start->pItems != NULL);
-    assert(start->pItems->pItem != NULL);
-    assert(start->pItems->pItem->pDual != NULL);
-    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
-    assert(start->pItems->pItem->pDual->pOne2 != NULL);
-    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
-    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+    assert(p_node_valid(p_Start_pItems(start)));
+    assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem)));
+    assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual)));
+    assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pTwo1)));
+    assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pOne2)));
+    assert(!p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pTwo2)));
+    assert(!p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pOne1)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;

spec/test_tree.cpp

@@ -0,0 +1,61 @@
+#include "testparser.h"
+#include <cassert>
+#include <cstring>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start start = p_result(context);
+    assert(start.pItems1().valid());
+    assert(start.pItems().valid());
+    Items items = start.pItems();
+    assert(items.pItem().valid());
+    assert(items.pItem().pToken1().valid());
+    assert_eq(TOKEN_a, items.pItem().pToken1().token());
+    assert_eq(11, items.pItem().pToken1().pvalue());
+    assert(items.pItemsMore().valid());
+    ItemsMore itemsmore = items.pItemsMore();
+    assert(itemsmore.pItem().valid());
+    assert(itemsmore.pItem().pItem().valid());
+    assert(itemsmore.pItem().pItem().pItem().valid());
+    assert(itemsmore.pItem().pItem().pItem().pToken1().valid());
+    assert_eq(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
+    assert_eq(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
+    assert(itemsmore.pItemsMore().valid());
+    itemsmore = itemsmore.pItemsMore();
+    assert(itemsmore.pItem().valid());
+    assert(itemsmore.pItem().pToken1().valid());
+    assert_eq(TOKEN_b, itemsmore.pItem().pToken1().token());
+    assert_eq(22, itemsmore.pItem().pToken1().pvalue());
+    assert(!itemsmore.pItemsMore().valid());
+
+    p_context_delete(context);
+
+    input = "";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(!start.pItems().valid());
+
+    p_context_delete(context);
+
+    input = "2 1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(start.pItems().valid());
+    assert(start.pItems().pItem().valid());
+    assert(start.pItems().pItem().pDual().valid());
+    assert(start.pItems().pItem().pDual().pTwo1().valid());
+    assert(start.pItems().pItem().pDual().pOne2().valid());
+    assert(!start.pItems().pItem().pDual().pTwo2().valid());
+    assert(!start.pItems().pItem().pDual().pOne1().valid());
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_tree.d

@@ -12,51 +12,51 @@ unittest
     string input = "a, ((b)), b";
     p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
-    Start * start = p_result(context);
-    assert(start.pItems1 !is null);
-    assert(start.pItems !is null);
-    Items * items = start.pItems;
-    assert(items.pItem !is null);
-    assert(items.pItem.pToken1 !is null);
+    Start start = p_result(context);
+    assert(start.pItems1.valid);
+    assert(start.pItems.valid);
+    Items items = start.pItems;
+    assert(items.pItem.valid);
+    assert(items.pItem.pToken1.valid);
     assert_eq(TOKEN_a, items.pItem.pToken1.token);
     assert_eq(11, items.pItem.pToken1.pvalue);
-    assert(items.pItemsMore !is null);
-    ItemsMore * itemsmore = items.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert(items.pItemsMore.valid);
+    ItemsMore itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem.valid);
+    assert(itemsmore.pItem.pItem.valid);
+    assert(itemsmore.pItem.pItem.pItem.valid);
+    assert(itemsmore.pItem.pItem.pItem.pToken1.valid);
     assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
     assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore !is null);
+    assert(itemsmore.pItemsMore.valid);
     itemsmore = itemsmore.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pToken1 !is null);
+    assert(itemsmore.pItem.valid);
+    assert(itemsmore.pItem.pToken1.valid);
     assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore is null);
+    assert(!itemsmore.pItemsMore.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start.pItems is null);
+    assert(!start.pItems.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start.pItems !is null);
-    assert(start.pItems.pItem !is null);
-    assert(start.pItems.pItem.pDual !is null);
-    assert(start.pItems.pItem.pDual.pTwo1 !is null);
-    assert(start.pItems.pItem.pDual.pOne2 !is null);
-    assert(start.pItems.pItem.pDual.pTwo2 is null);
-    assert(start.pItems.pItem.pDual.pOne1 is null);
+    assert(start.pItems.valid);
+    assert(start.pItems.pItem.valid);
+    assert(start.pItems.pItem.pDual.valid);
+    assert(start.pItems.pItem.pDual.pTwo1.valid);
+    assert(start.pItems.pItem.pDual.pOne2.valid);
+    assert(!start.pItems.pItem.pDual.pTwo2.valid);
+    assert(!start.pItems.pItem.pDual.pOne1.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_tree.rs

@@ -0,0 +1,46 @@
+use testparser::*;
+
+fn main() {
+    let input = b"a, ((b)), b";
+    let mut context = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems1().valid());
+        assert!(start.pItems().valid());
+        let items = start.pItems();
+        assert!(items.pItem().valid());
+        assert!(items.pItem().pToken1().valid());
+        assert_eq!(TOKEN_a, items.pItem().pToken1().token());
+        assert_eq!(11, items.pItem().pToken1().pvalue());
+        let itemsmore = items.pItemsMore();
+        assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
+        assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
+        assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
+        assert!(itemsmore.pItemsMore().valid());
+        let itemsmore = itemsmore.pItemsMore();
+        assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
+        assert!(!itemsmore.pItemsMore().valid());
+    }
+    p_context_delete(context);
+
+    /* Empty input yields a Start node with no Items child. */
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert!(!p_result(&context).pItems().valid());
+    p_context_delete(context);
+
+    /* Dual rule alternative field positions. */
+    let mut context = p_context_new(b"2 1");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems().pItem().pDual().pTwo1().valid());
+        assert!(start.pItems().pItem().pDual().pOne2().valid());
+        assert!(!start.pItems().pItem().pDual().pTwo2().valid());
+        assert!(!start.pItems().pItem().pDual().pOne1().valid());
+    }
+    p_context_delete(context);
+
+    println!("ok");
+}

spec/test_tree_delete_token_node_memory.c

@@ -9,12 +9,11 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
-    Start * start = p_result(context);
-    assert(start->a != NULL);
-    assert(*start->a->pvalue == 1);
-    assert(start->b != NULL);
-    assert(*start->b->pvalue == 2);
+    Start start = p_result(context);
+    assert(p_node_valid(p_Start_a(start)));
+    assert(*p_tree_walk_Start(start, a, pvalue) == 1);
+    assert(p_node_valid(p_Start_b(start)));
+    assert(*p_tree_walk_Start(start, b, pvalue) == 2);
 
-    p_tree_delete(start);
     p_context_delete(context);
 }

spec/test_tree_field_aliases.c

@@ -9,13 +9,12 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
+    Start start = p_result(context);
 
-    assert_eq(TOKEN_a, start->first->pToken->token);
-    assert_eq(TOKEN_b, start->second->pToken->token);
-    assert_eq(TOKEN_c, start->third->pToken->token);
+    assert_eq(TOKEN_a, p_tree_walk_Start(start, first, pToken, token));
+    assert_eq(TOKEN_b, p_tree_walk_Start(start, second, pToken, token));
+    assert_eq(TOKEN_c, p_tree_walk_Start(start, third, pToken, token));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;

spec/test_tree_field_aliases.d

@@ -12,11 +12,11 @@ unittest
     string input = "\na\nb\nc";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
+    Start start = p_result(context);
 
     assert_eq(TOKEN_a, start.first.pToken.token);
     assert_eq(TOKEN_b, start.second.pToken.token);
     assert_eq(TOKEN_c, start.third.pToken.token);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_tree_field_aliases.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"\na\nb\nc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert_eq!(TOKEN_a, start.first().pToken().token());
+        assert_eq!(TOKEN_b, start.second().pToken().token());
+        assert_eq!(TOKEN_c, start.third().pToken().token());
+    }
+    p_context_delete(context);
+}

spec/test_tree_invalid_positions.c

@@ -9,105 +9,113 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
+    Start start = p_result(context);
+    T t1 = p_Start_pT1(start);
+    Token k1 = p_T_pToken(t1);
+    A a1 = p_T_pA(t1);
 
-    assert_eq(2, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(2, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(3, start->pT1->pA->position.row);
-    assert_eq(3, start->pT1->pA->position.col);
-    assert_eq(3, start->pT1->pA->end_position.row);
-    assert_eq(8, start->pT1->pA->end_position.col);
-    assert_eq(2, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(3, start->pT1->end_position.row);
-    assert_eq(8, start->pT1->end_position.col);
+    assert_eq(2, p_node_position(k1).row);
+    assert_eq(1, p_node_position(k1).col);
+    assert_eq(2, p_node_end_position(k1).row);
+    assert_eq(1, p_node_end_position(k1).col);
+    assert(p_position_valid(p_node_position(a1)));
+    assert_eq(3, p_node_position(a1).row);
+    assert_eq(3, p_node_position(a1).col);
+    assert_eq(3, p_node_end_position(a1).row);
+    assert_eq(8, p_node_end_position(a1).col);
+    assert_eq(2, p_node_position(t1).row);
+    assert_eq(1, p_node_position(t1).col);
+    assert_eq(3, p_node_end_position(t1).row);
+    assert_eq(8, p_node_end_position(t1).col);
 
-    assert_eq(2, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(3, start->end_position.row);
-    assert_eq(8, start->end_position.col);
+    assert_eq(2, p_node_position(start).row);
+    assert_eq(1, p_node_position(start).col);
+    assert_eq(3, p_node_end_position(start).row);
+    assert_eq(8, p_node_end_position(start).col);
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "a\nbb";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
+    t1 = p_Start_pT1(start);
+    k1 = p_T_pToken(t1);
+    a1 = p_T_pA(t1);
 
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(1, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(2, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(2, start->pT1->end_position.col);
+    assert_eq(1, p_node_position(k1).row);
+    assert_eq(1, p_node_position(k1).col);
+    assert_eq(1, p_node_end_position(k1).row);
+    assert_eq(1, p_node_end_position(k1).col);
+    assert(p_position_valid(p_node_position(a1)));
+    assert_eq(2, p_node_position(a1).row);
+    assert_eq(1, p_node_position(a1).col);
+    assert_eq(2, p_node_end_position(a1).row);
+    assert_eq(2, p_node_end_position(a1).col);
+    assert_eq(1, p_node_position(t1).row);
+    assert_eq(1, p_node_position(t1).col);
+    assert_eq(2, p_node_end_position(t1).row);
+    assert_eq(2, p_node_end_position(t1).col);
 
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(2, start->end_position.col);
+    assert_eq(1, p_node_position(start).row);
+    assert_eq(1, p_node_position(start).col);
+    assert_eq(2, p_node_end_position(start).row);
+    assert_eq(2, p_node_end_position(start).col);
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "a\nc\nc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
+    t1 = p_Start_pT1(start);
+    k1 = p_T_pToken(t1);
+    a1 = p_T_pA(t1);
 
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(1, start->pT1->pA->position.col);
-    assert_eq(3, start->pT1->pA->end_position.row);
-    assert_eq(1, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(3, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
+    assert_eq(1, p_node_position(k1).row);
+    assert_eq(1, p_node_position(k1).col);
+    assert_eq(1, p_node_end_position(k1).row);
+    assert_eq(1, p_node_end_position(k1).col);
+    assert(p_position_valid(p_node_position(a1)));
+    assert_eq(2, p_node_position(a1).row);
+    assert_eq(1, p_node_position(a1).col);
+    assert_eq(3, p_node_end_position(a1).row);
+    assert_eq(1, p_node_end_position(a1).col);
+    assert_eq(1, p_node_position(t1).row);
+    assert_eq(1, p_node_position(t1).col);
+    assert_eq(3, p_node_end_position(t1).row);
+    assert_eq(1, p_node_end_position(t1).col);
 
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(3, start->end_position.row);
-    assert_eq(1, start->end_position.col);
+    assert_eq(1, p_node_position(start).row);
+    assert_eq(1, p_node_position(start).col);
+    assert_eq(3, p_node_end_position(start).row);
+    assert_eq(1, p_node_end_position(start).col);
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "a";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
+    t1 = p_Start_pT1(start);
+    k1 = p_T_pToken(t1);
+    a1 = p_T_pA(t1);
 
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(!p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(1, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
+    assert_eq(1, p_node_position(k1).row);
+    assert_eq(1, p_node_position(k1).col);
+    assert_eq(1, p_node_end_position(k1).row);
+    assert_eq(1, p_node_end_position(k1).col);
+    assert(!p_position_valid(p_node_position(a1)));
+    assert_eq(1, p_node_position(t1).row);
+    assert_eq(1, p_node_position(t1).col);
+    assert_eq(1, p_node_end_position(t1).row);
+    assert_eq(1, p_node_end_position(t1).col);
 
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(1, start->end_position.row);
-    assert_eq(1, start->end_position.col);
+    assert_eq(1, p_node_position(start).row);
+    assert_eq(1, p_node_position(start).col);
+    assert_eq(1, p_node_end_position(start).row);
+    assert_eq(1, p_node_end_position(start).col);
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;

spec/test_tree_invalid_positions.d

@@ -12,7 +12,7 @@ unittest
     string input = "\na\n  bb ccc";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start * start = p_result(context);
+    Start start = p_result(context);
 
     assert_eq(2, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -33,7 +33,7 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(8, start.end_position.col);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a\nbb";
     context = p_context_new(input);
@@ -59,7 +59,7 @@ unittest
     assert_eq(2, start.end_position.row);
     assert_eq(2, start.end_position.col);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a\nc\nc";
     context = p_context_new(input);
@@ -85,7 +85,7 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a";
     context = p_context_new(input);
@@ -107,5 +107,5 @@ unittest
     assert_eq(1, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_tree_invalid_positions.rs

@@ -0,0 +1,89 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"\na\n  bb ccc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let k1 = t1.pToken();
+        let a1 = t1.pA();
+        assert_eq!(2, k1.position().row);
+        assert_eq!(1, k1.position().col);
+        assert_eq!(2, k1.end_position().row);
+        assert_eq!(1, k1.end_position().col);
+        assert!(a1.position().valid());
+        assert_eq!(3, a1.position().row);
+        assert_eq!(3, a1.position().col);
+        assert_eq!(3, a1.end_position().row);
+        assert_eq!(8, a1.end_position().col);
+        assert_eq!(2, t1.position().row);
+        assert_eq!(1, t1.position().col);
+        assert_eq!(3, t1.end_position().row);
+        assert_eq!(8, t1.end_position().col);
+        assert_eq!(2, start.position().row);
+        assert_eq!(1, start.position().col);
+        assert_eq!(3, start.end_position().row);
+        assert_eq!(8, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a\nbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let k1 = t1.pToken();
+        let a1 = t1.pA();
+        assert_eq!(1, k1.position().row);
+        assert_eq!(1, k1.position().col);
+        assert!(a1.position().valid());
+        assert_eq!(2, a1.position().row);
+        assert_eq!(1, a1.position().col);
+        assert_eq!(2, a1.end_position().row);
+        assert_eq!(2, a1.end_position().col);
+        assert_eq!(1, t1.position().row);
+        assert_eq!(2, t1.end_position().row);
+        assert_eq!(2, t1.end_position().col);
+        assert_eq!(1, start.position().row);
+        assert_eq!(2, start.end_position().row);
+        assert_eq!(2, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a\nc\nc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let a1 = t1.pA();
+        assert!(a1.position().valid());
+        assert_eq!(2, a1.position().row);
+        assert_eq!(1, a1.position().col);
+        assert_eq!(3, a1.end_position().row);
+        assert_eq!(1, a1.end_position().col);
+        assert_eq!(1, t1.position().row);
+        assert_eq!(3, t1.end_position().row);
+        assert_eq!(1, t1.end_position().col);
+        assert_eq!(3, start.end_position().row);
+        assert_eq!(1, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let a1 = t1.pA();
+        assert!(!a1.position().valid());
+        assert_eq!(1, t1.position().row);
+        assert_eq!(1, t1.position().col);
+        assert_eq!(1, t1.end_position().row);
+        assert_eq!(1, t1.end_position().col);
+        assert_eq!(1, start.position().row);
+        assert_eq!(1, start.end_position().row);
+        assert_eq!(1, start.end_position().col);
+    }
+    p_context_delete(context);
+}

spec/test_tree_node_memory_remains.c

@@ -373,46 +373,45 @@ int main(int argc, char * argv[])
     context = p_context_new((const uint8_t *)input, strlen(input));
     size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(context);
-    PModuleItems * pmis = pmod->pModuleItems;
-    PFunctionDefinition ** pfds;
+    PModule pmod = p_result(context);
+    PModuleItems pmis = p_PModule_pModuleItems(pmod);
+    PFunctionDefinition * pfds;
     size_t n_pfds = 0u;
-    while (pmis != NULL)
+    while (p_node_valid(pmis))
     {
-        PModuleItem * pmi = pmis->pModuleItem;
-        if (pmi->pFunctionDefinition != NULL)
+        PModuleItem pmi = p_PModuleItems_pModuleItem(pmis);
+        if (p_node_valid(p_PModuleItem_pFunctionDefinition(pmi)))
         {
             n_pfds++;
         }
-        pmis = pmis->pModuleItems;
+        pmis = p_PModuleItems_pModuleItems(pmis);
     }
-    pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
-    pmis = pmod->pModuleItems;
+    pfds = (PFunctionDefinition *)malloc(n_pfds * sizeof(PFunctionDefinition));
+    pmis = p_PModule_pModuleItems(pmod);
     size_t pfd_i = n_pfds;
-    while (pmis != NULL)
+    while (p_node_valid(pmis))
     {
-        PModuleItem * pmi = pmis->pModuleItem;
-        PFunctionDefinition * pfd = pmi->pFunctionDefinition;
-        if (pfd != NULL)
+        PModuleItem pmi = p_PModuleItems_pModuleItem(pmis);
+        PFunctionDefinition pfd = p_PModuleItem_pFunctionDefinition(pmi);
+        if (p_node_valid(pfd))
         {
             pfd_i--;
             assert(pfd_i < n_pfds);
             pfds[pfd_i] = pfd;
         }
-        pmis = pmis->pModuleItems;
+        pmis = p_PModuleItems_pModuleItems(pmis);
     }
     assert_eq(51, n_pfds);
     for (size_t i = 0; i < n_pfds; i++)
     {
-        if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
-            (expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
+        if (strncmp(expected[i].name, (const char *)p_node_data(p_PFunctionDefinition_name(pfds[i]))->pvalue.s, strlen(expected[i].name)) != 0 ||
+            (expected[i].token != p_tree_walk_PFunctionDefinition(pfds[i], returntype, pType, pTypeBase, pToken1, token)))
         {
-            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
+            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, p_tree_walk_PFunctionDefinition(pfds[i], returntype, pType, pTypeBase, pToken1, token));
         }
     }
 
     free(pfds);
-    p_tree_delete(pmod);
     p_context_delete(context);
 
     return 0;

spec/test_tree_node_memory_remains.d

@@ -378,19 +378,19 @@ def main() -> int
     context = p_context_new(input);
     size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(context);
-    PModuleItems * pmis = pmod.pModuleItems;
-    PFunctionDefinition *[] pfds;
-    while (pmis !is null)
+    PModule pmod = p_result(context);
+    PModuleItems pmis = pmod.pModuleItems;
+    PFunctionDefinition[] pfds;
+    while (pmis.valid)
     {
-        PModuleItem * pmi = pmis.pModuleItem;
-        if (pmi is null)
+        PModuleItem pmi = pmis.pModuleItem;
+        if (!pmi.valid)
         {
             stderr.writeln("pmi is null!!!?");
             assert(0);
         }
-        PFunctionDefinition * pfd = pmi.pFunctionDefinition;
-        if (pfd !is null)
+        PFunctionDefinition pfd = pmi.pFunctionDefinition;
+        if (pfd.valid)
         {
             pfds = [pfd] ~ pfds;
         }
@@ -405,5 +405,5 @@ def main() -> int
             stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
         }
     }
-    p_tree_delete(pmod);
+    p_context_delete(context);
 }

spec/test_tree_node_memory_remains.rs

@@ -0,0 +1,85 @@
+use testparser::*;
+
+fn main() {
+    let entries: [(&str, &str, p_token_t); 51] = [
+        ("byte_val", "byte", TOKEN_byte),
+        ("short_val", "short", TOKEN_short),
+        ("int_val", "int", TOKEN_int),
+        ("long_val", "long", TOKEN_long),
+        ("ssize_t_val", "ssize_t", TOKEN_ssize_t),
+        ("byte_to_short", "short", TOKEN_short),
+        ("byte_to_int", "int", TOKEN_int),
+        ("byte_to_long", "long", TOKEN_long),
+        ("byte_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("short_to_byte", "byte", TOKEN_byte),
+        ("short_to_int", "int", TOKEN_int),
+        ("short_to_long", "long", TOKEN_long),
+        ("short_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("int_to_byte", "byte", TOKEN_byte),
+        ("int_to_short", "short", TOKEN_short),
+        ("int_to_long", "long", TOKEN_long),
+        ("int_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("long_to_byte", "byte", TOKEN_byte),
+        ("long_to_short", "short", TOKEN_short),
+        ("long_to_int", "int", TOKEN_int),
+        ("long_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("ssize_t_to_byte", "byte", TOKEN_byte),
+        ("ssize_t_to_short", "short", TOKEN_short),
+        ("ssize_t_to_int", "int", TOKEN_int),
+        ("ssize_t_to_long", "long", TOKEN_long),
+        ("ubyte_val", "ubyte", TOKEN_ubyte),
+        ("ushort_val", "ushort", TOKEN_ushort),
+        ("uint_val", "uint", TOKEN_uint),
+        ("ulong_val", "ulong", TOKEN_ulong),
+        ("size_t_val", "size_t", TOKEN_size_t),
+        ("ubyte_to_ushort", "ushort", TOKEN_ushort),
+        ("ubyte_to_uint", "uint", TOKEN_uint),
+        ("ubyte_to_ulong", "ulong", TOKEN_ulong),
+        ("ubyte_to_size_t", "size_t", TOKEN_size_t),
+        ("ushort_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("ushort_to_uint", "uint", TOKEN_uint),
+        ("ushort_to_ulong", "ulong", TOKEN_ulong),
+        ("ushort_to_size_t", "size_t", TOKEN_size_t),
+        ("uint_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("uint_to_ushort", "ushort", TOKEN_ushort),
+        ("uint_to_ulong", "ulong", TOKEN_ulong),
+        ("uint_to_size_t", "size_t", TOKEN_size_t),
+        ("ulong_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("ulong_to_ushort", "ushort", TOKEN_ushort),
+        ("ulong_to_uint", "uint", TOKEN_uint),
+        ("ulong_to_size_t", "size_t", TOKEN_size_t),
+        ("size_t_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("size_t_to_ushort", "ushort", TOKEN_ushort),
+        ("size_t_to_int", "int", TOKEN_int),
+        ("size_t_to_ulong", "ulong", TOKEN_ulong),
+        ("main", "int", TOKEN_int),
+    ];
+
+    let mut input = String::new();
+    for (name, ret, _) in entries.iter() {
+        input.push_str(&format!("def {}() -> {} {{\nreturn 0x42;\n}}\n", name, ret));
+    }
+
+    let mut c = p_context_new(input.as_bytes());
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let pmod = p_result(&c);
+        let mut pfds = Vec::new();
+        let mut pmis = pmod.pModuleItems();
+        while pmis.valid() {
+            let pmi = pmis.pModuleItem();
+            assert!(pmi.valid());
+            let pfd = pmi.pFunctionDefinition();
+            if pfd.valid() {
+                pfds.insert(0, pfd);
+            }
+            pmis = pmis.pModuleItems();
+        }
+        assert_eq!(51, pfds.len());
+        for i in 0..pfds.len() {
+            assert_eq!(entries[i].0, pfds[i].name().data().pvalue.s.as_str());
+            assert_eq!(entries[i].2, pfds[i].returntype().pType().pTypeBase().pToken1().token());
+        }
+    }
+    p_context_delete(c);
+}

spec/test_tree_ps.c

@@ -9,55 +9,52 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
-    PStartS * start = p_result(context);
-    assert(start->pItems1 != NULL);
-    assert(start->pItems != NULL);
-    PItemsS * items = start->pItems;
-    assert(items->pItem != NULL);
-    assert(items->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_a, items->pItem->pToken1->token);
-    assert_eq(11, items->pItem->pToken1->pvalue);
-    assert(items->pItemsMore != NULL);
-    PItemsMoreS * itemsmore = items->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore != NULL);
-    itemsmore = itemsmore->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore == NULL);
+    PStartS start = p_result(context);
+    assert(p_node_valid(p_PStartS_pItems1(start)));
+    assert(p_node_valid(p_PStartS_pItems(start)));
+    PItemsS items = p_PStartS_pItems(start);
+    assert(p_node_valid(p_PItemsS_pItem(items)));
+    assert(p_node_valid(p_tree_walk_PItemsS(items, pItem, pToken1)));
+    assert_eq(TOKEN_a, p_tree_walk_PItemsS(items, pItem, pToken1, token));
+    assert_eq(11, p_tree_walk_PItemsS(items, pItem, pToken1, pvalue));
+    assert(p_node_valid(p_PItemsS_pItemsMore(items)));
+    PItemsMoreS itemsmore = p_PItemsS_pItemsMore(items);
+    assert(p_node_valid(p_PItemsMoreS_pItem(itemsmore)));
+    assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem)));
+    assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem)));
+    assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1)));
+    assert_eq(TOKEN_b, p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1, token));
+    assert_eq(22, p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1, pvalue));
+    assert(p_node_valid(p_PItemsMoreS_pItemsMore(itemsmore)));
+    itemsmore = p_PItemsMoreS_pItemsMore(itemsmore);
+    assert(p_node_valid(p_PItemsMoreS_pItem(itemsmore)));
+    assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1)));
+    assert_eq(TOKEN_b, p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1, token));
+    assert_eq(22, p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1, pvalue));
+    assert(!p_node_valid(p_PItemsMoreS_pItemsMore(itemsmore)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start->pItems == NULL);
+    assert(!p_node_valid(p_PStartS_pItems(start)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     input = "2 1";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start->pItems != NULL);
-    assert(start->pItems->pItem != NULL);
-    assert(start->pItems->pItem->pDual != NULL);
-    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
-    assert(start->pItems->pItem->pDual->pOne2 != NULL);
-    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
-    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+    assert(p_node_valid(p_PStartS_pItems(start)));
+    assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem)));
+    assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual)));
+    assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pTwo1)));
+    assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pOne2)));
+    assert(!p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pTwo2)));
+    assert(!p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pOne1)));
 
-    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;

spec/test_tree_ps.d

@@ -12,51 +12,51 @@ unittest
     string input = "a, ((b)), b";
     p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
-    PStartS * start = p_result(context);
-    assert(start.pItems1 !is null);
-    assert(start.pItems !is null);
-    PItemsS * items = start.pItems;
-    assert(items.pItem !is null);
-    assert(items.pItem.pToken1 !is null);
+    PStartS start = p_result(context);
+    assert(start.pItems1.valid);
+    assert(start.pItems.valid);
+    PItemsS items = start.pItems;
+    assert(items.pItem.valid);
+    assert(items.pItem.pToken1.valid);
     assert_eq(TOKEN_a, items.pItem.pToken1.token);
     assert_eq(11, items.pItem.pToken1.pvalue);
-    assert(items.pItemsMore !is null);
-    PItemsMoreS * itemsmore = items.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert(items.pItemsMore.valid);
+    PItemsMoreS itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem.valid);
+    assert(itemsmore.pItem.pItem.valid);
+    assert(itemsmore.pItem.pItem.pItem.valid);
+    assert(itemsmore.pItem.pItem.pItem.pToken1.valid);
     assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
     assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore !is null);
+    assert(itemsmore.pItemsMore.valid);
     itemsmore = itemsmore.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pToken1 !is null);
+    assert(itemsmore.pItem.valid);
+    assert(itemsmore.pItem.pToken1.valid);
     assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore is null);
+    assert(!itemsmore.pItemsMore.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start.pItems is null);
+    assert(!start.pItems.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
-    assert(start.pItems !is null);
-    assert(start.pItems.pItem !is null);
-    assert(start.pItems.pItem.pDual !is null);
-    assert(start.pItems.pItem.pDual.pTwo1 !is null);
-    assert(start.pItems.pItem.pDual.pOne2 !is null);
-    assert(start.pItems.pItem.pDual.pTwo2 is null);
-    assert(start.pItems.pItem.pDual.pOne1 is null);
+    assert(start.pItems.valid);
+    assert(start.pItems.pItem.valid);
+    assert(start.pItems.pItem.pDual.valid);
+    assert(start.pItems.pItem.pDual.pTwo1.valid);
+    assert(start.pItems.pItem.pDual.pOne2.valid);
+    assert(!start.pItems.pItem.pDual.pTwo2.valid);
+    assert(!start.pItems.pItem.pDual.pOne1.valid);
 
-    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_tree_ps.rs

@@ -0,0 +1,46 @@
+use testparser::*;
+
+fn main() {
+    let input = b"a, ((b)), b";
+    let mut context = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems1().valid());
+        assert!(start.pItems().valid());
+        let items = start.pItems();
+        assert!(items.pItem().valid());
+        assert!(items.pItem().pToken1().valid());
+        assert_eq!(TOKEN_a, items.pItem().pToken1().token());
+        assert_eq!(11, items.pItem().pToken1().pvalue());
+        let itemsmore = items.pItemsMore();
+        assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
+        assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
+        assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
+        assert!(itemsmore.pItemsMore().valid());
+        let itemsmore = itemsmore.pItemsMore();
+        assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
+        assert!(!itemsmore.pItemsMore().valid());
+    }
+    p_context_delete(context);
+
+    /* Empty input yields a Start node with no Items child. */
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert!(!p_result(&context).pItems().valid());
+    p_context_delete(context);
+
+    /* Dual rule alternative field positions. */
+    let mut context = p_context_new(b"2 1");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems().pItem().pDual().pTwo1().valid());
+        assert!(start.pItems().pItem().pDual().pOne2().valid());
+        assert!(!start.pItems().pItem().pDual().pTwo2().valid());
+        assert!(!start.pItems().pItem().pDual().pOne1().valid());
+    }
+    p_context_delete(context);
+
+    println!("ok");
+}