.github/workflows/run-tests.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - name: Install dependencies (Linux)
       if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc valgrind
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
 
     - name: Install dependencies (macOS)
       if: runner.os == 'macOS'
@@ -31,9 +31,6 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby-version }}
 
-    - name: Set up Rust
-      uses: dtolnay/rust-toolchain@stable
-
     - name: Install dependencies
       run: bundle install
 

CHANGELOG.md

@@ -1,75 +1,3 @@
-## v5.1.0
-
-### New Features
-
-- Add a `node_id()` accessor to the C++ and D tree node handle types, for node
-  identity comparison. This matches the existing `p_node_id()` macro (C) and
-  `node_id()` method (Rust).
-
-## v5.0.0
-
-### New Features
-
-- Add Rust target language output.
-- Add Rust language detection in propane.vim.
-
-### API Changes
-
-- The matched text argument passed to lexer user code blocks is now named
-  `match_text` instead of `match`, since `match` is a keyword in Rust. The
-  `match_length` argument (C and C++) is unchanged.
-- Tree generation mode now stores all tree nodes in a compact arena owned by
-  the parser context (a flat node array plus a shared child-link array).
-  This replaces the previous design of one heap allocation per node with
-  layout-punned typed structs.
-- Tree nodes are now referenced by lightweight handles rather than pointers.
-  `p_result()` and the field accessors now return handle values in tree
-  generation mode.
-- The whole tree is freed together with the context by `p_context_delete()`.
-  The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
-  tree node handles are only valid while the context is alive.
-- Tree node field access changed per target language:
-  - C: per-field accessor functions (e.g. `p_Start_pItems(node)`) plus tree
-    walk macros (e.g. `p_tree_walk_Start(node, pItems, pItem, pToken1, token)`),
-    and generic accessors `p_node_valid()`, `p_node_position()`,
-    `p_node_end_position()`, `p_node_n_fields()`, `p_node_data()`, `p_node_id()`.
-  - C++: handle methods called with `()` (e.g. `node.pItems().pToken1().token()`),
-    plus the same C-style functions/macros for convenience.
-  - D: `@property` accessors preserving the previous field-access syntax
-    (e.g. `node.pItems.pToken1.token`); null checks use `.valid` instead of
-    `is null`.
-- Tree-mode parser rule user code: `$$` and `$1` etc. now yield node handles.
-  Reference child fields through the target-language accessors described above
-  rather than through struct pointer members.
-
-### Improvements
-
-- Improve D language detection in propane.vim
-- Speed up specs
-
-## v4.8.1
-
-### Fixes
-
-- Fix tree node struct type forward-declarations for C/C++
-
-## v4.8.0
-
-### New Features
-
-- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
-  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
-  given start rule, but instead of expecting the rest of the input to match
-  the start rule they allow specifying a set of tokens that may follow the
-  start rule.
-- Add `p_set_position()` API to set the current text position stored in the
-  context. Useful for setting the initial text position to something other
-  than `(1, 1)` for a nested parse operation.
-- Add `p_input_index()` API to get the current input text byte offset.
-- Add `p_set_input_index()` API to set the current input text byte offset.
-  Useful together with `p_set_position()` to rewind the input part-way through
-  a parse in order to re-read an earlier section of the input.
-
 ## v4.7.0
 
 ### New Features

README.md

@@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C, C++, D, or Rust language outputs
+  * targets C, C++, or D language outputs
   * optionally supports automatic full parse tree generation
   * supports starting parsing from multiple start rules
   * tracks input text start and end positions for all matched tokens/rules
@@ -69,7 +69,7 @@ token times /\*/;
 token power /\*\*/;
 token integer /\d+/ <<
   ulong v;
-  foreach (c; match_text)
+  foreach (c; match)
   {
     v *= 10;
     v += (c - '0');

Rakefile.rb

@@ -1,4 +1,3 @@
-require "fileutils"
 require "rake/clean"
 require "rspec/core/rake_task"
 require "simplecov"
@@ -12,10 +11,7 @@ end
 
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
-    ENV["partial_specs"] = "1"
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
-  else
-    FileUtils.rm_rf("coverage")
   end
 end
 task :spec do |task, args|
@@ -23,7 +19,7 @@ task :spec do |task, args|
     original_stdout = $stdout
     sio = StringIO.new
     $stdout = sio
-    SimpleCov.collate Dir["coverage/parts/*/.resultset.json"]
+    SimpleCov.collate Dir["coverage/.resultset.json"]
     $stdout = original_stdout
     sio.string.lines.each do |line|
       $stdout.write(line) unless line =~ /Coverage report generated for/
@@ -31,15 +27,6 @@ task :spec do |task, args|
   end
 end
 
-task :valgrind do
-  begin
-    ENV["spec-valgrind"] = "1"
-    Rake::Task[:spec].execute
-  ensure
-    ENV.delete("spec-valgrind")
-  end
-end
-
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.
 desc "Dist Specs"
@@ -56,4 +43,4 @@ task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
 
-task :all => [:valgrind, :dspec, :user_guide]
+task :all => [:spec, :dspec, :user_guide]

UPGRADING.md

@@ -1,92 +1,3 @@
-## v5.0.0
-
-The generated API for tree generation mode (`tree;`) has been changed
-significantly for this version.
-Aside from the lexer user code block matched text rename described below, the
-lexer/parser value APIs for non-tree grammars are unchanged.
-
-### Lexer user code block matched text
-
-The matched text argument passed to lexer user code blocks has been renamed
-from `match` to `match_text` for all target languages.
-
-- C, C++, and D: rename references to `match` in lexer user code blocks to
-  `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
-
-The `match_length` argument (C, C++) is unchanged.
-
-### Tree memory management
-
-- Remove all calls to `p_tree_delete()` / `p_tree_delete_XXX()`. Tree nodes now
-  live in the parser context and are freed by `p_context_delete()`.
-- Tree node handles (returned by `p_result()` and the field accessors) are only
-  valid while the context is alive. Do not use them after `p_context_delete()`.
-
-### Tree node field access
-
-Tree nodes are now referenced by handle values instead of pointers, and field
-access differs per target language:
-
-- C: replace `node->field` with the accessor function `p_TYPE_field(node)`, or
-  use the tree walk macro `p_tree_walk_TYPE(node, field1, field2, ...)`. Replace
-  `x != NULL` / `x == NULL` node checks with `p_node_valid(x)` /
-  `!p_node_valid(x)`. Read positions with `p_node_position(node)` /
-  `p_node_end_position(node)`, token payload with `p_TYPE_token(node)` /
-  `p_TYPE_pvalue(node)` or `p_node_data(node)->field`, and compare node identity
-  with `p_node_id(a) == p_node_id(b)`.
-- C++: replace `node->field` with the handle method `node.field()`. Use
-  `node.valid()`, `node.position()`, `node.token()`, `node.pvalue()`, and
-  `node.data()->field` for user token fields. (The C-style functions and macros
-  above are also available.)
-- D: replace pointer declarations (`Start * s`) with value handles (`Start s`)
-  and replace `x !is null` / `x is null` with `x.valid` / `!x.valid`. Field
-  access syntax (`node.field.field`) is otherwise unchanged.
-
-### Tree-mode parser rule user code
-
-In tree generation mode `$$` and `$1`, `$2`, ... now expand to node handles.
-Reference child fields through the target-language accessors above (for example
-`$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
-in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
-
-### Pointers into tree node storage
-
-Tree nodes previously each had their own allocation, so a pointer to a node
-stayed valid for the life of the tree. They are now held in a single array
-which is reallocated as it grows, so a pointer or reference into that array may
-be invalidated whenever a new node is created.
-
-New nodes are created while parsing, so this matters for a pointer taken in a
-tree-mode parser rule user code block, which runs before the parse has
-finished. Keep the node handle instead, which stores a node ID rather than an
-address and stays valid, and obtain the pointer from it when it is needed.
-
-For example, replace a saved pointer:
-
-```
-context_user_fields <<
-    p_node_data_t * saved;
->>
-Items -> Items a << ${context.saved} = p_node_data($$); >>
-```
-
-with a saved handle:
-
-```
-context_user_fields <<
-    Items saved_node;
->>
-Items -> Items a << ${context.saved_node} = $$; >>
-```
-
-```
-p_node_data_t * data = p_node_data(context->saved_node);
-```
-
-Once parsing has finished, no further nodes are created, so a pointer obtained
-after `p_parse()` returns stays valid until the context is deleted, as long as
-no further parsing is performed with the same context.
-
 ## v4.0.0
 
 ### API Changes

assets/parser.c.erb

@@ -68,18 +68,6 @@ const char * <%= @grammar.prefix %>token_names[] = {
     context->text_position.row = 1u;
     context->text_position.col = 1u;
     context->mode = <%= @lexer.mode_id("default") %>;
-<% if @grammar.tree %>
-
-    /* Reserve node ID 0 as the null tree node. */
-<%   if @cpp %>
-    context-><%= @grammar.prefix %>tree_nodes.resize(1);
-<%   else %>
-    context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
-    context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
-    memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
-    context-><%= @grammar.prefix %>tree_nodes_length = 1u;
-<%   end %>
-<% end %>
 
     return context;
 }
@@ -96,27 +84,9 @@ const char * <%= @grammar.prefix %>token_names[] = {
  */
 void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
 {
-<% if @grammar.tree && @grammar.free_token_node != "" %>
-<%   if @cpp %>
-    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
-<%   else %>
-    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
-<%   end %>
-    {
-        if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
-        {
-            <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
-<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
-        }
-    }
-<% end %>
 <% if @cpp %>
     delete context;
 <% else %>
-<%   if @grammar.tree %>
-    free(context-><%= @grammar.prefix %>tree_nodes);
-    free(context-><%= @grammar.prefix %>tree_children);
-<%   end %>
     free(context);
 <% end %>
 }
@@ -319,7 +289,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match_text
+ * @param match
  *   Matched text for this pattern.
  * @param match_length
  *   Matched text length.
@@ -330,7 +300,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   not explicitly return a token.
  */
 static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match_text,
+    lexer_user_code_id_t code_id, uint8_t const * match,
     size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -534,9 +504,9 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match_text = &context->input[context->input_index];
+            uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match, match_info.length, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -743,8 +713,8 @@ typedef struct
     size_t state_id;
 
 <% if @grammar.tree %>
-    /** Tree node ID. */
-    <%= @grammar.prefix %>node_id_t node_id;
+    /** tree node. */
+    void * tree_node;
 <% else %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
@@ -753,6 +723,18 @@ typedef struct
 <% end %>
 } state_value_t;
 
+<% if @grammar.tree %>
+/** Common tree node structure. */
+typedef struct TreeNode_s
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct TreeNode_s * fields[];
+} TreeNode;
+<% end %>
+
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
@@ -890,77 +872,6 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
-<% if @grammar.tree %>
-/* Tree arena helpers. */
-
-/**
- * Allocate a new (zeroed) tree node in the context arena.
- *
- * @return The new node ID.
- */
-static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
-{
-<% if @cpp %>
-    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
-    context-><%= @grammar.prefix %>tree_nodes.emplace_back();
-    return id;
-<% else %>
-    if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
-    {
-        size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
-        <%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
-        memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
-        free(context-><%= @grammar.prefix %>tree_nodes);
-        context-><%= @grammar.prefix %>tree_nodes = new_nodes;
-        context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
-    }
-    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
-    memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
-    context-><%= @grammar.prefix %>tree_nodes_length += 1u;
-    return id;
-<% end %>
-}
-
-/**
- * Reserve n contiguous (zeroed) child slots in the shared children array.
- *
- * @return The offset of the first reserved slot.
- */
-static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
-{
-<% if @cpp %>
-    <%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
-    context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
-    return offset;
-<% else %>
-    size_t offset = context-><%= @grammar.prefix %>tree_children_length;
-    size_t needed = offset + n;
-    if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
-    {
-        size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
-        while (new_capacity < needed)
-        {
-            new_capacity *= 2u;
-        }
-        <%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
-        if (context-><%= @grammar.prefix %>tree_children != NULL)
-        {
-            memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
-            free(context-><%= @grammar.prefix %>tree_children);
-        }
-        context-><%= @grammar.prefix %>tree_children = new_children;
-        context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
-    }
-    memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
-    context-><%= @grammar.prefix %>tree_children_length = needed;
-    return (<%= @grammar.prefix %>node_id_t)offset;
-<% end %>
-}
-
-/* Tree node field accessor functions. */
-<%= c_tree_accessor_defs %>
-<% end %>
-
 <% unless @grammar.tree %>
 /**
  * Get the rule position (start or end) for the currently matched rule.
@@ -1025,7 +936,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1075,7 +986,7 @@ static size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return Reduce table index to reduce with, or INVALID_ID if none.
+ * @return State to reduce to, or INVALID_ID if none.
  */
 static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1097,17 +1008,8 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @param start_state_id
+ * @start_state_id
  *   ID of the state in which to start.
- * @param start_rule_set_id
- *   Rule set ID for the requested start rule. Only used when
- *   @p follow_tokens is non-NULL, to gate follow-token shift success.
- * @param follow_tokens
- *   Optional array of caller-provided follow tokens (tokens expected to
- *   appear immediately after the start rule in some outer context). Used to
- *   drive the "parse inner" retry logic. May be NULL for a standard parse.
- * @param n_follow_tokens
- *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1120,17 +1022,14 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
-    size_t start_rule_set_id,
-    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
-    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t reduced_parser_node;
+    void * reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>position_t reduced_position;
     <%= @grammar.prefix %>position_t reduced_end_position;
@@ -1152,18 +1051,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
-        /* For a "parse inner" operation, determine once per iteration whether
-         * the current token is a member of the caller-provided follow token
-         * set. Used by both the shift-side and reduce-side retries below. */
-        bool token_is_follow = false;
-        for (size_t i = 0u; i < n_follow_tokens; i++)
-        {
-            if (token == follow_tokens[i])
-            {
-                token_is_follow = true;
-                break;
-            }
-        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1176,49 +1063,17 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
                 result = P_SUCCESS;
                 break;
             }
-            if ((shift_state == INVALID_ID) && token_is_follow)
-            {
-                /* For a "parse inner" operation, if the incoming token is one
-                 * of the caller's follow tokens, retry the shift as
-                 * TOKEN___EOF. Only consider the parse complete if the reduced
-                 * start rule is the only thing on the parse stack (i.e. the
-                 * initial state plus a single shifted start rule set entry). */
-                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
-                if ((retry_shift_state != INVALID_ID) &&
-                    (statevalues.length == 2u) &&
-                    (last_shifted_rule_set_id == start_rule_set_id))
-                {
-                    /* Successful parse via follow token. Rewind the input
-                     * position so that the follow token is not consumed from
-                     * the input stream and remains available for a subsequent
-                     * call to <%= @grammar.prefix %>lex() or a
-                     * <%= @grammar.prefix %>parse*() function. */
-                    context->input_index -= token_info.length;
-                    context->text_position = token_info.position;
-<% if @grammar.tree %>
-                    context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
-<% else %>
-                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
-<% end %>
-                    result = P_SUCCESS;
-                    break;
-                }
-            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. Track the last shifted rule set ID
-             * (INVALID_ID if we just shifted a token) so the follow-token
-             * shift retry can gate success on the reduced start rule being the
-             * only thing on top of the initial state. */
-            last_shifted_rule_set_id = reduced_rule_set;
+            /* We have something to shift. */
             state_values_stack_push(&statevalues);
             state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
             new_state_info->state_id = shift_state;
@@ -1226,8 +1081,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
-                <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
+<%   if @cpp %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
+<%   else %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
+<%   end %>
                 token_tree_node->position = token_info.position;
                 token_tree_node->end_position = token_info.end_position;
                 token_tree_node->n_fields = 0u;
@@ -1235,7 +1093,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                new_state_info->node_id = token_node_id;
+                new_state_info->tree_node = token_tree_node;
 <% else %>
                 new_state_info->position = token_info.position;
                 new_state_info->end_position = token_info.end_position;
@@ -1247,7 +1105,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                new_state_info->node_id = reduced_parser_node;
+                new_state_info->tree_node = reduced_parser_node;
 <% else %>
                 new_state_info->pvalue = reduced_parser_value;
                 new_state_info->position = reduced_position;
@@ -1262,69 +1120,56 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
-        if ((reduce_index == INVALID_ID) && token_is_follow)
-        {
-            /* For a "parse inner" operation, if the incoming token is one of
-             * the caller's follow tokens, retry the reduce lookup as
-             * TOKEN___EOF. Whatever reduce_index results (if any) is used
-             * regardless of which rule set it reduces to; this allows chains
-             * of reductions leading up to the start rule. */
-            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
-        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                /* Reserve child slots. New slots are zero-initialized
-                 * (null node ID) so absent optional children remain null. */
-                <%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
+                TreeNode * node = (TreeNode *)malloc(bytes);
+                memset(node, 0, bytes);
+                node->position = INVALID_POSITION;
+                node->end_position = INVALID_POSITION;
+                node->n_fields = n_fields;
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
+                        node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
                     }
                 }
-                <%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
-                <%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
-                node->position = INVALID_POSITION;
-                node->end_position = INVALID_POSITION;
-                node->child_offset = child_offset;
-                node->n_fields = n_fields;
-                node->is_token = 0u;
                 bool position_found = false;
-                for (uint16_t i = 0; i < n_fields; i++)
+                for (size_t i = 0; i < n_fields; i++)
                 {
-                    <%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
-                    if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
+                    TreeNode * child = node->fields[i];
+                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
                     {
                         if (!position_found)
                         {
-                            node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
+                            node->position = child->position;
                             position_found = true;
                         }
-                        node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
+                        node->end_position = child->end_position;
                     }
                 }
-                reduced_parser_node = node_id;
+                reduced_parser_node = node;
             }
             else
             {
-                reduced_parser_node = 0u;
+                reduced_parser_node = NULL;
             }
 <%   if @grammar.parser_user_code_used? %>
             if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1374,20 +1219,14 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
+    return parse_from(context, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
-}
-
-size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
-    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
-{
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
+    return parse_from(context, <%= i %>u);
 }
 <% end %>
 
@@ -1400,14 +1239,14 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
+    return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
+    return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
 }
 <%   end %>
 <% else %>
@@ -1436,58 +1275,6 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
     return context->text_position;
 }
 
-/**
- * Set the current text input position.
- *
- * This can be used to set the initial text position to something other than
- * (1, 1) for a nested parse operation so that error positions reported by
- * subsequent lexer/parser calls are relative to a larger enclosing document.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param position
- *   Text position to set.
- */
-void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
-{
-    context->text_position = position;
-}
-
-/**
- * Get the current input text byte offset.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return Current input text byte offset (measured from the start of the
- *   input text passed to <%= @grammar.prefix %>context_new()).
- */
-size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
-{
-    return context->input_index;
-}
-
-/**
- * Set the current input text byte offset.
- *
- * This moves the lexer's read cursor to the given byte offset (measured from
- * the start of the input text passed to <%= @grammar.prefix %>context_new()).
- * It can be used together with <%= @grammar.prefix %>set_position() to rewind
- * the input part-way through a parse in order to re-read an earlier section of
- * the input. The byte offset is not validated; the caller is responsible for
- * providing an offset within the bounds of the input text. A value previously
- * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param input_index
- *   Input text byte offset to set.
- */
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
-{
-    context->input_index = input_index;
-}
-
 /**
  * Get the user terminate code.
  *
@@ -1510,3 +1297,48 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
+<% if @grammar.tree %>
+
+static void tree_delete(TreeNode * node)
+{
+    if (node->is_token)
+    {
+        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+<%   if @cpp %>
+        delete token_tree_node;
+<%   else %>
+        free(token_tree_node);
+<%   end %>
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                tree_delete(node->fields[i]);
+            }
+        }
+        free(node);
+    }
+}
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   end %>
+<% end %>

assets/parser.d.erb

@@ -105,23 +105,22 @@ public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ?
 <% end %>
 
 <% if @grammar.tree %>
-/** Tree node ID type (index into the context node arena). ID 0 is null. */
-public alias <%= @grammar.prefix %>node_id_t = uint;
-
-/**
- * Tree node record.
- *
- * All tree nodes are stored contiguously in the context node arena. Child
- * links are stored in a shared children array: a node's children
- * occupy children[child_offset .. child_offset + n_fields]. Token payload
- * fields (token, pvalue, and any user fields) are only meaningful when
- * is_token is true.
- */
-private struct <%= @grammar.prefix %>node_data_t
+/** Common tree node structure. */
+private struct TreeNode
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-    <%= @grammar.prefix %>node_id_t child_offset;
+    ushort n_fields;
+    bool is_token;
+    void *[0] fields;
+}
+
+/** Tree node types. @{ */
+public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+{
+    /* TreeNode fields must be present in the same order here. */
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
     ushort n_fields;
     bool is_token;
     <%= @grammar.prefix %>token_t token;
@@ -129,96 +128,22 @@ private struct <%= @grammar.prefix %>node_data_t
 <%= @grammar.token_user_fields %>
 }
 
-/** Tree node handle types. @{ */
-
-/** Token tree node handle. */
-public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
-    private <%= @grammar.prefix %>context_t * __context;
-    private <%= @grammar.prefix %>node_id_t __id;
-
-    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
+<%     rule_set.tree_fields.each do |fields| %>
+    union
     {
-        this.__context = context;
-        this.__id = id;
-    }
-
-    /** Return whether this handle refers to a valid (non-null) node. */
-    @property bool valid()
-    {
-        return __id != 0u;
-    }
-
-    /** Return the node ID (for identity comparison). */
-    @property <%= @grammar.prefix %>node_id_t node_id()
-    {
-        return __id;
-    }
-
-    /** Access the underlying node record (token, pvalue, and user fields). */
-    @property ref <%= @grammar.prefix %>node_data_t __node()
-    {
-        return __context.<%= @grammar.prefix %>tree_nodes[__id];
-    }
-
-    alias __node this;
-}
-
-<%   tree_node_rule_sets.each do |rule_set| %>
-/** <%= rule_set.name %> tree node handle. */
-public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_suffix %>
-{
-    private <%= @grammar.prefix %>context_t * __context;
-    private <%= @grammar.prefix %>node_id_t __id;
-
-    this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
-    {
-        this.__context = context;
-        this.__id = id;
-    }
-
-    /** Return whether this handle refers to a valid (non-null) node. */
-    @property bool valid()
-    {
-        return __id != 0u;
-    }
-
-    /** Return the node ID (for identity comparison). */
-    @property <%= @grammar.prefix %>node_id_t node_id()
-    {
-        return __id;
-    }
-
-    /** Text position of the first code point spanned by this node. */
-    @property <%= @grammar.prefix %>position_t position()
-    {
-        return __context.<%= @grammar.prefix %>tree_nodes[__id].position;
-    }
-
-    /** Text position of the last code point spanned by this node. */
-    @property <%= @grammar.prefix %>position_t end_position()
-    {
-        return __context.<%= @grammar.prefix %>tree_nodes[__id].end_position;
-    }
-
-    /** Number of child fields in this node. */
-    @property ushort n_fields()
-    {
-        return __id ? __context.<%= @grammar.prefix %>tree_nodes[__id].n_fields : cast(ushort)0u;
-    }
-<%     rule_set.tree_fields.each_with_index do |fields, i| %>
 <%       fields.each do |field_name, type| %>
-
-    /** Access the <%= field_name %> child node. */
-    @property <%= type %> <%= field_name %>()
-    {
-        if (__id == 0u)
-        {
-            return <%= type %>(__context, 0u);
-        }
-        return <%= type %>(__context, __context.<%= @grammar.prefix %>tree_children[__context.<%= @grammar.prefix %>tree_nodes[__id].child_offset + <%= i %>u]);
-    }
+        <%= type %> * <%= field_name %>;
 <%       end %>
+    }
 <%     end %>
 }
 
@@ -271,13 +196,7 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t parse_result;
-
-    /** Tree node arena. Node ID 0 is reserved as the null node. */
-    <%= @grammar.prefix %>node_data_t[] <%= @grammar.prefix %>tree_nodes;
-
-    /** Shared tree child links. */
-    <%= @grammar.prefix %>node_id_t[] <%= @grammar.prefix %>tree_children;
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -334,6 +253,8 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *
  * @param input
  *   Text input.
+ * @param input_length
+ *   Text input length.
  *
  * @return Context structure for lexer/parser.
  */
@@ -347,11 +268,6 @@ private enum size_t INVALID_ID = cast(size_t)-1;
     context.text_position.row = 1u;
     context.text_position.col = 1u;
     context.mode = <%= @lexer.mode_id("default") %>;
-<% if @grammar.tree %>
-
-    /* Reserve node ID 0 as the null tree node. */
-    context.<%= @grammar.prefix %>tree_nodes = new <%= @grammar.prefix %>node_data_t[](1);
-<% end %>
 
     return context;
 }
@@ -364,16 +280,6 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  */
 void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
 {
-<% if @grammar.tree && @grammar.free_token_node != "" %>
-    foreach (ref node; context.<%= @grammar.prefix %>tree_nodes)
-    {
-        if (node.is_token)
-        {
-            <%= @grammar.prefix %>node_data_t * token_tree_node = &node;
-<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
-        }
-    }
-<% end %>
 }
 
 /**************************************************************************
@@ -572,7 +478,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match_text
+ * @param match
  *   Matched text for this pattern.
  * @param out_token_info
  *   Lexer token info in progress.
@@ -581,7 +487,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   not explicitly return a token.
  */
 private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, string match_text,
+    lexer_user_code_id_t code_id, string match,
     <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -777,9 +683,9 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
         }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
-            string match_text = context.input[context.input_index..(context.input_index + match_info.length)];
+            string match = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state.code_id, match_text, &token_info);
+                match_info.accepting_state.code_id, match, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -982,8 +888,8 @@ private struct state_value_t
     size_t state_id;
 
 <% if @grammar.tree %>
-    /** Tree node ID. */
-    <%= @grammar.prefix %>node_id_t node_id;
+    /** Tree node. */
+    void * tree_node;
 <% else %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
@@ -1101,7 +1007,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1151,7 +1057,7 @@ private size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return Reduce table index to reduce with, or INVALID_ID if none.
+ * @return State to reduce to, or INVALID_ID if none.
  */
 private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1173,16 +1079,8 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @param start_state_id
+ * @start_state_id
  *   ID of the state in which to start.
- * @param start_rule_set_id
- *   Rule set ID for the requested start rule. Only used when
- *   @p follow_tokens is non-empty, to gate follow-token shift success.
- * @param follow_tokens
- *   Optional slice of caller-provided follow tokens (tokens expected to
- *   appear immediately after the start rule in some outer context). Used to
- *   drive the "parse inner" retry logic. May be null/empty for a standard
- *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1195,18 +1093,15 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
-    size_t start_rule_set_id,
-    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
-    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t reduced_parser_node;
+    void * reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>position_t reduced_position;
     <%= @grammar.prefix %>position_t reduced_end_position;
@@ -1223,18 +1118,6 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             }
             token = token_info.token;
         }
-        /* For a "parse inner" operation, determine once per iteration whether
-         * the current token is a member of the caller-provided follow token
-         * set. Used by both the shift-side and reduce-side retries below. */
-        bool token_is_follow = false;
-        foreach (eof_token; follow_tokens)
-        {
-            if (token == eof_token)
-            {
-                token_is_follow = true;
-                break;
-            }
-        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1247,63 +1130,24 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* Successful parse. */
 <% if @grammar.tree %>
-                context.parse_result = statevalues[$-1].node_id;
+                context.parse_result = statevalues[$-1].tree_node;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
                 return P_SUCCESS;
             }
-            if ((shift_state == INVALID_ID) && token_is_follow)
-            {
-                /* For a "parse inner" operation, if the incoming token is one
-                 * of the caller's follow tokens, retry the shift as
-                 * TOKEN___EOF. Only consider the parse complete if the reduced
-                 * start rule is the only thing on the parse stack (i.e. the
-                 * initial state plus a single shifted start rule set entry). */
-                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
-                if ((retry_shift_state != INVALID_ID) &&
-                    (statevalues.length == 2u) &&
-                    (last_shifted_rule_set_id == start_rule_set_id))
-                {
-                    /* Successful parse via follow token. Rewind the input
-                     * position so that the follow token is not consumed from
-                     * the input stream and remains available for a subsequent
-                     * call to <%= @grammar.prefix %>lex() or a
-                     * <%= @grammar.prefix %>parse*() function. */
-                    context.input_index -= token_info.length;
-                    context.text_position = token_info.position;
-<% if @grammar.tree %>
-                    context.parse_result = statevalues[$-1].node_id;
-<% else %>
-                    context.parse_result = statevalues[$-1].pvalue;
-<% end %>
-                    return P_SUCCESS;
-                }
-            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. Track the last shifted rule set ID
-             * (INVALID_ID if we just shifted a token) so the follow-token
-             * shift retry can gate success on the reduced start rule being the
-             * only thing on top of the initial state. */
-            last_shifted_rule_set_id = reduced_rule_set;
+            /* We have something to shift. */
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.prefix %>node_id_t token_node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
-                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
-                <%= @grammar.prefix %>node_data_t * token_tree_node = &context.<%= @grammar.prefix %>tree_nodes[token_node_id];
-                token_tree_node.position = token_info.position;
-                token_tree_node.end_position = token_info.end_position;
-                token_tree_node.n_fields = 0u;
-                token_tree_node.is_token = true;
-                token_tree_node.token = token;
-                token_tree_node.pvalue = token_info.pvalue;
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                statevalues[$-1].node_id = token_node_id;
+                statevalues[$-1].tree_node = token_tree_node;
 <% else %>
                 statevalues[$-1].position = token_info.position;
                 statevalues[$-1].end_position = token_info.end_position;
@@ -1315,7 +1159,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                statevalues[$-1].node_id = reduced_parser_node;
+                statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
                 statevalues[$-1].position = reduced_position;
@@ -1329,71 +1173,61 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
-        if ((reduce_index == INVALID_ID) && token_is_follow)
-        {
-            /* For a "parse inner" operation, if the incoming token is one of
-             * the caller's follow tokens, retry the reduce lookup as
-             * TOKEN___EOF. Whatever reduce_index results (if any) is used
-             * regardless of which rule set it reduces to; this allows chains
-             * of reductions leading up to the start rule. */
-            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
-        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
 <% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = statevalues[$ - 1].node_id;
+                reduced_parser_node = statevalues[$ - 1].tree_node;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                ushort n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                /* Reserve child slots. New slots are zero-initialized
-                 * (null node ID) so absent optional children remain null. */
-                <%= @grammar.prefix %>node_id_t child_offset = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_children.length;
-                context.<%= @grammar.prefix %>tree_children.length += n_fields;
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
+                TreeNode * node = cast(TreeNode *)malloc(node_size);
+                GC.addRange(node, node_size);
+                node.position = <%= @grammar.prefix %>position_t.INVALID;
+                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
+                foreach (i; 0..n_fields)
+                {
+                    node.fields[i] = null;
+                }
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        context.<%= @grammar.prefix %>tree_children[child_offset + i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
+                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
                     }
                 }
                 else
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        context.<%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
+                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
                     }
                 }
-                <%= @grammar.prefix %>node_id_t node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
-                context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
-                <%= @grammar.prefix %>node_data_t * node = &context.<%= @grammar.prefix %>tree_nodes[node_id];
-                node.position = <%= @grammar.prefix %>position_t.INVALID;
-                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
-                node.child_offset = child_offset;
-                node.n_fields = n_fields;
-                node.is_token = false;
                 bool position_found = false;
                 foreach (i; 0..n_fields)
                 {
-                    <%= @grammar.prefix %>node_id_t child_id = context.<%= @grammar.prefix %>tree_children[child_offset + i];
-                    if (child_id != 0u && context.<%= @grammar.prefix %>tree_nodes[child_id].position.valid)
+                    TreeNode * child = cast(TreeNode *)node.fields[i];
+                    if (child && child.position.valid)
                     {
                         if (!position_found)
                         {
-                            node.position = context.<%= @grammar.prefix %>tree_nodes[child_id].position;
+                            node.position = child.position;
                             position_found = true;
                         }
-                        node.end_position = context.<%= @grammar.prefix %>tree_nodes[child_id].end_position;
+                        node.end_position = child.end_position;
                     }
                 }
-                reduced_parser_node = node_id;
+                reduced_parser_node = node;
             }
             else
             {
-                reduced_parser_node = 0u;
+                reduced_parser_node = null;
             }
 <%   if @grammar.parser_user_code_used? %>
             if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@@ -1437,20 +1271,14 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
+    return parse_from(context, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
-}
-
-public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
-    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
-{
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
+    return parse_from(context, <%= i %>u);
 }
 <% end %>
 
@@ -1463,14 +1291,14 @@ public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.p
  * @return Parse result value.
  */
 <% if @grammar.tree %>
-public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %>(context, context.parse_result);
+    return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %>(context, context.parse_result);
+    return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
 }
 <%   end %>
 <% else %>
@@ -1486,6 +1314,35 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
+
 /**
  * Get the current text input position.
  *
@@ -1499,58 +1356,6 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
     return context.text_position;
 }
 
-/**
- * Set the current text input position.
- *
- * This can be used to set the initial text position to something other than
- * (1, 1) for a nested parse operation so that error positions reported by
- * subsequent lexer/parser calls are relative to a larger enclosing document.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param position
- *   Text position to set.
- */
-public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
-{
-    context.text_position = position;
-}
-
-/**
- * Get the current input text byte offset.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return Current input text byte offset (measured from the start of the
- *   input text passed to <%= @grammar.prefix %>context_new()).
- */
-public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
-{
-    return context.input_index;
-}
-
-/**
- * Set the current input text byte offset.
- *
- * This moves the lexer's read cursor to the given byte offset (measured from
- * the start of the input text passed to <%= @grammar.prefix %>context_new()).
- * It can be used together with <%= @grammar.prefix %>set_position() to rewind
- * the input part-way through a parse in order to re-read an earlier section of
- * the input. The byte offset is not validated; the caller is responsible for
- * providing an offset within the bounds of the input text. A value previously
- * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param input_index
- *   Input text byte offset to set.
- */
-public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
-{
-    context.input_index = input_index;
-}
-
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -8,9 +8,6 @@
 
 #include <stdint.h>
 #include <stddef.h>
-<% if @cpp %>
-#include <vector>
-<% end %>
 
 /**************************************************************************
  * Public types
@@ -91,29 +88,47 @@ static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "defa
 <% end %>
 
 <% if @grammar.tree %>
-/** Tree node ID type (index into the context node arena). ID 0 is null. */
-typedef uint32_t <%= @grammar.prefix %>node_id_t;
-
-/**
- * Tree node record.
- *
- * All tree nodes are stored contiguously in the context node arena. Child
- * links are stored in a shared children array: a node's children
- * occupy children[child_offset .. child_offset + n_fields]. Token payload
- * fields (token, pvalue, and any user fields) are only meaningful when
- * is_token is nonzero.
- */
-typedef struct
+/** Tree node types. @{ */
+typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 {
+    <% # TreeNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-    <%= @grammar.prefix %>node_id_t child_offset;
     uint16_t n_fields;
     uint8_t is_token;
+<%= @grammar.token_user_fields %>
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
-<%= @grammar.token_user_fields %>
-} <%= @grammar.prefix %>node_data_t;
+} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+struct <%= name %>;
+<%   end %>
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
+{
+    <% # TreeNode fields must be present in the same order here. # %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+<%     rule_set.tree_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        struct <%= type %> * <%= field_name %>;
+<%       end %>
+    };
+<%     end %>
+} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
+
+<%   end %>
+/** @} */
 <% end %>
 
 /** Lexed token information. */
@@ -135,19 +150,13 @@ typedef struct
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.prefix %>token_info_t;
 
-typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t;
-
-<% if @grammar.tree %>
-<%= c_tree_handle_types_header %>
-
-<% end %>
 /**
  * Lexer and parser context.
  *
  * The user must allocate an instance of this structure and pass it to any
  * public API function.
  */
-struct <%= @grammar.prefix %>context_s
+typedef struct
 {
     /* Lexer context data. */
 
@@ -170,25 +179,7 @@ struct <%= @grammar.prefix %>context_s
 
     /** Parse result value. */
 <% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t parse_result;
-
-<%   if @cpp %>
-    /** Tree node arena. Node ID 0 is reserved as the null node. */
-    std::vector<<%= @grammar.prefix %>node_data_t> <%= @grammar.prefix %>tree_nodes;
-
-    /** Shared tree child links. */
-    std::vector<<%= @grammar.prefix %>node_id_t> <%= @grammar.prefix %>tree_children;
-<%   else %>
-    /** Tree node arena. Node ID 0 is reserved as the null node. */
-    <%= @grammar.prefix %>node_data_t * <%= @grammar.prefix %>tree_nodes;
-    size_t <%= @grammar.prefix %>tree_nodes_length;
-    size_t <%= @grammar.prefix %>tree_nodes_capacity;
-
-    /** Shared tree child links. */
-    <%= @grammar.prefix %>node_id_t * <%= @grammar.prefix %>tree_children;
-    size_t <%= @grammar.prefix %>tree_children_length;
-    size_t <%= @grammar.prefix %>tree_children_capacity;
-<%   end %>
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -200,11 +191,7 @@ struct <%= @grammar.prefix %>context_s
     size_t user_terminate_code;
 
 <%= @grammar.context_user_fields %>
-};
-
-<% if @grammar.tree %>
-<%= c_tree_types_header %>
-<% end %>
+} <%= @grammar.prefix %>context_t;
 
 /**************************************************************************
  * Public data
@@ -225,14 +212,12 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
-        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>
-<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
 <%   end %>
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
@@ -241,14 +226,15 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
 <%   end %>
 <% end %>
 
+<% if @grammar.tree %>
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
+<%   end %>
+<% end %>
+
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
-void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
-
-size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
-
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
-
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

extra/vim/syntax/propane.vim

@@ -7,46 +7,12 @@ if exists("b:current_syntax")
   finish
 endif
 
-" Guess the language of the user code blocks from their contents so that the
-" matching syntax file can be included below. b:propane_subtype may also be set
-" before this file is sourced to select the language explicitly.
 if !exists("b:propane_subtype")
-  " Rust markers. Each keyword requires the syntax that follows it in Rust so
-  " that a plain identifier of the same name in another language does not match
-  " (`int fn = 3;' in C, for example). Type names are only accepted within a
-  " `ptype' statement for the same reason.
-  let s:rust = '\<let\s\+\%(mut\s\+\)\?\w'
-  let s:rust .= '\|\<fn\s\+\w\+\s*('
-  let s:rust .= '\|&mut\>\|\<pub\s\+\w\|\<impl\s\+\w'
-  let s:rust .= '\|#\[\|\<use\s\+\%(std\|core\)::'
-  let s:rust .= '\|\<ptype\>[^;]*\<\%(isize\|usize\|i8\|i16\|i32\|i64\|i128'
-  let s:rust .= '\|u8\|u16\|u32\|u64\|u128\|f32\|f64\|String\)\>'
-  " D markers. These are spellings that have no valid C, C++, or Rust
-  " equivalent, so `import' is deliberately not among them: it is a D keyword
-  " but is also a C++20 module declaration.
-  let s:d = '\<foreach\%(_reverse\)\?\s*([^)]*;'
-  let s:d .= '\|\~=\|\<static\s\+if\s*(\|\<version\s*(\s*\w\+\s*)'
-  let s:d .= '\|\<scope\s*(\s*\%(exit\|failure\|success\)\s*)'
-  let s:d .= '\|\<\%(unittest\|mixin\|immutable\|__gshared\|invariant\)\>'
-  let s:d .= '\|\<alias\s\+\w\+\s*=\|\<enum\s\+\w\+\s*='
-  let s:d .= '\|@\%(property\|safe\|trusted\|system\|nogc\|disable\)\>'
-  let s:d .= '\|\<is\s\+null\>\|\<cast\s*(\s*\w\+\s*)'
-  let s:d .= '\|\<write\%(ln\|fln\|f\)\s*('
-  let s:d .= '\|\<\%(dchar\|dstring\|wstring\|cent\|ucent\)\>'
-  " A module import on its own is ambiguous between D and C++20, so only take
-  " it as D when nothing else in the file looks like C++.
-  let s:import = '\<import\s\+[A-Za-z_][A-Za-z0-9_.]*\s*;'
-  let s:cpp = '::\|\<template\s*<\|\<namespace\>\|\<nullptr\>\|#include\s*[<"]'
-  if search(s:rust, 'nw') > 0
-    let b:propane_subtype = "rust"
-  elseif search(s:d, 'nw') > 0
-    let b:propane_subtype = "d"
-  elseif search(s:import, 'nw') > 0 && search(s:cpp, 'nw') == 0
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
     let b:propane_subtype = "d"
   else
     let b:propane_subtype = "cpp"
   endif
-  unlet s:rust s:d s:import s:cpp
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"

lib/propane/generator.rb

@@ -18,8 +18,6 @@ class Propane
         elsif output_file =~ %r{\.(cc|cpp|cxx)$}
           @cpp = true
           "c"
-        elsif output_file.end_with?(".rs")
-          "rust"
         else
           raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
@@ -33,32 +31,15 @@ class Propane
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template_language = @language == "rust" ? "rs" : @language
-        template = Assets.get("parser.#{extension || template_language}.erb")
+        template = Assets.get("parser.#{extension || @language}.erb")
         if extension
           output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         else
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        # Rust has no #line directive support. For a Rust target the directives
-        # that the grammar embeds around user code blocks are replaced with
-        # comments naming the grammar file and line number the code came from,
-        # so that the origin of a section of user code can still be found by
-        # reading up from a compiler diagnostic pointing into the generated
-        # module.
-        user_code_origin = nil
         result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
-          if @language == "rust"
-            if md = line.match(/^#line (\d+) "([^"]*)"/)
-              user_code_origin = "#{md[2]} line #{md[1]}"
-              line.sub(/^#line \d+ "[^"]*"/, %[/* Begin user code from #{user_code_origin}. */])
-            elsif line == "#linereset\n"
-              %[/* End user code from #{user_code_origin}. */\n]
-            else
-              line
-            end
-          elsif line == "#linereset\n"
+          if line == "#linereset\n"
             %[#line #{i + 2} "#{output_file}"\n]
           else
             line
@@ -294,8 +275,6 @@ class Propane
           "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
         when "d"
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        when "rust"
-          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
       code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
@@ -305,8 +284,6 @@ class Propane
           "context->#{fieldname}"
         when "d"
           "context.#{fieldname}"
-        when "rust"
-          "context.#{fieldname}"
         end
       end
       code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
@@ -316,21 +293,16 @@ class Propane
           "token_tree_node->#{fieldname}"
         when "d"
           "token_tree_node.#{fieldname}"
-        when "rust"
-          "token_tree_node.#{fieldname}"
         end
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
           if @grammar.tree
-            typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
             case @language
             when "c"
-              tree_handle(typename, "_node_id")
+              "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
             when "d"
-              tree_handle(typename, "_node_id")
-            when "rust"
-              tree_handle(typename, "_node_id")
+              "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
             end
           else
             case @language
@@ -338,8 +310,6 @@ class Propane
               "_pvalue->v_#{rule.ptypename}"
             when "d"
               "_pvalue.v_#{rule.ptypename}"
-            when "rust"
-              "(*_pvalue.v_#{rule.ptypename}_mut())"
             end
           end
         end
@@ -374,8 +344,6 @@ class Propane
               "out_token_info->pvalue"
             when "d"
               "out_token_info.pvalue"
-            when "rust"
-              "out_token_info.pvalue"
             end
           else
             case @language
@@ -383,8 +351,6 @@ class Propane
               "out_token_info->pvalue.v_#{pattern.ptypename}"
             when "d"
               "out_token_info.pvalue.v_#{pattern.ptypename}"
-            when "rust"
-              "(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())"
             end
           end
         end
@@ -394,8 +360,6 @@ class Propane
             "out_token_info->position"
           when "d"
             "out_token_info.position"
-          when "rust"
-            "out_token_info.position"
           end
         end
         code = code.gsub(/\$\{end_position\}/) do |match|
@@ -404,8 +368,6 @@ class Propane
             "out_token_info->end_position"
           when "d"
             "out_token_info.end_position"
-          when "rust"
-            "out_token_info.end_position"
           end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
@@ -419,8 +381,6 @@ class Propane
             "context->mode = #{mode_id}u"
           when "d"
             "context.mode = #{mode_id}u"
-          when "rust"
-            "context.mode = #{mode_id}"
           end
         end
       end
@@ -442,7 +402,7 @@ class Propane
     def parser_component_reference(rule, index)
       component = rule.components[index - 1]
       if @grammar.tree
-        # In tree mode a component reference yields a handle to that
+        # In tree mode a component reference yields a pointer to that
         # component's tree node. An optional component propagates its target
         # node (or null), so use the optional target's node type.
         if component.is_a?(RuleSet) && component.optional?
@@ -452,11 +412,9 @@ class Propane
         typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
         case @language
         when "c"
-          tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
+          "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
         when "d"
-          tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
-        when "rust"
-          tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id")
+          "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
         end
       else
         case @language
@@ -464,358 +422,10 @@ class Propane
           "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
         when "d"
           "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
-        when "rust"
-          "statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()"
         end
       end
     end
 
-    # Construct a tree node handle expression for the target language.
-    #
-    # A handle is a small value pairing the parser context with a node ID
-    # (an index into the context's node arena). All handle types share this
-    # layout; the distinct types exist for documentation and, in C, to drive
-    # the tree walk macro's type threading.
-    #
-    # @param typename [String]
-    #   Handle type name.
-    # @param id_expr [String]
-    #   Expression yielding the node ID.
-    # @param parenthesize [Boolean]
-    #   Whether to parenthesize the expression. Parentheses are required where
-    #   the expression is substituted into a user code block, since the
-    #   expression could be followed there by a field access or appear in a
-    #   position where a bare Rust struct literal is not accepted. They are
-    #   unnecessary where the expression stands alone, and Rust warns about
-    #   them there, so this can be disabled for those uses.
-    #
-    # @return [String]
-    #   Handle constructor expression.
-    def tree_handle(typename, id_expr, parenthesize = true)
-      if @cpp
-        "(#{typename}{context, #{id_expr}})"
-      elsif @language == "c"
-        "((#{typename}){context, #{id_expr}})"
-      elsif @language == "rust"
-        expr = "#{typename} { context, id: #{id_expr} }"
-        parenthesize ? "(#{expr})" : expr
-      else
-        "#{typename}(context, #{id_expr})"
-      end
-    end
-
-    # Get the list of non-optional, non-internal rule sets that get a tree node
-    # handle type generated for them.
-    #
-    # @return [Array<Propane::RuleSet>]
-    #   Rule sets with generated tree node handle types.
-    def tree_node_rule_sets
-      @parser.rule_sets.reject do |name, rule_set|
-        name.start_with?("$") || rule_set.optional?
-      end.map {|name, rule_set| rule_set}
-    end
-
-    # Maximum number of chained fields supported by a single C tree walk macro
-    # invocation. Deeper navigation can be expressed by nesting walk calls.
-    C_TREE_WALK_MAX = 16
-
-    # Get the tree node handle type name for a node name.
-    #
-    # @param name [String]
-    #   Rule set name, or "Token".
-    #
-    # @return [String]
-    #   Handle type name.
-    def h_type(name)
-      "#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
-    end
-
-    # Get the list of all tree node handle type names (Token plus rule sets).
-    #
-    # @return [Array<String>]
-    #   Handle type names.
-    def tree_handle_types
-      [h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
-    end
-
-    # Enumerate the navigation fields of a rule set's tree node.
-    #
-    # @yield [rtype, field_name, child_type, slot]
-    #   Handle type name, field accessor name, child handle type, and child
-    #   slot index.
-    def each_tree_field(rule_set)
-      rtype = h_type(rule_set.name)
-      rule_set.tree_fields.each_with_index do |fields, slot|
-        fields.each do |field_name, child_type|
-          yield rtype, field_name, child_type, slot
-        end
-      end
-    end
-
-    # Generate the tree node handle type declarations for the header.
-    #
-    # These are emitted before the context structure definition so that a
-    # context_user_fields block can declare a field of a handle type.
-    #
-    # @return [String]
-    #   Handle type declarations.
-    def c_tree_handle_types_header
-      @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header
-    end
-
-    # Generate the remainder of the tree node section for the header.
-    #
-    # This is emitted after the context structure definition since it
-    # dereferences the context and so requires the complete type.
-    #
-    # @return [String]
-    #   Accessors, macros, and out-of-line handle method definitions.
-    def c_tree_types_header
-      @cpp ? cpp_tree_types_header : c_only_tree_types_header
-    end
-
-    # Generate the C (non-C++) tree node handle type section for the header.
-    def c_only_tree_handle_types_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Tree node handle types. @{ */"
-      tree_handle_types.each do |t|
-        out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
-      end
-      out << ""
-      out.join("\n")
-    end
-
-    def c_only_tree_types_header
-      out = []
-      out << c_common_accessors_header
-      out << "/** @} */"
-      out.join("\n")
-    end
-
-    # Generate the C-style (function + macro) tree node accessors shared by the
-    # C and C++ headers. In C++ these are provided in addition to the handle
-    # methods so that C-style code (and the tree walk macros) also works.
-    def c_common_accessors_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Generic tree node accessors (usable on any handle type). */"
-      out << "#define #{p}node_valid(h) ((h).__id != 0u)"
-      out << "#define #{p}node_id(h) ((h).__id)"
-      out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
-      out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
-      out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
-      out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
-      out << ""
-      out << "/** Tree node field accessor functions. */"
-      out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
-      out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
-        end
-      end
-      out << ""
-      out << c_tree_walk_macros
-      out.join("\n")
-    end
-
-    # Generate the C tree walk macro machinery.
-    def c_tree_walk_macros
-      p = @grammar.prefix
-      max = C_TREE_WALK_MAX
-      out = []
-      out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
-      out << "#define #{p}CAT_(a, b) a##b"
-      out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
-      out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
-      out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
-      argn = (1..max).map {|i| "_#{i}"}.join(", ")
-      rseq = (0..max).to_a.reverse.join(", ")
-      out << "#define #{p}ARG_N(#{argn}, N, ...) N"
-      out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
-      (1..max).each do |n|
-        fparams = (1..n).map {|k| "f#{k}"}.join(", ")
-        call = "h"
-        (1..n).each do |k|
-          texpr = "R"
-          (1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
-          call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
-        end
-        out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
-      end
-      out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
-      # Type transition map (navigation fields only).
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
-        end
-      end
-      # Per-handle-type walk entry points.
-      tree_handle_types.each do |t|
-        out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
-      end
-      out.join("\n")
-    end
-
-    # Generate the C tree node accessor function definitions for the source.
-    #
-    # @return [String]
-    #   Accessor function definitions.
-    def c_tree_accessor_defs
-      p = @grammar.prefix
-      tt = h_type("Token")
-      out = []
-      out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
-      out << "{"
-      out << "    return node.__context->#{p}tree_nodes[node.__id].token;"
-      out << "}"
-      out << ""
-      out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
-      out << "{"
-      out << "    return node.__context->#{p}tree_nodes[node.__id].pvalue;"
-      out << "}"
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << ""
-          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
-          out << "{"
-          out << "    #{child_type} result;"
-          out << "    result.__context = node.__context;"
-          out << "    if (node.__id == 0u)"
-          out << "    {"
-          out << "        result.__id = 0u;"
-          out << "        return result;"
-          out << "    }"
-          out << "    result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
-          out << "    return result;"
-          out << "}"
-        end
-      end
-      out.join("\n")
-    end
-
-    # Generate the C++ tree node handle class declarations for the header.
-    # Only valid() and node_id() are defined inline; every other method
-    # dereferences the context, which is still an incomplete type here, so
-    # those are declared and defined out of line once the context is
-    # complete.
-    def cpp_tree_handle_types_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Tree node handle types. @{ */"
-      tree_handle_types.each {|t| out << "struct #{t};"}
-      out << ""
-      tt = h_type("Token")
-      out << "struct #{tt}"
-      out << "{"
-      out << "    #{p}context_t * __context;"
-      out << "    #{p}node_id_t __id;"
-      out << "    bool valid() const { return __id != 0u; }"
-      out << "    #{p}node_id_t node_id() const { return __id; }"
-      out << "    #{p}node_data_t * data() const;"
-      out << "    #{p}position_t position() const;"
-      out << "    #{p}position_t end_position() const;"
-      out << "    uint16_t n_fields() const;"
-      out << "    #{p}token_t token() const;"
-      out << "    #{p}value_t pvalue() const;"
-      out << "};"
-      out << ""
-      tree_node_rule_sets.each do |rule_set|
-        rtype = h_type(rule_set.name)
-        out << "struct #{rtype}"
-        out << "{"
-        out << "    #{p}context_t * __context;"
-        out << "    #{p}node_id_t __id;"
-        out << "    bool valid() const { return __id != 0u; }"
-        out << "    #{p}node_id_t node_id() const { return __id; }"
-        out << "    #{p}node_data_t * data() const;"
-        out << "    #{p}position_t position() const;"
-        out << "    #{p}position_t end_position() const;"
-        out << "    uint16_t n_fields() const;"
-        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
-          out << "    #{child_type} #{field_name}() const;"
-        end
-        out << "};"
-        out << ""
-      end
-      out.join("\n")
-    end
-
-    # Generate the out-of-line C++ handle method definitions plus the C-style
-    # accessors. Emitted after the context structure definition.
-    def cpp_tree_types_header
-      p = @grammar.prefix
-      out = []
-      # Common node methods, now that the context type is complete.
-      ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht|
-        out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }"
-        out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }"
-        out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-        out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
-      end
-      tt = h_type("Token")
-      out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }"
-      out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
-      out << ""
-      # Out-of-line navigation method bodies (all handle types now complete).
-      tree_node_rule_sets.each do |rule_set|
-        rtype = h_type(rule_set.name)
-        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
-          out << "inline #{child_type} #{rtype}::#{field_name}() const"
-          out << "{"
-          out << "    if (__id == 0u)"
-          out << "    {"
-          out << "        return #{child_type}{__context, 0u};"
-          out << "    }"
-          out << "    return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
-          out << "}"
-        end
-      end
-      out << ""
-      out << "/*"
-      out << " * C-style function and macro accessors, provided in addition to the handle"
-      out << " * methods above so that C-style code and the tree walk macros also work."
-      out << " */"
-      out << c_common_accessors_header
-      out << "/** @} */"
-      out.join("\n")
-    end
-
-    # Rust keywords that must be escaped as raw identifiers when used as a
-    # generated identifier (e.g. a field alias named `type`).
-    RUST_KEYWORDS = %w[
-      as break const continue dyn else enum extern false fn for if impl in let
-      loop match mod move mut pub ref return static struct trait true type
-      unsafe use where while async await abstract become box do final macro
-      override priv typeof unsized virtual yield try gen
-    ]
-
-    # Escape a name as a Rust raw identifier if it is a reserved keyword.
-    #
-    # @param name [String]
-    #   Identifier name.
-    #
-    # @return [String]
-    #   Name, escaped as a raw identifier if necessary.
-    def rust_ident(name)
-      RUST_KEYWORDS.include?(name) ? "r##{name}" : name
-    end
-
-    # Map a ptype type string to a valid Rust type.
-    #
-    # The default ptype is a C "void *"; for Rust with no declared ptype we use
-    # the unit type instead.
-    #
-    # @param typestring [String]
-    #   ptype type string.
-    #
-    # @return [String]
-    #   Rust type string.
-    def rust_ptype(typestring)
-      typestring == "void *" ? "()" : typestring
-    end
-
     # Get the lex function to use.
     #
     # @return [String]
@@ -849,8 +459,6 @@ class Propane
           "uint8_t"
         when "d"
           "ubyte"
-        when "rust"
-          "u8"
         end
       elsif max <= 0xFFFF
         case @language
@@ -858,15 +466,11 @@ class Propane
           "uint16_t"
         when "d"
           "ushort"
-        when "rust"
-          "u16"
         end
       else
         case @language
         when "c"
           "uint32_t"
-        when "rust"
-          "u32"
         else
           "uint"
         end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "5.1.0"
+  VERSION = "4.7.0"
 end

spec/json_parser.c.propane

@@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n = 0.0;
     bool negative = false;
     size_t i = 0u;
-    if (match_text[i] == '-')
+    if (match[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match_text[i] && match_text[i] <= '9')
+    while ('0' <= match[i] && match[i] <= '9')
     {
         n *= 10.0;
-        n += (match_text[i] - '0');
+        n += (match[i] - '0');
         i++;
     }
-    if (match_text[i] == '.')
+    if (match[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
-            n += mult * (match_text[i] - '0');
+            n += mult * (match[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match_text[i] == 'e' || match_text[i] == 'E')
+    if (match[i] == 'e' || match[i] == 'E')
     {
         bool exp_negative = false;
         i++;
-        if (match_text[i] == '-')
+        if (match[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match_text[i] == '+')
+        else if (match[i] == '+')
         {
             i++;
         }
         long exp = 0.0;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
             exp *= 10;
-            exp += (match_text[i] - '0');
+            exp += (match[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
+  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {(char)match_text[0], 0};
+  char s[] = {(char)match[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_parser.d.propane

@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n;
     bool negative;
     size_t i = 0u;
-    if (match_text[i] == '-')
+    if (match[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match_text[i] && match_text[i] <= '9')
+    while ('0' <= match[i] && match[i] <= '9')
     {
         n *= 10.0;
-        n += (match_text[i] - '0');
+        n += (match[i] - '0');
         i++;
     }
-    if (match_text[i] == '.')
+    if (match[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
-            n += mult * (match_text[i] - '0');
+            n += mult * (match[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match_text[i] == 'e' || match_text[i] == 'E')
+    if (match[i] == 'e' || match[i] == 'E')
     {
         bool exp_negative;
         i++;
-        if (match_text[i] == '-')
+        if (match[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match_text[i] == '+')
+        else if (match[i] == '+')
         {
             i++;
         }
         long exp;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
             exp *= 10;
-            exp += (match_text[i] - '0');
+            exp += (match[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match_text[2..6] ~ "}";
+  string_value ~= "{" ~ match[2..6] ~ "}";
 >>
 string: /[^\\]/ <<
-  string_value ~= match_text;
+  string_value ~= match;
 >>
 Start -> Value <<
   $$ = $1;

spec/json_parser.rust.propane

@@ -1,176 +0,0 @@
-<<
-pub const JSON_OBJECT: usize = 0;
-pub const JSON_ARRAY: usize = 1;
-pub const JSON_NUMBER: usize = 2;
-pub const JSON_STRING: usize = 3;
-pub const JSON_TRUE: usize = 4;
-pub const JSON_FALSE: usize = 5;
-pub const JSON_NULL: usize = 6;
-
-#[derive(Clone, Default)]
-pub enum JSONValue {
-    #[default]
-    Null,
-    Object(Vec<(String, JSONValue)>),
-    Array(Vec<JSONValue>),
-    Number(f64),
-    StringVal(String),
-    True,
-    False,
-}
-
-impl JSONValue {
-    pub fn id(&self) -> usize {
-        match self {
-            JSONValue::Object(_) => JSON_OBJECT,
-            JSONValue::Array(_) => JSON_ARRAY,
-            JSONValue::Number(_) => JSON_NUMBER,
-            JSONValue::StringVal(_) => JSON_STRING,
-            JSONValue::True => JSON_TRUE,
-            JSONValue::False => JSON_FALSE,
-            JSONValue::Null => JSON_NULL,
-        }
-    }
-    pub fn number(&self) -> f64 {
-        if let JSONValue::Number(n) = self { *n } else { 0.0 }
-    }
-    pub fn string(&self) -> &str {
-        if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
-    }
-    pub fn object_len(&self) -> usize {
-        if let JSONValue::Object(e) = self { e.len() } else { 0 }
-    }
-    pub fn array_len(&self) -> usize {
-        if let JSONValue::Array(e) = self { e.len() } else { 0 }
-    }
-}
->>
-
-context_user_fields <<
-    pub string_value: String,
->>
-
-ptype JSONValue;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
-    $$ = JSONValue::Number(n);
->>
-token true <<
-  $$ = JSONValue::True;
->>
-token false <<
-  $$ = JSONValue::False;
->>
-token null <<
-  $$ = JSONValue::Null;
->>
-/"/ <<
-  $mode(string);
-  ${context.string_value} = String::new();
->>
-string: token string /"/ <<
-  $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
-  $mode(default);
->>
-string: /\\"/ <<
-  ${context.string_value}.push('"');
->>
-string: /\\\\/ <<
-  ${context.string_value}.push('\\');
->>
-string: /\\\// <<
-  ${context.string_value}.push('/');
->>
-string: /\\b/ <<
-  ${context.string_value}.push('\u{0008}');
->>
-string: /\\f/ <<
-  ${context.string_value}.push('\u{000C}');
->>
-string: /\\n/ <<
-  ${context.string_value}.push('\n');
->>
-string: /\\r/ <<
-  ${context.string_value}.push('\r');
->>
-string: /\\t/ <<
-  ${context.string_value}.push('\t');
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
-  ${context.string_value}.push_str(&s);
->>
-string: /[^\\]/ <<
-  ${context.string_value}.push(match_text[0] as char);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONValue::Object(Vec::new());
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  let mut obj = $1;
-  if let JSONValue::Object(kve) = $3 {
-      if let JSONValue::Object(entries) = &mut obj {
-          entries.extend(kve);
-      }
-  }
-  $$ = obj;
->>
-KeyValue -> string colon Value <<
-  let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
-  $$ = JSONValue::Object(vec![(name, $3)]);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONValue::Array(Vec::new());
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  let mut arr = $1;
-  if let JSONValue::Array(elems) = &mut arr {
-      elems.push($3);
-  }
-  $$ = arr;
->>

spec/macros.c.propane

@@ -1,25 +0,0 @@
-<<
-#include <stdlib.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
-void record(int v);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; record($$); >>
-MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -1,31 +0,0 @@
-<<
-import test_macros;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ <<
-    int n = 0;
-    foreach (c; match_text)
-    {
-        n *= 10;
-        n += (c - '0');
-    }
-    $$ = n;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; record($$); >>
-MacroStart -> macro macroname lbrace;

spec/macros.rust.propane

@@ -1,80 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    loop {
-        if context.expanding {
-            let ei = context.expand_i;
-            context.expand_i += 1;
-            if context.expand_i >= context.token_infos.len() {
-                context.expanding = false;
-            }
-            *out_token_info = context.token_infos[ei].clone();
-            return P_SUCCESS;
-        }
-
-        let lex_result = p_lex(context, out_token_info);
-        if lex_result != P_SUCCESS {
-            return lex_result;
-        }
-
-        if out_token_info.token == TOKEN_macro {
-            context.defining = true;
-        } else if out_token_info.token == TOKEN_macroname {
-            if !context.defining {
-                context.expanding = true;
-                context.expand_i = 0;
-                continue;
-            }
-        } else if out_token_info.token == TOKEN_lbrace {
-            if context.defining {
-                /* Capture the macro body tokens (up to the closing '}'). */
-                let mut infos: Vec<p_token_info_t> = Vec::new();
-                loop {
-                    let mut ti = p_token_info_t::default();
-                    assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
-                    if ti.token == TOKEN_rbrace {
-                        break;
-                    }
-                    infos.push(ti);
-                }
-                context.token_infos = infos;
-                context.defining = false;
-            }
-        } else {
-            context.defining = false;
-        }
-        return lex_result;
-    }
-}
->>
-
-context_user_fields <<
-    pub defining: bool,
-    pub expanding: bool,
-    pub expand_i: usize,
-    pub token_infos: Vec<p_token_info_t>,
-    pub nums: Vec<i64>,
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
-MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -1,19 +0,0 @@
-<<
-#include <stdlib.h>
-#include <string.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -1,17 +0,0 @@
-<<
-import test_parse_inner_nested;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.rust.propane

@@ -1,41 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    let result = p_lex(context, out_token_info);
-    if result != P_SUCCESS {
-        return result;
-    }
-    if out_token_info.token == TOKEN_lparen {
-        /* Reentrant nested parse of the parenthesized sub-expression. */
-        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
-        if inner_result != P_SUCCESS {
-            return inner_result;
-        }
-        let value = p_result_Start(context);
-        /* p_parse_inner rewound the input so ')' was not consumed; consume it. */
-        let mut rparen_info = p_token_info_t::default();
-        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
-        assert_eq!(TOKEN_rparen, rparen_info.token);
-        out_token_info.token = TOKEN_num;
-        out_token_info.pvalue = p_value(value);
-    }
-    P_SUCCESS
-}
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -1,17 +0,0 @@
-<<
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -1,17 +0,0 @@
-<<
-import test_parse_inner_nested_tree;
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/parse_inner_nested_tree.rust.propane

@@ -1,44 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    let result = p_lex(context, out_token_info);
-    if result != P_SUCCESS {
-        return result;
-    }
-    if out_token_info.token == TOKEN_lparen {
-        let start_position = out_token_info.position;
-        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
-        if inner_result != P_SUCCESS {
-            return inner_result;
-        }
-        /* Read the inner subtree's span before re-borrowing context to lex. */
-        let inner = p_result_Start(context);
-        assert!(inner.valid());
-        let inner_start_col = inner.position().col;
-        let inner_end_col = inner.end_position().col;
-        let mut rparen_info = p_token_info_t::default();
-        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
-        assert_eq!(TOKEN_rparen, rparen_info.token);
-        assert_eq!(start_position.col + 1, inner_start_col);
-        assert_eq!(rparen_info.position.col - 1, inner_end_col);
-        /* Synthesize a num token spanning the whole "( ... )" group. */
-        out_token_info.token = TOKEN_num;
-        out_token_info.position = start_position;
-        out_token_info.end_position = rparen_info.end_position;
-    }
-    P_SUCCESS
-}
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/rewind.c.propane

@@ -1,23 +0,0 @@
-<<
-#include <stdlib.h>
-#include <string.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
-void record(int value);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -1,20 +0,0 @@
-<<
-import test_rewind;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/rewind.rust.propane

@@ -1,67 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    loop {
-        let result = p_lex(context, out_token_info);
-        if result != P_SUCCESS {
-            return result;
-        }
-        if out_token_info.token == TOKEN_repeat {
-            let mut count_info = p_token_info_t::default();
-            assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
-            assert_eq!(TOKEN_num, count_info.token);
-            let mut brace_info = p_token_info_t::default();
-            assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
-            assert_eq!(TOKEN_lbrace, brace_info.token);
-            context.remaining = p_value_get(&count_info.pvalue);
-            context.body_index = p_input_index(context);
-            context.body_position = p_position(context);
-            continue;
-        }
-        if out_token_info.token == TOKEN_rbrace {
-            if context.remaining > 1 {
-                context.remaining -= 1;
-                let bi = context.body_index;
-                let bp = context.body_position;
-                p_set_input_index(context, bi);
-                p_set_position(context, bp);
-                continue;
-            }
-            context.remaining = 0;
-            continue;
-        }
-        if out_token_info.token == TOKEN_num {
-            context.num_cols.push(out_token_info.position.col);
-        }
-        return result;
-    }
-}
->>
-
-context_user_fields <<
-    pub nums: Vec<i64>,
-    pub num_cols: Vec<u32>,
-    pub remaining: i64,
-    pub body_index: usize,
-    pub body_position: p_position_t,
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << ${context.nums}.push($1 + $3); >>

spec/spec_helper.rb

@@ -15,10 +15,6 @@ unless ENV["dist_specs"]
       command_name "RSpec"
     end
     project_name "Propane"
-    # Keep this process's results separate from the propane subprocess results
-    # so that nothing has to merge on the fly; the spec Rake task collates all
-    # of the parts once the suite is done.
-    coverage_dir "coverage/parts/rspec"
     merge_timeout 3600
     formatter(MyFormatter)
   end

spec/test_basic_math_grammar.rs

@@ -1,16 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let cases: [(&[u8], u64); 4] = [
-        (b"1 + 2 * 3 + 4", 11),
-        (b"1 * 2 ** 4 * 3", 48),
-        (b"(1 + 2) * 3 + 4", 13),
-        (b"(2 * 2) ** 3 + 4 + 5", 73),
-    ];
-    for (input, expected) in cases {
-        let mut context = p_context_new(input);
-        assert_eq!(P_SUCCESS, p_parse(&mut context));
-        assert_eq!(expected, p_result(&context));
-        p_context_delete(context);
-    }
-}

spec/test_context_field_handle.c

@@ -1,39 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char input[128];
-    size_t i;
-    p_context_t * context;
-    Token token;
-
-    /* Enough tokens that the tree node arena is reallocated during the parse. */
-    memset(input, 0, sizeof(input));
-    for (i = 0u; i < 40u; i++)
-    {
-        input[i] = 'a';
-    }
-
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1u, context->have_first);
-    assert(p_node_valid(context->first_item));
-    token = p_Item_pToken1(context->first_item);
-    assert(p_node_valid(token));
-    assert_eq(TOKEN_a, p_Token_token(token));
-    assert_eq(7u, p_Token_pvalue(token));
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, p_node_position(context->first_item).row);
-    assert_eq(1u, p_node_position(context->first_item).col);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_context_field_handle.cpp

@@ -1,36 +0,0 @@
-#include "testparser.h"
-#include <cassert>
-#include <cstring>
-#include "testutils.h"
-
-int main()
-{
-    char input[128];
-
-    /* Enough tokens that the tree node arena is reallocated during the parse. */
-    memset(input, 0, sizeof(input));
-    for (size_t i = 0u; i < 40u; i++)
-    {
-        input[i] = 'a';
-    }
-
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1u, context->have_first);
-    assert(context->first_item.valid());
-    Token token = context->first_item.pToken1();
-    assert(token.valid());
-    assert_eq(TOKEN_a, token.token());
-    assert_eq(7u, token.pvalue());
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, context->first_item.position().row);
-    assert_eq(1u, context->first_item.position().col);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_context_field_handle.d

@@ -1,35 +0,0 @@
-import testparser;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* Enough tokens that the tree node array is reallocated during the parse. */
-    string input;
-    foreach (i; 0 .. 40)
-    {
-        input ~= "a";
-    }
-
-    p_context_t * context = p_context_new(input);
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1, context.have_first);
-    assert(context.first_item.valid);
-    Token token = context.first_item.pToken1;
-    assert(token.valid);
-    assert_eq(TOKEN_a, token.token);
-    assert_eq(7, token.pvalue);
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, context.first_item.position.row);
-    assert_eq(1u, context.first_item.position.col);
-
-    p_context_delete(context);
-}

spec/test_custom_lex_fn.rs

@@ -1,8 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"cbacba");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(0x932187932187, p_result(&c));
-    p_context_delete(c);
-}

spec/test_drop_code_block.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"    # comment 1\n#    comment 2\na\n");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_error_positions.rs

@@ -1,35 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"a 42");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a\n123\na  a");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(4, p_position(&c).col);
-    assert_eq!(TOKEN_a, p_token(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"12");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
-    assert_eq!(1, p_position(&c).row);
-    assert_eq!(1, p_position(&c).col);
-    assert_eq!(TOKEN_num, p_token(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a 12\n\nab");
-    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(2, p_position(&c).col);
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a 12\n\na\n\n77\na   \xAA");
-    assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
-    assert_eq!(6, p_position(&c).row);
-    assert_eq!(5, p_position(&c).col);
-    assert_eq!("a", p_token_names[TOKEN_a as usize]);
-    assert_eq!("num", p_token_names[TOKEN_num as usize]);
-    p_context_delete(c);
-}

spec/test_field_aliases.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"foo1\nbar2");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_input_index.c

@@ -1,60 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar (simple):
-     *   drop /\\s+/;
-     *   token a; token b;
-     *   Start -> a b;
-     *
-     * Verifies that p_input_index() reports the parser/lexer's current byte
-     * offset into the input text. */
-
-    /* Fresh context: input_index starts at 0. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        assert_eq(0u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* After each successful lex the byte offset advances past the token. */
-    {
-        char const * input = "a b";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(1u, p_input_index(context));
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        /* The dropped space between `a` and `b` advances input_index too. */
-        assert_eq(3u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* After a full successful parse, input_index has reached the end. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        assert(p_parse_Start(context) == P_SUCCESS);
-        assert_eq(2u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* When parse_inner completes via a follow token, the follow token is not
-     * consumed, so input_index points at the start of the follow token. */
-    {
-        char const * input = "abb";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(2u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_input_index.d

@@ -1,51 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_input_index.c for details on the grammar and cases. */
-
-    /* Fresh context: input_index starts at 0. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        assert(p_input_index(context) == 0);
-    }
-
-    /* After each successful lex the byte offset advances past the token. */
-    {
-        string input = "a b";
-        p_context_t * context = p_context_new(input);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(p_input_index(context) == 1);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(p_input_index(context) == 3);
-    }
-
-    /* After a full successful parse, input_index has reached the end. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        assert(p_parse_Start(context) == P_SUCCESS);
-        assert(p_input_index(context) == 2);
-    }
-
-    /* When parse_inner completes via a follow token, the follow token is not
-     * consumed, so input_index points at the start of the follow token. */
-    {
-        string input = "abb";
-        p_context_t * context = p_context_new(input);
-        p_token_t[] follow_tokens = [TOKEN_b];
-        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
-        assert(p_input_index(context) == 2);
-    }
-}

spec/test_input_index.rs

@@ -1,28 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let c = p_context_new(b"ab");
-    assert_eq!(0, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a b");
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(1, p_input_index(&c));
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(3, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    assert_eq!(2, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    let follow = [TOKEN_b];
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
-    assert_eq!(2, p_input_index(&c));
-    p_context_delete(c);
-}

spec/test_lexer.rs

@@ -1,49 +0,0 @@
-use testparser::*;
-
-fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
-    assert_eq!(row, ti.position.row);
-    assert_eq!(col, ti.position.col);
-    assert_eq!(erow, ti.end_position.row);
-    assert_eq!(ecol, ti.end_position.col);
-    assert_eq!(len, ti.length);
-    assert_eq!(token, ti.token);
-}
-
-fn main() {
-    let mut cp: p_code_point_t = 0;
-    let mut cpl: u8 = 0;
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
-    assert_eq!('5' as u32, cp);
-    assert_eq!(1, cpl);
-
-    assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
-    assert_eq!(0xA9, cp);
-    assert_eq!(2, cpl);
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
-    assert_eq!(0x1F9E1, cp);
-    assert_eq!(4, cpl);
-
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
-
-    let mut context = p_context_new(b"5 + 4 * \n677 + 567");
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"");
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
-    p_context_delete(context);
-}

spec/test_lexer_match_text.rs

@@ -1,8 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"identifier_123");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-}

spec/test_lexer_modes.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"abc \"a string\" def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abc \"abc def\" def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass2");
-    p_context_delete(context);
-}

spec/test_lexer_multiple_modes.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"abc.def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abc .  abc");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass2");
-    p_context_delete(context);
-}

spec/test_lexer_positions.rs

@@ -1,38 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"abc\n  defg hi\n!");
-    let mut ti = p_token_info_t::default();
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(1, c.last_start.row);
-    assert_eq!(1, c.last_start.col);
-    assert_eq!(1, c.last_end.row);
-    assert_eq!(3, c.last_end.col);
-    assert_eq!(c.last_start.row, ti.position.row);
-    assert_eq!(c.last_start.col, ti.position.col);
-    assert_eq!(c.last_end.row, ti.end_position.row);
-    assert_eq!(c.last_end.col, ti.end_position.col);
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(2, c.last_start.row);
-    assert_eq!(3, c.last_start.col);
-    assert_eq!(2, c.last_end.row);
-    assert_eq!(6, c.last_end.col);
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(2, c.last_start.row);
-    assert_eq!(8, c.last_start.col);
-    assert_eq!(2, c.last_end.row);
-    assert_eq!(9, c.last_end.col);
-
-    assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
-    assert_eq!(42, p_user_terminate_code(&c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(1, p_position(&c).col);
-
-    p_context_delete(c);
-}

spec/test_lexer_result_value.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"x");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(1, p_result(&context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"fabulous");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(8, p_result(&context));
-    p_context_delete(context);
-}

spec/test_lexer_unknown_character.rs

@@ -1,12 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"x");
-    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"123");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(123, p_result(&context));
-    p_context_delete(context);
-}

spec/test_macros.c

@@ -1,118 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-#include <assert.h>
-#include <stddef.h>
-#include <stdbool.h>
-
-static p_context_t * context;
-size_t n_tokens;
-p_token_info_t token_infos[10];
-
-/* Capture the macro body tokens (everything up to the closing '}') into
- * token_infos[]. Called from mylexfn() right after the definition's '{' has
- * been lexed, so the input cursor is positioned at the first body token. */
-static void capture_macro_body(void)
-{
-    n_tokens = 0u;
-    for (;;)
-    {
-        size_t result = p_lex(context, &token_infos[n_tokens]);
-        assert_eq(result, P_SUCCESS);
-        if (token_infos[n_tokens].token == TOKEN_rbrace)
-        {
-            break;
-        }
-        n_tokens++;
-        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
-    }
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static bool defining;
-    static bool expanding;
-    static size_t expand_i;
-
-    for (;;)
-    {
-        if (expanding)
-        {
-            size_t ei = expand_i++;
-            if (expand_i >= n_tokens)
-            {
-                expanding = false;
-            }
-            *out_token_info = token_infos[ei];
-            return P_SUCCESS;
-        }
-
-        size_t lex_result = p_lex(context, out_token_info);
-        if (lex_result != P_SUCCESS)
-        {
-            return lex_result;
-        }
-
-        switch (out_token_info->token)
-        {
-        case TOKEN_macro:
-            /* Start of a macro definition: "macro macroname { ... }". */
-            defining = true;
-            break;
-        case TOKEN_macroname:
-            if (!defining)
-            {
-                /* Use of a macro: replay its captured body tokens instead of
-                 * returning the macroname to the parser. */
-                expanding = true;
-                expand_i = 0u;
-                continue;
-            }
-            /* Definition name: pass through and keep waiting for '{'. */
-            break;
-        case TOKEN_lbrace:
-            if (defining)
-            {
-                /* Consume and store the macro body now, before the parser gets
-                 * a chance to read its lookahead token (which would otherwise
-                 * swallow the first body token). */
-                capture_macro_body();
-                defining = false;
-            }
-            break;
-        default:
-            defining = false;
-            break;
-        }
-        return lex_result;
-    }
-}
-
-size_t n_nums;
-int nums[10];
-
-void record(int v)
-{
-    nums[n_nums++] = v;
-}
-
-int main()
-{
-    char const * input =
-        "macro @m { 23 + 200 }\n"
-        "66 + 100\n"
-        "@m\n"
-        "33 + 55\n"
-        "@m\n";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    assert_eq(n_nums, 4);
-    assert_eq(nums[0], 166);
-    assert_eq(nums[1], 223);
-    assert_eq(nums[2], 88);
-    assert_eq(nums[3], 223);
-
-    return 0;
-}

spec/test_macros.d

@@ -1,116 +0,0 @@
-import testparser;
-import testutils;
-
-size_t n_tokens;
-p_token_info_t[10] token_infos;
-
-// Capture the macro body tokens (everything up to the closing '}') into
-// token_infos[]. Called from mylexfn() right after the definition's '{' has
-// been lexed, so the input cursor is positioned at the first body token.
-void capture_macro_body(p_context_t * context)
-{
-    n_tokens = 0u;
-    for (;;)
-    {
-        size_t result = p_lex(context, &token_infos[n_tokens]);
-        assert(result == P_SUCCESS);
-        if (token_infos[n_tokens].token == TOKEN_rbrace)
-        {
-            break;
-        }
-        n_tokens++;
-        assert(n_tokens < token_infos.length);
-    }
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static bool defining;
-    static bool expanding;
-    static size_t expand_i;
-
-    for (;;)
-    {
-        if (expanding)
-        {
-            size_t ei = expand_i++;
-            if (expand_i >= n_tokens)
-            {
-                expanding = false;
-            }
-            *out_token_info = token_infos[ei];
-            return P_SUCCESS;
-        }
-
-        size_t lex_result = p_lex(context, out_token_info);
-        if (lex_result != P_SUCCESS)
-        {
-            return lex_result;
-        }
-
-        switch (out_token_info.token)
-        {
-        case TOKEN_macro:
-            // Start of a macro definition: "macro macroname { ... }".
-            defining = true;
-            break;
-        case TOKEN_macroname:
-            if (!defining)
-            {
-                // Use of a macro: replay its captured body tokens instead of
-                // returning the macroname to the parser.
-                expanding = true;
-                expand_i = 0u;
-                continue;
-            }
-            // Definition name: pass through and keep waiting for '{'.
-            break;
-        case TOKEN_lbrace:
-            if (defining)
-            {
-                // Consume and store the macro body now, before the parser gets
-                // a chance to read its lookahead token (which would otherwise
-                // swallow the first body token).
-                capture_macro_body(context);
-                defining = false;
-            }
-            break;
-        default:
-            defining = false;
-            break;
-        }
-        return lex_result;
-    }
-}
-
-size_t n_nums;
-int[10] nums;
-
-void record(int v)
-{
-    nums[n_nums++] = v;
-}
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input =
-        "macro @m { 23 + 200 }\n" ~
-        "66 + 100\n" ~
-        "@m\n" ~
-        "33 + 55\n" ~
-        "@m\n";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    assert(n_nums == 4);
-    assert(nums[0] == 166);
-    assert(nums[1] == 223);
-    assert(nums[2] == 88);
-    assert(nums[3] == 223);
-}

spec/test_macros.rs

@@ -1,9 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n";
-    let mut c = p_context_new(input);
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(vec![166, 223, 88, 223], c.nums);
-    p_context_delete(c);
-}

spec/test_match_backslashes.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_multiple_parsers.rs

@@ -1,12 +0,0 @@
-use testparsermyp1 as m1;
-use testparsermyp2 as m2;
-
-fn main() {
-    let mut context1 = m1::myp1_context_new(b"a\n1");
-    assert_eq!(m1::MYP1_SUCCESS, m1::myp1_parse(&mut context1));
-    m1::myp1_context_delete(context1);
-
-    let mut context2 = m2::myp2_context_new(b"bcb");
-    assert_eq!(m2::MYP2_SUCCESS, m2::myp2_parse(&mut context2));
-    m2::myp2_context_delete(context2);
-}

spec/test_named_optional_rule_component_tree.c

@@ -9,42 +9,46 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start start = p_result(context);
-    assert(!p_node_valid(p_Start_a(start)));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
-    assert(!p_node_valid(p_Start_pR3(start)));
-    assert(!p_node_valid(p_Start_pR(start)));
-    assert(!p_node_valid(p_Start_r(start)));
+    Start * start = p_result(context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+    assert(start->r == NULL);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(p_node_valid(p_Start_a(start)));
-    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert(p_node_valid(p_Start_pR3(start)));
-    assert(p_node_valid(p_Start_pR(start)));
-    assert(p_node_valid(p_Start_r(start)));
-    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
-    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_r(start)));
-    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
+    assert(start->a != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->r != NULL);
+    assert(start->pR == start->pR3);
+    assert(start->pR == start->r);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(!p_node_valid(p_Start_a(start)));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert(p_node_valid(p_Start_r(start)));
-    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->r != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
+

spec/test_named_optional_rule_component_tree.d

@@ -12,40 +12,40 @@ unittest
     string input = "b";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start start = p_result(context);
-    assert(!start.pToken1.valid);
-    assert(start.pToken2.valid);
+    Start * start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(!start.pR3.valid);
-    assert(!start.pR.valid);
-    assert(!start.r.valid);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+    assert(start.r is null);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1.valid);
+    assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2.valid);
-    assert(start.pR3.valid);
-    assert(start.pR.valid);
-    assert(start.r.valid);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.r != null);
     assert(start.pR == start.pR3);
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(!start.pToken1.valid);
-    assert(start.pToken2.valid);
-    assert(start.pR.valid);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 }

spec/test_named_optional_rule_component_tree.rs

@@ -1,43 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"b");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(!start.a().valid());
-        assert!(start.pToken2().valid());
-        assert_eq!(TOKEN_b, start.pToken2().token());
-        assert!(!start.pR3().valid());
-        assert!(!start.pR().valid());
-        assert!(!start.r().valid());
-    }
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abcd");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(start.a().valid());
-        assert_eq!(TOKEN_a, start.pToken1().token());
-        assert!(start.pToken2().valid());
-        assert!(start.pR3().valid());
-        assert!(start.pR().valid());
-        assert!(start.r().valid());
-        assert_eq!(start.pR().node_id(), start.pR3().node_id());
-        assert_eq!(start.pR().node_id(), start.r().node_id());
-        assert_eq!(TOKEN_c, start.pR().pToken1().token());
-    }
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"bdc");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(!start.a().valid());
-        assert!(start.pToken2().valid());
-        assert!(start.r().valid());
-        assert_eq!(TOKEN_d, start.pR().pToken1().token());
-    }
-    p_context_delete(context);
-}

spec/test_optional_rule_component.rs

@@ -1,9 +0,0 @@
-use testparser::*;
-
-fn main() {
-    for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] {
-        let mut context = p_context_new(input);
-        assert_eq!(P_SUCCESS, p_parse(&mut context));
-        p_context_delete(context);
-    }
-}

spec/test_optional_rule_component_tree.c

@@ -9,39 +9,43 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
-    Start start = p_result(context);
-    assert(!p_node_valid(p_Start_pToken1(start)));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
-    assert(!p_node_valid(p_Start_pR3(start)));
-    assert(!p_node_valid(p_Start_pR(start)));
+    Start * start = p_result(context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     input = "abcd";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(p_node_valid(p_Start_pToken1(start)));
-    assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert(p_node_valid(p_Start_pR3(start)));
-    assert(p_node_valid(p_Start_pR(start)));
-    assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
-    assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
+    assert(start->pToken1 != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->pR == start->pR3);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     input = "bdc";
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(!p_node_valid(p_Start_pToken1(start)));
-    assert(p_node_valid(p_Start_pToken2(start)));
-    assert(p_node_valid(p_Start_pR(start)));
-    assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->pR != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;
 }
+

spec/test_optional_rule_component_tree.d

@@ -12,38 +12,37 @@ unittest
     string input = "b";
     p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
-    Start start = p_result(context);
-    assert(!start.pToken1.valid);
-    assert(start.pToken2.valid);
+    Start * start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
-    assert(!start.pR3.valid);
-    assert(!start.pR.valid);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(start.pToken1.valid);
+    assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2.valid);
-    assert(start.pR3.valid);
-    assert(start.pR.valid);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
     assert(start.pR == start.pR3);
-    assert_eq(start.pR.node_id, start.pR3.node_id);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     start = p_result(context);
-    assert(!start.pToken1.valid);
-    assert(start.pToken2.valid);
-    assert(start.pR.valid);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
 
-    p_context_delete(context);
+    p_tree_delete(start);
 }

spec/test_optional_rule_component_tree.rs

@@ -1,40 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"b");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(!start.pToken1().valid());
-        assert!(start.pToken2().valid());
-        assert_eq!(TOKEN_b, start.pToken2().token());
-        assert!(!start.pR3().valid());
-        assert!(!start.pR().valid());
-    }
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abcd");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(start.pToken1().valid());
-        assert_eq!(TOKEN_a, start.pToken1().token());
-        assert!(start.pToken2().valid());
-        assert!(start.pR3().valid());
-        assert!(start.pR().valid());
-        assert_eq!(start.pR().node_id(), start.pR3().node_id());
-        assert_eq!(TOKEN_c, start.pR().pToken1().token());
-    }
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"bdc");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let start = p_result(&context);
-        assert!(!start.pToken1().valid());
-        assert!(start.pToken2().valid());
-        assert!(start.pR().valid());
-        assert_eq!(TOKEN_d, start.pR().pToken1().token());
-    }
-    p_context_delete(context);
-}

spec/test_parse_inner.c

@@ -1,73 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar (chain reduce):
-     *   Start -> Y      << $$ = $1; >>
-     *   Y     -> a      << $$ = $1; >>
-     *   token a         << $$ = 1; >>
-     *
-     * The reduce lookahead for both `Y -> a` and `Start -> Y` is only $EOF,
-     * so `p_parse_Start("ab")` fails at token `b`. p_parse_inner_Start with
-     * `b` as a follow token should succeed via the reduce-side retry chain
-     * (Y then Start) followed by the shift-side retry hitting $EOF at the
-     * final state. */
-
-    /* Standard parse succeeds on complete input. */
-    char const * input = "a";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert_eq(1u, (size_t)p_result_Start(context));
-    p_context_delete(context);
-
-    /* Standard parse fails when there's an unexpected trailing token. */
-    input = "ab";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-    p_context_delete(context);
-
-    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
-     * followed by the shift-side retry hitting $EOF at the final state. */
-    {
-        input = "ab";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(1u, (size_t)p_result_Start(context));
-        p_context_delete(context);
-    }
-
-    /* parse_inner with an empty (NULL) follow-token vector behaves like a
-     * standard parse. */
-    input = "ab";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_inner_Start(context, NULL, 0u) == P_UNEXPECTED_TOKEN);
-    p_context_delete(context);
-
-    /* parse_inner behaves like a standard parse when the input matches the
-     * grammar fully. */
-    input = "a";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    {
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(1u, (size_t)p_result_Start(context));
-    }
-    p_context_delete(context);
-
-    /* parse_inner with a non-matching follow token still fails. The grammar
-     * can't consume `b` and it isn't listed as a follow token, so the retries
-     * do not fire. */
-    {
-        input = "ab";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN___EOF };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_UNEXPECTED_TOKEN);
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_parse_inner.d

@@ -1,51 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_parse_inner.c for details on the grammar and cases. */
-
-    /* Standard parse succeeds on complete input. */
-    string input = "a";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert(p_result_Start(context) == 1);
-
-    /* Standard parse fails when there's an unexpected trailing token. */
-    input = "ab";
-    context = p_context_new(input);
-    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-
-    /* parse_inner succeeds via a chain of reduce retries (Y, then Start),
-     * followed by the shift-side retry hitting $EOF at the final state. */
-    input = "ab";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_b = [TOKEN_b];
-    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
-    assert(p_result_Start(context) == 1);
-
-    /* parse_inner with a null follow-token slice behaves like a standard
-     * parse. */
-    input = "ab";
-    context = p_context_new(input);
-    assert(p_parse_inner_Start(context, null) == P_UNEXPECTED_TOKEN);
-
-    /* parse_inner behaves like a standard parse when the input matches the
-     * grammar fully. */
-    input = "a";
-    context = p_context_new(input);
-    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
-    assert(p_result_Start(context) == 1);
-
-    /* parse_inner with a non-matching follow token still fails. */
-    input = "ab";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_eof = [TOKEN___EOF];
-    assert(p_parse_inner_Start(context, follow_tokens_eof) == P_UNEXPECTED_TOKEN);
-}

spec/test_parse_inner.rs

@@ -1,30 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"a");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    assert_eq!(1, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
-    assert_eq!(1, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[]));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a");
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
-    assert_eq!(1, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF]));
-    p_context_delete(c);
-}

spec/test_parse_inner_nested.c

@@ -1,78 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-/* Grammar (integer evaluator; parentheses handled by the lex function):
- *   ptype int;
- *   lex_fn mylexfn;
- *   token lparen /\(/; token rparen /\)/; token plus /\+/;
- *   token num /\d+/          << ... atoi ... >>
- *   Start -> Expr            << $$ = $1; >>
- *   Expr  -> num             << $$ = $1; >>
- *   Expr  -> Expr plus num   << $$ = $1 + $3; >>
- *
- * The tokens lparen and rparen appear in no grammar rule. Instead, when the
- * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start)
- * of the parenthesized sub-expression -- reentrantly, while the outer parse is
- * still suspended in this callback -- reads the computed value with
- * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the
- * input, and hands a single synthesized num token carrying that value back to
- * the outer parse. Nested groups recurse this process to arbitrary depth. */
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    size_t result = p_lex(context, out_token_info);
-    if (result != P_SUCCESS)
-    {
-        return result;
-    }
-    if (out_token_info->token == TOKEN_lparen)
-    {
-        /* Nested parse of the parenthesized sub-expression, stopping at the
-         * closing ')' follow token. This re-enters the parser while the outer
-         * parse is suspended in this lex callback. */
-        p_token_t follow_tokens[] = { TOKEN_rparen };
-        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
-        if (inner_result != P_SUCCESS)
-        {
-            return inner_result;
-        }
-        int value = p_result_Start(context);
-        /* p_parse_inner rewound the input so that ')' was not consumed; consume
-         * it now. */
-        p_token_info_t rparen_info;
-        size_t rparen_result = p_lex(context, &rparen_info);
-        assert(rparen_result == P_SUCCESS);
-        assert(rparen_info.token == TOKEN_rparen);
-        /* Replace the '(' token with a synthesized num carrying the nested
-         * parse result. */
-        out_token_info->token = TOKEN_num;
-        out_token_info->pvalue = p_value(value);
-    }
-    return P_SUCCESS;
-}
-
-static int eval(char const * input)
-{
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    int value = p_result(context);
-    p_context_delete(context);
-    return value;
-}
-
-int main()
-{
-    /* No parentheses: plain outer parse. */
-    assert_eq(5u, (size_t)eval("2 + 3"));
-    /* A single group evaluated by the nested parse. */
-    assert_eq(3u, (size_t)eval("(1 + 2)"));
-    /* A group in the middle of an outer expression. */
-    assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5"));
-    /* Nested groups: the nested parse re-enters itself. */
-    assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))"));
-    assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))"));
-
-    return 0;
-}

spec/test_parse_inner_nested.d

@@ -1,64 +0,0 @@
-import testparser;
-import testutils;
-
-/* Grammar: see test_parse_inner_nested.c. */
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    size_t result = p_lex(context, out_token_info);
-    if (result != P_SUCCESS)
-    {
-        return result;
-    }
-    if (out_token_info.token == TOKEN_lparen)
-    {
-        /* Nested parse of the parenthesized sub-expression, stopping at the
-         * closing ')' follow token. This re-enters the parser while the outer
-         * parse is suspended in this lex callback. */
-        p_token_t[] follow_tokens = [TOKEN_rparen];
-        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
-        if (inner_result != P_SUCCESS)
-        {
-            return inner_result;
-        }
-        int value = p_result_Start(context);
-        /* p_parse_inner rewound the input so that ')' was not consumed; consume
-         * it now. */
-        p_token_info_t rparen_info;
-        size_t rparen_result = p_lex(context, &rparen_info);
-        assert(rparen_result == P_SUCCESS);
-        assert(rparen_info.token == TOKEN_rparen);
-        /* Replace the '(' token with a synthesized num carrying the nested
-         * parse result. */
-        out_token_info.token = TOKEN_num;
-        out_token_info.pvalue = p_value(value);
-    }
-    return P_SUCCESS;
-}
-
-int eval(string input)
-{
-    p_context_t * context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    int value = p_result(context);
-    p_context_delete(context);
-    return value;
-}
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* No parentheses: plain outer parse. */
-    assert_eq(5, eval("2 + 3"));
-    /* A single group evaluated by the nested parse. */
-    assert_eq(3, eval("(1 + 2)"));
-    /* A group in the middle of an outer expression. */
-    assert_eq(14, eval("2 + (3 + 4) + 5"));
-    /* Nested groups: the nested parse re-enters itself. */
-    assert_eq(37, eval("2 + (10 + (20 + 5))"));
-    assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))"));
-}

spec/test_parse_inner_nested.rs

@@ -1,17 +0,0 @@
-use testparser::*;
-
-fn eval(input: &[u8]) -> i64 {
-    let mut c = p_context_new(input);
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    let v = p_result(&c);
-    p_context_delete(c);
-    v
-}
-
-fn main() {
-    assert_eq!(5, eval(b"2 + 3"));
-    assert_eq!(3, eval(b"(1 + 2)"));
-    assert_eq!(14, eval(b"2 + (3 + 4) + 5"));
-    assert_eq!(37, eval(b"2 + (10 + (20 + 5))"));
-    assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))"));
-}

spec/test_parse_inner_nested_tree.c

@@ -1,91 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-/* Grammar: see the D variant / spec. Tree generation mode; parentheses handled
- * by the lex function. Tree nodes live in the context arena. */
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    size_t result = p_lex(context, out_token_info);
-    if (result != P_SUCCESS)
-    {
-        return result;
-    }
-    if (out_token_info->token == TOKEN_lparen)
-    {
-        p_position_t start_position = out_token_info->position;
-        /* Reentrant nested parse of the parenthesized sub-expression. */
-        p_token_t follow_tokens[] = { TOKEN_rparen };
-        size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u);
-        if (inner_result != P_SUCCESS)
-        {
-            return inner_result;
-        }
-        Start inner = p_result_Start(context);
-        assert(p_node_valid(inner));
-        /* p_parse_inner rewound the input so that ')' was not consumed; consume
-         * it now. */
-        p_token_info_t rparen_info;
-        size_t rparen_result = p_lex(context, &rparen_info);
-        assert(rparen_result == P_SUCCESS);
-        assert(rparen_info.token == TOKEN_rparen);
-        /* The subtree covers the region strictly between the parentheses. */
-        assert_eq((size_t)(start_position.col + 1u), (size_t)p_node_position(inner).col);
-        assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)p_node_end_position(inner).col);
-        /* The inner subtree is discarded (the lexer synthesizes a num token in
-         * its place), but its nodes remain in the shared context arena and are
-         * freed with the context. */
-        out_token_info->token = TOKEN_num;
-        out_token_info->position = start_position;
-        out_token_info->end_position = rparen_info.end_position;
-    }
-    return P_SUCCESS;
-}
-
-int main()
-{
-    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
-     * lexer into a single num token spanning its group. */
-    char const * input = "(3 + 4) + (5 + 6)";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-
-    Start tree = p_result(context);
-    assert(p_node_valid(tree));
-
-    /* Start -> Expr, where the top Expr is "Expr plus num". */
-    Expr top = p_Start_pExpr(tree);
-    assert(p_node_valid(top));
-    assert(p_node_valid(p_Expr_pExpr(top)));
-    assert(p_node_valid(p_Expr_pToken2(top)));
-    assert(p_node_valid(p_Expr_pToken3(top)));
-
-    /* The '+' joining the two groups is at column 9. */
-    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken2(top)).row);
-    assert_eq(9u, (size_t)p_node_position(p_Expr_pToken2(top)).col);
-
-    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
-    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken3(top)).row);
-    assert_eq(11u, (size_t)p_node_position(p_Expr_pToken3(top)).col);
-    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken3(top)).row);
-    assert_eq(17u, (size_t)p_node_end_position(p_Expr_pToken3(top)).col);
-
-    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
-     * columns 1..7. */
-    Expr left = p_Expr_pExpr(top);
-    assert(p_node_valid(p_Expr_pToken1(left)));
-    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).row);
-    assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).col);
-    assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken1(left)).row);
-    assert_eq(7u, (size_t)p_node_end_position(p_Expr_pToken1(left)).col);
-
-    /* The whole tree spans columns 1..17. */
-    assert_eq(1u, (size_t)p_node_position(tree).col);
-    assert_eq(17u, (size_t)p_node_end_position(tree).col);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_parse_inner_nested_tree.d

@@ -1,92 +0,0 @@
-import testparser;
-import testutils;
-
-/* Grammar: see test_parse_inner_nested_tree.c. */
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    size_t result = p_lex(context, out_token_info);
-    if (result != P_SUCCESS)
-    {
-        return result;
-    }
-    if (out_token_info.token == TOKEN_lparen)
-    {
-        p_position_t start_position = out_token_info.position;
-        /* Reentrant nested parse of the parenthesized sub-expression. */
-        p_token_t[] follow_tokens = [TOKEN_rparen];
-        size_t inner_result = p_parse_inner_Start(context, follow_tokens);
-        if (inner_result != P_SUCCESS)
-        {
-            return inner_result;
-        }
-        Start inner = p_result_Start(context);
-        assert(inner.valid);
-        /* p_parse_inner rewound the input so that ')' was not consumed; consume
-         * it now. */
-        p_token_info_t rparen_info;
-        size_t rparen_result = p_lex(context, &rparen_info);
-        assert(rparen_result == P_SUCCESS);
-        assert(rparen_info.token == TOKEN_rparen);
-        /* The subtree covers the region strictly between the parentheses. */
-        assert_eq(start_position.col + 1u, inner.position.col);
-        assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
-        /* The inner subtree is discarded (the lexer synthesizes a num token in
-         * its place), but its nodes remain in the shared context arena and are
-         * freed with the context. */
-        /* Synthesize a num token spanning the entire "( ... )" group. */
-        out_token_info.token = TOKEN_num;
-        out_token_info.position = start_position;
-        out_token_info.end_position = rparen_info.end_position;
-    }
-    return P_SUCCESS;
-}
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the
-     * lexer into a single num token spanning its group. */
-    string input = "(3 + 4) + (5 + 6)";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-
-    Start tree = p_result(context);
-    assert(tree.valid);
-
-    /* Start -> Expr, where the top Expr is "Expr plus num". */
-    Expr top = tree.pExpr;
-    assert(top.valid);
-    assert(top.pExpr.valid);
-    assert(top.pToken2.valid);
-    assert(top.pToken3.valid);
-
-    /* The '+' joining the two groups is at column 9. */
-    assert_eq(1u, top.pToken2.position.row);
-    assert_eq(9u, top.pToken2.position.col);
-
-    /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
-    assert_eq(1u, top.pToken3.position.row);
-    assert_eq(11u, top.pToken3.position.col);
-    assert_eq(1u, top.pToken3.end_position.row);
-    assert_eq(17u, top.pToken3.end_position.col);
-
-    /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
-     * columns 1..7. */
-    Expr left = top.pExpr;
-    assert(left.pToken1.valid);
-    assert_eq(1u, left.pToken1.position.row);
-    assert_eq(1u, left.pToken1.position.col);
-    assert_eq(1u, left.pToken1.end_position.row);
-    assert_eq(7u, left.pToken1.end_position.col);
-
-    /* The whole tree spans columns 1..17. */
-    assert_eq(1u, tree.position.col);
-    assert_eq(17u, tree.end_position.col);
-
-    p_context_delete(context);
-}

spec/test_parse_inner_nested_tree.rs

@@ -1,36 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"(3 + 4) + (5 + 6)");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    {
-        let tree = p_result(&c);
-        assert!(tree.valid());
-
-        /* Start -> Expr, where the top Expr is "Expr plus num". */
-        let top = tree.pExpr();
-        assert!(top.valid());
-        assert!(top.pExpr().valid());
-        assert!(top.pToken2().valid());
-        assert!(top.pToken3().valid());
-
-        /* The '+' joining the two groups is at column 9. */
-        assert_eq!(1, top.pToken2().position().row);
-        assert_eq!(9, top.pToken2().position().col);
-
-        /* Right operand: synthesized num for "(5 + 6)", columns 11..17. */
-        assert_eq!(11, top.pToken3().position().col);
-        assert_eq!(17, top.pToken3().end_position().col);
-
-        /* Left operand: synthesized num for "(3 + 4)", columns 1..7. */
-        let left = top.pExpr();
-        assert!(left.pToken1().valid());
-        assert_eq!(1, left.pToken1().position().col);
-        assert_eq!(7, left.pToken1().end_position().col);
-
-        /* The whole tree spans columns 1..17. */
-        assert_eq!(1, tree.position().col);
-        assert_eq!(17, tree.end_position().col);
-    }
-    p_context_delete(c);
-}

spec/test_parse_inner_recursive.c

@@ -1,77 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar (recursive):
-     *   Start -> a Start b   << $$ = $2; >>
-     *   Start -> c           << $$ = $1; >>
-     *   token a << $$ = 1; >>
-     *   token b << $$ = 2; >>
-     *   token c << $$ = 3; >>
-     *
-     * Here `Start` can appear in the middle of another `Start` rule, so the
-     * inner-parse follow-token success must be blocked whenever an unfinished
-     * outer `Start -> a Start b` remains on the parse stack (i.e. the parse
-     * stack contains more than just the initial state and the reduced start
-     * rule set). */
-
-    /* Standard parse of `c` succeeds. */
-    char const * input = "c";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert_eq(3u, (size_t)p_result_Start(context));
-    p_context_delete(context);
-
-    /* Standard parse of `acb` succeeds (full outer rule). */
-    input = "acb";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert_eq(3u, (size_t)p_result_Start(context));
-    p_context_delete(context);
-
-    /* Standard parse of `ac` fails (`b` missing). */
-    input = "ac";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-    p_context_delete(context);
-
-    /* parse_inner with `ac` and follow token `b` also fails: even though the
-     * inner `Start -> c` reduces and `Start` is shifted, the outer
-     * `Start -> a Start . b` is still on the stack (stack length > 2), so the
-     * "reduced start rule is the only thing on the parse stack" invariant
-     * blocks the shift-side follow-token success. */
-    {
-        input = "ac";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b, TOKEN___EOF };
-        assert(p_parse_inner_Start(context, follow_tokens, 2u) == P_UNEXPECTED_TOKEN);
-        p_context_delete(context);
-    }
-
-    /* parse_inner with `acb` (complete outer rule) succeeds via the standard
-     * path. */
-    {
-        input = "acb";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(3u, (size_t)p_result_Start(context));
-        p_context_delete(context);
-    }
-
-    /* parse_inner with just `c` succeeds via the standard path even when a
-     * follow-token vector is supplied. */
-    {
-        input = "c";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(3u, (size_t)p_result_Start(context));
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_parse_inner_recursive.d

@@ -1,49 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_parse_inner_recursive.c for details on the grammar. */
-
-    /* Standard parse of `c` succeeds. */
-    string input = "c";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert(p_result_Start(context) == 3);
-
-    /* Standard parse of `acb` succeeds. */
-    input = "acb";
-    context = p_context_new(input);
-    assert(p_parse_Start(context) == P_SUCCESS);
-    assert(p_result_Start(context) == 3);
-
-    /* Standard parse of `ac` fails. */
-    input = "ac";
-    context = p_context_new(input);
-    assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-
-    /* parse_inner with `ac` fails: outer rule still on the stack. */
-    input = "ac";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_bothway = [TOKEN_b, TOKEN___EOF];
-    assert(p_parse_inner_Start(context, follow_tokens_bothway) == P_UNEXPECTED_TOKEN);
-
-    /* parse_inner with `acb` succeeds via the standard path. */
-    input = "acb";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_b = [TOKEN_b];
-    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
-    assert(p_result_Start(context) == 3);
-
-    /* parse_inner with just `c` succeeds via the standard path. */
-    input = "c";
-    context = p_context_new(input);
-    assert(p_parse_inner_Start(context, follow_tokens_b) == P_SUCCESS);
-    assert(p_result_Start(context) == 3);
-}

spec/test_parse_inner_recursive.rs

@@ -1,31 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"c");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    assert_eq!(3, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"acb");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    assert_eq!(3, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ac");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ac");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF]));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"acb");
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
-    assert_eq!(3, p_result_Start(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"c");
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
-    assert_eq!(3, p_result_Start(&c));
-    p_context_delete(c);
-}

spec/test_parse_inner_shared.c

@@ -1,104 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar:
-     *   start Start;
-     *   start R1;
-     *   Start -> R1 a;
-     *   Start -> R2 b;
-     *   R1 -> a b   << $$ = 11; >>
-     *   R2 -> a b   << $$ = 22; >>
-     *   token a; token b;
-     *
-     * The rules `R1 -> a b` and `R2 -> a b` produce identical input. Within
-     * parse_Start, the generated parser differentiates the reduce by
-     * lookahead: `a` selects R1 (because `Start -> R1 a`) and `b` selects R2
-     * (because `Start -> R2 b`). Within parse_R1, the reduce is unconditional
-     * on any lookahead. This test exercises p_parse_inner_R1() to confirm
-     * that reductions to R1 succeed even when the incoming follow token is
-     * not the natural lookahead used by parse_Start's disambiguation. */
-
-    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead in the
-     * shared "a b" state. */
-    char const * input = "aba";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    input = "abb";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_Start(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    /* Standard parse of R1 succeeds on "ab". */
-    input = "ab";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_R1(context) == P_SUCCESS);
-    assert_eq(11u, (size_t)p_result_R1(context));
-    p_context_delete(context);
-
-    /* Standard parse of R1 fails on "abb" (unexpected trailing token). */
-    input = "abb";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
-    p_context_delete(context);
-
-    /* parse_inner_R1("abb", [b]) succeeds: even though `b` is the lookahead
-     * that parse_Start uses to select R2 over R1 in the ambiguous state, from
-     * R1's start state the reduce to R1 is unconditional, and the follow-
-     * token shift retry at the R1-accepting state completes the parse.
-     *
-     * The follow token that completed the parse must not be consumed from
-     * the input: p_position() should point to the follow token, and a
-     * subsequent p_lex() should return it. */
-    {
-        input = "abb";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(11u, (size_t)p_result_R1(context));
-        /* Follow token `b` is at column 3 (1-based). */
-        p_position_t pos = p_position(context);
-        assert_eq(1u, (size_t)pos.row);
-        assert_eq(3u, (size_t)pos.col);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        assert_eq(1u, (size_t)token_info.position.row);
-        assert_eq(3u, (size_t)token_info.position.col);
-        p_context_delete(context);
-    }
-
-    /* parse_inner_R1("aba", [a]) also succeeds: `a` is the follow token
-     * parse_Start uses to select R1, and it works here as a follow token
-     * too. */
-    {
-        input = "aba";
-        context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_a };
-        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(11u, (size_t)p_result_R1(context));
-        /* Follow token `a` is at column 3 (1-based) and remains in the
-         * input. */
-        p_position_t pos = p_position(context);
-        assert_eq(1u, (size_t)pos.row);
-        assert_eq(3u, (size_t)pos.col);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        p_context_delete(context);
-    }
-
-    /* parse_inner_R1("ab", NULL) behaves like p_parse_R1("ab"). */
-    input = "ab";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse_inner_R1(context, NULL, 0u) == P_SUCCESS);
-    assert_eq(11u, (size_t)p_result_R1(context));
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_parse_inner_shared.d

@@ -1,72 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_parse_inner_shared.c for details on the grammar. */
-
-    /* Sanity-check that parse_Start resolves R1 vs R2 via lookahead. */
-    string input = "aba";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse_Start(context) == P_SUCCESS);
-
-    input = "abb";
-    context = p_context_new(input);
-    assert(p_parse_Start(context) == P_SUCCESS);
-
-    /* Standard parse of R1 succeeds on "ab". */
-    input = "ab";
-    context = p_context_new(input);
-    assert(p_parse_R1(context) == P_SUCCESS);
-    assert(p_result_R1(context) == 11);
-
-    /* Standard parse of R1 fails on "abb". */
-    input = "abb";
-    context = p_context_new(input);
-    assert(p_parse_R1(context) == P_UNEXPECTED_TOKEN);
-
-    /* parse_inner_R1("abb", [b]) succeeds: `b` is the lookahead that
-     * parse_Start would use to select R2 over R1, but from R1's own start
-     * state R1 reduces unconditionally, and the follow-token shift retry at
-     * the R1-accepting state completes the parse.
-     *
-     * The follow token that completed the parse must not be consumed: a
-     * subsequent p_lex() should return it. */
-    input = "abb";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_b = [TOKEN_b];
-    assert(p_parse_inner_R1(context, follow_tokens_b) == P_SUCCESS);
-    assert(p_result_R1(context) == 11);
-    p_position_t pos = p_position(context);
-    assert(pos.row == 1);
-    assert(pos.col == 3);
-    p_token_info_t token_info;
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_b);
-    assert(token_info.position.row == 1);
-    assert(token_info.position.col == 3);
-
-    /* parse_inner_R1("aba", [a]) also succeeds. */
-    input = "aba";
-    context = p_context_new(input);
-    p_token_t[] follow_tokens_a = [TOKEN_a];
-    assert(p_parse_inner_R1(context, follow_tokens_a) == P_SUCCESS);
-    assert(p_result_R1(context) == 11);
-    pos = p_position(context);
-    assert(pos.row == 1);
-    assert(pos.col == 3);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_a);
-
-    /* parse_inner_R1("ab", null) behaves like p_parse_R1("ab"). */
-    input = "ab";
-    context = p_context_new(input);
-    assert(p_parse_inner_R1(context, null) == P_SUCCESS);
-    assert(p_result_R1(context) == 11);
-}

spec/test_parse_inner_shared.rs

@@ -1,47 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"aba");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
-    assert_eq!(11, p_result_R1(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
-    assert_eq!(11, p_result_R1(&c));
-    let pos = p_position(&c);
-    assert_eq!(1, pos.row);
-    assert_eq!(3, pos.col);
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(3, ti.position.col);
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"aba");
-    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a]));
-    assert_eq!(11, p_result_R1(&c));
-    let pos = p_position(&c);
-    assert_eq!(3, pos.col);
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[]));
-    assert_eq!(11, p_result_R1(&c));
-    p_context_delete(c);
-}

spec/test_parse_inner_tree.c

@@ -1,68 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* See the D variant / grammar comments for details. In tree mode the tree
-     * nodes live in the context arena and are freed with p_context_delete(). */
-
-    /* Baseline: p_parse_R1 works on "ab" and the returned tree is
-     * well-formed. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        assert(p_parse_R1(context) == P_SUCCESS);
-        R1 tree = p_result_R1(context);
-        assert(p_node_valid(tree));
-        assert(p_node_valid(p_R1_pToken1(tree)));
-        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
-        assert(p_node_valid(p_R1_pToken2(tree)));
-        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
-        p_context_delete(context);
-    }
-
-    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
-     * the parse, returns a well-formed tree, and leaves the follow token
-     * unconsumed. */
-    {
-        char const * input = "abb";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
-
-        /* Tree is well-formed. */
-        R1 tree = p_result_R1(context);
-        assert(p_node_valid(tree));
-        assert(p_node_valid(p_R1_pToken1(tree)));
-        assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
-        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).row);
-        assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).col);
-        assert(p_node_valid(p_R1_pToken2(tree)));
-        assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
-        assert_eq(1u, (size_t)p_node_position(p_R1_pToken2(tree)).row);
-        assert_eq(2u, (size_t)p_node_position(p_R1_pToken2(tree)).col);
-
-        /* The R1 tree covers positions 1..2 - the third `b` at column 3 is
-         * the follow token and is not part of the tree. */
-        assert_eq(1u, (size_t)p_node_position(tree).row);
-        assert_eq(1u, (size_t)p_node_position(tree).col);
-        assert_eq(1u, (size_t)p_node_end_position(tree).row);
-        assert_eq(2u, (size_t)p_node_end_position(tree).col);
-
-        /* Follow token remains in the input. */
-        p_position_t pos = p_position(context);
-        assert_eq(1u, (size_t)pos.row);
-        assert_eq(3u, (size_t)pos.col);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        assert_eq(1u, (size_t)token_info.position.row);
-        assert_eq(3u, (size_t)token_info.position.col);
-
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_parse_inner_tree.d

@@ -1,68 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_parse_inner_tree.c for details on the grammar and cases. */
-
-    /* Baseline: p_parse_R1 works on "ab". */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        assert(p_parse_R1(context) == P_SUCCESS);
-        R1 tree = p_result_R1(context);
-        assert(tree.valid);
-        assert(tree.pToken1.valid);
-        assert(tree.pToken1.token == TOKEN_a);
-        assert(tree.pToken2.valid);
-        assert(tree.pToken2.token == TOKEN_b);
-        p_context_delete(context);
-    }
-
-    /* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
-     * the parse, returns a well-formed tree, and leaves the follow token
-     * unconsumed. */
-    {
-        string input = "abb";
-        p_context_t * context = p_context_new(input);
-        p_token_t[] follow_tokens = [TOKEN_b];
-        assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
-
-        /* Tree is well-formed. */
-        R1 tree = p_result_R1(context);
-        assert(tree.valid);
-        assert(tree.pToken1.valid);
-        assert(tree.pToken1.token == TOKEN_a);
-        assert(tree.pToken1.position.row == 1);
-        assert(tree.pToken1.position.col == 1);
-        assert(tree.pToken2.valid);
-        assert(tree.pToken2.token == TOKEN_b);
-        assert(tree.pToken2.position.row == 1);
-        assert(tree.pToken2.position.col == 2);
-
-        /* The R1 tree covers positions 1..2. The third `b` at column 3 is
-         * the follow token and is not part of the tree. */
-        assert(tree.position.row == 1);
-        assert(tree.position.col == 1);
-        assert(tree.end_position.row == 1);
-        assert(tree.end_position.col == 2);
-
-        /* Follow token remains in the input. */
-        p_position_t pos = p_position(context);
-        assert(pos.row == 1);
-        assert(pos.col == 3);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(token_info.position.row == 1);
-        assert(token_info.position.col == 3);
-
-        p_context_delete(context);
-    }
-}

spec/test_parse_inner_tree.rs

@@ -1,36 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
-    {
-        let tree = p_result_R1(&c);
-        assert!(tree.valid());
-        assert!(tree.pToken1().valid());
-        assert_eq!(TOKEN_a, tree.pToken1().token());
-        assert!(tree.pToken2().valid());
-        assert_eq!(TOKEN_b, tree.pToken2().token());
-    }
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
-    {
-        let tree = p_result_R1(&c);
-        assert!(tree.valid());
-        assert_eq!(TOKEN_a, tree.pToken1().token());
-        assert_eq!(1, tree.pToken1().position().row);
-        assert_eq!(1, tree.pToken1().position().col);
-        assert_eq!(TOKEN_b, tree.pToken2().token());
-        assert_eq!(2, tree.pToken2().position().col);
-        assert_eq!(1, tree.position().col);
-        assert_eq!(2, tree.end_position().col);
-    }
-    let pos = p_position(&c);
-    assert_eq!(3, pos.col);
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(3, ti.position.col);
-    p_context_delete(c);
-}

spec/test_parser_identical_rules_lookahead.rs

@@ -1,9 +0,0 @@
-use testparser::*;
-
-fn main() {
-    for input in [&b"aba"[..], &b"abb"[..]] {
-        let mut context = p_context_new(input);
-        assert_eq!(P_SUCCESS, p_parse(&mut context));
-        p_context_delete(context);
-    }
-}

spec/test_parser_rule_from_multiple_states.rs

@@ -1,18 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"a");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context));
-    assert_eq!(1, p_position(&context).row);
-    assert_eq!(2, p_position(&context).col);
-    assert_eq!(TOKEN___EOF, p_token(&context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"a b");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"bb");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    p_context_delete(context);
-}

spec/test_parser_rule_user_code.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    p_context_delete(context);
-}

spec/test_parser_user_code_tree.c

@@ -28,10 +28,11 @@ int main()
     assert_eq(11, context->alias_a_value);
     assert_eq(22, context->alias_b_value);
 
-    Start start = p_result(context);
-    assert(p_node_valid(p_Start_pA(start)));
-    assert(p_node_valid(p_Start_pB(start)));
-    assert(!p_node_valid(p_Start_pC(start)));
+    Start * start = p_result(context);
+    assert(start->pA != NULL);
+    assert(start->pB != NULL);
+    assert(start->pC == NULL);
+    p_tree_delete(start);
     p_context_delete(context);
 
     return 0;

spec/test_parser_user_code_tree.d

@@ -31,9 +31,9 @@ unittest
     assert_eq(11, context.alias_a_value);
     assert_eq(22, context.alias_b_value);
 
-    Start start = p_result(context);
-    assert(start.pA.valid);
-    assert(start.pB.valid);
-    assert(!start.pC.valid);
-    p_context_delete(context);
+    Start * start = p_result(context);
+    assert(start.pA !is null);
+    assert(start.pB !is null);
+    assert(start.pC is null);
+    p_tree_delete(start);
 }

spec/test_parser_user_code_tree.rs

@@ -1,24 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-
-    assert_eq!(3, context.start_n_fields);
-    assert_eq!(11, context.start_a_value);
-    assert_eq!(11, context.a_value);
-    assert_eq!(22, context.b_value);
-    assert_eq!(TOKEN_b, context.b_token);
-    assert_eq!(1, context.c_is_null);
-    assert_eq!(1, context.c_field_is_null);
-    assert_eq!(11, context.alias_a_value);
-    assert_eq!(22, context.alias_b_value);
-
-    {
-        let start = p_result(&context);
-        assert!(start.pA().valid());
-        assert!(start.pB().valid());
-        assert!(!start.pC().valid());
-    }
-    p_context_delete(context);
-}

spec/test_parsing_json.rs

@@ -1,31 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"{}");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(JSON_OBJECT, p_result(&c).id());
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"[]");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(JSON_ARRAY, p_result(&c).id());
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"-45.6");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(JSON_NUMBER, p_result(&c).id());
-    assert_eq!(-45.6, p_result(&c).number());
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"{\"hi\":true}");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(JSON_OBJECT, p_result(&c).id());
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    let v = p_result(&c);
-    assert_eq!(JSON_ARRAY, v.id());
-    assert_eq!(5, v.array_len());
-    p_context_delete(c);
-}

spec/test_parsing_lists.rs

@@ -1,11 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)];
-    for (input, expected) in cases {
-        let mut context = p_context_new(input);
-        assert_eq!(P_SUCCESS, p_parse(&mut context));
-        assert_eq!(expected, p_result(&context));
-        p_context_delete(context);
-    }
-}

spec/test_pattern.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"abcdef");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"defabcdef");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass2");
-    p_context_delete(context);
-}

spec/test_positions.rs

@@ -1,19 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"    Hello\n\n        4200\n");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-
-    println!();
-
-    let mut c = p_context_new(b"\n tok2");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-
-    println!();
-
-    let mut c = p_context_new(b"  tok1");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_return_token_from_pattern.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"defghidef");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    p_context_delete(context);
-}

spec/test_rewind.c

@@ -1,124 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-/* Grammar (statement list of additions; a "repeat" directive handled entirely
- * by the lex function):
- *   ptype int;
- *   lex_fn mylexfn;
- *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
- *   token plus /\+/; token num /\d+/ << ... atoi ... >>
- *   Start      -> Statements;
- *   Statements -> ;
- *   Statements -> Statement Statements;
- *   Statement  -> Add;
- *   Add        -> num plus num  << record($1 + $3); >>
- *
- * Scenario: a "repeat <count> { <body> }" directive that expands its body
- * <count> times, similar to loop unrolling in a configuration DSL. The tokens
- * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
- * interprets the directive and feeds the body's tokens to the parser <count>
- * times. Rather than buffering the body tokens, the lex function records the
- * input byte offset and text position at the start of the body (with
- * p_input_index() and p_position()) and, each time it reaches the closing '}',
- * rewinds the lexer back to that point (with p_set_input_index() and
- * p_set_position()) to re-read the body from the original input. Rewinding the
- * text position as well as the byte offset means each expansion reports the
- * same token positions as the first. */
-
-static int nums[16];
-static size_t n_nums;
-static uint32_t num_cols[16];
-static size_t n_num_cols;
-
-void record(int value)
-{
-    nums[n_nums++] = value;
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static int remaining;
-    static size_t body_index;
-    static p_position_t body_position;
-
-    for (;;)
-    {
-        size_t result = p_lex(context, out_token_info);
-        if (result != P_SUCCESS)
-        {
-            return result;
-        }
-
-        if (out_token_info->token == TOKEN_repeat)
-        {
-            /* Consume "repeat <count> {" and remember where the body begins. */
-            p_token_info_t count_info;
-            size_t count_result = p_lex(context, &count_info);
-            assert(count_result == P_SUCCESS);
-            assert(count_info.token == TOKEN_num);
-            p_token_info_t brace_info;
-            size_t brace_result = p_lex(context, &brace_info);
-            assert(brace_result == P_SUCCESS);
-            assert(brace_info.token == TOKEN_lbrace);
-            remaining = p_value_get(&count_info.pvalue);
-            body_index = p_input_index(context);
-            body_position = p_position(context);
-            continue;
-        }
-        if (out_token_info->token == TOKEN_rbrace)
-        {
-            /* End of the body. If more expansions remain, rewind the lexer to
-             * the start of the body and re-read it; otherwise fall through to
-             * the input following the '}'. */
-            if (remaining > 1)
-            {
-                remaining--;
-                p_set_input_index(context, body_index);
-                p_set_position(context, body_position);
-                continue;
-            }
-            remaining = 0;
-            continue;
-        }
-        if (out_token_info->token == TOKEN_num)
-        {
-            num_cols[n_num_cols++] = out_token_info->position.col;
-        }
-        return result;
-    }
-}
-
-int main()
-{
-    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
-     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
-    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    /* The additions were recorded once per body expansion, then once more for
-     * the trailing statement. */
-    assert_eq(4u, n_nums);
-    assert_eq(30u, (size_t)nums[0]);
-    assert_eq(30u, (size_t)nums[1]);
-    assert_eq(30u, (size_t)nums[2]);
-    assert_eq(10u, (size_t)nums[3]);
-
-    /* Each body expansion reported the same columns for its num tokens (12 and
-     * 17), because the text position was rewound along with the byte offset.
-     * The trailing statement's nums are at columns 22 and 26. */
-    assert_eq(8u, n_num_cols);
-    assert_eq(12u, (size_t)num_cols[0]);
-    assert_eq(17u, (size_t)num_cols[1]);
-    assert_eq(12u, (size_t)num_cols[2]);
-    assert_eq(17u, (size_t)num_cols[3]);
-    assert_eq(12u, (size_t)num_cols[4]);
-    assert_eq(17u, (size_t)num_cols[5]);
-    assert_eq(22u, (size_t)num_cols[6]);
-    assert_eq(26u, (size_t)num_cols[7]);
-
-    return 0;
-}

spec/test_rewind.d

@@ -1,101 +0,0 @@
-import testparser;
-import testutils;
-
-/* Grammar and scenario: see test_rewind.c. */
-
-int[16] nums;
-size_t n_nums;
-uint[16] num_cols;
-size_t n_num_cols;
-
-void record(int value)
-{
-    nums[n_nums++] = value;
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static int remaining;
-    static size_t body_index;
-    static p_position_t body_position;
-
-    for (;;)
-    {
-        size_t result = p_lex(context, out_token_info);
-        if (result != P_SUCCESS)
-        {
-            return result;
-        }
-
-        if (out_token_info.token == TOKEN_repeat)
-        {
-            /* Consume "repeat <count> {" and remember where the body begins. */
-            p_token_info_t count_info;
-            size_t count_result = p_lex(context, &count_info);
-            assert(count_result == P_SUCCESS);
-            assert(count_info.token == TOKEN_num);
-            p_token_info_t brace_info;
-            size_t brace_result = p_lex(context, &brace_info);
-            assert(brace_result == P_SUCCESS);
-            assert(brace_info.token == TOKEN_lbrace);
-            remaining = p_value_get(&count_info.pvalue);
-            body_index = p_input_index(context);
-            body_position = p_position(context);
-            continue;
-        }
-        if (out_token_info.token == TOKEN_rbrace)
-        {
-            /* End of the body. If more expansions remain, rewind the lexer to
-             * the start of the body and re-read it; otherwise fall through to
-             * the input following the '}'. */
-            if (remaining > 1)
-            {
-                remaining--;
-                p_set_input_index(context, body_index);
-                p_set_position(context, body_position);
-                continue;
-            }
-            remaining = 0;
-            continue;
-        }
-        if (out_token_info.token == TOKEN_num)
-        {
-            num_cols[n_num_cols++] = out_token_info.position.col;
-        }
-        return result;
-    }
-}
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
-     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
-    string input = "repeat 3 { 10 + 20 } 5 + 5";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    assert_eq(4u, n_nums);
-    assert_eq(30, nums[0]);
-    assert_eq(30, nums[1]);
-    assert_eq(30, nums[2]);
-    assert_eq(10, nums[3]);
-
-    /* Each body expansion reported the same columns for its num tokens (12 and
-     * 17), because the text position was rewound along with the byte offset.
-     * The trailing statement's nums are at columns 22 and 26. */
-    assert_eq(8u, n_num_cols);
-    assert_eq(12u, num_cols[0]);
-    assert_eq(17u, num_cols[1]);
-    assert_eq(12u, num_cols[2]);
-    assert_eq(17u, num_cols[3]);
-    assert_eq(12u, num_cols[4]);
-    assert_eq(17u, num_cols[5]);
-    assert_eq(22u, num_cols[6]);
-    assert_eq(26u, num_cols[7]);
-}

spec/test_rewind.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
-     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
-    let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-
-    assert_eq!(vec![30, 30, 30, 10], c.nums);
-    assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols);
-
-    p_context_delete(c);
-}

spec/test_set_position.c

@@ -1,109 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar (simple):
-     *   token a; token b;
-     *   Start -> a b;
-     *
-     * Verifies that p_set_position() overrides the default (1, 1) starting
-     * position so that lexed tokens and error positions are reported
-     * relative to the caller-supplied position. */
-
-    /* Baseline: without p_set_position(), positions start at (1, 1). */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_position_t pos = p_position(context);
-        assert_eq(1u, (size_t)pos.row);
-        assert_eq(1u, (size_t)pos.col);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(1u, (size_t)token_info.position.row);
-        assert_eq(1u, (size_t)token_info.position.col);
-        p_context_delete(context);
-    }
-
-    /* p_set_position() overrides the initial position; subsequent lex calls
-     * report token positions relative to the set position. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_position_t initial = {5u, 20u};
-        p_set_position(context, initial);
-        p_position_t pos = p_position(context);
-        assert_eq(5u, (size_t)pos.row);
-        assert_eq(20u, (size_t)pos.col);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(5u, (size_t)token_info.position.row);
-        assert_eq(20u, (size_t)token_info.position.col);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        assert_eq(5u, (size_t)token_info.position.row);
-        assert_eq(21u, (size_t)token_info.position.col);
-        p_context_delete(context);
-    }
-
-    /* p_set_position() before a full parse: successful parse still works and
-     * text_position tracking is relative to the set starting point. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_position_t initial = {3u, 7u};
-        p_set_position(context, initial);
-        assert(p_parse_Start(context) == P_SUCCESS);
-        p_context_delete(context);
-    }
-
-    /* p_set_position() before a parse that fails: the reported error
-     * position is relative to the set starting point. */
-    {
-        char const * input = "aa";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_position_t initial = {10u, 2u};
-        p_set_position(context, initial);
-        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-        p_position_t err_pos = p_position(context);
-        /* Error is at the second `a`, which is one column past the initial
-         * column. */
-        assert_eq(10u, (size_t)err_pos.row);
-        assert_eq(3u, (size_t)err_pos.col);
-        p_context_delete(context);
-    }
-
-    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
-     * p_set_position(), it re-reads an earlier section of the input: both
-     * tokens are lexed, then the cursor and text position are rewound to the
-     * start so that the same tokens are produced again with the same reported
-     * positions. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_info_t token_info;
-        size_t start_index = p_input_index(context);
-        p_position_t start_position = p_position(context);
-        assert_eq(0u, start_index);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        assert_eq(2u, p_input_index(context));
-        /* Rewind and re-read from the start. */
-        p_set_input_index(context, start_index);
-        p_set_position(context, start_position);
-        assert_eq(0u, p_input_index(context));
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(1u, (size_t)token_info.position.row);
-        assert_eq(1u, (size_t)token_info.position.col);
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_set_position.d

@@ -1,90 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_set_position.c for details. */
-
-    /* Baseline: without p_set_position(), positions start at (1, 1). */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        p_position_t pos = p_position(context);
-        assert(pos.row == 1);
-        assert(pos.col == 1);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(token_info.position.row == 1);
-        assert(token_info.position.col == 1);
-    }
-
-    /* p_set_position() overrides the initial position. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        p_set_position(context, p_position_t(5u, 20u));
-        p_position_t pos = p_position(context);
-        assert(pos.row == 5);
-        assert(pos.col == 20);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(token_info.position.row == 5);
-        assert(token_info.position.col == 20);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(token_info.position.row == 5);
-        assert(token_info.position.col == 21);
-    }
-
-    /* p_set_position() before a full parse still parses successfully. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        p_set_position(context, p_position_t(3u, 7u));
-        assert(p_parse_Start(context) == P_SUCCESS);
-    }
-
-    /* p_set_position() before a parse that fails: error position is
-     * relative to the set starting point. */
-    {
-        string input = "aa";
-        p_context_t * context = p_context_new(input);
-        p_set_position(context, p_position_t(10u, 2u));
-        assert(p_parse_Start(context) == P_UNEXPECTED_TOKEN);
-        p_position_t err_pos = p_position(context);
-        assert(err_pos.row == 10);
-        assert(err_pos.col == 3);
-    }
-
-    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
-     * p_set_position(), it re-reads an earlier section of the input. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        p_token_info_t token_info;
-        size_t start_index = p_input_index(context);
-        p_position_t start_position = p_position(context);
-        assert(start_index == 0);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(p_input_index(context) == 2);
-        /* Rewind and re-read from the start. */
-        p_set_input_index(context, start_index);
-        p_set_position(context, start_position);
-        assert(p_input_index(context) == 0);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(token_info.position.row == 1);
-        assert(token_info.position.col == 1);
-    }
-}

spec/test_set_position.rs

@@ -1,64 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut ti = p_token_info_t::default();
-
-    /* Baseline: default (1, 1). */
-    let mut c = p_context_new(b"ab");
-    let pos = p_position(&c);
-    assert_eq!(1, pos.row);
-    assert_eq!(1, pos.col);
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(1, ti.position.row);
-    assert_eq!(1, ti.position.col);
-    p_context_delete(c);
-
-    /* p_set_position overrides the initial position. */
-    let mut c = p_context_new(b"ab");
-    p_set_position(&mut c, p_position_t { row: 5, col: 20 });
-    assert_eq!(p_position_t { row: 5, col: 20 }, p_position(&c));
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(5, ti.position.row);
-    assert_eq!(20, ti.position.col);
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(5, ti.position.row);
-    assert_eq!(21, ti.position.col);
-    p_context_delete(c);
-
-    /* Set position before a full parse. */
-    let mut c = p_context_new(b"ab");
-    p_set_position(&mut c, p_position_t { row: 3, col: 7 });
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    p_context_delete(c);
-
-    /* Set position before a failing parse: error position is relative. */
-    let mut c = p_context_new(b"aa");
-    p_set_position(&mut c, p_position_t { row: 10, col: 2 });
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
-    let ep = p_position(&c);
-    assert_eq!(10, ep.row);
-    assert_eq!(3, ep.col);
-    p_context_delete(c);
-
-    /* p_set_input_index rewinds the byte cursor to re-read a section. */
-    let mut c = p_context_new(b"ab");
-    let start_index = p_input_index(&c);
-    let start_position = p_position(&c);
-    assert_eq!(0, start_index);
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(2, p_input_index(&c));
-    p_set_input_index(&mut c, start_index);
-    p_set_position(&mut c, start_position);
-    assert_eq!(0, p_input_index(&c));
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(1, ti.position.row);
-    assert_eq!(1, ti.position.col);
-    p_context_delete(c);
-}

spec/test_start_rule.rs

@@ -1,2 +0,0 @@
-fn main() {
-}

spec/test_start_rule_tree.c

@@ -9,10 +9,11 @@ int main()
     p_context_t * context;
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(context));
-    Top top = p_result(context);
-    assert(p_node_valid(p_Top_pToken(top)));
-    assert_eq(TOKEN_hi, p_tree_walk_Top(top, pToken, token));
+    Top * top = p_result(context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
 
+    p_tree_delete(top);
     p_context_delete(context);
 
     return 0;

spec/test_start_rule_tree.d

@@ -13,7 +13,7 @@ unittest
     p_context_t * context;
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
-    Top top = p_result(context);
-    assert(top.pToken.valid);
+    Top * top = p_result(context);
+    assert(top.pToken !is null);
     assert_eq(TOKEN_hi, top.pToken.token);
 }

spec/test_start_rule_tree.rs

@@ -1,12 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"hi");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    {
-        let top = p_result(&context);
-        assert!(top.pToken().valid());
-        assert_eq!(TOKEN_hi, top.pToken().token());
-    }
-    p_context_delete(context);
-}

spec/test_starting_rules.rs

@@ -1,18 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"bbbb");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(8, p_result(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"bbbb");
-    assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
-    assert_eq!(8, p_result_Bs(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"c");
-    assert_eq!(P_SUCCESS, p_parse_R(&mut c));
-    assert_eq!(3, p_result_R(&c));
-    p_context_delete(c);
-}