v4.3.0

.rspec

@@ -1,3 +1,2 @@
---format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,3 +1,57 @@
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
+## v4.0.0
+
+### New Features
+
+- Add `context_user_fields` statement to allow custom context user fields.
+- Add `token_user_fields` statement to allow custom token user fields.
+- Add `on_token_node` statement to allow custom code when constructing token nodes.
+- Add `free_token_node` statement to allow custom code when freeing token nodes.
+- Add `p_context_delete()`.
+- Allow `drop` patterns to execute lexer user code blocks.
+
+### Breaking Changes
+
+- Replace `p_context_init()` with `p_context_new()` and `p_context_delete()`.
+- Renamed `p_free_tree()` to `p_tree_delete()`.
+- The `free_token_node` statement now takes a user code block instead of a
+  function name parameter.
+
+## v3.0.0
+
+### New Features
+
+- Add support for multiple starting rules (#38)
+- Add `p_free_tree()` functions to reclaim generated tree memory
+- Add `free_token_node` grammar statement to reclaim user-allocated memory stored in a Token tree node `pvalue` field
+- Add valgrind memory leak tests to unit tests
+- Fix build issues for C++ to officially support C++ target output
+
+### Improvements
+
+- Document `p_lex()` and `p_token_info_t` in user guide (#37)
+
+### Breaking Changes
+
+- Rename AST generation mode to tree generation mode (see [UPGRADING.md](UPGRADING.md))
+
 ## v2.3.0
 
 ### New Features

Gemfile.lock

@@ -2,39 +2,41 @@ GEM
   remote: https://rubygems.org/
   specs:
     base64 (0.3.0)
-    date (3.4.1)
+    date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (5.0.2)
-    psych (5.2.6)
+    erb (6.0.4)
+    psych (5.3.1)
       date
       stringio
-    rake (13.3.0)
-    rdoc (6.14.2)
+    rake (13.4.2)
+    rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
+      tsort
     redcarpet (3.6.1)
-    rspec (3.13.1)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.5)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.5)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.4)
+    rspec-support (3.13.7)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    stringio (3.1.7)
+    stringio (3.2.0)
     syntax (1.2.2)
+    tsort (0.2.0)
 
 PLATFORMS
   ruby
@@ -49,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.11

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -6,8 +6,10 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C or D language outputs
-  * optionally supports automatic full AST generation
+  * targets C, C++, or D language outputs
+  * optionally supports automatic full parse tree generation
+  * supports starting parsing from multiple start rules
+  * tracks input text start and end positions for all matched tokens/rules
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 

Rakefile.rb

@@ -1,5 +1,7 @@
 require "rake/clean"
 require "rspec/core/rake_task"
+require "simplecov"
+require "stringio"
 
 CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
 
@@ -12,6 +14,18 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
   end
 end
+task :spec do |task, args|
+  unless ENV["dist_specs"]
+    original_stdout = $stdout
+    sio = StringIO.new
+    $stdout = sio
+    SimpleCov.collate Dir["coverage/.resultset.json"]
+    $stdout = original_stdout
+    sio.string.lines.each do |line|
+      $stdout.write(line) unless line =~ /Coverage report generated for/
+    end
+  end
+end
 
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.

UPGRADING.md

@@ -0,0 +1,21 @@
+## v4.0.0
+
+### API Changes
+
+- Replace any calls to `p_context_init()` with `p_context_new()`.
+- Replace any references to the address of a statically allocated context
+  structure with the pointer returned from `p_context_init()` (e.g. `&context`
+  -> `context`).
+- Add a call to `p_context_delete()` (for C or C++) after lexing/parsing to
+  reclaim context memory.
+- Rename `p_free_tree()` calls to `p_tree_delete()`.
+- Change `free_token_node` statement calls from taking a function name argument
+  to taking a user code block.
+
+## v3.0.0
+
+### Grammar Changes
+
+- Rename `ast;` statement to `tree;`.
+- Rename `ast_prefix;` statement to `tree_prefix;`.
+- Rename `ast_suffix;` statement to `tree_suffix;`.

assets/parser.c.erb

@@ -43,30 +43,52 @@ const char * <%= @grammar.prefix %>token_names[] = {
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
  * @param input_length
  *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length)
 {
-    /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext;
-    memset(&newcontext, 0, sizeof(newcontext));
+<% if @cpp %>
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t();
+<% else %>
+    <%= @grammar.prefix %>context_t * context = (<%= @grammar.prefix %>context_t *)calloc(1, sizeof(<%= @grammar.prefix %>context_t));
+<% end %>
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.input_length = input_length;
-    newcontext.text_position.row = 1u;
-    newcontext.text_position.col = 1u;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context->input = input;
+    context->input_length = input_length;
+    context->text_position.row = 1u;
+    context->text_position.col = 1u;
+    context->mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * For C++, destructors will be called for any context user fields. However, if
+ * pointers are used to store allocated resources, the user should free them
+ * before calling this function.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+<% if @cpp %>
+    delete context;
+<% else %>
+    free(context);
+<% end %>
 }
 
 /**************************************************************************
@@ -638,7 +660,7 @@ typedef struct
      * reduce action.
      */
     parser_state_id_t n_states;
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 
     /**
      * Map of rule components to rule set child fields.
@@ -646,7 +668,7 @@ typedef struct
     uint16_t const * rule_set_node_field_index_map;
 
     /**
-     * Number of rule set AST node fields.
+     * Number of rule set tree node fields.
      */
     uint16_t rule_set_node_field_array_size;
 
@@ -688,19 +710,23 @@ typedef struct
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
 
-<% if @grammar.ast %>
-    /** AST node. */
-    void * ast_node;
+<% if @grammar.tree %>
+    /** tree node. */
+    void * tree_node;
 <% end %>
 } state_value_t;
 
-/** Common AST node structure. */
-typedef struct
+<% if @grammar.tree %>
+/** Common tree node structure. */
+typedef struct TreeNode_s
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-    void * fields[];
-} ASTNode;
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct TreeNode_s * fields[];
+} TreeNode;
+<% end %>
 
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
@@ -709,7 +735,7 @@ static const shift_t parser_shift_table[] = {
 <%   end %>
 };
 
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 <%   @grammar.rules.each do |rule| %>
 <%     unless rule.flat_rule_set_node_field_index_map? %>
 const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
@@ -724,14 +750,14 @@ static const reduce_t parser_reduce_table[] = {
         <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
         <%= reduce[:rule_id] %>u, /* Rule ID */
         <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
-<%     if @grammar.ast %>
+<%     if @grammar.tree %>
         <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
         NULL, /* No rule set node field index map (flat map) */
 <%       else %>
         &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
+        <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
         <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
 <%     else %>
         <%= reduce[:n_states] %>u},
@@ -805,7 +831,7 @@ static void state_values_stack_push(state_values_stack_t * stack)
     if (current_length >= current_capacity)
     {
         size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
+        state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
         memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
@@ -839,7 +865,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
-<% unless @grammar.ast %>
+<% unless @grammar.tree %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -922,6 +948,8 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
+ * @start_state_id
+ *   ID of the state in which to start.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -934,25 +962,26 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
-<% if @grammar.ast %>
+<% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
     state_values_stack_push(&statevalues);
+    state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
     size_t result;
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -971,8 +1000,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
-<% if @grammar.ast %>
-                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
+<% if @grammar.tree %>
+                context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -988,13 +1017,20 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-<% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
-                token_ast_node->position = token_info.position;
-                token_ast_node->end_position = token_info.end_position;
-                token_ast_node->token = token;
-                token_ast_node->pvalue = token_info.pvalue;
-                state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
+<% if @grammar.tree %>
+<%   if @cpp %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
+<%   else %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
+<%   end %>
+                token_tree_node->position = token_info.position;
+                token_tree_node->end_position = token_info.end_position;
+                token_tree_node->n_fields = 0u;
+                token_tree_node->is_token = 1u;
+                token_tree_node->token = token;
+                token_tree_node->pvalue = token_info.pvalue;
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
+                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
 <% end %>
@@ -1003,8 +1039,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             else
             {
                 /* We shifted a RuleSet. */
-<% if @grammar.ast %>
-                state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node;
+<% if @grammar.tree %>
+                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
                 <%= @grammar.prefix %>value_t new_parse_result;
@@ -1020,39 +1056,38 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-<% if @grammar.ast %>
+<% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = state_values_stack_index(&statevalues, -1)->ast_node;
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                ASTNode * node = (ASTNode *)malloc(sizeof(ASTNode) + n_fields * sizeof(void *));
+                size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
+                TreeNode * node = (TreeNode *)malloc(bytes);
+                memset(node, 0, bytes);
                 node->position = INVALID_POSITION;
                 node->end_position = INVALID_POSITION;
-                for (size_t i = 0; i < n_fields; i++)
-                {
-                    node->fields[i] = NULL;
-                }
+                node->n_fields = n_fields;
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
                     }
                 }
                 bool position_found = false;
                 for (size_t i = 0; i < n_fields; i++)
                 {
-                    ASTNode * child = (ASTNode *)node->fields[i];
+                    TreeNode * child = node->fields[i];
                     if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
                     {
                         if (!position_found)
@@ -1074,6 +1109,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
             {
+                state_values_stack_free(&statevalues);
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
@@ -1097,6 +1133,19 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
     return result;
 }
 
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, 0u);
+}
+
+<% @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, <%= i %>u);
+}
+<% end %>
+
 /**
  * Get the parse result value.
  *
@@ -1105,18 +1154,29 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  *
  * @return Parse result value.
  */
-<% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% if @grammar.tree %>
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
+}
+<%   end %>
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
-<% end %>
 {
-<% if @grammar.ast %>
-    return context->parse_result;
-<% else %>
     return context->parse_result.v_<%= start_rule_type[0] %>;
-<% end %>
 }
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return context->parse_result.v_<%= start_rule_type(i)[0] %>;
+}
+<%   end %>
+<% end %>
 
 /**
  * Get the current text input position.
@@ -1153,3 +1213,48 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
+<% if @grammar.tree %>
+
+static void tree_delete(TreeNode * node)
+{
+    if (node->is_token)
+    {
+        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+<%   if @cpp %>
+        delete token_tree_node;
+<%   else %>
+        free(token_tree_node);
+<%   end %>
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                tree_delete(node->fields[i]);
+            }
+        }
+        free(node);
+    }
+}
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   end %>
+<% end %>

assets/parser.d.erb

@@ -9,7 +9,7 @@ module <%= @grammar.modulename %>;
 <% end %>
 
 import core.memory;
-import core.stdc.stdlib : malloc;
+import core.stdc.stdlib : malloc, free;
 
 /**************************************************************************
  * User code blocks
@@ -75,7 +75,7 @@ public struct <%= @grammar.prefix %>position_t
     }
 }
 
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 /** Parser values type. */
 public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
 <% else %>
@@ -86,35 +86,50 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
 <% end %>
 
-<% if @grammar.ast %>
-/** Common AST node structure. */
-private struct ASTNode
+<% if @grammar.tree %>
+/** Common tree node structure. */
+private struct TreeNode
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     void *[0] fields;
 }
 
-/** AST node types. @{ */
-public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
+/** Tree node types. @{ */
+public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 {
-    /* ASTNode fields must be present in the same order here. */
+    /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
+<%= @grammar.token_user_fields %>
 }
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
 <%     next if rule_set.optional? %>
-public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
+public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-<%     rule_set.ast_fields.each do |fields| %>
+    ushort n_fields;
+    bool is_token;
+<%     rule_set.tree_fields.each do |fields| %>
     union
     {
 <%       fields.each do |field_name, type| %>
@@ -172,8 +187,8 @@ public struct <%= @grammar.prefix %>context_t
     /* Parser context data. */
 
     /** Parse result value. */
-<% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+<% if @grammar.tree %>
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -183,6 +198,8 @@ public struct <%= @grammar.prefix %>context_t
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 }
 
 /**************************************************************************
@@ -222,26 +239,39 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
+ * @param input_length
+ *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, string input)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(string input)
 {
     /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext;
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t;
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.text_position.row = 1u;
-    newcontext.text_position.col = 1u;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context.input = input;
+    context.text_position.row = 1u;
+    context.text_position.col = 1u;
+    context.mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
 }
 
 /**************************************************************************
@@ -797,7 +827,7 @@ private struct reduce_t
      * reduce action.
      */
     parser_state_id_t n_states;
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 
     /**
      * Map of rule components to rule set child fields.
@@ -805,7 +835,7 @@ private struct reduce_t
     immutable(ushort) * rule_set_node_field_index_map;
 
     /**
-     * Number of rule set AST node fields.
+     * Number of rule set tree node fields.
      */
     ushort rule_set_node_field_array_size;
 
@@ -847,9 +877,9 @@ private struct state_value_t
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
 
-<% if @grammar.ast %>
-    /** AST node. */
-    void * ast_node;
+<% if @grammar.tree %>
+    /** Tree node. */
+    void * tree_node;
 <% end %>
 
     this(size_t state_id)
@@ -865,7 +895,7 @@ private immutable shift_t[] parser_shift_table = [
 <%   end %>
 ];
 
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 <%   @grammar.rules.each do |rule| %>
 <%     unless rule.flat_rule_set_node_field_index_map? %>
 immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
@@ -880,14 +910,14 @@ private immutable reduce_t[] parser_reduce_table = [
         <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
         <%= reduce[:rule_id] %>u, /* Rule ID */
         <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
-<%     if @grammar.ast %>
+<%     if @grammar.tree %>
         <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
         null, /* No rule set node field index map (flat map) */
 <%       else %>
         &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
+        <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
         <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
 <%     else %>
         <%= reduce[:n_states] %>u), /* Number of states */
@@ -902,7 +932,7 @@ private immutable parser_state_t[] parser_state_table = [
 <%   end %>
 ];
 
-<% unless @grammar.ast %>
+<% unless @grammar.tree %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -985,6 +1015,8 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
+ * @start_state_id
+ *   ID of the state in which to start.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -997,13 +1029,14 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
+    statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
-<% if @grammar.ast %>
+<% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
     <%= @grammar.prefix %>value_t reduced_parser_value;
@@ -1012,7 +1045,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
@@ -1030,8 +1063,8 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
-<% if @grammar.ast %>
-                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
+<% if @grammar.tree %>
+                context.parse_result = statevalues[$-1].tree_node;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -1045,9 +1078,10 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-<% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
-                statevalues[$-1].ast_node = token_ast_node;
+<% if @grammar.tree %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
+                statevalues[$-1].tree_node = token_tree_node;
 <% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
@@ -1056,8 +1090,8 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             else
             {
                 /* We shifted a RuleSet. */
-<% if @grammar.ast %>
-                statevalues[$-1].ast_node = reduced_parser_node;
+<% if @grammar.tree %>
+                statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
                 <%= @grammar.prefix %>value_t new_parse_result;
@@ -1072,19 +1106,21 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-<% if @grammar.ast %>
+<% if @grammar.tree %>
             if (parser_reduce_table[reduce_index].propagate_optional_target)
             {
-                reduced_parser_node = statevalues[$ - 1].ast_node;
+                reduced_parser_node = statevalues[$ - 1].tree_node;
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                size_t node_size = ASTNode.sizeof + n_fields * (void *).sizeof;
-                ASTNode * node = cast(ASTNode *)malloc(node_size);
+                size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
+                TreeNode * node = cast(TreeNode *)malloc(node_size);
                 GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
                 foreach (i; 0..n_fields)
                 {
                     node.fields[i] = null;
@@ -1093,20 +1129,20 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
                     }
                 }
                 else
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
                     }
                 }
                 bool position_found = false;
                 foreach (i; 0..n_fields)
                 {
-                    ASTNode * child = cast(ASTNode *)node.fields[i];
+                    TreeNode * child = cast(TreeNode *)node.fields[i];
                     if (child && child.position.valid)
                     {
                         if (!position_found)
@@ -1147,6 +1183,19 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
     }
 }
 
+public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, 0u);
+}
+
+<% @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, <%= i %>u);
+}
+<% end %>
+
 /**
  * Get the parse result value.
  *
@@ -1155,18 +1204,58 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  *
  * @return Parse result value.
  */
-<% if @grammar.ast %>
-public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% if @grammar.tree %>
+public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
+}
+<%   end %>
 <% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
-<% end %>
 {
-<% if @grammar.ast %>
-    return context.parse_result;
-<% else %>
     return context.parse_result.v_<%= start_rule_type[0] %>;
-<% end %>
 }
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return context.parse_result.v_<%= start_rule_type(i)[0] %>;
+}
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
 
 /**
  * Get the current text input position.

assets/parser.h.erb

@@ -58,7 +58,7 @@ typedef struct
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>
 
-<% if @grammar.ast %>
+<% if @grammar.tree %>
 /** Parser values type. */
 typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
 <% else %>
@@ -69,18 +69,29 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
 <% end %>
 
-<% if @grammar.ast %>
-/** AST node types. @{ */
-typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
+<% if @grammar.tree %>
+/** Tree node types. @{ */
+typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 {
-    /* ASTNode fields must be present in the same order here. */
+    <% # TreeNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+<%= @grammar.token_user_fields %>
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
+} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
@@ -91,11 +102,14 @@ struct <%= name %>;
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
 <%     next if rule_set.optional? %>
-typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
+typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
+    <% # TreeNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-<%     rule_set.ast_fields.each do |fields| %>
+    uint16_t n_fields;
+    uint8_t is_token;
+<%     rule_set.tree_fields.each do |fields| %>
     union
     {
 <%       fields.each do |field_name, type| %>
@@ -103,7 +117,7 @@ typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 <%       end %>
     };
 <%     end %>
-} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>;
+} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
 
 <%   end %>
 /** @} */
@@ -156,8 +170,8 @@ typedef struct
     /* Parser context data. */
 
     /** Parse result value. */
-<% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+<% if @grammar.tree %>
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -167,6 +181,8 @@ typedef struct
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 } <%= @grammar.prefix %>context_t;
 
 /**************************************************************************
@@ -176,7 +192,9 @@ typedef struct
 /** Token names. */
 extern const char * <%= @grammar.prefix %>token_names[];
 
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length);
+
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
         <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
@@ -184,11 +202,27 @@ size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t inp
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
 
-<% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% if @grammar.tree %>
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
+<%   end %>
 <% end %>
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);

extra/vim/syntax/propane.vim

@@ -20,7 +20,7 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
+syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -11,25 +11,40 @@ class Propane
         @log = StringIO.new
       end
       @language =
-        if output_file =~ /\.([a-z]+)$/
-          $1
-        else
+        if output_file.end_with?(".d")
           "d"
+        elsif output_file.end_with?(".c")
+          "c"
+        elsif output_file =~ %r{\.(cc|cpp|cxx)$}
+          @cpp = true
+          "c"
+        else
+          raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
       @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [@language]
+      extensions = [nil]
       if @language == "c"
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension}.erb")
+        template = Assets.get("parser.#{extension || @language}.erb")
+        if extension
+          output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        else
+          output_file = @output_file
+        end
         erb = ERB.new(template, trim_mode: "<>")
-        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            "#line #{i + 2}\n"
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -67,12 +82,15 @@ class Propane
         end
         tokens_by_name[token.name] = token
       end
-      # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
-        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
+      # Create real start rule(s).
+      real_start_rules = @grammar.start_rules.map do |start_rule|
+        unless @grammar.rules.find {|rule| rule.name == start_rule}
+          raise Error.new("Start rule `#{start_rule}` not found")
+        end
+        Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil)
       end
-      # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
+      # Add real start rules before user-given rules.
+      @grammar.rules = real_start_rules + @grammar.rules
       # Generate and add rules for optional components.
       generate_optional_component_rules!(tokens_by_name)
       # Build rule sets.
@@ -258,6 +276,24 @@ class Propane
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
+      code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "context->#{fieldname}"
+        when "d"
+          "context.#{fieldname}"
+        end
+      end
+      code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "token_tree_node->#{fieldname}"
+        when "d"
+          "token_tree_node.#{fieldname}"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
           case @language
@@ -291,7 +327,7 @@ class Propane
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          if @grammar.ast
+          if @grammar.tree
             case @language
             when "c"
               "out_token_info->pvalue"
@@ -324,13 +360,21 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]
     #   Start rule parser value type name and type string.
-    def start_rule_type
+    def start_rule_type(start_rule_index = 0)
       start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rule
+        rule.name == @grammar.start_rules[start_rule_index]
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -5,21 +5,27 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
-    attr_reader :ast
-    attr_reader :ast_prefix
-    attr_reader :ast_suffix
+    attr_reader :context_user_fields
+    attr_reader :lex_fn
+    attr_reader :tree
+    attr_reader :tree_prefix
+    attr_reader :tree_suffix
+    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
-    attr_reader :rules
-    attr_reader :start_rule
+    attr_accessor :rules
+    attr_reader :start_rules
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
     attr_reader :prefix
+    attr_reader :on_token_node
+    attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
-      @start_rule = "Start"
+      @start_rules = []
       @tokens = []
       @rules = []
       @code_blocks = {}
@@ -29,10 +35,15 @@ class Propane
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
-      @ast = false
-      @ast_prefix = ""
-      @ast_suffix = ""
+      @tree = false
+      @tree_prefix = ""
+      @tree_suffix = ""
+      @free_token_node = ""
+      @context_user_fields = nil
+      @on_token_node = ""
+      @token_user_fields = nil
       parse_grammar!
+      @start_rules << "Start" if @start_rules.empty?
     end
 
     def ptype
@@ -59,10 +70,15 @@ class Propane
       if parse_white_space!
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
-      elsif parse_ast_statement!
-      elsif parse_ast_prefix_statement!
-      elsif parse_ast_suffix_statement!
+      elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
+      elsif parse_tree_statement!
+      elsif parse_tree_prefix_statement!
+      elsif parse_tree_suffix_statement!
+      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
+      elsif parse_on_token_node_statement!
+      elsif parse_token_user_fields_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
       elsif parse_start_statement!
@@ -94,21 +110,37 @@ class Propane
       consume!(/#.*\n/)
     end
 
-    def parse_ast_statement!
-      if consume!(/ast\s*;/)
-        @ast = true
+    def parse_context_user_fields_statement!
+      if md = consume!(/context_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @context_user_fields ||= ""
+        @context_user_fields += code
       end
     end
 
-    def parse_ast_prefix_statement!
-      if md = consume!(/ast_prefix\s+(\w+)\s*;/)
-        @ast_prefix = md[1]
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
       end
     end
 
-    def parse_ast_suffix_statement!
-      if md = consume!(/ast_suffix\s+(\w+)\s*;/)
-        @ast_suffix = md[1]
+    def parse_tree_statement!
+      if consume!(/tree\s*;/)
+        @tree = true
+      end
+    end
+
+    def parse_tree_prefix_statement!
+      if md = consume!(/tree_prefix\s+(\w+)\s*;/)
+        @tree_prefix = md[1]
+      end
+    end
+
+    def parse_tree_suffix_statement!
+      if md = consume!(/tree_suffix\s+(\w+)\s*;/)
+        @tree_suffix = md[1]
       end
     end
 
@@ -122,12 +154,40 @@ class Propane
       end
     end
 
+    def parse_on_token_node_statement!
+      if md = consume!(/on_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @on_token_node += code
+      end
+    end
+
+    def parse_token_user_fields_statement!
+      if md = consume!(/token_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @token_user_fields ||= ""
+        @token_user_fields += code
+      end
+    end
+
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @free_token_node += code
+      end
+    end
+
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
         name = "default"
         if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
           end
           name = md[1]
         end
@@ -141,8 +201,8 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
           end
           ptypename = md[1]
         end
@@ -165,8 +225,8 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
           end
           ptypename = md[1]
         end
@@ -185,8 +245,10 @@ class Propane
           raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
         end
         consume!(/\s+/)
-        consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline)
         @modeline = nil
         true
       end
@@ -195,12 +257,12 @@ class Propane
     def parse_rule_statement!
       if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
         rule_name, ptypename = *md[1, 2]
-        if @ast && ptypename
-          raise Error.new("Multiple ptypes are unsupported in AST mode")
+        if @tree && ptypename
+          raise Error.new("Multiple ptypes are unsupported in tree mode")
         end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        if @ast
+        if @tree
           consume!(/;/, "expected `;'")
         else
           unless code = parse_code_block!
@@ -217,8 +279,8 @@ class Propane
       if pattern = parse_pattern!
         consume!(/\s+/)
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
           end
           ptypename = md[1]
         end
@@ -232,16 +294,23 @@ class Propane
     end
 
     def parse_start_statement!
-      if md = consume!(/start\s+(\w+)\s*;/)
-        @start_rule = md[1]
+      if md = consume!(/start\s+([\w\s]*);/)
+        start_rules = md[1].split(/\s+/).map(&:strip)
+        start_rules.each do |start_rule|
+          @start_rules << start_rule unless @start_rules.include?(start_rule)
+        end
       end
     end
 
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        if code.start_with?("\n")
+          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+        else
+          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -272,6 +341,8 @@ class Propane
             end
           elsif md = consume!(%r{(.)})
             pattern += md[1]
+          elsif @input == "" || @input.start_with?("\n")
+            raise Error.new("Line #{@line_number}: Unterminated pattern; expected `/`")
           end
         end
         pattern
@@ -280,9 +351,12 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        if code.start_with?("\n")
+          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+        else
+          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+        end
         code
       end
     end

lib/propane/parser.rb

@@ -16,11 +16,20 @@ class Propane
       @warnings = Set.new
       @errors = Set.new
       @options = options
-      start_item = Item.new(grammar.rules.first, 0)
-      eval_item_sets = Set[ItemSet.new([start_item])]
+      start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule|
+        Item.new(start_rule, 0)
+      end
+      start_item_sets = start_items.map {|item| ItemSet.new([item])}
+      eval_item_sets = Set[*start_item_sets]
 
       while eval_item_sets.size > 0
-        item_set = eval_item_sets.first
+        item_set =
+          if start_item_sets.size > 0
+            # Ensure we evaluate start_item_sets first in order
+            start_item_sets.slice!(0)
+          else
+            eval_item_sets.first
+          end
         eval_item_sets.delete(item_set)
         unless @item_sets_set.include?(item_set)
           item_set.id = @item_sets.size

lib/propane/rule.rb

@@ -36,7 +36,7 @@ class Propane
 
     # @return [Array<Integer>]
     #   Map this rule's components to their positions in the parent RuleSet's
-    #   node field pointer array. This is used for AST construction.
+    #   node field pointer array. This is used for tree construction.
     attr_accessor :rule_set_node_field_index_map
 
     # Construct a Rule.

lib/propane/rule_set.rb

@@ -4,8 +4,8 @@ class Propane
   class RuleSet
 
     # @return [Array<Hash>]
-    #   AST fields.
-    attr_reader :ast_fields
+    #   tree fields.
+    attr_reader :tree_fields
 
     # @return [Integer]
     #   ID of the RuleSet.
@@ -100,28 +100,28 @@ class Propane
 
     # Finalize a RuleSet after adding all Rules to it.
     def finalize(grammar)
-      if grammar.ast
-        build_ast_fields(grammar)
+      if grammar.tree
+        build_tree_fields(grammar)
       end
     end
 
     private
 
-    # Build the set of AST fields for this RuleSet.
+    # Build the set of tree fields for this RuleSet.
     #
     # This is an Array of Hashes. Each entry in the Array corresponds to a
-    # field location in the AST node. The entry is a Hash. It could have one or
+    # field location in the tree node. The entry is a Hash. It could have one or
     # two keys. It will always have the field name with a positional suffix as
     # a key. It may also have the field name without the positional suffix if
     # that field only exists in one position across all Rules in the RuleSet.
     #
     # @return [void]
-    def build_ast_fields(grammar)
-      field_ast_node_indexes = {}
+    def build_tree_fields(grammar)
+      field_tree_node_indexes = {}
       field_indexes_across_all_rules = {}
-      # Stores the index into @ast_fields by field alias name.
+      # Stores the index into @tree_fields by field alias name.
       field_aliases = {}
-      @ast_fields = []
+      @tree_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
           if component.is_a?(RuleSet) && component.optional?
@@ -132,25 +132,25 @@ class Propane
           else
             node_name = component.name
           end
-          struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+          struct_name = "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
           field_name = "p#{node_name}#{i + 1}"
-          unless field_ast_node_indexes[field_name]
-            field_ast_node_indexes[field_name] = @ast_fields.size
-            @ast_fields << {field_name => struct_name}
+          unless field_tree_node_indexes[field_name]
+            field_tree_node_indexes[field_name] = @tree_fields.size
+            @tree_fields << {field_name => struct_name}
           end
           rule.aliases.each do |alias_name, index|
             if index == i
-              alias_ast_fields_index = field_ast_node_indexes[field_name]
-              if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index
-                raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
+              alias_tree_fields_index = field_tree_node_indexes[field_name]
+              if field_aliases[alias_name] && field_aliases[alias_name] != alias_tree_fields_index
+                raise Error.new("Error: conflicting tree node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
               end
-              field_aliases[alias_name] = alias_ast_fields_index
-              @ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1]
+              field_aliases[alias_name] = alias_tree_fields_index
+              @tree_fields[alias_tree_fields_index][alias_name] = @tree_fields[alias_tree_fields_index].first[1]
             end
           end
           field_indexes_across_all_rules[node_name] ||= Set.new
-          field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
-          rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
+          field_indexes_across_all_rules[node_name] << field_tree_node_indexes[field_name]
+          rule.rule_set_node_field_index_map[i] = field_tree_node_indexes[field_name]
         end
       end
       field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
@@ -158,8 +158,8 @@ class Propane
           # If this field was only seen in one position across all rules,
           # then add an alias to the positional field name that does not
           # include the position.
-          @ast_fields[indexes_across_all_rules.first]["p#{node_name}"] =
-            "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+          @tree_fields[indexes_across_all_rules.first]["p#{node_name}"] =
+            "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
         end
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "2.3.0"
+  VERSION = "4.3.0"
 end

spec/json_parser.c.propane

@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {match[0], 0};
+  char s[] = {(char)match[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_types.c

@@ -5,7 +5,7 @@
 
 JSONValue * JSONValue_new(size_t id)
 {
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
     jv->id = id;
     return jv;
 }
@@ -29,7 +29,7 @@ void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
         }
     }
     size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
     if (size > 0)
     {
         memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
@@ -52,7 +52,7 @@ void JSONArray_append(JSONValue * array, JSONValue * value)
 {
     size_t const size = array->array.size;
     size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
     if (array->array.size > 0)
     {
         memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);

spec/json_types.h

@@ -11,6 +11,12 @@
 #define JSON_FALSE 5u
 #define JSON_NULL 6u
 
+typedef struct JSONObjectEntry_s
+{
+    char const * name;
+    struct JSONValue_s * value;
+} JSONObjectEntry;
+
 typedef struct JSONValue_s
 {
     size_t id;
@@ -19,11 +25,7 @@ typedef struct JSONValue_s
         struct
         {
             size_t size;
-            struct
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
+            JSONObjectEntry * entries;
         } object;
         struct
         {

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
     end
 
     it "supports mode labels" do
@@ -144,7 +144,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +197,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -14,6 +14,7 @@ describe Propane do
   end
 
   def run_propane(options = {})
+    options[:language] ||= "d"
     @statics[:build_test_id] ||= 0
     @statics[:build_test_id] += 1
     if ENV["dist_specs"]
@@ -65,26 +66,45 @@ EOF
   end
 
   def compile(test_files, options = {})
-    test_files = Array(test_files)
+    test_files = Array(test_files).map do |test_file|
+      if !File.exist?(test_file) && test_file.end_with?(".cpp")
+        test_file.sub(%r{\.cpp$}, ".c")
+      else
+        test_file
+      end
+    end
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
       "spec/run/testparser#{name}.#{options[:language]}"
     end
     case options[:language]
     when "c"
-      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
+      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+    when "cpp"
+      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      result = system(*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
     end
+    result = system(*command)
     expect(result).to be_truthy
   end
 
-  def run_test
+  def run_test(options)
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
-    Results.new(stdout, stderr, status)
+    results = Results.new(stdout, stderr, status)
+    stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+    vgout = stdout + stderr
+    File.binwrite("spec/run/.vgout", vgout)
+    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
+      bytes = $1.to_i
+      if bytes > 0
+        raise "Valgrind detected memory leak"
+      end
+    end
+    results
   end
 
   def lines(str)
@@ -224,20 +244,20 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  it "errors when an alias is in different positions for different rules in a rule set when AST mode is enabled" do
+  it "errors when an alias is in different positions for different rules in a rule set when tree mode is enabled" do
     write_grammar <<EOF
-ast;
+tree;
 token a;
 token b;
 Start -> a:foo b;
 Start -> b b:foo;
 EOF
     results = run_propane(extra_args: %w[-w], capture: true)
-    expect(results.stderr).to match %r{Error: conflicting AST node field positions for alias `foo`}
+    expect(results.stderr).to match %r{Error: conflicting tree node field positions for alias `foo`}
     expect(results.status).to_not eq 0
   end
 
-  it "does not error when an alias is in different positions for different rules in a rule set when AST mode is not enabled" do
+  it "does not error when an alias is in different positions for different rules in a rule set when tree mode is not enabled" do
     write_grammar <<EOF
 token a;
 token b;
@@ -261,7 +281,7 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c].each do |language|
+  %w[d c cpp].each do |language|
 
     context "#{language.upcase} language" do
 
@@ -277,14 +297,14 @@ Foo -> plus <<>>
 EOF
         run_propane(language: language)
         compile("spec/test_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "detects a lexer error when an unknown character is seen" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype int;
 token int /\\d+/ <<
@@ -315,7 +335,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -338,7 +358,7 @@ EOF
 
       it "generates a parser that does basic math - user guide example" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <math.h>
@@ -409,7 +429,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -435,7 +455,7 @@ R2 -> a b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
@@ -450,13 +470,13 @@ R1 -> b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
       it "executes user code when matching lexer token" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -485,7 +505,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "abc!",
@@ -498,7 +518,7 @@ EOF
 
       it "supports a pattern statement" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -521,7 +541,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -534,7 +554,7 @@ EOF
 
       it "supports returning tokens from pattern code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -563,7 +583,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -574,7 +594,7 @@ EOF
 
       it "supports lexer modes" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -621,7 +641,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "begin string mode",
@@ -635,7 +655,7 @@ EOF
 
       it "multiple lexer modes may apply to a pattern" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -679,7 +699,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_multiple_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "ident: d",
@@ -691,7 +711,7 @@ EOF
 
       it "executes user code associated with a parser rule" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -716,7 +736,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "A!",
@@ -727,7 +747,7 @@ EOF
 
       it "parses lists" do
         write_grammar <<EOF
-ptype #{language == "c" ? "uint32_t" : "uint"};
+ptype #{language == "d" ? "uint" : "uint32_t"};
 token a;
 Start -> As << $$ = $1; >>
 As -> << $$ = 0u; >>
@@ -735,7 +755,7 @@ As -> As a << $$ = $1 + 1u; >>
 EOF
         run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         expect(results.stderr).to eq ""
       end
@@ -762,14 +782,14 @@ EOF
 
       it "provides matched text to user code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
 #include <stdlib.h>
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * t = malloc(match_length + 1);
+  char * t = (char *)malloc(match_length + 1);
   strncpy(t, (char *)match, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
@@ -789,7 +809,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "Matched token is identifier_123",
@@ -799,7 +819,7 @@ EOF
 
       it "allows storing a result value for the lexer" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype uint64_t;
 token word /[a-z]+/ <<
@@ -822,7 +842,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -837,13 +857,14 @@ Start -> a num;
 EOF
         run_propane(language: language)
         compile("spec/test_error_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/json_parser.#{ext}.propane"))
         run_propane(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end
@@ -865,7 +886,7 @@ Start -> b c b;
 EOF
         run_propane(name: "myp2", language: language)
         compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -884,7 +905,7 @@ Any -> c;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -902,14 +923,14 @@ Any -> ;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "matches backslash escape sequences" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
   #include <stdio.h>
@@ -946,7 +967,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -974,9 +995,9 @@ EOF
         run_propane(language: language)
       end
 
-      it "generates an AST" do
+      it "generates a tree" do
         write_grammar <<EOF
-ast;
+tree;
 
 ptype int;
 
@@ -1010,17 +1031,17 @@ One -> one;
 Two -> two;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "supports AST node prefix and suffix" do
+      it "supports tree node prefix and suffix" do
         write_grammar <<EOF
-ast;
-ast_prefix P ;
-ast_suffix  S;
+tree;
+tree_prefix P ;
+tree_suffix  S;
 
 ptype int;
 
@@ -1054,8 +1075,8 @@ One -> one;
 Two -> two;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast_ps.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_ps.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1070,15 +1091,15 @@ EOF
         compile("spec/test_start_rule.#{language}", language: language)
       end
 
-      it "allows specifying a different start rule with AST generation" do
+      it "allows specifying a different start rule with tree generation" do
         write_grammar <<EOF
-ast;
+tree;
 token hi;
 start Top;
 Top -> hi;
 EOF
         run_propane(language: language)
-        compile("spec/test_start_rule_ast.#{language}", language: language)
+        compile("spec/test_start_rule_tree.#{language}", language: language)
       end
 
       it "allows marking a rule component as optional" do
@@ -1123,13 +1144,13 @@ Start -> a? b R? <<
   printf("b: %d\\n", $2);
   printf("R: %s\\n", $3 == NULL ? "" : $3);
 >>
-R -> c d << $$ = "cd"; >>
-R (string) -> d c << $$ = "dc"; >>
+R -> c d << $$ = (char *)"cd"; >>
+R (string) -> d c << $$ = (char *)"dc"; >>
 EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -1145,10 +1166,10 @@ EOF
         ])
       end
 
-      it "allows marking a rule component as optional in AST generation mode" do
+      it "allows marking a rule component as optional in tree generation mode" do
         if language == "d"
           write_grammar <<EOF
-ast;
+tree;
 
 <<
 import std.stdio;
@@ -1164,7 +1185,7 @@ R -> d c;
 EOF
         else
           write_grammar <<EOF
-ast;
+tree;
 
 <<
 #include <stdio.h>
@@ -1180,16 +1201,16 @@ R -> d c;
 EOF
         end
         run_propane(language: language)
-        compile("spec/test_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        compile("spec/test_optional_rule_component_tree.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "allows naming an optional rule component in AST generation mode" do
+      it "allows naming an optional rule component in tree generation mode" do
         if language == "d"
           write_grammar <<EOF
-ast;
+tree;
 
 <<
 import std.stdio;
@@ -1205,7 +1226,7 @@ R -> d c;
 EOF
         else
           write_grammar <<EOF
-ast;
+tree;
 
 <<
 #include <stdio.h>
@@ -1221,15 +1242,15 @@ R -> d c;
 EOF
         end
         run_propane(language: language)
-        compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        compile("spec/test_named_optional_rule_component_tree.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "stores token and rule positions in AST nodes" do
+      it "stores token and rule positions in tree nodes" do
         write_grammar <<EOF
-ast;
+tree;
 
 token a;
 token bb;
@@ -1241,15 +1262,15 @@ T -> bb;
 T -> c;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast_token_positions.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_token_positions.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "stores invalid positions for empty rule matches" do
         write_grammar <<EOF
-ast;
+tree;
 
 token a;
 token bb;
@@ -1261,15 +1282,15 @@ T -> a A;
 A -> bb? c?;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast_invalid_positions.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_invalid_positions.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "allows specifying field aliases in AST mode" do
+      it "allows specifying field aliases in tree mode" do
         write_grammar <<EOF
-ast;
+tree;
 
 token a;
 token b;
@@ -1281,15 +1302,15 @@ T -> b;
 T -> c;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast_field_aliases.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "aliases the correct field when multiple rules are in a rule set in AST mode" do
+      it "aliases the correct field when multiple rules are in a rule set in tree mode" do
         write_grammar <<EOF
-ast;
+tree;
 
 token a;
 token b;
@@ -1304,13 +1325,13 @@ T -> b;
 T -> c;
 EOF
         run_propane(language: language)
-        compile("spec/test_ast_field_aliases.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
-      it "allows specifying field aliases when AST mode is not enabled" do
+      it "allows specifying field aliases when tree mode is not enabled" do
         if language == "d"
           write_grammar <<EOF
 <<
@@ -1332,9 +1353,9 @@ EOF
 #include <stdio.h>
 #include <string.h>
 >>
-ptype char const *;
+ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * s = malloc(match_length + 1);
+  char * s = (char *)malloc(match_length + 1);
   strncpy(s, (char const *)match, match_length);
   s[match_length] = 0;
   $$ = s;
@@ -1343,18 +1364,20 @@ drop /\\s+/;
 Start -> id:first id:second <<
   printf("first is %s\\n", ${first});
   printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
 >>
 EOF
         end
         run_propane(language: language)
         compile("spec/test_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
 
-      it "aliases the correct field when multiple rules are in a rule set when AST mode is not enabled" do
+      it "aliases the correct field when multiple rules are in a rule set when tree mode is not enabled" do
         if language == "d"
           write_grammar <<EOF
 <<
@@ -1379,9 +1402,9 @@ EOF
 #include <stdio.h>
 #include <string.h>
 >>
-ptype char const *;
+ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * s = malloc(match_length + 1);
+  char * s = (char *)malloc(match_length + 1);
   strncpy(s, (char const *)match, match_length);
   s[match_length] = 0;
   $$ = s;
@@ -1392,26 +1415,372 @@ Start -> Foo;
 Start -> id:first id:second <<
   printf("first is %s\\n", ${first});
   printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
 >>
 Foo -> ;
 EOF
         end
         run_propane(language: language)
         compile("spec/test_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
 
-      it "does not free memory allocated for AST nodes" do
-        write_grammar(File.read("spec/ast_node_memory_remains.#{language}.propane"))
+      # D garbage collector was freeing memory for nodes when it should not
+      # have been. This was due to pointers being in the TreeNode that was
+      # allocated by C malloc() function instead of allocated using D.
+      it "does not free memory allocated for tree nodes" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/tree_node_memory_remains.#{ext}.propane"))
         run_propane(language: language)
-        compile("spec/test_ast_node_memory_remains.#{language}", language: language)
-        results = run_test
+        compile("spec/test_tree_node_memory_remains.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "allows multiple starting rules" do
+    write_grammar <<EOF
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a b R;
+Start -> Bs:bs << $$ = $1; >>
+R -> c:c << $$ = $1; >>
+Bs -> << $$ = 0; >>
+Bs -> b:b Bs:bs << $$ = $1 + $2; >>
+start Start R Bs;
+EOF
+        run_propane(language: language)
+        compile("spec/test_starting_rules.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows multiple starting rules in tree mode" do
+    write_grammar <<EOF
+tree;
+ptype int;
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> a b R;
+Start -> Bs:bs;
+R -> c:c;
+Bs -> ;
+Bs -> b:b Bs:bs;
+start Start R Bs;
+EOF
+        run_propane(language: language)
+        compile("spec/test_starting_rules_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      if %w[c cpp].include?(language)
+        it "allows a user function to free token node memory in tree mode" do
+          write_grammar <<EOF
+tree;
+free_token_node <<
+    free(${token.pvalue});
+>>
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+EOF
+          run_propane(language: language)
+          compile("spec/test_tree_delete_token_node_memory.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
+
+      it "executes code blocks associated with drop statements" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  stderr.write("comment: ", match);
+>>
+token a;
+Start -> a;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+>>
+token a;
+Start -> a;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_drop_code_block.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to match %r{comment: # comment 1\n.*comment: #    comment 2}m
+        expect(results.status).to eq 0
+      end
+
+      it "allows user-defined context fields" do
+        if language == "d"
+          write_grammar <<EOF
+context_user_fields <<
+    string comments;
+    uint acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} ~= match;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <string.h>
+#include <stdlib.h>
+>>
+context_user_fields <<
+    char * comments;
+    unsigned int acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    size_t cur_len = 0u;
+    if (${context.comments} != NULL)
+        cur_len = strlen(${context.comments});
+    char * commentsnew = (char *)malloc(cur_len + match_length + 1);
+    if (${context.comments} != NULL)
+        memcpy(commentsnew, ${context.comments}, cur_len);
+    memcpy(&commentsnew[cur_len], match, match_length);
+    commentsnew[cur_len + match_length] = '\\0';
+    if (${context.comments} != NULL)
+    {
+        free(${context.comments});
+    }
+    ${context.comments} = commentsnew;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_user_context_fields.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to include %r{comments: # comment 1\n#    comment 2}
+        expect(results.stderr).to include %r{acount: 11\n}
+        expect(results.status).to eq 0
+      end
+
+      it "allows custom token user fields" do
+        if language == "d"
+          write_grammar <<EOF
+context_user_fields <<
+    string comments;
+>>
+token_user_fields <<
+    string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} ~= match;
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        elsif language == "c"
+          write_grammar <<EOF
+<<
+#include <string.h>
+#include <stdlib.h>
+>>
+context_user_fields <<
+    char * comments;
+>>
+token_user_fields <<
+    char * comments;
+>>
+free_token_node <<
+    free(${token.comments});
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = (char *)malloc(1);
+    ${context.comments}[0] = '\\0';
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    size_t cur_len = 0u;
+    if (${context.comments} != NULL)
+        cur_len = strlen(${context.comments});
+    char * commentsnew = (char *)malloc(cur_len + match_length + 1);
+    if (${context.comments} != NULL)
+        memcpy(commentsnew, ${context.comments}, cur_len);
+    memcpy(&commentsnew[cur_len], match, match_length);
+    commentsnew[cur_len + match_length] = '\\0';
+    if (${context.comments} != NULL)
+    {
+        free(${context.comments});
+    }
+    ${context.comments} = commentsnew;
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        else # C++
+          write_grammar <<EOF
+<<header
+#include <string>
+>>
+context_user_fields <<
+    std::string comments;
+>>
+token_user_fields <<
+    std::string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_token_user_fields.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/spec_helper.rb

@@ -2,6 +2,10 @@ unless ENV["dist_specs"]
   require "bundler/setup"
   require "simplecov"
 
+  class MyFormatter
+    def format(*args)
+    end
+  end
   SimpleCov.start do
     add_filter "/spec/"
     add_filter "/.bundle/"
@@ -12,6 +16,7 @@ unless ENV["dist_specs"]
     end
     project_name "Propane"
     merge_timeout 3600
+    formatter(MyFormatter)
   end
 
   RSpec.configure do |config|

spec/test_basic_math_grammar.c

@@ -5,25 +5,29 @@
 int main()
 {
     char const * input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
+    p_context_delete(context);
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
+    p_context_delete(context);
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
+    p_context_delete(context);
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
+    p_context_delete(context);
 
     return 0;
 }

spec/test_basic_math_grammar.d

@@ -10,23 +10,23 @@ int main()
 unittest
 {
     string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
 }

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_drop_code_block.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_drop_code_block.d

@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_error_positions.c

@@ -5,38 +5,43 @@
 int main()
 {
     char const * input = "a 42";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "a\n123\na  a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 4);
-    assert(p_token(&context) == TOKEN_a);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 4);
+    assert(p_token(context) == TOKEN_a);
+    p_context_delete(context);
 
     input = "12";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 1);
-    assert(p_token(&context) == TOKEN_num);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 1);
+    assert(p_token(context) == TOKEN_num);
+    p_context_delete(context);
 
     input = "a 12\n\nab";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 2);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 2);
+    p_context_delete(context);
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 6);
-    assert(p_position(&context).col == 5);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context).row == 6);
+    assert(p_position(context).col == 5);
 
     assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
     assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_error_positions.d

@@ -9,31 +9,31 @@ int main()
 unittest
 {
     string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(3, 4));
-    assert(p_token(&context) == TOKEN_a);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(3, 4));
+    assert(p_token(context) == TOKEN_a);
 
     input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 1));
-    assert(p_token(&context) == TOKEN_num);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 1));
+    assert(p_token(context) == TOKEN_num);
 
     input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(3, 2));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(6, 5));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context) == p_position_t(6, 5));
 
     assert(p_token_names[TOKEN_a] == "a");
     assert(p_token_names[TOKEN_num] == "num");

spec/test_field_aliases.c

@@ -6,8 +6,9 @@
 int main()
 {
     char const * input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
     return 0;
 }

spec/test_field_aliases.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_lexer.c

@@ -38,73 +38,75 @@ int main()
 
     p_token_info_t token_info;
     char const * input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 3u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 5u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 7u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 5u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 7u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 10u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
-    p_context_init(&context, (uint8_t const *)"", 0u);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)"", 0u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer.d

@@ -44,26 +44,26 @@ unittest
 {
     p_token_info_t token_info;
     string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    context = p_context_new("");
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_lexer_match_text.c

@@ -6,10 +6,11 @@
 int main()
 {
     char const * input = "identifier_123";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_match_text.d

@@ -9,8 +9,8 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 }

spec/test_lexer_modes.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abc \"a string\" def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abc \"abc def\" def";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_multiple_modes.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abc.def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abc .  abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_multiple_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc.def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc .  abc`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_result_value.c

@@ -5,15 +5,17 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "fabulous";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_result_value.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = `fabulous`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
 }

spec/test_lexer_unknown_character.c

@@ -5,14 +5,16 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    p_context_delete(context);
 
     input = "123";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_unknown_character.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
 
     input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
 }

spec/test_match_backslashes.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_match_backslashes.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_multiple_parsers.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new((uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
+    myp1_context_delete(context1);
 
     char const * input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new((uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
+    myp2_context_delete(context2);
 
     return 0;
 }

spec/test_multiple_parsers.d

@@ -10,12 +10,12 @@ int main()
 unittest
 {
     string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new(input1);
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
 
     string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new(input2);
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
 }

spec/test_named_optional_rule_component_tree.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start->a == NULL);
     assert(start->pToken2 != NULL);
     assert_eq(TOKEN_b, start->pToken2->token);
@@ -17,10 +17,13 @@ int main()
     assert(start->pR == NULL);
     assert(start->r == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->a != NULL);
     assert_eq(TOKEN_a, start->pToken1->token);
     assert(start->pToken2 != NULL);
@@ -31,15 +34,21 @@ int main()
     assert(start->pR == start->r);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->a == NULL);
     assert(start->pToken2 != NULL);
     assert(start->r != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }
 

spec/test_named_optional_rule_component_tree.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
@@ -21,10 +20,12 @@ unittest
     assert(start.pR is null);
     assert(start.r is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
     assert(start.pToken2 != null);
@@ -35,12 +36,16 @@ unittest
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_optional_rule_component.c

@@ -5,17 +5,20 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_optional_rule_component.d

@@ -9,15 +9,15 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_optional_rule_component_tree.c

@@ -6,20 +6,23 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start->pToken1 == NULL);
     assert(start->pToken2 != NULL);
     assert_eq(TOKEN_b, start->pToken2->token);
     assert(start->pR3 == NULL);
     assert(start->pR == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->pToken1 != NULL);
     assert_eq(TOKEN_a, start->pToken1->token);
     assert(start->pToken2 != NULL);
@@ -28,15 +31,21 @@ int main()
     assert(start->pR == start->pR3);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->pToken1 == NULL);
     assert(start->pToken2 != NULL);
     assert(start->pR != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }
 

spec/test_optional_rule_component_tree.d

@@ -10,20 +10,21 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
     assert(start.pR3 is null);
     assert(start.pR is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
     assert(start.pToken2 != null);
@@ -32,12 +33,16 @@ unittest
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_parser_identical_rules_lookahead.c

@@ -5,13 +5,15 @@
 int main()
 {
     char const * input = "aba";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_identical_rules_lookahead.d

@@ -9,11 +9,11 @@ int main()
 unittest
 {
     string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_from_multiple_states.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 2);
-    assert(context.token == TOKEN___EOF);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 2);
+    assert(context->token == TOKEN___EOF);
+    p_context_delete(context);
 
     input = "a b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "bb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_from_multiple_states.d

@@ -9,17 +9,17 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 2));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_user_code.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_user_code.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "ab";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parsing_json.c

@@ -6,51 +6,58 @@
 int main()
 {
     char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_OBJECT);
+    p_context_delete(context);
 
     input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_ARRAY);
+    p_context_delete(context);
 
     input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == -45.6);
+    p_context_delete(context);
 
     input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == 0.02);
+    p_context_delete(context);
 
     input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    JSONValue * o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(1, o->object.size);
     assert(strcmp(o->object.entries[0].name, "hi") == 0);
     assert(o->object.entries[0].value->id == JSON_TRUE);
+    p_context_delete(context);
 
     input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(2, o->object.size);
     assert(strcmp(o->object.entries[0].name, "ff") == 0);
     assert(o->object.entries[0].value->id == JSON_FALSE);
     assert(strcmp(o->object.entries[1].name, "nn") == 0);
     assert(o->object.entries[1].value->id == JSON_NULL);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_json.d

@@ -10,45 +10,45 @@ int main()
 unittest
 {
     string input = ``;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
 
     input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONArray)p_result(context));
 
     input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == -45.6);
 
     input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == 0.02);
 
     input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    JSONObject o = cast(JSONObject)p_result(context);
     assert(o.value["hi"]);
     assert(cast(JSONTrue)o.value["hi"]);
 
     input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    o = cast(JSONObject)p_result(context);
     assert(o.value["ff"]);
     assert(cast(JSONFalse)o.value["ff"]);
     assert(o.value["nn"]);

spec/test_parsing_lists.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
+    p_context_delete(context);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_lists.d

@@ -9,18 +9,18 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = "";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
 }

spec/test_pattern.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "defabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_pattern.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = "defabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_return_token_from_pattern.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_return_token_from_pattern.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_start_rule_ast.c

@@ -1,17 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "hi";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Top * top = p_result(&context);
-    assert(top->pToken != NULL);
-    assert_eq(TOKEN_hi, top->pToken->token);
-
-    return 0;
-}

spec/test_start_rule_tree.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "hi";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
+
+    p_tree_delete(top);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_start_rule_tree.d

@@ -10,10 +10,10 @@ int main()
 unittest
 {
     string input = "hi";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Top * top = p_result(&context);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
     assert(top.pToken !is null);
     assert_eq(TOKEN_hi, top.pToken.token);
 }

spec/test_starting_rules.c

@@ -0,0 +1,30 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "bbbb";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
+    assert_eq(8, result);
+    p_context_delete(context);
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
+    assert_eq(8, result);
+    p_context_delete(context);
+
+    input = "c";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
+    assert_eq(3, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_starting_rules.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "bbbb";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
+    assert(result == 8);
+
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
+    assert(result == 8);
+
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
+    assert(result == 3);
+}

spec/test_starting_rules_tree.c

@@ -0,0 +1,40 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "bbbb";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert_not_null(start->bs);
+    assert_not_null(start->bs->b);
+    assert_not_null(start->bs->bs->b);
+    assert_not_null(start->bs->bs->bs->b);
+    assert_not_null(start->bs->bs->bs->bs->b);
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
+    assert_not_null(bs->b);
+    assert_not_null(bs->bs->b);
+    assert_not_null(bs->bs->bs->b);
+    assert_not_null(bs->bs->bs->bs->b);
+    p_tree_delete_Bs(bs);
+    p_context_delete(context);
+
+    input = "c";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
+    assert_not_null(r->c);
+    p_tree_delete_R(r);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_starting_rules_tree.d

@@ -0,0 +1,41 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "bbbb";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.bs);
+    assert(start.bs.b);
+    assert(start.bs.bs.b);
+    assert(start.bs.bs.bs.b);
+    assert(start.bs.bs.bs.bs.b);
+
+    p_tree_delete(start);
+
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
+    assert(bs.b);
+    assert(bs.bs.b);
+    assert(bs.bs.bs.b);
+    assert(bs.bs.bs.bs.b);
+
+    p_tree_delete_Bs(bs);
+
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
+    assert(r.c);
+
+    p_tree_delete_R(r);
+}

spec/test_token_user_fields.c

@@ -0,0 +1,46 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input =
+        "# c1\n"
+        "#  c2\n"
+        "\n"
+        "first\n"
+        "\n   \n  \n"
+        "  # s1\n"
+        "   #   s2\n"
+        "second\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start->pIDs);
+    assert(start->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
+#else
+    assert(start->pIDs->id->comments);
+    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
+#endif
+    assert(start->pIDs->pIDs);
+    assert(start->pIDs->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
+#else
+    assert(start->pIDs->pIDs->id->comments);
+    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
+#endif
+
+#ifndef __cplusplus
+    free(context->comments);
+#endif
+    p_context_delete(context);
+    p_tree_delete(start);
+
+    return 0;
+}

spec/test_token_user_fields.d

@@ -0,0 +1,31 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "# c1\n" ~
+        "#  c2\n" ~
+        "\n" ~
+        "first\n" ~
+        "\n   \n  \n" ~
+        "  # s1\n" ~
+        "   #   s2\n" ~
+        "second\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.pIDs);
+    assert(start.pIDs.id);
+    assert(start.pIDs.id.comments == "# c1\n#  c2\n");
+    assert(start.pIDs.pIDs);
+    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
+
+    p_tree_delete(start);
+}

spec/test_tree.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
     assert(start->pItems1 != NULL);
     assert(start->pItems != NULL);
     Items * items = start->pItems;
@@ -33,16 +33,22 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems != NULL);
     assert(start->pItems->pItem != NULL);
     assert(start->pItems->pItem->pDual != NULL);
@@ -51,5 +57,8 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
     assert(start.pItems1 !is null);
     assert(start.pItems !is null);
     Items * items = start.pItems;
@@ -37,16 +36,20 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems !is null);
     assert(start.pItems.pItem !is null);
     assert(start.pItems.pItem.pDual !is null);
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_delete_token_node_memory.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
+    assert(start->a != NULL);
+    assert(*start->a->pvalue == 1);
+    assert(start->b != NULL);
+    assert(*start->b->pvalue == 2);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+}

spec/test_tree_field_aliases.c

@@ -6,14 +6,17 @@
 int main()
 {
     char const * input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start->first->pToken->token);
     assert_eq(TOKEN_b, start->second->pToken->token);
     assert_eq(TOKEN_c, start->third->pToken->token);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree_field_aliases.d

@@ -10,12 +10,13 @@ int main()
 unittest
 {
     string input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start.first.pToken.token);
     assert_eq(TOKEN_b, start.second.pToken.token);
     assert_eq(TOKEN_c, start.third.pToken.token);
+
+    p_tree_delete(start);
 }

spec/test_tree_invalid_positions.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(2, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -30,10 +30,13 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(8, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "a\nbb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -54,10 +57,13 @@ int main()
     assert_eq(2, start->end_position.row);
     assert_eq(2, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "a\nc\nc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -78,10 +84,13 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -98,5 +107,8 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree_invalid_positions.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(2, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -34,10 +33,12 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(8, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nbb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -58,10 +59,12 @@ unittest
     assert_eq(2, start.end_position.row);
     assert_eq(2, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nc\nc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -82,10 +85,12 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -101,4 +106,6 @@ unittest
     assert_eq(1, start.position.col);
     assert_eq(1, start.end_position.row);
     assert_eq(1, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_tree_node_memory_remains.c

@@ -369,11 +369,11 @@ int main(int argc, char * argv[])
         {"size_t_to_ulong", TOKEN_ulong},
         {"main", TOKEN_int},
     };
-    p_context_t context;
-    p_context_init(&context, (const uint8_t *)input, strlen(input));
-    size_t result = p_parse(&context);
+    p_context_t * context;
+    context = p_context_new((const uint8_t *)input, strlen(input));
+    size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
+    PModule * pmod = p_result(context);
     PModuleItems * pmis = pmod->pModuleItems;
     PFunctionDefinition ** pfds;
     size_t n_pfds = 0u;
@@ -386,7 +386,7 @@ int main(int argc, char * argv[])
         }
         pmis = pmis->pModuleItems;
     }
-    pfds = malloc(n_pfds * sizeof(PModuleItems *));
+    pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
     pmis = pmod->pModuleItems;
     size_t pfd_i = n_pfds;
     while (pmis != NULL)
@@ -411,5 +411,9 @@ int main(int argc, char * argv[])
         }
     }
 
+    free(pfds);
+    p_tree_delete(pmod);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree_node_memory_remains.d

@@ -374,11 +374,11 @@ def main() -> int
         Expected("size_t_to_ulong", TOKEN_ulong),
         Expected("main", TOKEN_int),
     ];
-    p_context_t context;
-    p_context_init(&context, input);
-    size_t result = p_parse(&context);
+    p_context_t * context;
+    context = p_context_new(input);
+    size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
+    PModule * pmod = p_result(context);
     PModuleItems * pmis = pmod.pModuleItems;
     PFunctionDefinition *[] pfds;
     while (pmis !is null)
@@ -405,4 +405,5 @@ def main() -> int
             stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
         }
     }
+    p_tree_delete(pmod);
 }

spec/test_tree_ps.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    PStartS * start = p_result(context);
     assert(start->pItems1 != NULL);
     assert(start->pItems != NULL);
     PItemsS * items = start->pItems;
@@ -33,16 +33,22 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems != NULL);
     assert(start->pItems->pItem != NULL);
     assert(start->pItems->pItem->pDual != NULL);
@@ -51,5 +57,8 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree_ps.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    PStartS * start = p_result(context);
     assert(start.pItems1 !is null);
     assert(start.pItems !is null);
     PItemsS * items = start.pItems;
@@ -37,16 +36,20 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems !is null);
     assert(start.pItems.pItem !is null);
     assert(start.pItems.pItem.pDual !is null);
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_token_positions.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -43,10 +43,13 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(3, start->pT1->pToken->position.row);
     assert_eq(3, start->pT1->pToken->position.col);
@@ -80,5 +83,8 @@ int main()
     assert_eq(7, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
+    p_tree_delete(start);
+    p_context_delete(context);
+
     return 0;
 }

spec/test_tree_token_positions.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -47,10 +46,12 @@ unittest
     assert_eq(1, start.end_position.row);
     assert_eq(6, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(3, start.pT1.pToken.position.row);
     assert_eq(3, start.pT1.pToken.position.col);
@@ -83,4 +84,6 @@ unittest
     assert_eq(3, start.position.col);
     assert_eq(7, start.end_position.row);
     assert_eq(6, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_user_code.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abcabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_user_code.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = "abcabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_user_context_fields.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    fprintf(stderr, "comments: %s", context->comments);
+    fprintf(stderr, "acount: %u\n", context->acount);
+    free(context->comments);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_user_context_fields.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    stderr.writeln("comments: ", context.comments);
+    stderr.writeln("acount: ", context.acount);
+}

spec/test_user_terminate.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input = "aacc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 4200);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_user_terminate.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = "aacc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 4200);
 }