v4.3.0

CHANGELOG.md

@@ -1,3 +1,39 @@
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
+## v4.0.0
+
+### New Features
+
+- Add `context_user_fields` statement to allow custom context user fields.
+- Add `token_user_fields` statement to allow custom token user fields.
+- Add `on_token_node` statement to allow custom code when constructing token nodes.
+- Add `free_token_node` statement to allow custom code when freeing token nodes.
+- Add `p_context_delete()`.
+- Allow `drop` patterns to execute lexer user code blocks.
+
+### Breaking Changes
+
+- Replace `p_context_init()` with `p_context_new()` and `p_context_delete()`.
+- Renamed `p_free_tree()` to `p_tree_delete()`.
+- The `free_token_node` statement now takes a user code block instead of a
+  function name parameter.
+
 ## v3.0.0
 
 ### New Features
@@ -12,9 +48,9 @@
 
 - Document `p_lex()` and `p_token_info_t` in user guide (#37)
 
-### Breaking changes
+### Breaking Changes
 
-- Rename AST generation mode to tree generation mode (see [UPGRADING.md]())
+- Rename AST generation mode to tree generation mode (see [UPGRADING.md](UPGRADING.md))
 
 ## v2.3.0
 

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
+    erb (6.0.4)
     psych (5.3.1)
       date
       stringio
-    rake (13.3.1)
+    rake (13.4.2)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.11

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

UPGRADING.md

@@ -1,6 +1,20 @@
+## v4.0.0
+
+### API Changes
+
+- Replace any calls to `p_context_init()` with `p_context_new()`.
+- Replace any references to the address of a statically allocated context
+  structure with the pointer returned from `p_context_init()` (e.g. `&context`
+  -> `context`).
+- Add a call to `p_context_delete()` (for C or C++) after lexing/parsing to
+  reclaim context memory.
+- Rename `p_free_tree()` calls to `p_tree_delete()`.
+- Change `free_token_node` statement calls from taking a function name argument
+  to taking a user code block.
+
 ## v3.0.0
 
-### Grammar changes
+### Grammar Changes
 
 - Rename `ast;` statement to `tree;`.
 - Rename `ast_prefix;` statement to `tree_prefix;`.

assets/parser.c.erb

@@ -43,30 +43,52 @@ const char * <%= @grammar.prefix %>token_names[] = {
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
  * @param input_length
  *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length)
 {
-    /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext;
-    memset(&newcontext, 0, sizeof(newcontext));
+<% if @cpp %>
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t();
+<% else %>
+    <%= @grammar.prefix %>context_t * context = (<%= @grammar.prefix %>context_t *)calloc(1, sizeof(<%= @grammar.prefix %>context_t));
+<% end %>
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.input_length = input_length;
-    newcontext.text_position.row = 1u;
-    newcontext.text_position.col = 1u;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context->input = input;
+    context->input_length = input_length;
+    context->text_position.row = 1u;
+    context->text_position.col = 1u;
+    context->mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * For C++, destructors will be called for any context user fields. However, if
+ * pointers are used to store allocated resources, the user should free them
+ * before calling this function.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+<% if @cpp %>
+    delete context;
+<% else %>
+    free(context);
+<% end %>
 }
 
 /**************************************************************************
@@ -694,6 +716,7 @@ typedef struct
 <% end %>
 } state_value_t;
 
+<% if @grammar.tree %>
 /** Common tree node structure. */
 typedef struct TreeNode_s
 {
@@ -703,6 +726,7 @@ typedef struct TreeNode_s
     uint8_t is_token;
     struct TreeNode_s * fields[];
 } TreeNode;
+<% end %>
 
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
@@ -957,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -994,13 +1018,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
+<%   if @cpp %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
+<%   else %>
                 <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
+<%   end %>
                 token_tree_node->position = token_info.position;
                 token_tree_node->end_position = token_info.end_position;
                 token_tree_node->n_fields = 0u;
                 token_tree_node->is_token = 1u;
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
@@ -1186,14 +1215,17 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 }
 <% if @grammar.tree %>
 
-static void free_tree_node(TreeNode * node)
+static void tree_delete(TreeNode * node)
 {
     if (node->is_token)
     {
-<%   if @grammar.free_token_node %>
-        <%= @grammar.free_token_node %>((<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *) node);
+        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+<%   if @cpp %>
+        delete token_tree_node;
+<%   else %>
+        free(token_tree_node);
 <%   end %>
-        /* TODO: free value_t */
     }
     else if (node->n_fields > 0u)
     {
@@ -1201,28 +1233,28 @@ static void free_tree_node(TreeNode * node)
         {
             if (node->fields[i] != NULL)
             {
-                free_tree_node(node->fields[i]);
+                tree_delete(node->fields[i]);
             }
         }
+        free(node);
     }
-    free(node);
 }
 
 /**
  * Free all tree node memory.
  */
-void <%= @grammar.prefix %>free_tree(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
 {
-    free_tree_node((TreeNode *)tree);
+    tree_delete((TreeNode *)tree);
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 /**
  * Free all tree node memory.
  */
-void <%= @grammar.prefix %>free_tree_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
 {
-    free_tree_node((TreeNode *)tree);
+    tree_delete((TreeNode *)tree);
 }
 <%   end %>
 <% end %>

assets/parser.d.erb

@@ -9,7 +9,7 @@ module <%= @grammar.modulename %>;
 <% end %>
 
 import core.memory;
-import core.stdc.stdlib : malloc;
+import core.stdc.stdlib : malloc, free;
 
 /**************************************************************************
  * User code blocks
@@ -86,6 +86,14 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -94,6 +102,8 @@ private struct TreeNode
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     void *[0] fields;
 }
 
@@ -103,8 +113,11 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
     /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
+<%= @grammar.token_user_fields %>
 }
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -114,6 +127,8 @@ public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
 <%     rule_set.tree_fields.each do |fields| %>
     union
     {
@@ -183,6 +198,8 @@ public struct <%= @grammar.prefix %>context_t
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 }
 
 /**************************************************************************
@@ -222,26 +239,39 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
+ * @param input_length
+ *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, string input)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(string input)
 {
     /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext;
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t;
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.text_position.row = 1u;
-    newcontext.text_position.col = 1u;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context.input = input;
+    context.text_position.row = 1u;
+    context.text_position.col = 1u;
+    context.mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
 }
 
 /**************************************************************************
@@ -1015,7 +1045,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
@@ -1049,7 +1079,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;
@@ -1088,6 +1119,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
                 foreach (i; 0..n_fields)
                 {
                     node.fields[i] = null;
@@ -1195,6 +1228,35 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
+
 /**
  * Get the current text input position.
  *

assets/parser.h.erb

@@ -69,6 +69,14 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -80,6 +88,7 @@ typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
     <%= @grammar.prefix %>position_t end_position;
     uint16_t n_fields;
     uint8_t is_token;
+<%= @grammar.token_user_fields %>
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
@@ -172,6 +181,8 @@ typedef struct
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 } <%= @grammar.prefix %>context_t;
 
 /**************************************************************************
@@ -181,7 +192,9 @@ typedef struct
 /** Token names. */
 extern const char * <%= @grammar.prefix %>token_names[];
 
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length);
+
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
         <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
@@ -206,9 +219,9 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
 <% end %>
 
 <% if @grammar.tree %>
-void <%= @grammar.prefix %>free_tree(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-void <%= @grammar.prefix %>free_tree_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
 <%   end %>
 <% end %>
 

doc/user_guide.md

@@ -151,8 +151,35 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
+#### C/C++
+
+The lexer code block is passed the following arguments:
+
+  * `match` - a pointer points to the text matched by the lexer pattern.
+  * `match_length` - length of the matched text.
+
 Example:
 
+```
+ptype long;
+
+token integer /\d+/ <<
+  long v = 0;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+```
+
+#### D
+
+The lexer code block is passed the following arguments:
+
+  * `match` - a slice containing the text matched by the lexer pattern.
+
 ```
 ptype ulong;
 
@@ -167,7 +194,7 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `token` or pattern expression.
+Lexer code blocks appear following a `drop`, `token`, or pattern expression.
 User code in a lexer code block will be executed when the lexer matches the
 given pattern.
 Assignment to the `$$` symbol will associate a parser value with the lexed
@@ -194,7 +221,409 @@ Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
 
-##> Tree generation mode - the `tree` statement
+##> `context_user_fields` statement - adding custom fields to the context
+
+Propane uses a context structure for lexer and parser operations.
+Custom fields may be added to the context structure by using the grammar
+`context_user_fields` statement.
+This allows lexer pattern or parser rule code blocks to access user-defined
+fields within the context structure.
+
+Example:
+
+```
+context_user_fields <<
+    int mycontextval;
+>>
+```
+
+Lexer user code blocks or parser user code blocks can access user-defined
+context fields by using the `${context.<field>}` syntax.
+
+C++ example:
+
+```
+context_user_fields <<
+    std::string comments;
+>>
+drop /#(.*)\n/ <<
+    /* Accumulate comments before the next parser tree node. */
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+```
+
+If a pointer to any allocated memory is stored in a user-defined context field,
+it is up to the user to free any memory when the program is finished using the
+context structure.
+
+##> `drop` statement - ignoring input patterns
+
+A `drop` statement can be used to specify a lexer pattern that when matched
+should result in the matched input being dropped and lexing continuing after
+the matched input.
+
+A common use for a `drop` statement would be to ignore whitespace sequences in
+the user input.
+
+Example:
+
+```
+drop /\s+/;
+```
+
+See also ${#Regular expression syntax}.
+
+## `free_token_node` statement - freeing user-allocated memory in token node fields
+
+If user lexer code block allocates memory to store in a token node's `pvalue`
+or any custom token user fields store pointers to allocated memory, the
+`free_token_node` grammar statement can be used to provide a code block which
+can be used to free memory properly.
+
+Example freeing `pvalue` (C):
+
+```
+tree;
+free_token_node <<
+    free(${token.pvalue});
+>>
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+```
+
+Example freeing custom token user fields (C):
+
+```
+token_user_fields <<
+    char * comments;
+>>
+on_token_node <<
+    ${token.comments} = (char *)malloc(some_len);
+>>
+free_token_node <<
+    free(${token.comments});
+>>
+```
+
+The `free_token_node` statement user code block is not emitted for D language
+since D has a garbage collector.
+
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
+##> `module` statement - specifying the generated parser module name
+
+The `module` statement can be used to specify the module name for a generated
+D module.
+
+```
+module proj.parser;
+```
+
+If a module statement is not present, then the generated D module will not
+contain a module statement and the default module name will be used.
+
+##> `on_tree_node` statement -  custom initialization of a token tree node
+
+The `on_token_node` statement can be used to provide code that initializes
+any token user fields when a token tree node instance is created.
+
+For example (C++):
+
+```
+context_user_fields <<
+    std::string comments;
+>>
+token_user_fields <<
+    std::string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+drop /#(.*)\n/ <<
+    /* Accumulate comments before the next parser tree node. */
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+```
+
+##> `prefix` statement - specifying the generated API prefix
+
+By default the public API (types, constants, and functions) of the generated
+lexer and parser uses a prefix of `p_`.
+
+This prefix can be changed with the `prefix` statement.
+
+Example:
+
+```
+prefix myparser_;
+```
+
+With a parser generated with this `prefix` statement, instead of calling
+`p_context_new()` you would call `myparser_context_new()`.
+
+The `prefix` statement can be optionally used if you would like to change the
+prefix used by your generated lexer and parser to something other than the
+default.
+
+It can also be used when generating multiple lexers/parsers to be used in the
+same program to avoid symbol collisions.
+
+##> `ptype` statement - specifying parser value types
+
+The `ptype` statement is used to define parser value type(s).
+Example:
+
+```
+ptype void *;
+```
+
+This defines the default parser value type to be `void *` (this is, in fact,
+the default parser value type if the grammar file does not specify otherwise).
+
+Each defined lexer token type and parser rule has an associated parser value
+type.
+When the lexer runs, each lexed token has a parser value associated with it.
+When the parser runs, each instance of a reduced rule has a parser value
+associated with it.
+Propane supports using different parser value types for different rules and
+token types.
+The example `ptype` statement above defines the default parser value type.
+A parser value type name can optionally be specified following the `ptype`
+keyword.
+For example:
+
+```
+ptype Value;
+ptype array = Value[];
+ptype dict = Value[string];
+
+Object -> lbrace rbrace << $$ = new Value(); >>
+
+Values (array) -> Value << $$ = [$1]; >>
+Values -> Values comma Value << $$ = $1 ~ [$3]; >>
+
+KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
+```
+
+In this example, the default parser value type is `Value`.
+A parser value type named `array` is defined to mean `Value[]`.
+A parser value type named `dict` is defined to mean `Value[string]`.
+Any defined tokens or rules that do not specify a parser value type will have
+the default parser value type associated with them.
+To associate a different parser value type with a token or rule, write the
+parser value type name in parentheses following the name of the token or rule.
+In this example:
+
+  * a reduced `Object`'s parser value has a type of `Value`.
+  * a reduced `Values`'s parser value has a type of `Value[]`.
+  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
+
+When tree generation mode is active, the `ptype` functionality works differently.
+In this mode, only one `ptype` is used by the parser.
+Lexer user code blocks may assign a parse value to the generated `Token` node
+by assigning to `$$` within a lexer code block.
+The type of the parse value `$$` is given by the global `ptype` type.
+
+##> `start` statement - specifying the parser start rule name
+
+The start rule can be changed from the default of `Start` by using the `start`
+statement.
+Example:
+
+```
+start MyStartRule;
+```
+
+Multiple start rules can be specified, either with multiple `start` statements
+or one `start` statement listing multiple start rules.
+Example:
+
+```
+start Module ModuleItem Statement Expression;
+```
+
+When multiple start rules are specified, multiple `p_parse_*()` functions,
+`p_result_*()`, and `p_tree_delete_*()` functions (in tree mode) are generated.
+A default `p_parse()`, `p_result()`, `p_tree_delete()` are generated corresponding
+to the first start rule.
+Additionally, each start rule causes the generation of another version of each
+of these functions, for example `p_parse_Statement()`, `p_result_Statement()`,
+and `p_tree_delete_Statement()`.
+
+##> `token` statement - specifying tokens
+
+The `token` statement allows defining a lexer token and a pattern to match that
+token.
+The name of the token must be specified immediately following the `token`
+keyword.
+A regular expression pattern may optionally follow the token name.
+If a regular expression pattern is not specified, the name of the token is
+taken to be the pattern.
+See also: ${#Regular expression syntax}.
+
+Example:
+
+```
+token for;
+```
+
+In this example, the token name is `for` and the pattern to match it is
+`/for/`.
+
+Example:
+
+```
+token lbrace /\{/;
+```
+
+In this example, the token name is `lbrace` and a single left curly brace will
+match it.
+
+The `token` statement can also include a user code block.
+The user code block will be executed whenever the token is matched by the
+lexer.
+
+Example:
+
+```
+token if << writeln("'if' keyword lexed"); >>
+```
+
+The `token` statement is actually a shortcut statement for a combination of a
+`tokenid` statement and a pattern statement.
+To define a lexer token without an associated pattern to match it, use a
+`tokenid` statement.
+To define a lexer pattern that may or may not result in a matched token, use
+a pattern statement.
+
+##> `tokenid` statement - defining tokens without a matching pattern
+
+The `tokenid` statement can be used to define a token without associating it
+with a lexer pattern that matches it.
+
+Example:
+
+```
+tokenid string;
+```
+
+The `tokenid` statement can be useful when defining a token that may optionally
+be returned by user code associated with a pattern.
+
+It is also useful when lexer modes and multiple lexer patterns are required to
+build up a full token.
+A common example is parsing a string.
+See the ${#Lexer modes} chapter for more information.
+
+##> `token_user_fields` statement - adding custom token fields
+
+When tree generation mode is active, Propane generates a tree node structure
+and a token node structure for each matching rule and token instance in the
+input string.
+The user may add custom fields to token tree nodes using the `token_user_fields`
+statement.
+The code block supplied to the `token_user_fields` is inserted in the `struct`
+generated by the parser to hold a token tree node.
+
+Example (D/C++):
+
+```
+token_user_fields <<
+    string mytokenval;
+>>
+```
+
+The `on_token_node` statement can be used to provide code that initializes
+any token user fields when a token tree node instance is created.
+
+For example (C++):
+
+```
+context_user_fields <<
+    std::string comments;
+>>
+token_user_fields <<
+    std::string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+drop /#(.*)\n/ <<
+    /* Accumulate comments before the next parser tree node. */
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+```
+
+If a pointer to any allocated memory is stored in a user-defined context field,
+the `free_token_node` statement can be used to supply a code block which
+will be executed immediately before the token node is freed.
+For C++, the `delete` statement is used to free the token tree node, so the
+destructor for any custom token user fields will be called.
+
+##> `tree` statement - tree generation mode
 
 To activate tree generation mode, place the `tree` statement in your grammar file:
 
@@ -257,10 +686,9 @@ example parse:
 
 ```
 string input = "a, ((b)), b";
-p_context_t context;
-p_context_init(&context, input);
-assert_eq(P_SUCCESS, p_parse(&context));
-Start * start = p_result(&context);
+p_context_t * context = p_context_new(input);
+assert_eq(P_SUCCESS, p_parse(context));
+Start * start = p_result(context);
 assert(start.pItems1 !is null);
 assert(start.pItems !is null);
 Items * items = start.pItems;
@@ -308,10 +736,9 @@ Then the types would be used as such instead:
 
 ```
 string input = "a, ((b)), b";
-p_context_t context;
-p_context_init(&context, input);
-assert_eq(P_SUCCESS, p_parse(&context));
-ABCStartXYZ * start = p_result(&context);
+p_context_t * context = p_context_new(input);
+assert_eq(P_SUCCESS, p_parse(context));
+ABCStartXYZ * start = p_result(context);
 assert(start.pItems1 !is null);
 assert(start.pItems !is null);
 ABCItemsXYZ * items = start.pItems;
@@ -327,102 +754,7 @@ assert(itemsmore.pItem.pItem.pItem !is null);
 assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 ```
 
-## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
-
-If user lexer code block allocates memory to store in a token node's `pvalue`,
-the `free_token_node` grammar statement can be used to specify the name of a
-function which will be called during the `p_free_tree()` call to free the memory
-associated with a token node.
-
-Example:
-
-```
-<<
-static void free_token(Token * token)
-{
-    free(token->pvalue);
-}
->>
-tree;
-free_token_node free_token;
-ptype int *;
-token a <<
-  $$ = (int *)malloc(sizeof(int));
-  *$$ = 1;
->>
-token b <<
-  $$ = (int *)malloc(sizeof(int));
-  *$$ = 2;
->>
-Start -> a:a b:b;
-```
-
-##> Specifying tokens - the `token` statement
-
-The `token` statement allows defining a lexer token and a pattern to match that
-token.
-The name of the token must be specified immediately following the `token`
-keyword.
-A regular expression pattern may optionally follow the token name.
-If a regular expression pattern is not specified, the name of the token is
-taken to be the pattern.
-See also: ${#Regular expression syntax}.
-
-Example:
-
-```
-token for;
-```
-
-In this example, the token name is `for` and the pattern to match it is
-`/for/`.
-
-Example:
-
-```
-token lbrace /\{/;
-```
-
-In this example, the token name is `lbrace` and a single left curly brace will
-match it.
-
-The `token` statement can also include a user code block.
-The user code block will be executed whenever the token is matched by the
-lexer.
-
-Example:
-
-```
-token if << writeln("'if' keyword lexed"); >>
-```
-
-The `token` statement is actually a shortcut statement for a combination of a
-`tokenid` statement and a pattern statement.
-To define a lexer token without an associated pattern to match it, use a
-`tokenid` statement.
-To define a lexer pattern that may or may not result in a matched token, use
-a pattern statement.
-
-##> Defining tokens without a matching pattern - the `tokenid` statement
-
-The `tokenid` statement can be used to define a token without associating it
-with a lexer pattern that matches it.
-
-Example:
-
-```
-tokenid string;
-```
-
-The `tokenid` statement can be useful when defining a token that may optionally
-be returned by user code associated with a pattern.
-
-It is also useful when lexer modes and multiple lexer patterns are required to
-build up a full token.
-A common example is parsing a string.
-See the ${#Lexer modes} chapter for more information.
-
-##> Specifying a lexer pattern - the pattern statement
+##> Specifying a lexer pattern
 
 A pattern statement is used to define a lexer pattern that can execute user
 code but may not result in a matched token.
@@ -437,23 +769,6 @@ This can be especially useful with ${#Lexer modes}.
 
 See also ${#Regular expression syntax}.
 
-##> Ignoring input sections - the `drop` statement
-
-A `drop` statement can be used to specify a lexer pattern that when matched
-should result in the matched input being dropped and lexing continuing after
-the matched input.
-
-A common use for a `drop` statement would be to ignore whitespace sequences in
-the user input.
-
-Example:
-
-```
-drop /\s+/;
-```
-
-See also ${#Regular expression syntax}.
-
 ##> Regular expression syntax
 
 A regular expression ("regex") is used to define lexer patterns in `token`,
@@ -592,63 +907,7 @@ token dot /\./ <<
 default, identonly: drop /\s+/;
 ```
 
-##> Specifying parser value types - the `ptype` statement
-
-The `ptype` statement is used to define parser value type(s).
-Example:
-
-```
-ptype void *;
-```
-
-This defines the default parser value type to be `void *` (this is, in fact,
-the default parser value type if the grammar file does not specify otherwise).
-
-Each defined lexer token type and parser rule has an associated parser value
-type.
-When the lexer runs, each lexed token has a parser value associated with it.
-When the parser runs, each instance of a reduced rule has a parser value
-associated with it.
-Propane supports using different parser value types for different rules and
-token types.
-The example `ptype` statement above defines the default parser value type.
-A parser value type name can optionally be specified following the `ptype`
-keyword.
-For example:
-
-```
-ptype Value;
-ptype array = Value[];
-ptype dict = Value[string];
-
-Object -> lbrace rbrace << $$ = new Value(); >>
-
-Values (array) -> Value << $$ = [$1]; >>
-Values -> Values comma Value << $$ = $1 ~ [$3]; >>
-
-KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
-```
-
-In this example, the default parser value type is `Value`.
-A parser value type named `array` is defined to mean `Value[]`.
-A parser value type named `dict` is defined to mean `Value[string]`.
-Any defined tokens or rules that do not specify a parser value type will have
-the default parser value type associated with them.
-To associate a different parser value type with a token or rule, write the
-parser value type name in parentheses following the name of the token or rule.
-In this example:
-
-  * a reduced `Object`'s parser value has a type of `Value`.
-  * a reduced `Values`'s parser value has a type of `Value[]`.
-  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
-
-When tree generation mode is active, the `ptype` functionality works differently.
-In this mode, only one `ptype` is used by the parser.
-Lexer user code blocks may assign a parse value to the generated `Token` node
-by assigning to `$$` within a lexer code block.
-The type of the parse value `$$` is given by the global `ptype` type.
-
-##> Specifying a parser rule - the rule statement
+##> Specifying parser rules
 
 Rule statements create parser rules which define the grammar that will be
 parsed by the generated parser.
@@ -729,67 +988,6 @@ can be used to produce the parser value for the accepted rule.
 Parser rule code blocks are not allowed and not used when tree generation mode
 is active.
 
-##> Specifying the parser start rule name - the `start` statement
-
-The start rule can be changed from the default of `Start` by using the `start`
-statement.
-Example:
-
-```
-start MyStartRule;
-```
-
-Multiple start rules can be specified, either with multiple `start` statements
-or one `start` statement listing multiple start rules.
-Example:
-
-```
-start Module ModuleItem Statement Expression;
-```
-
-When multiple start rules are specified, multiple `p_parse_*()` functions,
-`p_result_*()`, and `p_free_tree_*()` functions (in tree mode) are generated.
-A default `p_parse()`, `p_result()`, `p_free_tree()` are generated corresponding
-to the first start rule.
-Additionally, each start rule causes the generation of another version of each
-of these functions, for example `p_parse_Statement()`, `p_result_Statement()`,
-and `p_free_tree_Statement()`.
-
-##> Specifying the parser module name - the `module` statement
-
-The `module` statement can be used to specify the module name for a generated
-D module.
-
-```
-module proj.parser;
-```
-
-If a module statement is not present, then the generated D module will not
-contain a module statement and the default module name will be used.
-
-##> Specifying the generated API prefix - the `prefix` statement
-
-By default the public API (types, constants, and functions) of the generated
-lexer and parser uses a prefix of `p_`.
-
-This prefix can be changed with the `prefix` statement.
-
-Example:
-
-```
-prefix myparser_;
-```
-
-With a parser generated with this `prefix` statement, instead of calling
-`p_context_init()` you would call `myparser_context_init()`.
-
-The `prefix` statement can be optionally used if you would like to change the
-prefix used by your generated lexer and parser to something other than the
-default.
-
-It can also be used when generating multiple lexers/parsers to be used in the
-same program to avoid symbol collisions.
-
 ##> User termination of the lexer or parser
 
 Propane supports allowing lexer or parser user code blocks to terminate
@@ -864,6 +1062,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -963,27 +1172,30 @@ fields `pExpB`, `pExpB3`, and `right` will all point to the same child node
 
 ##> Functions
 
-### `p_context_init`
+### `p_context_new`
 
-The `p_context_init()` function must be called to initialize the context
-structure.
+The `p_context_new()` function must be called to allocate and initialize the
+context structure.
 The input to be used for lexing/parsing is passed in when initializing the
 context structure.
 
 C example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
+p_context_t * context = p_context_new(input, input_length);
 ```
 
 D example:
 
 ```
-p_context_t context;
-p_context_init(&context, input);
+p_context_t * context = p_context_new(input);
 ```
 
+### `p_context_delete`
+
+The `p_context_delete()` function must be called to deinitialize and deallocate
+a context structure allocated by `p_context_init()`.
+
 ### `p_lex`
 
 The `p_lex()` function is the main entry point to the lexer.
@@ -995,10 +1207,9 @@ generated lexer in a standalone mode.
 Example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
+p_context_t * context = p_context_new(input, input_length);
 p_token_info_t token_info;
-size_t result = p_lex(&context, &token_info);
+size_t result = p_lex(context, &token_info);
 switch (result)
 {
 case P_DECODE_ERROR:
@@ -1030,9 +1241,8 @@ It must be passed a pointer to an initialized context structure.
 Example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
+p_context_t * context = p_context_new(input, input_length);
+size_t result = p_parse(context);
 ```
 
 When multiple start rules are specified, a separate parse function is generated
@@ -1040,7 +1250,7 @@ for each which starts parsing at the given rule.
 For example, if `Statement` is specified as a start rule:
 
 ```
-size_t result = p_parse_Statement(&context);
+size_t result = p_parse_Statement(context);
 ```
 
 In this case, the parser will start parsing with the `Statement` rule.
@@ -1071,12 +1281,11 @@ The `p_result()` function can be used to retrieve the final parse value after
 Example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_SUCCESS)
+p_context_t * context = p_context_new(input, input_length);
+size_t result = p_parse(context);
+if (p_parse(context) == P_SUCCESS)
 {
-    result = p_result(&context);
+    result = p_result(context);
 }
 ```
 
@@ -1088,12 +1297,11 @@ for each which returns the parse result for the corresponding rule.
 For example, if `Statement` is specified as a start rule:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse_Statement(&context) == P_SUCCESS)
+p_context_t * context = p_context_new(input, input_length);
+size_t result = p_parse(context);
+if (p_parse_Statement(context) == P_SUCCESS)
 {
-    result = p_result_Statement(&context);
+    result = p_result_Statement(context);
 }
 ```
 
@@ -1108,12 +1316,11 @@ an error occurred.
 Example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+p_context_t * context = p_context_new(input, input_length);
+size_t result = p_parse(context);
+if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
-    p_position_t error_position = p_position(&context);
+    p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
         error_position.row + 1, error_position.col + 1);
 }
@@ -1130,9 +1337,9 @@ They have no particular meaning to Propane.
 Example:
 
 ```
-if (p_parse(&context) == P_USER_TERMINATED)
+if (p_parse(context) == P_USER_TERMINATED)
 {
-    size_t user_terminate_code = p_user_terminate_code(&context);
+    size_t user_terminate_code = p_user_terminate_code(context);
 }
 ```
 
@@ -1146,9 +1353,9 @@ indicate what token the parser was not expecting.
 Example:
 
 ```
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
-    p_token_t unexpected_token = p_token(&context);
+    p_token_t unexpected_token = p_token(context);
 }
 ```
 
@@ -1172,30 +1379,42 @@ assert(code_point == 0x1F9E1u);
 assert(code_point_length == 4u);
 ```
 
-### `p_free_tree`
+### `p_tree_delete`
 
-The `p_free_tree()` function can be used to free the memory used by the tree.
+The `p_tree_delete()` function can be used to free the memory used by the tree.
 It should be passed the same value that is returned by `p_result()`.
 
-The `p_free_tree()` function is only available for C/C++ output targets.
-
 Note that if any lexer user code block allocates memory to store in a token's
-`pvalue`, in order to properly free this memory a `free_token_node` function
-should be specified in the grammar file.
-If specified, the `free_token_node` function will be called during the
-`p_free_tree()` process to allow user code to free any memory associated with
+`pvalue`, in order to properly free this memory the `free_token_node` statement
+should be used to provide a code block that frees this memory.
+If specified, the `free_token_node` code block will be executed during the
+`p_tree_delete()` process to allow user code to free any memory associated with
 a token node's `pvalue`.
 
-When multiple start rules are specified, a separate `p_free_tree` function is
+When multiple start rules are specified, a separate `p_tree_delete` function is
 generated for each which frees the tree resulting from parsing the given rule.
 For example, if `Statement` is specified as a start rule:
 
 ```
-p_free_tree_Statement(statement_tree);
+p_tree_delete_Statement(statement_tree);
 ```
 
 In this case, Propane will free a `Statement` tree structure returned by the
-`p_parse_Statement(&context)` function.
+`p_parse_Statement(context)` function.
+
+### `p_value`
+
+When tree generation mode is not active, the `p_value_t` union can hold one of
+several different possible value types.
+
+The `p_value(v)` function returns an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is set for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
 
 ##> Data
 
@@ -1207,14 +1426,13 @@ It is indexed by the token ID.
 C example:
 
 ```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+p_context_t * context = p_context_new(input, input_length);
+size_t result = p_parse(context);
+if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
-    p_position_t error_position = p_position(&context);
+    p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
-        p_token_names[context.token],
+        p_token_names[context->token],
         error_position.row + 1, error_position.col + 1);
 }
 ```

extra/vim/syntax/propane.vim

@@ -20,7 +20,7 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node module prefix ptype start token tokenid tree tree_prefix tree_suffix
+syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -13,8 +13,13 @@ class Propane
       @language =
         if output_file.end_with?(".d")
           "d"
-        else
+        elsif output_file.end_with?(".c")
           "c"
+        elsif output_file =~ %r{\.(cc|cpp|cxx)$}
+          @cpp = true
+          "c"
+        else
+          raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
       @options = options
       process_grammar!
@@ -33,7 +38,13 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            "#line #{i + 2}\n"
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -265,6 +276,24 @@ class Propane
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
+      code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "context->#{fieldname}"
+        when "d"
+          "context.#{fieldname}"
+        end
+      end
+      code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "token_tree_node->#{fieldname}"
+        when "d"
+          "token_tree_node.#{fieldname}"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
           case @language
@@ -331,6 +360,14 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -5,6 +5,8 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -17,8 +19,11 @@ class Propane
     attr_reader :code_blocks
     attr_reader :ptypes
     attr_reader :prefix
+    attr_reader :on_token_node
+    attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -33,7 +38,10 @@ class Propane
       @tree = false
       @tree_prefix = ""
       @tree_suffix = ""
-      @free_token_node = nil
+      @free_token_node = ""
+      @context_user_fields = nil
+      @on_token_node = ""
+      @token_user_fields = nil
       parse_grammar!
       @start_rules << "Start" if @start_rules.empty?
     end
@@ -62,11 +70,15 @@ class Propane
       if parse_white_space!
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
+      elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
       elsif parse_free_token_node_statement!
       elsif parse_module_statement!
+      elsif parse_on_token_node_statement!
+      elsif parse_token_user_fields_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
       elsif parse_start_statement!
@@ -98,6 +110,22 @@ class Propane
       consume!(/#.*\n/)
     end
 
+    def parse_context_user_fields_statement!
+      if md = consume!(/context_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @context_user_fields ||= ""
+        @context_user_fields += code
+      end
+    end
+
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true
@@ -116,12 +144,6 @@ class Propane
       end
     end
 
-    def parse_free_token_node_statement!
-      if md = consume!(/free_token_node\s+(\w+)\s*;/)
-        @free_token_node = md[1]
-      end
-    end
-
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -132,6 +154,34 @@ class Propane
       end
     end
 
+    def parse_on_token_node_statement!
+      if md = consume!(/on_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @on_token_node += code
+      end
+    end
+
+    def parse_token_user_fields_statement!
+      if md = consume!(/token_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @token_user_fields ||= ""
+        @token_user_fields += code
+      end
+    end
+
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @free_token_node += code
+      end
+    end
+
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
         name = "default"
@@ -195,8 +245,10 @@ class Propane
           raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
         end
         consume!(/\s+/)
-        consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline)
         @modeline = nil
         true
       end
@@ -253,8 +305,12 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        if code.start_with?("\n")
+          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+        else
+          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -285,6 +341,8 @@ class Propane
             end
           elsif md = consume!(%r{(.)})
             pattern += md[1]
+          elsif @input == "" || @input.start_with?("\n")
+            raise Error.new("Line #{@line_number}: Unterminated pattern; expected `/`")
           end
         end
         pattern
@@ -293,9 +351,12 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        if code.start_with?("\n")
+          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+        else
+          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+        end
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "3.0.0"
+  VERSION = "4.3.0"
 end

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
     end
 
     it "supports mode labels" do
@@ -144,7 +144,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +197,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -14,6 +14,7 @@ describe Propane do
   end
 
   def run_propane(options = {})
+    options[:language] ||= "d"
     @statics[:build_test_id] ||= 0
     @statics[:build_test_id] += 1
     if ENV["dist_specs"]
@@ -94,15 +95,13 @@ EOF
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
     results = Results.new(stdout, stderr, status)
-    if %w[c cpp].include?(options[:language])
-      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
-      vgout = stdout + stderr
-      File.binwrite("spec/run/.vgout", vgout)
-      vgout.scan(/lost: (\d+) bytes/) do |match|
-        bytes = $1.to_i
-        if bytes > 0
-          raise "Valgrind detected memory leak"
-        end
+    stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+    vgout = stdout + stderr
+    File.binwrite("spec/run/.vgout", vgout)
+    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
+      bytes = $1.to_i
+      if bytes > 0
+        raise "Valgrind detected memory leak"
       end
     end
     results
@@ -1430,6 +1429,9 @@ EOF
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
 
+      # D garbage collector was freeing memory for nodes when it should not
+      # have been. This was due to pointers being in the TreeNode that was
+      # allocated by C malloc() function instead of allocated using D.
       it "does not free memory allocated for tree nodes" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/tree_node_memory_remains.#{ext}.propane"))
@@ -1483,15 +1485,11 @@ EOF
 
       if %w[c cpp].include?(language)
         it "allows a user function to free token node memory in tree mode" do
-    write_grammar <<EOF
-<<
-static void free_token(Token * token)
-{
-    free(token->pvalue);
-}
->>
+          write_grammar <<EOF
 tree;
-free_token_node free_token;
+free_token_node <<
+    free(${token.pvalue});
+>>
 ptype int *;
 token a <<
   $$ = (int *)malloc(sizeof(int));
@@ -1504,12 +1502,285 @@ token b <<
 Start -> a:a b:b;
 EOF
           run_propane(language: language)
-          compile("spec/test_free_tree_token_node_memory.#{language}", language: language)
+          compile("spec/test_tree_delete_token_node_memory.#{language}", language: language)
           results = run_test(language: language)
           expect(results.stderr).to eq ""
           expect(results.status).to eq 0
         end
       end
+
+      it "executes code blocks associated with drop statements" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  stderr.write("comment: ", match);
+>>
+token a;
+Start -> a;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+>>
+token a;
+Start -> a;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_drop_code_block.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to match %r{comment: # comment 1\n.*comment: #    comment 2}m
+        expect(results.status).to eq 0
+      end
+
+      it "allows user-defined context fields" do
+        if language == "d"
+          write_grammar <<EOF
+context_user_fields <<
+    string comments;
+    uint acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} ~= match;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <string.h>
+#include <stdlib.h>
+>>
+context_user_fields <<
+    char * comments;
+    unsigned int acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    size_t cur_len = 0u;
+    if (${context.comments} != NULL)
+        cur_len = strlen(${context.comments});
+    char * commentsnew = (char *)malloc(cur_len + match_length + 1);
+    if (${context.comments} != NULL)
+        memcpy(commentsnew, ${context.comments}, cur_len);
+    memcpy(&commentsnew[cur_len], match, match_length);
+    commentsnew[cur_len + match_length] = '\\0';
+    if (${context.comments} != NULL)
+    {
+        free(${context.comments});
+    }
+    ${context.comments} = commentsnew;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_user_context_fields.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to include %r{comments: # comment 1\n#    comment 2}
+        expect(results.stderr).to include %r{acount: 11\n}
+        expect(results.status).to eq 0
+      end
+
+      it "allows custom token user fields" do
+        if language == "d"
+          write_grammar <<EOF
+context_user_fields <<
+    string comments;
+>>
+token_user_fields <<
+    string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} ~= match;
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        elsif language == "c"
+          write_grammar <<EOF
+<<
+#include <string.h>
+#include <stdlib.h>
+>>
+context_user_fields <<
+    char * comments;
+>>
+token_user_fields <<
+    char * comments;
+>>
+free_token_node <<
+    free(${token.comments});
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = (char *)malloc(1);
+    ${context.comments}[0] = '\\0';
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    size_t cur_len = 0u;
+    if (${context.comments} != NULL)
+        cur_len = strlen(${context.comments});
+    char * commentsnew = (char *)malloc(cur_len + match_length + 1);
+    if (${context.comments} != NULL)
+        memcpy(commentsnew, ${context.comments}, cur_len);
+    memcpy(&commentsnew[cur_len], match, match_length);
+    commentsnew[cur_len + match_length] = '\\0';
+    if (${context.comments} != NULL)
+    {
+        free(${context.comments});
+    }
+    ${context.comments} = commentsnew;
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        else # C++
+          write_grammar <<EOF
+<<header
+#include <string>
+>>
+context_user_fields <<
+    std::string comments;
+>>
+token_user_fields <<
+    std::string comments;
+>>
+on_token_node <<
+    ${token.comments} = ${context.comments};
+    ${context.comments} = "";
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_token_user_fields.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_basic_math_grammar.c

@@ -5,25 +5,29 @@
 int main()
 {
     char const * input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
+    p_context_delete(context);
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
+    p_context_delete(context);
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
+    p_context_delete(context);
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
+    p_context_delete(context);
 
     return 0;
 }

spec/test_basic_math_grammar.d

@@ -10,23 +10,23 @@ int main()
 unittest
 {
     string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
 }

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_drop_code_block.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_drop_code_block.d

@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_error_positions.c

@@ -5,38 +5,43 @@
 int main()
 {
     char const * input = "a 42";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "a\n123\na  a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 4);
-    assert(p_token(&context) == TOKEN_a);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 4);
+    assert(p_token(context) == TOKEN_a);
+    p_context_delete(context);
 
     input = "12";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 1);
-    assert(p_token(&context) == TOKEN_num);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 1);
+    assert(p_token(context) == TOKEN_num);
+    p_context_delete(context);
 
     input = "a 12\n\nab";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 2);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 2);
+    p_context_delete(context);
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 6);
-    assert(p_position(&context).col == 5);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context).row == 6);
+    assert(p_position(context).col == 5);
 
     assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
     assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_error_positions.d

@@ -9,31 +9,31 @@ int main()
 unittest
 {
     string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(3, 4));
-    assert(p_token(&context) == TOKEN_a);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(3, 4));
+    assert(p_token(context) == TOKEN_a);
 
     input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 1));
-    assert(p_token(&context) == TOKEN_num);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 1));
+    assert(p_token(context) == TOKEN_num);
 
     input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(3, 2));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(6, 5));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context) == p_position_t(6, 5));
 
     assert(p_token_names[TOKEN_a] == "a");
     assert(p_token_names[TOKEN_num] == "num");

spec/test_field_aliases.c

@@ -6,8 +6,9 @@
 int main()
 {
     char const * input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
     return 0;
 }

spec/test_field_aliases.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_lexer.c

@@ -38,73 +38,75 @@ int main()
 
     p_token_info_t token_info;
     char const * input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 3u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 5u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 7u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 5u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 7u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 2u);
     assert(token_info.position.col == 10u);
     assert(token_info.end_position.row == 2u);
     assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
-    p_context_init(&context, (uint8_t const *)"", 0u);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)"", 0u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
     assert(token_info.position.col == 1u);
     assert(token_info.end_position.row == 1u);
     assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer.d

@@ -44,26 +44,26 @@ unittest
 {
     p_token_info_t token_info;
     string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    context = p_context_new("");
+    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_lexer_match_text.c

@@ -6,10 +6,11 @@
 int main()
 {
     char const * input = "identifier_123";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_match_text.d

@@ -9,8 +9,8 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 }

spec/test_lexer_modes.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abc \"a string\" def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abc \"abc def\" def";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_multiple_modes.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abc.def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abc .  abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_multiple_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc.def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc .  abc`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_result_value.c

@@ -5,15 +5,17 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "fabulous";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_result_value.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = `fabulous`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
 }

spec/test_lexer_unknown_character.c

@@ -5,14 +5,16 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    p_context_delete(context);
 
     input = "123";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_unknown_character.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
 
     input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
 }

spec/test_match_backslashes.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_match_backslashes.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_multiple_parsers.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new((uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
+    myp1_context_delete(context1);
 
     char const * input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new((uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
+    myp2_context_delete(context2);
 
     return 0;
 }

spec/test_multiple_parsers.d

@@ -10,12 +10,12 @@ int main()
 unittest
 {
     string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new(input1);
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
 
     string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new(input2);
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
 }

spec/test_named_optional_rule_component_tree.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start->a == NULL);
     assert(start->pToken2 != NULL);
     assert_eq(TOKEN_b, start->pToken2->token);
@@ -17,12 +17,13 @@ int main()
     assert(start->pR == NULL);
     assert(start->r == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->a != NULL);
     assert_eq(TOKEN_a, start->pToken1->token);
     assert(start->pToken2 != NULL);
@@ -33,18 +34,20 @@ int main()
     assert(start->pR == start->r);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->a == NULL);
     assert(start->pToken2 != NULL);
     assert(start->r != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_named_optional_rule_component_tree.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
@@ -21,10 +20,12 @@ unittest
     assert(start.pR is null);
     assert(start.r is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
     assert(start.pToken2 != null);
@@ -35,12 +36,16 @@ unittest
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_optional_rule_component.c

@@ -5,17 +5,20 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_optional_rule_component.d

@@ -9,15 +9,15 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_optional_rule_component_tree.c

@@ -6,22 +6,23 @@
 int main()
 {
     char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start->pToken1 == NULL);
     assert(start->pToken2 != NULL);
     assert_eq(TOKEN_b, start->pToken2->token);
     assert(start->pR3 == NULL);
     assert(start->pR == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->pToken1 != NULL);
     assert_eq(TOKEN_a, start->pToken1->token);
     assert(start->pToken2 != NULL);
@@ -30,18 +31,20 @@ int main()
     assert(start->pR == start->pR3);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start->pToken1 == NULL);
     assert(start->pToken2 != NULL);
     assert(start->pR != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_optional_rule_component_tree.d

@@ -10,20 +10,21 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert_eq(TOKEN_b, start.pToken2.token);
     assert(start.pR3 is null);
     assert(start.pR is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 != null);
     assert_eq(TOKEN_a, start.pToken1.token);
     assert(start.pToken2 != null);
@@ -32,12 +33,16 @@ unittest
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
     assert(start.pToken1 is null);
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_parser_identical_rules_lookahead.c

@@ -5,13 +5,15 @@
 int main()
 {
     char const * input = "aba";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_identical_rules_lookahead.d

@@ -9,11 +9,11 @@ int main()
 unittest
 {
     string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_from_multiple_states.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 2);
-    assert(context.token == TOKEN___EOF);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 2);
+    assert(context->token == TOKEN___EOF);
+    p_context_delete(context);
 
     input = "a b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "bb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_from_multiple_states.d

@@ -9,17 +9,17 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 2));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_user_code.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_user_code.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "ab";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parsing_json.c

@@ -6,51 +6,58 @@
 int main()
 {
     char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_OBJECT);
+    p_context_delete(context);
 
     input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_ARRAY);
+    p_context_delete(context);
 
     input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == -45.6);
+    p_context_delete(context);
 
     input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == 0.02);
+    p_context_delete(context);
 
     input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    JSONValue * o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(1, o->object.size);
     assert(strcmp(o->object.entries[0].name, "hi") == 0);
     assert(o->object.entries[0].value->id == JSON_TRUE);
+    p_context_delete(context);
 
     input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(2, o->object.size);
     assert(strcmp(o->object.entries[0].name, "ff") == 0);
     assert(o->object.entries[0].value->id == JSON_FALSE);
     assert(strcmp(o->object.entries[1].name, "nn") == 0);
     assert(o->object.entries[1].value->id == JSON_NULL);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_json.d

@@ -10,45 +10,45 @@ int main()
 unittest
 {
     string input = ``;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
 
     input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONArray)p_result(context));
 
     input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == -45.6);
 
     input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == 0.02);
 
     input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    JSONObject o = cast(JSONObject)p_result(context);
     assert(o.value["hi"]);
     assert(cast(JSONTrue)o.value["hi"]);
 
     input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    o = cast(JSONObject)p_result(context);
     assert(o.value["ff"]);
     assert(cast(JSONFalse)o.value["ff"]);
     assert(o.value["nn"]);

spec/test_parsing_lists.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
+    p_context_delete(context);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_lists.d

@@ -9,18 +9,18 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = "";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
 }

spec/test_pattern.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "defabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_pattern.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = "defabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_return_token_from_pattern.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_return_token_from_pattern.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_start_rule_tree.c

@@ -6,14 +6,15 @@
 int main()
 {
     char const * input = "hi";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Top * top = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
     assert(top->pToken != NULL);
     assert_eq(TOKEN_hi, top->pToken->token);
 
-    p_free_tree(top);
+    p_tree_delete(top);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_start_rule_tree.d

@@ -10,10 +10,10 @@ int main()
 unittest
 {
     string input = "hi";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Top * top = p_result(&context);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
     assert(top.pToken !is null);
     assert_eq(TOKEN_hi, top.pToken.token);
 }

spec/test_starting_rules.c

@@ -6,22 +6,25 @@
 int main()
 {
     char const * input = "bbbb";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    int result = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
     assert_eq(8, result);
+    p_context_delete(context);
 
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse_Bs(&context) == P_SUCCESS);
-    result = p_result_Bs(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
     assert_eq(8, result);
+    p_context_delete(context);
 
     input = "c";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse_R(&context) == P_SUCCESS);
-    result = p_result_R(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
     assert_eq(3, result);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_starting_rules.d

@@ -10,20 +10,20 @@ int main()
 unittest
 {
     string input = "bbbb";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    int result = p_result(&context);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
     assert(result == 8);
 
-    p_context_init(&context, input);
-    assert(p_parse_Bs(&context) == P_SUCCESS);
-    result = p_result_Bs(&context);
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
     assert(result == 8);
 
     input = "c";
-    p_context_init(&context, input);
-    assert(p_parse_R(&context) == P_SUCCESS);
-    result = p_result_R(&context);
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
     assert(result == 3);
 }

spec/test_starting_rules_tree.c

@@ -6,32 +6,35 @@
 int main()
 {
     char const * input = "bbbb";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert_not_null(start->bs);
     assert_not_null(start->bs->b);
     assert_not_null(start->bs->bs->b);
     assert_not_null(start->bs->bs->bs->b);
     assert_not_null(start->bs->bs->bs->bs->b);
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse_Bs(&context) == P_SUCCESS);
-    Bs * bs = p_result_Bs(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
     assert_not_null(bs->b);
     assert_not_null(bs->bs->b);
     assert_not_null(bs->bs->bs->b);
     assert_not_null(bs->bs->bs->bs->b);
-    p_free_tree_Bs(bs);
+    p_tree_delete_Bs(bs);
+    p_context_delete(context);
 
     input = "c";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse_R(&context) == P_SUCCESS);
-    R * r = p_result_R(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
     assert_not_null(r->c);
-    p_free_tree_R(r);
+    p_tree_delete_R(r);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_starting_rules_tree.d

@@ -10,27 +10,32 @@ int main()
 unittest
 {
     string input = "bbbb";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
     assert(start.bs);
     assert(start.bs.b);
     assert(start.bs.bs.b);
     assert(start.bs.bs.bs.b);
     assert(start.bs.bs.bs.bs.b);
 
-    p_context_init(&context, input);
-    assert(p_parse_Bs(&context) == P_SUCCESS);
-    Bs * bs = p_result_Bs(&context);
+    p_tree_delete(start);
+
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
     assert(bs.b);
     assert(bs.bs.b);
     assert(bs.bs.bs.b);
     assert(bs.bs.bs.bs.b);
 
+    p_tree_delete_Bs(bs);
+
     input = "c";
-    p_context_init(&context, input);
-    assert(p_parse_R(&context) == P_SUCCESS);
-    R * r = p_result_R(&context);
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
     assert(r.c);
+
+    p_tree_delete_R(r);
 }

spec/test_token_user_fields.c

@@ -0,0 +1,46 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input =
+        "# c1\n"
+        "#  c2\n"
+        "\n"
+        "first\n"
+        "\n   \n  \n"
+        "  # s1\n"
+        "   #   s2\n"
+        "second\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start->pIDs);
+    assert(start->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
+#else
+    assert(start->pIDs->id->comments);
+    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
+#endif
+    assert(start->pIDs->pIDs);
+    assert(start->pIDs->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
+#else
+    assert(start->pIDs->pIDs->id->comments);
+    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
+#endif
+
+#ifndef __cplusplus
+    free(context->comments);
+#endif
+    p_context_delete(context);
+    p_tree_delete(start);
+
+    return 0;
+}

spec/test_token_user_fields.d

@@ -0,0 +1,31 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "# c1\n" ~
+        "#  c2\n" ~
+        "\n" ~
+        "first\n" ~
+        "\n   \n  \n" ~
+        "  # s1\n" ~
+        "   #   s2\n" ~
+        "second\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.pIDs);
+    assert(start.pIDs.id);
+    assert(start.pIDs.id.comments == "# c1\n#  c2\n");
+    assert(start.pIDs.pIDs);
+    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
+
+    p_tree_delete(start);
+}

spec/test_tree.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
     assert(start->pItems1 != NULL);
     assert(start->pItems != NULL);
     Items * items = start->pItems;
@@ -33,20 +33,22 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems != NULL);
     assert(start->pItems->pItem != NULL);
     assert(start->pItems->pItem->pDual != NULL);
@@ -55,7 +57,8 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
     assert(start.pItems1 !is null);
     assert(start.pItems !is null);
     Items * items = start.pItems;
@@ -37,16 +36,20 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems !is null);
     assert(start.pItems.pItem !is null);
     assert(start.pItems.pItem.pDual !is null);
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_delete_token_node_memory.c

@@ -6,14 +6,15 @@
 int main()
 {
     char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
     assert(start->a != NULL);
     assert(*start->a->pvalue == 1);
     assert(start->b != NULL);
     assert(*start->b->pvalue == 2);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 }

spec/test_tree_field_aliases.c

@@ -6,16 +6,17 @@
 int main()
 {
     char const * input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start->first->pToken->token);
     assert_eq(TOKEN_b, start->second->pToken->token);
     assert_eq(TOKEN_c, start->third->pToken->token);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree_field_aliases.d

@@ -10,12 +10,13 @@ int main()
 unittest
 {
     string input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start.first.pToken.token);
     assert_eq(TOKEN_b, start.second.pToken.token);
     assert_eq(TOKEN_c, start.third.pToken.token);
+
+    p_tree_delete(start);
 }

spec/test_tree_invalid_positions.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(2, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -30,12 +30,13 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(8, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a\nbb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -56,12 +57,13 @@ int main()
     assert_eq(2, start->end_position.row);
     assert_eq(2, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a\nc\nc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -82,12 +84,13 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -104,7 +107,8 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree_invalid_positions.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(2, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -34,10 +33,12 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(8, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nbb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -58,10 +59,12 @@ unittest
     assert_eq(2, start.end_position.row);
     assert_eq(2, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nc\nc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -82,10 +85,12 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -101,4 +106,6 @@ unittest
     assert_eq(1, start.position.col);
     assert_eq(1, start.end_position.row);
     assert_eq(1, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_tree_node_memory_remains.c

@@ -369,11 +369,11 @@ int main(int argc, char * argv[])
         {"size_t_to_ulong", TOKEN_ulong},
         {"main", TOKEN_int},
     };
-    p_context_t context;
-    p_context_init(&context, (const uint8_t *)input, strlen(input));
-    size_t result = p_parse(&context);
+    p_context_t * context;
+    context = p_context_new((const uint8_t *)input, strlen(input));
+    size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
+    PModule * pmod = p_result(context);
     PModuleItems * pmis = pmod->pModuleItems;
     PFunctionDefinition ** pfds;
     size_t n_pfds = 0u;
@@ -412,7 +412,8 @@ int main(int argc, char * argv[])
     }
 
     free(pfds);
-    p_free_tree(pmod);
+    p_tree_delete(pmod);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree_node_memory_remains.d

@@ -374,11 +374,11 @@ def main() -> int
         Expected("size_t_to_ulong", TOKEN_ulong),
         Expected("main", TOKEN_int),
     ];
-    p_context_t context;
-    p_context_init(&context, input);
-    size_t result = p_parse(&context);
+    p_context_t * context;
+    context = p_context_new(input);
+    size_t result = p_parse(context);
     assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
+    PModule * pmod = p_result(context);
     PModuleItems * pmis = pmod.pModuleItems;
     PFunctionDefinition *[] pfds;
     while (pmis !is null)
@@ -405,4 +405,5 @@ def main() -> int
             stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
         }
     }
+    p_tree_delete(pmod);
 }

spec/test_tree_ps.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    PStartS * start = p_result(context);
     assert(start->pItems1 != NULL);
     assert(start->pItems != NULL);
     PItemsS * items = start->pItems;
@@ -33,20 +33,22 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start->pItems != NULL);
     assert(start->pItems->pItem != NULL);
     assert(start->pItems->pItem->pDual != NULL);
@@ -55,7 +57,8 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree_ps.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    PStartS * start = p_result(context);
     assert(start.pItems1 !is null);
     assert(start.pItems !is null);
     PItemsS * items = start.pItems;
@@ -37,16 +36,20 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
     assert(start.pItems !is null);
     assert(start.pItems.pItem !is null);
     assert(start.pItems.pItem.pDual !is null);
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_token_positions.c

@@ -6,10 +6,10 @@
 int main()
 {
     char const * input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(1, start->pT1->pToken->position.row);
     assert_eq(1, start->pT1->pToken->position.col);
@@ -43,12 +43,13 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(3, start->pT1->pToken->position.row);
     assert_eq(3, start->pT1->pToken->position.col);
@@ -82,7 +83,8 @@ int main()
     assert_eq(7, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
-    p_free_tree(start);
+    p_tree_delete(start);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_tree_token_positions.d

@@ -10,10 +10,9 @@ int main()
 unittest
 {
     string input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
 
     assert_eq(1, start.pT1.pToken.position.row);
     assert_eq(1, start.pT1.pToken.position.col);
@@ -47,10 +46,12 @@ unittest
     assert_eq(1, start.end_position.row);
     assert_eq(6, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
 
     assert_eq(3, start.pT1.pToken.position.row);
     assert_eq(3, start.pT1.pToken.position.col);
@@ -83,4 +84,6 @@ unittest
     assert_eq(3, start.position.col);
     assert_eq(7, start.end_position.row);
     assert_eq(6, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_user_code.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abcabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_user_code.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = "abcabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_user_context_fields.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+
+    fprintf(stderr, "comments: %s", context->comments);
+    fprintf(stderr, "acount: %u\n", context->acount);
+    free(context->comments);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_user_context_fields.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    stderr.writeln("comments: ", context.comments);
+    stderr.writeln("acount: ", context.acount);
+}

spec/test_user_terminate.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input = "aacc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 4200);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_user_terminate.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = "aacc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 4200);
 }

spec/test_user_terminate_lexer.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 8675309);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 8675309);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_user_terminate_lexer.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 8675309);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 8675309);
 }