v4.2.0

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
 ## v4.0.0
 
 ### New Features

Gemfile.lock

@@ -5,7 +5,7 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
+    erb (6.0.2)
     psych (5.3.1)
       date
       stringio
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.c.erb

@@ -981,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;

assets/parser.d.erb

@@ -9,7 +9,7 @@ module <%= @grammar.modulename %>;
 <% end %>
 
 import core.memory;
-import core.stdc.stdlib : malloc;
+import core.stdc.stdlib : malloc, free;
 
 /**************************************************************************
  * User code blocks
@@ -86,6 +86,14 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -94,6 +102,8 @@ private struct TreeNode
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     void *[0] fields;
 }
 
@@ -103,6 +113,8 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
     /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 <%= @grammar.token_user_fields %>
@@ -115,6 +127,8 @@ public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
 <%     rule_set.tree_fields.each do |fields| %>
     union
     {
@@ -225,6 +239,7 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
+ * Allocate and initialize lexer/parser context structure.
  *
  * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
@@ -249,6 +264,16 @@ private enum size_t INVALID_ID = cast(size_t)-1;
     return context;
 }
 
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+}
+
 /**************************************************************************
  * Decoder
  *************************************************************************/
@@ -1020,7 +1045,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
@@ -1054,7 +1079,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
@@ -1094,6 +1119,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
                 foreach (i; 0..n_fields)
                 {
                     node.fields[i] = null;
@@ -1201,6 +1228,35 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
+
 /**
  * Get the current text input position.
  *

assets/parser.h.erb

@@ -69,6 +69,14 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -316,6 +316,61 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -1007,6 +1062,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1130,8 +1196,6 @@ p_context_t * context = p_context_new(input);
 The `p_context_delete()` function must be called to deinitialize and deallocate
 a context structure allocated by `p_context_init()`.
 
-This function is not available for D language since D has a garbage collector.
-
 ### `p_lex`
 
 The `p_lex()` function is the main entry point to the lexer.
@@ -1320,8 +1384,6 @@ assert(code_point_length == 4u);
 The `p_tree_delete()` function can be used to free the memory used by the tree.
 It should be passed the same value that is returned by `p_result()`.
 
-The `p_tree_delete()` function is only available for C/C++ output targets.
-
 Note that if any lexer user code block allocates memory to store in a token's
 `pvalue`, in order to properly free this memory the `free_token_node` statement
 should be used to provide a code block that frees this memory.
@@ -1340,6 +1402,20 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
+### `p_value`
+
+When tree generation mode is not active, the `p_value_t` union can hold one of
+several different possible value types.
+
+The `p_value(v)` function returns an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is set for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
+
 ##> Data
 
 ### `p_token_names`

lib/propane/generator.rb

@@ -354,6 +354,14 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -69,6 +70,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -117,6 +119,12 @@ class Propane
       end
     end
 
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.0.0"
+  VERSION = "4.2.0"
 end

spec/propane_spec.rb

@@ -95,17 +95,15 @@ EOF
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
     results = Results.new(stdout, stderr, status)
-    if %w[c cpp].include?(options[:language])
     stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
     vgout = stdout + stderr
     File.binwrite("spec/run/.vgout", vgout)
-      vgout.scan(/lost: (\d+) bytes/) do |match|
+    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
       bytes = $1.to_i
       if bytes > 0
         raise "Valgrind detected memory leak"
       end
     end
-    end
     results
   end
 
@@ -1431,6 +1429,9 @@ EOF
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
 
+      # D garbage collector was freeing memory for nodes when it should not
+      # have been. This was due to pointers being in the TreeNode that was
+      # allocated by C malloc() function instead of allocated using D.
       it "does not free memory allocated for tree nodes" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/tree_node_memory_remains.#{ext}.propane"))
@@ -1623,7 +1624,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         elsif language == "c"
           write_grammar <<EOF
@@ -1665,7 +1666,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         else # C++
           write_grammar <<EOF
@@ -1690,7 +1691,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         end
         run_propane(language: language)
@@ -1698,6 +1699,88 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_named_optional_rule_component_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.pToken1 is null);
@@ -21,6 +20,8 @@ unittest
     assert(start.pR is null);
     assert(start.r is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -35,6 +36,8 @@ unittest
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -43,4 +46,6 @@ unittest
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_optional_rule_component_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.pToken1 is null);
@@ -20,6 +19,8 @@ unittest
     assert(start.pR3 is null);
     assert(start.pR is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -32,6 +33,8 @@ unittest
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -40,4 +43,6 @@ unittest
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_starting_rules_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "bbbb";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.bs);
@@ -20,6 +19,8 @@ unittest
     assert(start.bs.bs.bs.b);
     assert(start.bs.bs.bs.bs.b);
 
+    p_tree_delete(start);
+
     context = p_context_new(input);
     assert(p_parse_Bs(context) == P_SUCCESS);
     Bs * bs = p_result_Bs(context);
@@ -28,9 +29,13 @@ unittest
     assert(bs.bs.bs.b);
     assert(bs.bs.bs.bs.b);
 
+    p_tree_delete_Bs(bs);
+
     input = "c";
     context = p_context_new(input);
     assert(p_parse_R(context) == P_SUCCESS);
     R * r = p_result_R(context);
     assert(r.c);
+
+    p_tree_delete_R(r);
 }

spec/test_token_user_fields.c

@@ -19,6 +19,22 @@ int main()
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
+    assert(start->pIDs);
+    assert(start->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
+#else
+    assert(start->pIDs->id->comments);
+    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
+#endif
+    assert(start->pIDs->pIDs);
+    assert(start->pIDs->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
+#else
+    assert(start->pIDs->pIDs->id->comments);
+    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
+#endif
 
 #ifndef __cplusplus
     free(context->comments);

spec/test_token_user_fields.d

@@ -17,8 +17,15 @@ unittest
         "  # s1\n" ~
         "   #   s2\n" ~
         "second\n";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
+    assert(start.pIDs);
+    assert(start.pIDs.id);
+    assert(start.pIDs.id.comments == "# c1\n#  c2\n");
+    assert(start.pIDs.pIDs);
+    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
+
+    p_tree_delete(start);
 }

spec/test_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     Start * start = p_result(context);
     assert(start.pItems1 !is null);
@@ -37,12 +36,16 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_field_aliases.d

@@ -10,12 +10,13 @@ int main()
 unittest
 {
     string input = "\na\nb\nc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start.first.pToken.token);
     assert_eq(TOKEN_b, start.second.pToken.token);
     assert_eq(TOKEN_c, start.third.pToken.token);
+
+    p_tree_delete(start);
 }

spec/test_tree_invalid_positions.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "\na\n  bb ccc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
@@ -34,6 +33,8 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(8, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nbb";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -58,6 +59,8 @@ unittest
     assert_eq(2, start.end_position.row);
     assert_eq(2, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nc\nc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -82,6 +85,8 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -101,4 +106,6 @@ unittest
     assert_eq(1, start.position.col);
     assert_eq(1, start.end_position.row);
     assert_eq(1, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_tree_node_memory_remains.d

@@ -405,4 +405,5 @@ def main() -> int
             stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
         }
     }
+    p_tree_delete(pmod);
 }

spec/test_tree_ps.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     PStartS * start = p_result(context);
     assert(start.pItems1 !is null);
@@ -37,12 +36,16 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_token_positions.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "abbccc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
@@ -47,6 +46,8 @@ unittest
     assert_eq(1, start.end_position.row);
     assert_eq(6, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -83,4 +84,6 @@ unittest
     assert_eq(3, start.position.col);
     assert_eq(7, start.end_position.row);
     assert_eq(6, start.end_position.col);
+
+    p_tree_delete(start);
 }