wip

Add free_token_node grammar statement
Turn on valgrind for C, C++ target specs
2026-02-08 22:22:27 -05:00 · 2026-02-08 20:33:00 -05:00 · 2026-02-08 19:24:11 -05:00 · 2026-02-08 00:02:24 -05:00
17 changed files with 274 additions and 67 deletions
--- a/assets/parser.c.erb
+++ b/assets/parser.c.erb
@ -695,11 +695,13 @@ typedef struct
 } state_value_t;

 /** Common AST node structure. */
-typedef struct
+typedef struct ASTNode_s
 {
    <%= @grammar.prefix %>position_t position;
    <%= @grammar.prefix %>position_t end_position;
-    void * fields[];
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct ASTNode_s * fields[];
 } ASTNode;

 /** Parser shift table. */
@ -972,7 +974,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
            {
                /* Successful parse. */
 <% if @grammar.ast %>
-                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node;
 <% else %>
                context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@ -992,6 +994,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
                token_ast_node->position = token_info.position;
                token_ast_node->end_position = token_info.end_position;
+                token_ast_node->n_fields = 0u;
+                token_ast_node->is_token = 1u;
                token_ast_node->token = token;
                token_ast_node->pvalue = token_info.pvalue;
                state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@ -1028,31 +1032,30 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
            else if (parser_reduce_table[reduce_index].n_states > 0)
            {
                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                ASTNode * node = (ASTNode *)malloc(sizeof(ASTNode) + n_fields * sizeof(void *));
+                size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *);
+                ASTNode * node = (ASTNode *)malloc(bytes);
+                memset(node, 0, bytes);
                node->position = INVALID_POSITION;
                node->end_position = INVALID_POSITION;
-                for (size_t i = 0; i < n_fields; i++)
-                {
-                    node->fields[i] = NULL;
-                }
+                node->n_fields = n_fields;
                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                {
                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                    {
-                        node->fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                    }
                }
                else
                {
                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                    {
-                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                    }
                }
                bool position_found = false;
                for (size_t i = 0; i < n_fields; i++)
                {
-                    ASTNode * child = (ASTNode *)node->fields[i];
+                    ASTNode * child = node->fields[i];
                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
                    {
                        if (!position_found)
@ -1074,6 +1077,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
            memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
            {
+                state_values_stack_free(&statevalues);
                return P_USER_TERMINATED;
            }
            reduced_parser_value = reduced_parser_value2;
@ -1106,13 +1110,13 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 * @return Parse result value.
 */
 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context->parse_result;
+    return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result;
 <% else %>
    return context->parse_result.v_<%= start_rule_type[0] %>;
 <% end %>
@ -1153,3 +1157,35 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
    return context->token;
 }
+<% if @grammar.ast %>
+
+static void free_ast_node(ASTNode * node)
+{
+    if (node->is_token)
+    {
+<%   if @grammar.free_token_node %>
+        <%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
+<%   end %>
+        /* TODO: free value_t */
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                free_ast_node(node->fields[i]);
+            }
+        }
+    }
+    free(node);
+}
+
+/**
+ * Free all AST node memory.
+ */
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast)
+{
+    free_ast_node((ASTNode *)ast);
+}
+<% end %>
--- a/assets/parser.d.erb
+++ b/assets/parser.d.erb
@ -173,7 +173,7 @@ public struct <%= @grammar.prefix %>context_t

    /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
    <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@ -1031,7 +1031,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
            {
                /* Successful parse. */
 <% if @grammar.ast %>
-                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
+                context.parse_result = statevalues[$-1].ast_node;
 <% else %>
                context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@ -1156,13 +1156,13 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
 * @return Parse result value.
 */
 <% if @grammar.ast %>
-public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context.parse_result;
+    return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result;
 <% else %>
    return context.parse_result.v_<%= start_rule_type[0] %>;
 <% end %>
--- a/assets/parser.h.erb
+++ b/assets/parser.h.erb
@ -75,9 +75,11 @@ typedef union
 /** AST node types. @{ */
 typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
-    /* ASTNode fields must be present in the same order here. */
+    <% # ASTNode fields must be present in the same order here. # %>
    <%= @grammar.prefix %>position_t position;
    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
    <%= @grammar.prefix %>token_t token;
    <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
@ -93,8 +95,11 @@ struct <%= name %>;
 <%     next if rule_set.optional? %>
 typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
+    <% # ASTNode fields must be present in the same order here. # %>
    <%= @grammar.prefix %>position_t position;
    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
 <%     rule_set.ast_fields.each do |fields| %>
    union
    {
@ -157,7 +162,7 @@ typedef struct

    /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
    <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@ -186,7 +191,9 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);

 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <% end %>
--- a/doc/user_guide.md
+++ b/doc/user_guide.md
@ -326,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null);
 assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 ```

+## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
+
+If user lexer code block allocates memory to store in a token node's `pvalue`,
+the `free_token_node` grammar statement can be used to specify the name of a
+function which will be called during the `p_free_ast()` call to free the memory
+associated with a token node.
+
+Example:
+
+```
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+```
+
 ##> Specifying tokens - the `token` statement

 The `token` statement allows defining a lexer token and a pattern to match that
@ -1098,6 +1128,20 @@ assert(code_point == 0x1F9E1u);
 assert(code_point_length == 4u);
 ```

+### `p_free_ast`
+
+The `p_free_ast()` function can be used to free the memory used by the AST.
+It should be passed the same value that is returned by `p_result()`.
+
+The `p_free_ast()` function is only available for C/C++ output targets.
+
+Note that if any lexer user code block allocates memory to store in a token's
+`pvalue`, in order to properly free this memory a `free_token_node` function
+should be specified in the grammar file.
+If specified, the `free_token_node` function will be called during the
+`p_free_ast()` process to allow user code to free any memory associated with
+a token node's `pvalue`.
+
 ##> Data

 ### `p_token_names`
--- a/lib/propane/generator.rb
+++ b/lib/propane/generator.rb
@ -71,12 +71,14 @@ class Propane
        end
        tokens_by_name[token.name] = token
      end
-      # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
-        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
+      # Check for user start rules.
+      @grammar.start_rules.each_with_index do |start_rule, i|
+        unless @grammar.rules.find {|rule| rule.name == start_rule}
+          raise Error.new("Start rule `#{start_rule}` not found")
+        end
+        # Add "real" start rule.
+        @grammar.rules.unshift(Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil))
      end
-      # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
      # Generate and add rules for optional components.
      generate_optional_component_rules!(tokens_by_name)
      # Build rule sets.
@ -332,9 +334,9 @@ class Propane
    #
    # @return [Array<String>]
    #   Start rule parser value type name and type string.
-    def start_rule_type
+    def start_rule_type(start_rule_index = 0)
      start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rule
+        rule.name == @grammar.start_rules[start_rule_index]
      end
      [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
    end
--- a/lib/propane/grammar.rb
+++ b/lib/propane/grammar.rb
@ -8,10 +8,11 @@ class Propane
    attr_reader :ast
    attr_reader :ast_prefix
    attr_reader :ast_suffix
+    attr_reader :free_token_node
    attr_reader :modulename
    attr_reader :patterns
    attr_reader :rules
-    attr_reader :start_rule
+    attr_reader :start_rules
    attr_reader :tokens
    attr_reader :code_blocks
    attr_reader :ptypes
@ -19,7 +20,7 @@ class Propane

    def initialize(input)
      @patterns = []
-      @start_rule = "Start"
+      @start_rules = []
      @tokens = []
      @rules = []
      @code_blocks = {}
@ -32,7 +33,9 @@ class Propane
      @ast = false
      @ast_prefix = ""
      @ast_suffix = ""
+      @free_token_node = nil
      parse_grammar!
+      @start_rules << "Start" if @start_rules.empty?
    end

    def ptype
@ -62,6 +65,7 @@ class Propane
      elsif parse_ast_statement!
      elsif parse_ast_prefix_statement!
      elsif parse_ast_suffix_statement!
+      elsif parse_free_token_node_statement!
      elsif parse_module_statement!
      elsif parse_ptype_statement!
      elsif parse_pattern_statement!
@ -112,6 +116,12 @@ class Propane
      end
    end

+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\s+(\w+)\s*;/)
+        @free_token_node = md[1]
+      end
+    end
+
    def parse_module_statement!
      if consume!(/module\s+/)
        md = consume!(/([\w.]+)\s*/, "expected module name")
@ -233,7 +243,7 @@ class Propane

    def parse_start_statement!
      if md = consume!(/start\s+(\w+)\s*;/)
-        @start_rule = md[1]
+        @start_rules << md[1] unless @start_rules.include?(md[1])
      end
    end

--- a/spec/propane_spec.rb
+++ b/spec/propane_spec.rb
@ -78,9 +78,9 @@ EOF
    end
    case options[:language]
    when "c"
-      command = [*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
    when "cpp"
-      command = [*%w[g++ -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
    when "d"
      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
    end
@ -88,12 +88,24 @@ EOF
    expect(result).to be_truthy
  end

-  def run_test
+  def run_test(options)
    stdout, stderr, status = Open3.capture3("spec/run/testparser")
    File.binwrite("spec/run/.stderr", stderr)
    File.binwrite("spec/run/.stdout", stdout)
    stderr.sub!(/^.*modules passed unittests\n/, "")
-    Results.new(stdout, stderr, status)
+    results = Results.new(stdout, stderr, status)
+    if %w[c cpp].include?(options[:language])
+      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+      vgout = stdout + stderr
+      File.binwrite("spec/run/.vgout", vgout)
+      vgout.scan(/lost: (\d+) bytes/) do |match|
+        bytes = $1.to_i
+        if bytes > 0
+          raise "Valgrind detected memory leak"
+        end
+      end
+    end
+    results
  end

  def lines(str)
@ -286,7 +298,7 @@ Foo -> plus <<>>
 EOF
        run_propane(language: language)
        compile("spec/test_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -324,7 +336,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -418,7 +430,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -444,7 +456,7 @@ R2 -> a b;
 EOF
        run_propane(language: language)
        compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
      end

@ -459,7 +471,7 @@ R1 -> b;
 EOF
        run_propane(language: language)
        compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
      end

@ -494,7 +506,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "abc!",
@ -530,7 +542,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "def!",
@ -572,7 +584,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "def!",
@ -630,7 +642,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "begin string mode",
@ -688,7 +700,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_lexer_multiple_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "ident: d",
@ -725,7 +737,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "A!",
@ -744,7 +756,7 @@ As -> As a << $$ = $1 + 1u; >>
 EOF
        run_propane(language: language)
        compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        expect(results.stderr).to eq ""
      end
@ -798,7 +810,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "Matched token is identifier_123",
@ -831,7 +843,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -846,7 +858,7 @@ Start -> a num;
 EOF
        run_propane(language: language)
        compile("spec/test_error_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -875,7 +887,7 @@ Start -> b c b;
 EOF
        run_propane(name: "myp2", language: language)
        compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -894,7 +906,7 @@ Any -> c;
 EOF
        run_propane(language: language)
        compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -912,7 +924,7 @@ Any -> ;
 EOF
        run_propane(language: language)
        compile("spec/test_user_terminate.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -956,7 +968,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
@ -1021,7 +1033,7 @@ Two -> two;
 EOF
        run_propane(language: language)
        compile("spec/test_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1065,7 +1077,7 @@ Two -> two;
 EOF
        run_propane(language: language)
        compile("spec/test_ast_ps.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1139,7 +1151,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_optional_rule_component.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
@ -1191,7 +1203,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1232,7 +1244,7 @@ EOF
        end
        run_propane(language: language)
        compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1252,7 +1264,7 @@ T -> c;
 EOF
        run_propane(language: language)
        compile("spec/test_ast_token_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1272,7 +1284,7 @@ A -> bb? c?;
 EOF
        run_propane(language: language)
        compile("spec/test_ast_invalid_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1292,7 +1304,7 @@ T -> c;
 EOF
        run_propane(language: language)
        compile("spec/test_ast_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1315,7 +1327,7 @@ T -> c;
 EOF
        run_propane(language: language)
        compile("spec/test_ast_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@ -1342,7 +1354,7 @@ EOF
 #include <stdio.h>
 #include <string.h>
 >>
-ptype char const *;
+ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
  char * s = (char *)malloc(match_length + 1);
  strncpy(s, (char const *)match, match_length);
@ -1353,12 +1365,14 @@ drop /\\s+/;
 Start -> id:first id:second <<
  printf("first is %s\\n", ${first});
  printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
 >>
 EOF
        end
        run_propane(language: language)
        compile("spec/test_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
        expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1389,7 +1403,7 @@ EOF
 #include <stdio.h>
 #include <string.h>
 >>
-ptype char const *;
+ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
  char * s = (char *)malloc(match_length + 1);
  strncpy(s, (char const *)match, match_length);
@ -1402,13 +1416,15 @@ Start -> Foo;
 Start -> id:first id:second <<
  printf("first is %s\\n", ${first});
  printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
 >>
 Foo -> ;
 EOF
        end
        run_propane(language: language)
        compile("spec/test_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
        expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1419,10 +1435,40 @@ EOF
        write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane"))
        run_propane(language: language)
        compile("spec/test_ast_node_memory_remains.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
+
+      if %w[c cpp].include?(language)
+        it "allows a user function to free token node memory in AST mode" do
+    write_grammar <<EOF
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+EOF
+          run_propane(language: language)
+          compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
    end
  end
 end
--- a/spec/test_ast.c
+++ b/spec/test_ast.c
@ -33,12 +33,16 @@ int main()
    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
    assert(itemsmore->pItemsMore == NULL);

+    p_free_ast(start);
+
    input = "";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert_eq(P_SUCCESS, p_parse(&context));
    start = p_result(&context);
    assert(start->pItems == NULL);

+    p_free_ast(start);
+
    input = "2 1";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert_eq(P_SUCCESS, p_parse(&context));
@ -51,5 +55,7 @@ int main()
    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
    assert(start->pItems->pItem->pDual->pOne1 == NULL);

+    p_free_ast(start);
+
    return 0;
 }
--- a/spec/test_ast_field_aliases.c
+++ b/spec/test_ast_field_aliases.c
@ -15,5 +15,7 @@ int main()
    assert_eq(TOKEN_b, start->second->pToken->token);
    assert_eq(TOKEN_c, start->third->pToken->token);

+    p_free_ast(start);
+
    return 0;
 }
--- a/spec/test_ast_invalid_positions.c
+++ b/spec/test_ast_invalid_positions.c
@ -30,6 +30,8 @@ int main()
    assert_eq(3, start->end_position.row);
    assert_eq(8, start->end_position.col);

+    p_free_ast(start);
+
    input = "a\nbb";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -54,6 +56,8 @@ int main()
    assert_eq(2, start->end_position.row);
    assert_eq(2, start->end_position.col);

+    p_free_ast(start);
+
    input = "a\nc\nc";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -78,6 +82,8 @@ int main()
    assert_eq(3, start->end_position.row);
    assert_eq(1, start->end_position.col);

+    p_free_ast(start);
+
    input = "a";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -98,5 +104,7 @@ int main()
    assert_eq(1, start->end_position.row);
    assert_eq(1, start->end_position.col);

+    p_free_ast(start);
+
    return 0;
 }
--- a/spec/test_ast_node_memory_remains.c
+++ b/spec/test_ast_node_memory_remains.c
@ -411,5 +411,8 @@ int main(int argc, char * argv[])
        }
    }

+    free(pfds);
+    p_free_ast(pmod);
+
    return 0;
 }
--- a/spec/test_ast_ps.c
+++ b/spec/test_ast_ps.c
@ -33,12 +33,16 @@ int main()
    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
    assert(itemsmore->pItemsMore == NULL);

+    p_free_ast(start);
+
    input = "";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert_eq(P_SUCCESS, p_parse(&context));
    start = p_result(&context);
    assert(start->pItems == NULL);

+    p_free_ast(start);
+
    input = "2 1";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert_eq(P_SUCCESS, p_parse(&context));
@ -51,5 +55,7 @@ int main()
    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
    assert(start->pItems->pItem->pDual->pOne1 == NULL);

+    p_free_ast(start);
+
    return 0;
 }
--- a/spec/test_ast_token_positions.c
+++ b/spec/test_ast_token_positions.c
@ -43,6 +43,8 @@ int main()
    assert_eq(1, start->end_position.row);
    assert_eq(6, start->end_position.col);

+    p_free_ast(start);
+
    input = "\n\n  bb\nc\ncc\n\n     a";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -80,5 +82,7 @@ int main()
    assert_eq(7, start->end_position.row);
    assert_eq(6, start->end_position.col);

+    p_free_ast(start);
+
    return 0;
 }
--- a/spec/test_free_ast_token_node_memory.c
+++ b/spec/test_free_ast_token_node_memory.c
@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->a != NULL);
+    assert(*start->a->pvalue == 1);
+    assert(start->b != NULL);
+    assert(*start->b->pvalue == 2);
+
+    p_free_ast(start);
+}
--- a/spec/test_named_optional_rule_component_ast.c
+++ b/spec/test_named_optional_rule_component_ast.c
@ -17,6 +17,8 @@ int main()
    assert(start->pR == NULL);
    assert(start->r == NULL);

+    p_free_ast(start);
+
    input = "abcd";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -31,6 +33,8 @@ int main()
    assert(start->pR == start->r);
    assert_eq(TOKEN_c, start->pR->pToken1->token);

+    p_free_ast(start);
+
    input = "bdc";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -40,6 +44,8 @@ int main()
    assert(start->r != NULL);
    assert_eq(TOKEN_d, start->pR->pToken1->token);

+    p_free_ast(start);
+
    return 0;
 }

--- a/spec/test_optional_rule_component_ast.c
+++ b/spec/test_optional_rule_component_ast.c
@ -16,6 +16,8 @@ int main()
    assert(start->pR3 == NULL);
    assert(start->pR == NULL);

+    p_free_ast(start);
+
    input = "abcd";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -28,6 +30,8 @@ int main()
    assert(start->pR == start->pR3);
    assert_eq(TOKEN_c, start->pR->pToken1->token);

+    p_free_ast(start);
+
    input = "bdc";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_SUCCESS);
@ -37,6 +41,8 @@ int main()
    assert(start->pR != NULL);
    assert_eq(TOKEN_d, start->pR->pToken1->token);

+    p_free_ast(start);
+
    return 0;
 }

--- a/spec/test_start_rule_ast.c
+++ b/spec/test_start_rule_ast.c
@ -13,5 +13,7 @@ int main()
    assert(top->pToken != NULL);
    assert_eq(TOKEN_hi, top->pToken->token);

+    p_free_ast(top);
+
    return 0;
 }
Author	SHA1	Message	Date
Josh Holtrop	7122bdc2db	wip	2026-02-08 22:22:27 -05:00
Josh Holtrop	cb426b4be1	Add free_token_node grammar statement	2026-02-08 20:33:00 -05:00
Josh Holtrop	addf27d837	Turn on valgrind for C, C++ target specs	2026-02-08 19:24:11 -05:00
Josh Holtrop	9c03d20083	Add p_free_ast() to free AST node memory for C targets	2026-02-08 00:02:24 -05:00