Add free_token_node grammar statement

assets/parser.c.erb

@@ -1163,6 +1163,9 @@ static void free_ast_node(ASTNode * node)
 {
     if (node->is_token)
     {
+<%   if @grammar.free_token_node %>
+        <%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
+<%   end %>
         /* TODO: free value_t */
     }
     else if (node->n_fields > 0u)

doc/user_guide.md

@@ -326,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null);
 assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 ```
 
+## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
+
+If user lexer code block allocates memory to store in a token node's `pvalue`,
+the `free_token_node` grammar statement can be used to specify the name of a
+function which will be called during the `p_free_ast()` call to free the memory
+associated with a token node.
+
+Example:
+
+```
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+```
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -1098,6 +1128,20 @@ assert(code_point == 0x1F9E1u);
 assert(code_point_length == 4u);
 ```
 
+### `p_free_ast`
+
+The `p_free_ast()` function can be used to free the memory used by the AST.
+It should be passed the same value that is returned by `p_result()`.
+
+The `p_free_ast()` function is only available for C/C++ output targets.
+
+Note that if any lexer user code block allocates memory to store in a token's
+`pvalue`, in order to properly free this memory a `free_token_node` function
+should be specified in the grammar file.
+If specified, the `free_token_node` function will be called during the
+`p_free_ast()` process to allow user code to free any memory associated with
+a token node's `pvalue`.
+
 ##> Data
 
 ### `p_token_names`

lib/propane/grammar.rb

@@ -8,6 +8,7 @@ class Propane
     attr_reader :ast
     attr_reader :ast_prefix
     attr_reader :ast_suffix
+    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
@@ -32,6 +33,7 @@ class Propane
       @ast = false
       @ast_prefix = ""
       @ast_suffix = ""
+      @free_token_node = nil
       parse_grammar!
     end
 
@@ -62,6 +64,7 @@ class Propane
       elsif parse_ast_statement!
       elsif parse_ast_prefix_statement!
       elsif parse_ast_suffix_statement!
+      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
@@ -112,6 +115,12 @@ class Propane
       end
     end
 
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\s+(\w+)\s*;/)
+        @free_token_node = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")

spec/propane_spec.rb

@@ -1439,6 +1439,36 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      if %w[c cpp].include?(language)
+        it "allows a user function to free token node memory in AST mode" do
+    write_grammar <<EOF
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+EOF
+          run_propane(language: language)
+          compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
     end
   end
 end

spec/test_free_ast_token_node_memory.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->a != NULL);
+    assert(*start->a->pvalue == 1);
+    assert(start->b != NULL);
+    assert(*start->b->pvalue == 2);
+
+    p_free_ast(start);
+}