Combine free_token_user_fields and free_token_node statements

assets/parser.c.erb

@@ -1220,10 +1220,7 @@ static void tree_delete(TreeNode * node)
     if (node->is_token)
     {
         <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
-<%   if @grammar.free_token_node %>
-        <%= @grammar.free_token_node %>(token_tree_node);
-<%   end %>
-<%= expand_code(@grammar.free_token_user_fields, false, nil, nil) %>
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
 <%   if @cpp %>
         delete token_tree_node;
 <%   else %>

doc/user_guide.md

@@ -297,7 +297,7 @@ drop /#(.*)\n/ <<
 ```
 
 If a pointer to any allocated memory is stored in a user-defined context field,
-the `free_token_user_fields` statement can be used to supply a code block which
+the `free_token_node` statement can be used to supply a code block which
 will be executed immediately before the token node is freed.
 For C++, the `delete` statement is used to free the token tree node, so the
 destructor for any custom token user fields will be called.
@@ -326,25 +326,6 @@ drop /#(.*)\n/ <<
 >>
 ```
 
-### Freeing allocated memory in a custom token user field - the `free_token_user_fields` statement
-
-The `free_token_user_fields` statement allows the user to provide a code block
-which will be executed immediately prior to freeing the token tree node.
-
-For example (C):
-
-```
-token_user_fields <<
-    char * comments;
->>
-on_token_node <<
-    ${token.comments} = (char *)malloc(some_len);
->>
-free_token_user_fields <<
-    free(${token.comments});
->>
-```
-
 ##> Tree generation mode - the `tree` statement
 
 To activate tree generation mode, place the `tree` statement in your grammar file:
@@ -478,22 +459,18 @@ assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 
 ## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
 
-If user lexer code block allocates memory to store in a token node's `pvalue`,
-the `free_token_node` grammar statement can be used to specify the name of a
-function which will be called during the `p_tree_delete()` call to free the memory
-associated with a token node.
+If user lexer code block allocates memory to store in a token node's `pvalue`
+or any custom token user fields store pointers to allocated memory, the
+`free_token_node` grammar statement can be used to provide a code block which
+can be used to free memory properly.
 
-Example:
+Example freeing `pvalue` (C):
 
 ```
-<<
-static void free_token(Token * token)
-{
-    free(token->pvalue);
-}
->>
 tree;
-free_token_node free_token;
+free_token_node <<
+    free(${token.pvalue});
+>>
 ptype int *;
 token a <<
   $$ = (int *)malloc(sizeof(int));
@@ -506,6 +483,23 @@ token b <<
 Start -> a:a b:b;
 ```
 
+Example freeing custom token user fields (C):
+
+```
+token_user_fields <<
+    char * comments;
+>>
+on_token_node <<
+    ${token.comments} = (char *)malloc(some_len);
+>>
+free_token_node <<
+    free(${token.comments});
+>>
+```
+
+The `free_token_node` statement user code block is not emitted for D language
+since D has a garbage collector.
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -1329,9 +1323,9 @@ It should be passed the same value that is returned by `p_result()`.
 The `p_tree_delete()` function is only available for C/C++ output targets.
 
 Note that if any lexer user code block allocates memory to store in a token's
-`pvalue`, in order to properly free this memory a `free_token_node` function
-should be specified in the grammar file.
-If specified, the `free_token_node` function will be called during the
+`pvalue`, in order to properly free this memory the `free_token_node` statement
+should be used to provide a code block that frees this memory.
+If specified, the `free_token_node` code block will be executed during the
 `p_tree_delete()` process to allow user code to free any memory associated with
 a token node's `pvalue`.
 

lib/propane/grammar.rb

@@ -20,7 +20,6 @@ class Propane
     attr_reader :prefix
     attr_reader :on_token_node
     attr_reader :token_user_fields
-    attr_reader :free_token_user_fields
 
     def initialize(input)
       @patterns = []
@@ -37,11 +36,10 @@ class Propane
       @tree = false
       @tree_prefix = ""
       @tree_suffix = ""
-      @free_token_node = nil
+      @free_token_node = ""
       @context_user_fields = nil
       @on_token_node = ""
       @token_user_fields = nil
-      @free_token_user_fields = ""
       parse_grammar!
       @start_rules << "Start" if @start_rules.empty?
     end
@@ -78,7 +76,6 @@ class Propane
       elsif parse_module_statement!
       elsif parse_on_token_node_statement!
       elsif parse_token_user_fields_statement!
-      elsif parse_free_token_user_fields_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
       elsif parse_start_statement!
@@ -138,12 +135,6 @@ class Propane
       end
     end
 
-    def parse_free_token_node_statement!
-      if md = consume!(/free_token_node\s+(\w+)\s*;/)
-        @free_token_node = md[1]
-      end
-    end
-
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -173,12 +164,12 @@ class Propane
       end
     end
 
-    def parse_free_token_user_fields_statement!
-      if md = consume!(/free_token_user_fields\b\s*/)
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\b\s*/)
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block")
         end
-        @free_token_user_fields += code
+        @free_token_node += code
       end
     end
 

spec/propane_spec.rb

@@ -1485,14 +1485,10 @@ EOF
       if %w[c cpp].include?(language)
         it "allows a user function to free token node memory in tree mode" do
           write_grammar <<EOF
-<<
-static void free_token(Token * token)
-{
-    free(token->pvalue);
-}
->>
 tree;
-free_token_node free_token;
+free_token_node <<
+    free(${token.pvalue});
+>>
 ptype int *;
 token a <<
   $$ = (int *)malloc(sizeof(int));
@@ -1641,7 +1637,7 @@ context_user_fields <<
 token_user_fields <<
     char * comments;
 >>
-free_token_user_fields <<
+free_token_node <<
     free(${token.comments});
 >>
 on_token_node <<