diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 5da3ceb..4736949 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1219,11 +1219,8 @@ static void tree_delete(TreeNode * node) { if (node->is_token) { - <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *) node; -<% if @grammar.free_token_node %> - <%= @grammar.free_token_node %>(token_tree_node); -<% end %> -<%= expand_code(@grammar.free_token_user_fields, false, nil, nil) %> + <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node; +<%= expand_code(@grammar.free_token_node, false, nil, nil) %> <% if @cpp %> delete token_tree_node; <% else %> diff --git a/doc/user_guide.md b/doc/user_guide.md index c8b5de9..9c14d4f 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -297,7 +297,7 @@ drop /#(.*)\n/ << ``` If a pointer to any allocated memory is stored in a user-defined context field, -the `free_token_user_fields` statement can be used to supply a code block which +the `free_token_node` statement can be used to supply a code block which will be executed immediately before the token node is freed. For C++, the `delete` statement is used to free the token tree node, so the destructor for any custom token user fields will be called. @@ -326,25 +326,6 @@ drop /#(.*)\n/ << >> ``` -### Freeing allocated memory in a custom token user field - the `free_token_user_fields` statement - -The `free_token_user_fields` statement allows the user to provide a code block -which will be executed immediately prior to freeing the token tree node. - -For example (C): - -``` -token_user_fields << - char * comments; ->> -on_token_node << - ${token.comments} = (char *)malloc(some_len); ->> -free_token_user_fields << - free(${token.comments}); ->> -``` - ##> Tree generation mode - the `tree` statement To activate tree generation mode, place the `tree` statement in your grammar file: @@ -478,22 +459,18 @@ assert(itemsmore.pItem.pItem.pItem.pToken1 !is null); ## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement -If user lexer code block allocates memory to store in a token node's `pvalue`, -the `free_token_node` grammar statement can be used to specify the name of a -function which will be called during the `p_tree_delete()` call to free the memory -associated with a token node. +If user lexer code block allocates memory to store in a token node's `pvalue` +or any custom token user fields store pointers to allocated memory, the +`free_token_node` grammar statement can be used to provide a code block which +can be used to free memory properly. -Example: +Example freeing `pvalue` (C): ``` -<< -static void free_token(Token * token) -{ - free(token->pvalue); -} ->> tree; -free_token_node free_token; +free_token_node << + free(${token.pvalue}); +>> ptype int *; token a << $$ = (int *)malloc(sizeof(int)); @@ -506,6 +483,23 @@ token b << Start -> a:a b:b; ``` +Example freeing custom token user fields (C): + +``` +token_user_fields << + char * comments; +>> +on_token_node << + ${token.comments} = (char *)malloc(some_len); +>> +free_token_node << + free(${token.comments}); +>> +``` + +The `free_token_node` statement user code block is not emitted for D language +since D has a garbage collector. + ##> Specifying tokens - the `token` statement The `token` statement allows defining a lexer token and a pattern to match that @@ -1329,9 +1323,9 @@ It should be passed the same value that is returned by `p_result()`. The `p_tree_delete()` function is only available for C/C++ output targets. Note that if any lexer user code block allocates memory to store in a token's -`pvalue`, in order to properly free this memory a `free_token_node` function -should be specified in the grammar file. -If specified, the `free_token_node` function will be called during the +`pvalue`, in order to properly free this memory the `free_token_node` statement +should be used to provide a code block that frees this memory. +If specified, the `free_token_node` code block will be executed during the `p_tree_delete()` process to allow user code to free any memory associated with a token node's `pvalue`. diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index dd318cb..095a418 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -20,7 +20,6 @@ class Propane attr_reader :prefix attr_reader :on_token_node attr_reader :token_user_fields - attr_reader :free_token_user_fields def initialize(input) @patterns = [] @@ -37,11 +36,10 @@ class Propane @tree = false @tree_prefix = "" @tree_suffix = "" - @free_token_node = nil + @free_token_node = "" @context_user_fields = nil @on_token_node = "" @token_user_fields = nil - @free_token_user_fields = "" parse_grammar! @start_rules << "Start" if @start_rules.empty? end @@ -78,7 +76,6 @@ class Propane elsif parse_module_statement! elsif parse_on_token_node_statement! elsif parse_token_user_fields_statement! - elsif parse_free_token_user_fields_statement! elsif parse_ptype_statement! elsif parse_pattern_statement! elsif parse_start_statement! @@ -138,12 +135,6 @@ class Propane end end - def parse_free_token_node_statement! - if md = consume!(/free_token_node\s+(\w+)\s*;/) - @free_token_node = md[1] - end - end - def parse_module_statement! if consume!(/module\s+/) md = consume!(/([\w.]+)\s*/, "expected module name") @@ -173,12 +164,12 @@ class Propane end end - def parse_free_token_user_fields_statement! - if md = consume!(/free_token_user_fields\b\s*/) + def parse_free_token_node_statement! + if md = consume!(/free_token_node\b\s*/) unless code = parse_code_block! raise Error.new("Line #{@line_number}: expected code block") end - @free_token_user_fields += code + @free_token_node += code end end diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 409cf20..0d9b3c4 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1485,14 +1485,10 @@ EOF if %w[c cpp].include?(language) it "allows a user function to free token node memory in tree mode" do write_grammar <pvalue); -} ->> tree; -free_token_node free_token; +free_token_node << + free(${token.pvalue}); +>> ptype int *; token a << $$ = (int *)malloc(sizeof(int)); @@ -1641,7 +1637,7 @@ context_user_fields << token_user_fields << char * comments; >> -free_token_user_fields << +free_token_node << free(${token.comments}); >> on_token_node <<