From cb426b4be16f79af8d984ebbe4f7b5621555cd90 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Sun, 8 Feb 2026 20:33:00 -0500 Subject: [PATCH] Add free_token_node grammar statement --- assets/parser.c.erb | 3 ++ doc/user_guide.md | 44 ++++++++++++++++++++++++++ lib/propane/grammar.rb | 9 ++++++ spec/propane_spec.rb | 30 ++++++++++++++++++ spec/test_free_ast_token_node_memory.c | 19 +++++++++++ 5 files changed, 105 insertions(+) create mode 100644 spec/test_free_ast_token_node_memory.c diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 96c184b..45002b8 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1163,6 +1163,9 @@ static void free_ast_node(ASTNode * node) { if (node->is_token) { +<% if @grammar.free_token_node %> + <%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node); +<% end %> /* TODO: free value_t */ } else if (node->n_fields > 0u) diff --git a/doc/user_guide.md b/doc/user_guide.md index b0099c6..0e4cb45 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -326,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null); assert(itemsmore.pItem.pItem.pItem.pToken1 !is null); ``` +## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement + +If user lexer code block allocates memory to store in a token node's `pvalue`, +the `free_token_node` grammar statement can be used to specify the name of a +function which will be called during the `p_free_ast()` call to free the memory +associated with a token node. + +Example: + +``` +<< +static void free_token(Token * token) +{ + free(token->pvalue); +} +>> +ast; +free_token_node free_token; +ptype int *; +token a << + $$ = (int *)malloc(sizeof(int)); + *$$ = 1; +>> +token b << + $$ = (int *)malloc(sizeof(int)); + *$$ = 2; +>> +Start -> a:a b:b; +``` + ##> Specifying tokens - the `token` statement The `token` statement allows defining a lexer token and a pattern to match that @@ -1098,6 +1128,20 @@ assert(code_point == 0x1F9E1u); assert(code_point_length == 4u); ``` +### `p_free_ast` + +The `p_free_ast()` function can be used to free the memory used by the AST. +It should be passed the same value that is returned by `p_result()`. + +The `p_free_ast()` function is only available for C/C++ output targets. + +Note that if any lexer user code block allocates memory to store in a token's +`pvalue`, in order to properly free this memory a `free_token_node` function +should be specified in the grammar file. +If specified, the `free_token_node` function will be called during the +`p_free_ast()` process to allow user code to free any memory associated with +a token node's `pvalue`. + ##> Data ### `p_token_names` diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index a3fc6ed..f12c330 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -8,6 +8,7 @@ class Propane attr_reader :ast attr_reader :ast_prefix attr_reader :ast_suffix + attr_reader :free_token_node attr_reader :modulename attr_reader :patterns attr_reader :rules @@ -32,6 +33,7 @@ class Propane @ast = false @ast_prefix = "" @ast_suffix = "" + @free_token_node = nil parse_grammar! end @@ -62,6 +64,7 @@ class Propane elsif parse_ast_statement! elsif parse_ast_prefix_statement! elsif parse_ast_suffix_statement! + elsif parse_free_token_node_statement! elsif parse_module_statement! elsif parse_ptype_statement! elsif parse_pattern_statement! @@ -112,6 +115,12 @@ class Propane end end + def parse_free_token_node_statement! + if md = consume!(/free_token_node\s+(\w+)\s*;/) + @free_token_node = md[1] + end + end + def parse_module_statement! if consume!(/module\s+/) md = consume!(/([\w.]+)\s*/, "expected module name") diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 73e40d9..4e83d7a 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1439,6 +1439,36 @@ EOF expect(results.stderr).to eq "" expect(results.status).to eq 0 end + + if %w[c cpp].include?(language) + it "allows a user function to free token node memory in AST mode" do + write_grammar <pvalue); +} +>> +ast; +free_token_node free_token; +ptype int *; +token a << + $$ = (int *)malloc(sizeof(int)); + *$$ = 1; +>> +token b << + $$ = (int *)malloc(sizeof(int)); + *$$ = 2; +>> +Start -> a:a b:b; +EOF + run_propane(language: language) + compile("spec/test_free_ast_token_node_memory.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + end end end end diff --git a/spec/test_free_ast_token_node_memory.c b/spec/test_free_ast_token_node_memory.c new file mode 100644 index 0000000..7dd8fbf --- /dev/null +++ b/spec/test_free_ast_token_node_memory.c @@ -0,0 +1,19 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +int main() +{ + char const * input = "ab"; + p_context_t context; + p_context_init(&context, (uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(&context)); + Start * start = p_result(&context); + assert(start->a != NULL); + assert(*start->a->pvalue == 1); + assert(start->b != NULL); + assert(*start->b->pvalue == 2); + + p_free_ast(start); +}