From 98390429a833add07bbb527e79ea5744d3ff3284 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Mon, 6 Jul 2026 23:27:44 -0400 Subject: [PATCH] Allow parser user code blocks in tree mode --- CHANGELOG.md | 6 +++ assets/parser.c.erb | 11 ++++- assets/parser.d.erb | 10 +++- doc/user_guide.md | 43 +++++++++++++++-- lib/propane/generator.rb | 80 +++++++++++++++++++++++-------- lib/propane/grammar.rb | 12 ++--- spec/propane_spec.rb | 72 ++++++++++++++++++++++++++++ spec/test_parser_user_code_tree.c | 39 +++++++++++++++ spec/test_parser_user_code_tree.d | 39 +++++++++++++++ 9 files changed, 281 insertions(+), 31 deletions(-) create mode 100644 spec/test_parser_user_code_tree.c create mode 100644 spec/test_parser_user_code_tree.d diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1277d..c7a4f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## Unreleased + +### New Features + +- Support parser rule user code blocks in tree generation mode. + ## v4.6.0 ### New Features diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 6451924..32b4fca 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -923,7 +923,9 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * memset(&empty_pos, 0, sizeof(empty_pos)); return empty_pos; } +<% end %> +<% if !@grammar.tree || @grammar.parser_user_code_used? %> /** * Execute user code associated with a parser rule. * @@ -934,7 +936,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * * @retval P_USER_TERMINATED * User requested to terminate parsing. */ -static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context) +static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context) { switch (rule) { @@ -1169,6 +1171,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start { reduced_parser_node = NULL; } +<% if @grammar.parser_user_code_used? %> + if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED) + { + state_values_stack_free(&statevalues); + return P_USER_TERMINATED; + } +<% end %> <% else %> <%= @grammar.prefix %>value_t reduced_parser_value2; memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2)); diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 91b5217..c2d4aba 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -994,7 +994,9 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state } return <%= @grammar.prefix %>position_t.INVALID; } +<% end %> +<% if !@grammar.tree || @grammar.parser_user_code_used? %> /** * Execute user code associated with a parser rule. * @@ -1005,7 +1007,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state * @retval P_USER_TERMINATED * User requested to terminate parsing. */ -private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context) +private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context) { switch (rule) { @@ -1227,6 +1229,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star { reduced_parser_node = null; } +<% if @grammar.parser_user_code_used? %> + if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED) + { + return P_USER_TERMINATED; + } +<% end %> <% else %> <%= @grammar.prefix %>value_t reduced_parser_value2; if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED) diff --git a/doc/user_guide.md b/doc/user_guide.md index 1e025db..a5afb82 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -270,9 +270,38 @@ For C targets this can be accomplished with `if (p_position_valid(${$.position}))` and for D targets this can be accomplished with `if (${$.position}.valid)`. -Parser rule code blocks are not available in tree generation mode. In tree generation mode, a full parse tree is automatically constructed in memory for user code to traverse after parsing is complete. +Parser rule code blocks are still supported in tree generation mode, but they +behave differently than when tree generation mode is not active. +The code block for a rule is executed after the rule has been matched and its +tree node has been fully formed. +Within the code block, `$$` refers to the tree node for the reduced rule, typed +as a pointer to that rule's generated tree node structure. +The tree nodes for the rule components are accessed positionally with `$1`, +`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure +for that component (a rule node or a `Token` node). +Field aliases (see the "Specifying parser rules" section) may also be used to +reference a component tree node by name; a field alias behaves identically to +the positional reference for that component. +The positional position expansions (`${$.position}`, `${N.position}`, etc...) +are not available in tree generation mode; the `position` and `end_position` +fields of the tree nodes can be accessed directly instead. + +Example: + +``` +tree; + +Assignment -> ident equals Expr << + /* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is + * the Expr rule node. */ + printf("assignment on row %d, col %d\n", + $$->position.row, $$->position.col); + printf("target identifier ends on row %d, col %d\n", + $1->end_position.row, $1->end_position.col); +>> +``` ##> `context_user_fields` statement - adding custom fields to the context @@ -710,7 +739,9 @@ It is recommended to place this statement early in the grammar. In tree generation mode various aspects of propane's behavior are changed: * Only one `ptype` is allowed. - * Parser user code blocks are not supported. + * Parser user code blocks execute after the rule's tree node has been formed + and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule + code blocks" section). * Structure types are generated to represent the parsed tokens and rules as defined in the grammar. * The parse result from `p_result()` points to a `Start` struct containing @@ -1059,8 +1090,12 @@ The `$$` symbol accesses the output parser value for this rule. The above examples demonstrate how the parser values for the rule components can be used to produce the parser value for the accepted rule. -Parser rule code blocks are not allowed and not used when tree generation mode -is active. +In tree generation mode, parser rule code blocks access the reduced rule tree +node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser +rule code blocks" section). +Field aliases may still be used in tree generation mode to reference a component +tree node by name, behaving identically to the corresponding positional +reference. ##> User termination of the lexer or parser diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 3df5c31..52c17dd 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -185,7 +185,8 @@ class Propane end end @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number) - @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number) + optcode = @grammar.tree ? nil : "$$ = $1;\n" + @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number) optional_rules_added << component end end @@ -296,21 +297,24 @@ class Propane end if parser code = code.gsub(/\$\$/) do |match| - case @language - when "c" - "_pvalue->v_#{rule.ptypename}" - when "d" - "_pvalue.v_#{rule.ptypename}" + if @grammar.tree + case @language + when "c" + "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)" + when "d" + "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)" + end + else + case @language + when "c" + "_pvalue->v_#{rule.ptypename}" + when "d" + "_pvalue.v_#{rule.ptypename}" + end end end code = code.gsub(/\$(\d+)/) do |match| - index = $1.to_i - case @language - when "c" - "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}" - when "d" - "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}" - end + parser_component_reference(rule, $1.to_i) end code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match| index = $1.to_i @@ -323,12 +327,11 @@ class Propane code = code.gsub(/\$\{(\w+)\}/) do |match| aliasname = $1 if index = rule.aliases[aliasname] - case @language - when "c" - "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}" - when "d" - "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}" - end + # Field aliases are just a named reference to a positional rule + # component, so reuse the same expansion as `$1', `$2', etc. Note + # that rule.aliases stores a 0-based component index, so add 1 to + # convert it to the 1-based index used for positional references. + parser_component_reference(rule, index + 1) else raise Error.new("Field alias '#{aliasname}' not found") end @@ -384,6 +387,45 @@ class Propane code end + # Expand a positional reference to a parser rule component. + # + # This is used to expand `$1', `$2', etc. as well as field aliases (which + # are just named references to a positional rule component). + # + # @param rule [Rule] + # The Rule containing the user code. + # @param index [Integer] + # 1-based index of the rule component to reference. + # + # @return [String] + # Expanded rule component reference. + def parser_component_reference(rule, index) + component = rule.components[index - 1] + if @grammar.tree + # In tree mode a component reference yields a pointer to that + # component's tree node. An optional component propagates its target + # node (or null), so use the optional target's node type. + if component.is_a?(RuleSet) && component.optional? + component = component.option_target + end + node_name = component.is_a?(Token) ? "Token" : component.name + typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}" + case @language + when "c" + "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)" + when "d" + "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)" + end + else + case @language + when "c" + "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}" + when "d" + "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}" + end + end + end + # Get the lex function to use. # # @return [String] diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index cde6004..1fbc612 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -58,6 +58,10 @@ class Propane @tokens.size + 1 end + def parser_user_code_used? + @rules.any? {|r| r.code} + end + private def parse_grammar! @@ -263,12 +267,8 @@ class Propane end md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list") components = md[1].strip.split(/\s+/) - if @tree - consume!(/;/, "expected `;'") - else - unless code = parse_code_block! - consume!(/;/, "expected `;' or code block") - end + unless code = parse_code_block! + consume!(/;/, "expected `;' or code block") end @rules << Rule.new(rule_name, components, code, ptypename, @line_number) @modeline = nil diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 4fa9256..1542b39 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -745,6 +745,78 @@ EOF ]) end + it "executes user code associated with a parser rule in tree mode" do + case language + when "c", "cpp" + write_grammar <> +ptype int; +token a << $$ = 11; >> +token b << $$ = 22; >> +Start -> A:ay B:bee C << + ${context.start_n_fields} = $$->n_fields; + ${context.start_a_value} = $$->pA->pToken1->pvalue; + ${context.a_value} = $1->pToken1->pvalue; + ${context.b_value} = $2->pToken1->pvalue; + ${context.b_token} = $2->pToken1->token; + ${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0; + ${context.alias_a_value} = ${ay}->pToken1->pvalue; + ${context.alias_b_value} = ${bee}->pToken1->pvalue; +>> +A -> a; +B -> b; +C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >> +EOF + when "d" + write_grammar <> +ptype int; +token a << $$ = 11; >> +token b << $$ = 22; >> +Start -> A:ay B:bee C << + ${context.start_n_fields} = $$.n_fields; + ${context.start_a_value} = $$.pA.pToken1.pvalue; + ${context.a_value} = $1.pToken1.pvalue; + ${context.b_value} = $2.pToken1.pvalue; + ${context.b_token} = $2.pToken1.token; + ${context.c_field_is_null} = ($$.pC is null) ? 1 : 0; + ${context.alias_a_value} = ${ay}.pToken1.pvalue; + ${context.alias_b_value} = ${bee}.pToken1.pvalue; +>> +A -> a; +B -> b; +C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >> +EOF + end + run_propane(language: language) + compile("spec/test_parser_user_code_tree.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "parses lists" do write_grammar < +#include +#include "testutils.h" + +int main() +{ + char const * input = "ab"; + p_context_t * context; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(context)); + + /* The parser user code recorded values accessed via $$, $1, and $2 while + * the tree node for the Start rule was being formed. */ + assert_eq(3, context->start_n_fields); + assert_eq(11, context->start_a_value); + assert_eq(11, context->a_value); + assert_eq(22, context->b_value); + assert_eq(TOKEN_b, context->b_token); + + /* The empty-matched rule C has a null $$ tree node, and its field in the + * Start node is null as well. */ + assert_eq(1, context->c_is_null); + assert_eq(1, context->c_field_is_null); + + /* Field aliases reference the same component tree nodes as the positional + * references. */ + assert_eq(11, context->alias_a_value); + assert_eq(22, context->alias_b_value); + + Start * start = p_result(context); + assert(start->pA != NULL); + assert(start->pB != NULL); + assert(start->pC == NULL); + p_tree_delete(start); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_parser_user_code_tree.d b/spec/test_parser_user_code_tree.d new file mode 100644 index 0000000..8272eb5 --- /dev/null +++ b/spec/test_parser_user_code_tree.d @@ -0,0 +1,39 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + string input = "ab"; + p_context_t * context = p_context_new(input); + assert_eq(P_SUCCESS, p_parse(context)); + + /* The parser user code recorded values accessed via $$, $1, and $2 while + * the tree node for the Start rule was being formed. */ + assert_eq(3, context.start_n_fields); + assert_eq(11, context.start_a_value); + assert_eq(11, context.a_value); + assert_eq(22, context.b_value); + assert_eq(TOKEN_b, context.b_token); + + /* The empty-matched rule C has a null $$ tree node, and its field in the + * Start node is null as well. */ + assert_eq(1, context.c_is_null); + assert_eq(1, context.c_field_is_null); + + /* Field aliases reference the same component tree nodes as the positional + * references. */ + assert_eq(11, context.alias_a_value); + assert_eq(22, context.alias_b_value); + + Start * start = p_result(context); + assert(start.pA !is null); + assert(start.pB !is null); + assert(start.pC is null); + p_tree_delete(start); +}