Allow parser user code blocks in tree mode

CHANGELOG.md

@@ -1,3 +1,9 @@
+## Unreleased
+
+### New Features
+
+- Support parser rule user code blocks in tree generation mode.
+
 ## v4.6.0
 
 ### New Features

assets/parser.c.erb

@@ -923,7 +923,9 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
     memset(&empty_pos, 0, sizeof(empty_pos));
     return empty_pos;
 }
+<% end %>
 
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -934,7 +936,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1169,6 +1171,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 reduced_parser_node = NULL;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                state_values_stack_free(&statevalues);
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));

assets/parser.d.erb

@@ -994,7 +994,9 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
     }
     return <%= @grammar.prefix %>position_t.INVALID;
 }
+<% end %>
 
+<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -1005,7 +1007,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
     switch (rule)
     {
@@ -1227,6 +1229,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 reduced_parser_node = null;
             }
+<%   if @grammar.parser_user_code_used? %>
+            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+<%   end %>
 <% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)

doc/user_guide.md

@@ -270,9 +270,38 @@ For C targets this can be accomplished with
 `if (p_position_valid(${$.position}))` and for D targets this can be
 accomplished with `if (${$.position}.valid)`.
 
-Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
+Parser rule code blocks are still supported in tree generation mode, but they
+behave differently than when tree generation mode is not active.
+The code block for a rule is executed after the rule has been matched and its
+tree node has been fully formed.
+Within the code block, `$$` refers to the tree node for the reduced rule, typed
+as a pointer to that rule's generated tree node structure.
+The tree nodes for the rule components are accessed positionally with `$1`,
+`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure
+for that component (a rule node or a `Token` node).
+Field aliases (see the "Specifying parser rules" section) may also be used to
+reference a component tree node by name; a field alias behaves identically to
+the positional reference for that component.
+The positional position expansions (`${$.position}`, `${N.position}`, etc...)
+are not available in tree generation mode; the `position` and `end_position`
+fields of the tree nodes can be accessed directly instead.
+
+Example:
+
+```
+tree;
+
+Assignment -> ident equals Expr <<
+    /* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
+     * the Expr rule node. */
+    printf("assignment on row %d, col %d\n",
+        $$->position.row, $$->position.col);
+    printf("target identifier ends on row %d, col %d\n",
+        $1->end_position.row, $1->end_position.col);
+>>
+```
 
 ##> `context_user_fields` statement - adding custom fields to the context
 
@@ -710,7 +739,9 @@ It is recommended to place this statement early in the grammar.
 In tree generation mode various aspects of propane's behavior are changed:
 
   * Only one `ptype` is allowed.
-  * Parser user code blocks are not supported.
+  * Parser user code blocks execute after the rule's tree node has been formed
+  and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule
+  code blocks" section).
   * Structure types are generated to represent the parsed tokens and rules as
   defined in the grammar.
   * The parse result from `p_result()` points to a `Start` struct containing
@@ -1059,8 +1090,12 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
 
-Parser rule code blocks are not allowed and not used when tree generation mode
+In tree generation mode, parser rule code blocks access the reduced rule tree
-is active.
+node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser
+rule code blocks" section).
+Field aliases may still be used in tree generation mode to reference a component
+tree node by name, behaving identically to the corresponding positional
+reference.
 
 ##> User termination of the lexer or parser
 

lib/propane/generator.rb

@@ -185,7 +185,8 @@ class Propane
                 end
               end
               @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optcode = @grammar.tree ? nil : "$$ = $1;\n"
+              @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
               optional_rules_added << component
             end
           end
@@ -296,6 +297,14 @@ class Propane
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
+          if @grammar.tree
+            case @language
+            when "c"
+              "((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            when "d"
+              "(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
+            end
+          else
             case @language
             when "c"
               "_pvalue->v_#{rule.ptypename}"
@@ -303,14 +312,9 @@ class Propane
               "_pvalue.v_#{rule.ptypename}"
             end
           end
-        code = code.gsub(/\$(\d+)/) do |match|
-          index = $1.to_i
-          case @language
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
         end
+        code = code.gsub(/\$(\d+)/) do |match|
+          parser_component_reference(rule, $1.to_i)
         end
         code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
           index = $1.to_i
@@ -323,12 +327,11 @@ class Propane
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
-            case @language
+            # Field aliases are just a named reference to a positional rule
-            when "c"
+            # component, so reuse the same expansion as `$1', `$2', etc. Note
-              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
+            # that rule.aliases stores a 0-based component index, so add 1 to
-            when "d"
+            # convert it to the 1-based index used for positional references.
-              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
+            parser_component_reference(rule, index + 1)
-            end
           else
             raise Error.new("Field alias '#{aliasname}' not found")
           end
@@ -384,6 +387,45 @@ class Propane
       code
     end
 
+    # Expand a positional reference to a parser rule component.
+    #
+    # This is used to expand `$1', `$2', etc. as well as field aliases (which
+    # are just named references to a positional rule component).
+    #
+    # @param rule [Rule]
+    #   The Rule containing the user code.
+    # @param index [Integer]
+    #   1-based index of the rule component to reference.
+    #
+    # @return [String]
+    #   Expanded rule component reference.
+    def parser_component_reference(rule, index)
+      component = rule.components[index - 1]
+      if @grammar.tree
+        # In tree mode a component reference yields a pointer to that
+        # component's tree node. An optional component propagates its target
+        # node (or null), so use the optional target's node type.
+        if component.is_a?(RuleSet) && component.optional?
+          component = component.option_target
+        end
+        node_name = component.is_a?(Token) ? "Token" : component.name
+        typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
+        case @language
+        when "c"
+          "((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
+        when "d"
+          "(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
+        end
+      else
+        case @language
+        when "c"
+          "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
+        when "d"
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
+        end
+      end
+    end
+
     # Get the lex function to use.
     #
     # @return [String]

lib/propane/grammar.rb

@@ -58,6 +58,10 @@ class Propane
       @tokens.size + 1
     end
 
+    def parser_user_code_used?
+      @rules.any? {|r| r.code}
+    end
+
     private
 
     def parse_grammar!
@@ -263,13 +267,9 @@ class Propane
         end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        if @tree
-          consume!(/;/, "expected `;'")
-        else
         unless code = parse_code_block!
           consume!(/;/, "expected `;' or code block")
         end
-        end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @modeline = nil
         true

spec/propane_spec.rb

@@ -745,6 +745,78 @@ EOF
         ])
       end
 
+      it "executes user code associated with a parser rule in tree mode" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$->n_fields;
+  ${context.start_a_value} = $$->pA->pToken1->pvalue;
+  ${context.a_value} = $1->pToken1->pvalue;
+  ${context.b_value} = $2->pToken1->pvalue;
+  ${context.b_token} = $2->pToken1->token;
+  ${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}->pToken1->pvalue;
+  ${context.alias_b_value} = ${bee}->pToken1->pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
+EOF
+        when "d"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    int start_n_fields;
+    int start_a_value;
+    int a_value;
+    int b_value;
+    p_token_t b_token;
+    int c_is_null;
+    int c_field_is_null;
+    int alias_a_value;
+    int alias_b_value;
+>>
+ptype int;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields;
+  ${context.start_a_value} = $$.pA.pToken1.pvalue;
+  ${context.a_value} = $1.pToken1.pvalue;
+  ${context.b_value} = $2.pToken1.pvalue;
+  ${context.b_token} = $2.pToken1.token;
+  ${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
+  ${context.alias_a_value} = ${ay}.pToken1.pvalue;
+  ${context.alias_b_value} = ${bee}.pToken1.pvalue;
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_parser_user_code_tree.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "parses lists" do
         write_grammar <<EOF
 ptype #{language == "d" ? "uint" : "uint32_t"};

spec/test_parser_user_code_tree.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context->start_n_fields);
+    assert_eq(11, context->start_a_value);
+    assert_eq(11, context->a_value);
+    assert_eq(22, context->b_value);
+    assert_eq(TOKEN_b, context->b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context->c_is_null);
+    assert_eq(1, context->c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context->alias_a_value);
+    assert_eq(22, context->alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start->pA != NULL);
+    assert(start->pB != NULL);
+    assert(start->pC == NULL);
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_parser_user_code_tree.d

@@ -0,0 +1,39 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The parser user code recorded values accessed via $$, $1, and $2 while
+     * the tree node for the Start rule was being formed. */
+    assert_eq(3, context.start_n_fields);
+    assert_eq(11, context.start_a_value);
+    assert_eq(11, context.a_value);
+    assert_eq(22, context.b_value);
+    assert_eq(TOKEN_b, context.b_token);
+
+    /* The empty-matched rule C has a null $$ tree node, and its field in the
+     * Start node is null as well. */
+    assert_eq(1, context.c_is_null);
+    assert_eq(1, context.c_field_is_null);
+
+    /* Field aliases reference the same component tree nodes as the positional
+     * references. */
+    assert_eq(11, context.alias_a_value);
+    assert_eq(22, context.alias_b_value);
+
+    Start * start = p_result(context);
+    assert(start.pA !is null);
+    assert(start.pB !is null);
+    assert(start.pC is null);
+    p_tree_delete(start);
+}