Add AST generation - #22

lib/propane/generator.rb

@@ -119,6 +119,9 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
+      rule_sets.each do |name, rule_set|
+        rule_set.finalize
+      end
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
@@ -228,11 +231,20 @@ class Propane
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "out_token_info->pvalue.v_#{pattern.ptypename}"
-          when "d"
-            "out_token_info.pvalue.v_#{pattern.ptypename}"
+          if @grammar.ast
+            case @language
+            when "c"
+              "out_token_info->pvalue"
+            when "d"
+              "out_token_info.pvalue"
+            end
+          else
+            case @language
+            when "c"
+              "out_token_info->pvalue.v_#{pattern.ptypename}"
+            when "d"
+              "out_token_info.pvalue.v_#{pattern.ptypename}"
+            end
           end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|

lib/propane/grammar.rb

@@ -5,6 +5,7 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :ast
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
@@ -24,6 +25,7 @@ class Propane
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
+      @ast = false
       parse_grammar!
     end
 
@@ -51,6 +53,7 @@ class Propane
       if parse_white_space!
       elsif parse_comment_line!
       elsif @mode.nil? && parse_mode_label!
+      elsif parse_ast_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
@@ -82,6 +85,12 @@ class Propane
       consume!(/#.*\n/)
     end
 
+    def parse_ast_statement!
+      if consume!(/ast\s*;/)
+        @ast = true
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -96,6 +105,9 @@ class Propane
       if consume!(/ptype\s+/)
         name = "default"
         if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           name = md[1]
         end
         md = consume!(/([^;]+);/, "expected parser result type expression")
@@ -108,12 +120,15 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         pattern = parse_pattern! || name
         consume!(/\s+/)
         unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+          consume!(/;/, "expected `;' or code block")
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
@@ -129,6 +144,9 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         consume!(/;/, "expected `;'");
@@ -156,10 +174,17 @@ class Propane
     def parse_rule_statement!
       if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
         rule_name, ptypename = *md[1, 2]
+        if @ast && ptypename
+          raise Error.new("Multiple ptypes are unsupported in AST mode")
+        end
         md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+        if @ast
+          consume!(/;/, "expected `;'")
+        else
+          unless code = parse_code_block!
+            consume!(/;/, "expected `;' or code block")
+          end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @mode = nil
@@ -171,6 +196,9 @@ class Propane
       if pattern = parse_pattern!
         consume!(/\s+/)
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         unless code = parse_code_block!

lib/propane/parser.rb

@@ -63,11 +63,11 @@ class Propane
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
+            [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
               rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
           when Hash
             ra.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id,
+              {token_id: token.id, rule_id: rule.id, rule: rule,
                rule_set_id: rule.rule_set.id, n_states: rule.components.size}
             end
           else

lib/propane/rule.rb

@@ -30,6 +30,11 @@ class Propane
     #   The RuleSet that this Rule is a part of.
     attr_accessor :rule_set
 
+    # @return [Array<Integer>]
+    #   Map this rule's components to their positions in the parent RuleSet's
+    #   node field pointer array. This is used for AST construction.
+    attr_accessor :rule_set_node_field_index_map
+
     # Construct a Rule.
     #
     # @param name [String]
@@ -45,6 +50,7 @@ class Propane
     def initialize(name, components, code, ptypename, line_number)
       @name = name
       @components = components
+      @rule_set_node_field_index_map = components.map {0}
       @code = code
       @ptypename = ptypename
       @line_number = line_number
@@ -68,6 +74,17 @@ class Propane
       "#{@name} -> #{@components.map(&:name).join(" ")}"
     end
 
+    # Check whether the rule set node field index map is just a 1:1 mapping.
+    #
+    # @return [Boolean]
+    #   Boolean indicating whether the rule set node field index map is just a
+    #   1:1 mapping.
+    def flat_rule_set_node_field_index_map?
+      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
+        v == i
+      end
+    end
+
   end
 
 end

lib/propane/rule_set.rb

@@ -1,5 +1,6 @@
 class Propane
 
+  # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
     # @return [Integer]
@@ -75,6 +76,42 @@ class Propane
       @_start_token_set
     end
 
+    # Build the set of AST fields for this RuleSet.
+    #
+    # The keys are the field names and the values are the AST node structure
+    # names.
+    #
+    # @return [Hash]
+    #   AST fields.
+    def ast_fields
+      @_ast_fields ||=
+        begin
+          field_indexes = {}
+          fields = {}
+          @rules.each do |rule|
+            rule.components.each_with_index do |component, i|
+              if component.is_a?(Token)
+                node_name = "Token"
+              else
+                node_name = component.name
+              end
+              field_name = "p#{node_name}#{i + 1}"
+              unless field_indexes[field_name]
+                field_indexes[field_name] = fields.size
+                fields[field_name] = node_name
+              end
+              rule.rule_set_node_field_index_map[i] = field_indexes[field_name]
+            end
+          end
+          fields
+        end
+    end
+
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize
+      ast_fields
+    end
+
   end
 
 end