Allow defining multiple parser types and assigning parser types to tokens and rules

lib/propane/generator.rb

@@ -38,7 +38,7 @@ class Propane
         raise Error.new("No patterns found for default mode")
       end
       # Add EOF token.
-      @grammar.tokens << Token.new("$EOF", nil)
+      @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
       @grammar.tokens.each_with_index do |token, token_id|
         # Assign token ID.
@@ -54,7 +54,7 @@ class Propane
         raise Error.new("Start rule not found")
       end
       # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil))
+      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|

lib/propane/grammar.rb

@@ -2,13 +2,15 @@ class Propane
 
   class Grammar
 
+    IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z_0-9]*/
+
     attr_reader :classname
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
     attr_reader :tokens
     attr_reader :code_blocks
-    attr_reader :ptype
+    attr_reader :ptypes
 
     def initialize(input)
       @patterns = []
@@ -19,10 +21,14 @@ class Propane
       @next_line_number = @line_number
       @mode = nil
       @input = input.gsub("\r\n", "\n")
-      @ptype = "void *"
+      @ptypes = {"default" => "void *"}
       parse_grammar!
     end
 
+    def ptype
+      @ptypes["default"]
+    end
+
     private
 
     def parse_grammar!
@@ -53,7 +59,7 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/([a-zA-Z_][a-zA-Z_0-9]*)\s*:/)
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
         @mode = md[1]
       end
     end
@@ -88,24 +94,28 @@ class Propane
 
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
-        md = consume!(/([^;]+);/, "expected result type expression")
-        @ptype = md[1].strip
+        name = "default"
+        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          name = md[1]
+        end
+        md = consume!(/([^;]+);/, "expected parser result type expression")
+        @ptypes[name] = md[1].strip
       end
     end
 
     def parse_token_statement!
       if consume!(/token\s+/)
-        md = consume!(/([a-zA-Z_][a-zA-Z_0-9]*)/, "expected token name")
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
-        if consume!(/\s+/)
-          pattern = parse_pattern!
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
         end
-        pattern ||= name
+        pattern = parse_pattern! || name
         consume!(/\s+/)
         unless code = parse_code_block!
           consume!(/;/, "expected pattern or `;' or code block")
         end
-        token = Token.new(name, @line_number)
+        token = Token.new(name, ptypename, @line_number)
         @tokens << token
         pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode)
         @patterns << pattern
@@ -115,12 +125,14 @@ class Propane
     end
 
     def parse_tokenid_statement!
-      if md = consume!(/tokenid\s+(\S+?)\s*;/m)
+      if md = consume!(/tokenid\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
-        unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-          raise Error.new("Invalid token name #{name.inspect}")
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
         end
-        token = Token.new(name, @line_number)
+        consume!(/;/, "expected `;'");
+        token = Token.new(name, ptypename, @line_number)
         @tokens << token
         @mode = nil
         true
@@ -142,13 +154,14 @@ class Propane
     end
 
     def parse_rule_statement!
-      if md = consume!(/(\S+)\s*->\s*([^\n]*?)(?:;|<<\n(.*?)^>>\n)/m)
-        rule_name, components, code = *md[1, 3]
-        unless rule_name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-          raise Error.new("Invalid rule name #{name.inspect}")
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
+        rule_name, ptypename = *md[1, 2]
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        components = md[1].strip.split(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected pattern or `;' or code block")
         end
-        components = components.strip.split(/\s+/)
-        @rules << Rule.new(rule_name, components, code, @line_number)
+        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @mode = nil
         true
       end

lib/propane/rule.rb

@@ -14,6 +14,10 @@ class Propane
     #   Rule ID.
     attr_accessor :id
 
+    # @return [String, nil]
+    #   Parser type name.
+    attr_reader :ptypename
+
     # @return [Integer]
     #   Line number where the rule was defined in the input grammar.
     attr_reader :line_number
@@ -34,12 +38,15 @@ class Propane
     #   Rule components.
     # @param code [String]
     #   User code associated with the rule.
+    # @param ptypename [String, nil]
+    #   Parser type name.
     # @param line_number [Integer]
     #   Line number where the rule was defined in the input grammar.
-    def initialize(name, components, code, line_number)
+    def initialize(name, components, code, ptypename, line_number)
       @name = name
       @components = components
       @code = code
+      @ptypename = ptypename
       @line_number = line_number
     end
 

lib/propane/token.rb

@@ -18,6 +18,10 @@ class Propane
     #   Token name.
     attr_reader :name
 
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_reader :ptypename
+
     # @return [Integer, nil]
     #   Token ID.
     attr_accessor :id
@@ -28,14 +32,15 @@ class Propane
 
     # Construct a Token.
     #
-    # @param options [Hash]
-    #   Optional parameters.
-    # @option options [String, nil] :name
+    # @param name [String, nil]
     #   Token name.
-    # @option options [Integer, nil] :line_number
+    # @param ptypename [String, nil]
+    #   Parser value type for this token.
+    # @param line_number [Integer, nil]
     #   Line number where the token was defined in the input grammar.
-    def initialize(name, line_number)
+    def initialize(name, ptypename, line_number)
       @name = name
+      @ptypename = ptypename
       @line_number = line_number
     end
 

spec/propane/grammar_spec.rb

@@ -33,6 +33,7 @@ EOF
       expect(grammar.classname).to eq "Foobar"
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
+      expect(grammar.ptypes).to eq("default" => "XYZ  *")
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
@@ -173,5 +174,42 @@ EOF
       expect(o).to_not be_nil
       expect(o.mode).to eq "m3"
     end
+
+    it "allows assigning ptypes to tokens and rules" do
+      input = <<EOF
+ptype Subnode *;
+ptype string = char *;
+ptype integer = int;
+ptype node = Node *;
+
+token abc(string);
+token bar;
+tokenid int(integer);
+
+Start (node) -> R;
+R -> abc int;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "abc"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.tokens.find {|token| token.name == "bar"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "int"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "integer"
+
+      o = grammar.rules.find {|rule| rule.name == "Start"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "node"
+
+      o = grammar.rules.find {|rule| rule.name == "R"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+    end
   end
 end