diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 297d9cc..1d9c9cc 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -38,7 +38,7 @@ class Propane raise Error.new("No patterns found for default mode") end # Add EOF token. - @grammar.tokens << Token.new("$EOF", nil) + @grammar.tokens << Token.new("$EOF", nil, nil) tokens_by_name = {} @grammar.tokens.each_with_index do |token, token_id| # Assign token ID. @@ -54,7 +54,7 @@ class Propane raise Error.new("Start rule not found") end # Add "real" start rule. - @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil)) + @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil)) rule_sets = {} rule_set_id = @grammar.tokens.size @grammar.rules.each_with_index do |rule, rule_id| diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index d9f9ade..2c2c8ac 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -2,13 +2,15 @@ class Propane class Grammar + IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z_0-9]*/ + attr_reader :classname attr_reader :modulename attr_reader :patterns attr_reader :rules attr_reader :tokens attr_reader :code_blocks - attr_reader :ptype + attr_reader :ptypes def initialize(input) @patterns = [] @@ -19,10 +21,14 @@ class Propane @next_line_number = @line_number @mode = nil @input = input.gsub("\r\n", "\n") - @ptype = "void *" + @ptypes = {"default" => "void *"} parse_grammar! end + def ptype + @ptypes["default"] + end + private def parse_grammar! @@ -53,7 +59,7 @@ class Propane end def parse_mode_label! - if md = consume!(/([a-zA-Z_][a-zA-Z_0-9]*)\s*:/) + if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/) @mode = md[1] end end @@ -88,24 +94,28 @@ class Propane def parse_ptype_statement! if consume!(/ptype\s+/) - md = consume!(/([^;]+);/, "expected result type expression") - @ptype = md[1].strip + name = "default" + if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/) + name = md[1] + end + md = consume!(/([^;]+);/, "expected parser result type expression") + @ptypes[name] = md[1].strip end end def parse_token_statement! if consume!(/token\s+/) - md = consume!(/([a-zA-Z_][a-zA-Z_0-9]*)/, "expected token name") + md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name") name = md[1] - if consume!(/\s+/) - pattern = parse_pattern! + if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/) + ptypename = md[1] end - pattern ||= name + pattern = parse_pattern! || name consume!(/\s+/) unless code = parse_code_block! consume!(/;/, "expected pattern or `;' or code block") end - token = Token.new(name, @line_number) + token = Token.new(name, ptypename, @line_number) @tokens << token pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode) @patterns << pattern @@ -115,12 +125,14 @@ class Propane end def parse_tokenid_statement! - if md = consume!(/tokenid\s+(\S+?)\s*;/m) + if md = consume!(/tokenid\s+/) + md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name") name = md[1] - unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/ - raise Error.new("Invalid token name #{name.inspect}") + if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/) + ptypename = md[1] end - token = Token.new(name, @line_number) + consume!(/;/, "expected `;'"); + token = Token.new(name, ptypename, @line_number) @tokens << token @mode = nil true @@ -142,13 +154,14 @@ class Propane end def parse_rule_statement! - if md = consume!(/(\S+)\s*->\s*([^\n]*?)(?:;|<<\n(.*?)^>>\n)/m) - rule_name, components, code = *md[1, 3] - unless rule_name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/ - raise Error.new("Invalid rule name #{name.inspect}") + if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/) + rule_name, ptypename = *md[1, 2] + md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list") + components = md[1].strip.split(/\s+/) + unless code = parse_code_block! + consume!(/;/, "expected pattern or `;' or code block") end - components = components.strip.split(/\s+/) - @rules << Rule.new(rule_name, components, code, @line_number) + @rules << Rule.new(rule_name, components, code, ptypename, @line_number) @mode = nil true end diff --git a/lib/propane/rule.rb b/lib/propane/rule.rb index 62c6758..0b005d3 100644 --- a/lib/propane/rule.rb +++ b/lib/propane/rule.rb @@ -14,6 +14,10 @@ class Propane # Rule ID. attr_accessor :id + # @return [String, nil] + # Parser type name. + attr_reader :ptypename + # @return [Integer] # Line number where the rule was defined in the input grammar. attr_reader :line_number @@ -34,12 +38,15 @@ class Propane # Rule components. # @param code [String] # User code associated with the rule. + # @param ptypename [String, nil] + # Parser type name. # @param line_number [Integer] # Line number where the rule was defined in the input grammar. - def initialize(name, components, code, line_number) + def initialize(name, components, code, ptypename, line_number) @name = name @components = components @code = code + @ptypename = ptypename @line_number = line_number end diff --git a/lib/propane/token.rb b/lib/propane/token.rb index 6c21554..abe81f9 100644 --- a/lib/propane/token.rb +++ b/lib/propane/token.rb @@ -18,6 +18,10 @@ class Propane # Token name. attr_reader :name + # @return [String, nil] + # Parser value type name. + attr_reader :ptypename + # @return [Integer, nil] # Token ID. attr_accessor :id @@ -28,14 +32,15 @@ class Propane # Construct a Token. # - # @param options [Hash] - # Optional parameters. - # @option options [String, nil] :name + # @param name [String, nil] # Token name. - # @option options [Integer, nil] :line_number + # @param ptypename [String, nil] + # Parser value type for this token. + # @param line_number [Integer, nil] # Line number where the token was defined in the input grammar. - def initialize(name, line_number) + def initialize(name, ptypename, line_number) @name = name + @ptypename = ptypename @line_number = line_number end diff --git a/spec/propane/grammar_spec.rb b/spec/propane/grammar_spec.rb index 285d50e..03ee210 100644 --- a/spec/propane/grammar_spec.rb +++ b/spec/propane/grammar_spec.rb @@ -33,6 +33,7 @@ EOF expect(grammar.classname).to eq "Foobar" expect(grammar.modulename).to eq "a.b" expect(grammar.ptype).to eq "XYZ *" + expect(grammar.ptypes).to eq("default" => "XYZ *") o = grammar.tokens.find {|token| token.name == "while"} expect(o).to_not be_nil @@ -173,5 +174,42 @@ EOF expect(o).to_not be_nil expect(o.mode).to eq "m3" end + + it "allows assigning ptypes to tokens and rules" do + input = < R; +R -> abc int; +EOF + grammar = Grammar.new(input) + + o = grammar.tokens.find {|token| token.name == "abc"} + expect(o).to_not be_nil + expect(o.ptypename).to eq "string" + + o = grammar.tokens.find {|token| token.name == "bar"} + expect(o).to_not be_nil + expect(o.ptypename).to be_nil + + o = grammar.tokens.find {|token| token.name == "int"} + expect(o).to_not be_nil + expect(o.ptypename).to eq "integer" + + o = grammar.rules.find {|rule| rule.name == "Start"} + expect(o).to_not be_nil + expect(o.ptypename).to eq "node" + + o = grammar.rules.find {|rule| rule.name == "R"} + expect(o).to_not be_nil + expect(o.ptypename).to be_nil + end end end