Parse grammar input by multiline regex

lib/imbecile/grammar.rb

@@ -15,37 +15,45 @@ module Imbecile
 
     def initialize(input)
       @tokens = []
-      token_names = Set.new
-      input.each_line.each_with_index do |line, line_index|
-        line = line.chomp
-        line_number = line_index + 1
-        if line =~ /^\s*#/
+      @token_names = Set.new
+      input = input.gsub("\r\n", "\n")
+      while !input.empty?
+        consume(input)
+      end
+    end
+
+    private
+
+    def consume(input)
+      if input.slice!(/\A\s+/)
+        # Skip white space.
+      elsif input.slice!(/\A#.*\n/)
         # Skip comment lines.
-        elsif line =~ /^\s*$/
-          # Skip blank lines.
-        elsif line =~ /^\s*module\s+(\S+)$/
+      elsif input.slice!(/\Amodule\s+(\S+)\n/)
         @modulename = $1
-        elsif line =~ /^\s*class\s+(\S+)$/
+      elsif input.slice!(/\Aclass\s+(\S+)\n/)
         @classname = $1
-        elsif line =~ /^\s*token\s+(\S+)(?:\s+(\S+))?$/
+      elsif input.slice!(/\Atoken\s+(\S+)(?:\s+(\S+))?\n/)
         name, pattern = $1, $2
-          if pattern.to_s == ""
+        if pattern.nil?
           pattern = name
         end
         unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-            raise Error.new("Invalid token name #{name} on line #{line_number}")
+          raise Error.new("Invalid token name #{name}")
         end
-          if token_names.include?(name)
-            raise Error.new("Duplicate token name #{name} on line #{line_number}")
+        if @token_names.include?(name)
+          raise Error.new("Duplicate token name #{name}")
         end
         @tokens << Token.new(name, pattern, @tokens.size)
-          token_names << name
-        elsif line =~ /^\s*drop\s+(\S+)$/
+        @token_names << name
+      elsif input.slice!(/\Adrop\s+(\S+)\n/)
         pattern = $1
         @tokens << Token.new(nil, pattern, @tokens.size)
       else
-          raise Error.new("Unexpected input on line #{line_number}: #{line}")
+        if input.size > 25
+          input = input.slice(0..20) + "..."
         end
+        raise Error.new("Unexpected grammar input: #{input}")
       end
     end