Add semicolon to end of all grammar statements

lib/propane/grammar.rb

@@ -25,12 +25,12 @@ class Propane
           # Skip white space.
         elsif sliced = input.slice!(/\A#.*\n/)
           # Skip comment lines.
-        elsif sliced = input.slice!(/\Amodule\s+(\S+)\n/)
+        elsif sliced = input.slice!(/\Amodule\s+(\S+)\s*;/)
           @modulename = $1
-        elsif sliced = input.slice!(/\Aclass\s+(\S+)\n/)
+        elsif sliced = input.slice!(/\Aclass\s+(\S+)\s*;/)
           @classname = $1
-        elsif sliced = input.slice!(/\Atoken\s+(\S+)(?:\s+(\S+))?\n/)
+        elsif sliced = input.slice!(/\Atoken\s+(\S+?)(?:\s+(.+?))?\s*(?:;|<<\n(.*?)^>>\n)/m)
-          name, pattern = $1, $2
+          name, pattern, code = $1, $2, $3
           if pattern.nil?
             pattern = name
           end
@@ -38,7 +38,7 @@ class Propane
             raise Error.new("Invalid token name #{name.inspect}")
           end
           @tokens << Token.new(name: name, pattern: pattern, id: @tokens.size, line_number: line_number)
-        elsif sliced = input.slice!(/\Adrop\s+(\S+)\n/)
+        elsif sliced = input.slice!(/\Adrop\s+(\S+)\s*;/)
           pattern = $1
           @drop_tokens << Token.new(pattern: pattern, line_number: line_number)
         elsif sliced = input.slice!(/\A(\S+)\s*->\s*(.*?)(?:;|<<\n(.*?)^>>\n)/m)

spec/propane/lexer/dfa_spec.rb

@@ -59,7 +59,7 @@ end
 describe Propane::Lexer::DFA do
   it "lexes a simple token" do
     expect(run(<<EOF, "foo")).to eq [["foo", "foo"]]
-token foo
+token foo;
 EOF
   end
 
@@ -69,8 +69,8 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo
+token foo;
-token bar
+token bar;
 EOF
   end
 
@@ -79,17 +79,17 @@ EOF
       ["identifier", "foobar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo
+token foo;
-token bar
+token bar;
-token identifier [a-z]+
+token identifier [a-z]+;
 EOF
     expected = [
       ["plusplus", "++"],
       ["plus", "+"],
     ]
     expect(run(<<EOF, "+++")).to eq expected
-token plus \\+
+token plus \\+;
-token plusplus \\+\\+
+token plusplus \\+\\+;
 EOF
   end
 
@@ -100,9 +100,9 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo
+token foo;
-token bar
+token bar;
-token WS \\s+
+token WS \\s+;
 EOF
   end
 
@@ -113,9 +113,9 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo
+token foo;
-token bar
+token bar;
-drop \\s+
+drop \\s+;
 EOF
   end
 end

spec/propane_spec.rb

@@ -27,10 +27,10 @@ describe Propane do
 
   it "generates a D lexer" do
     write_grammar <<EOF
-token int \\d+
+token int \\d+;
-token plus \\+
+token plus \\+;
-token times \\*
+token times \\*;
-drop \\s+
+drop \\s+;
 Start -> Foo;
 Foo -> int <<
 >>
@@ -44,10 +44,10 @@ EOF
 
   it "generates a parser" do
     write_grammar <<EOF
-token plus \\+
+token plus \\+;
-token times \\*
+token times \\*;
-token zero 0
+token zero 0;
-token one 1
+token one 1;
 Start -> E;
 E -> E times B;
 E -> E plus B;
@@ -60,7 +60,7 @@ EOF
 
   it "generates an SLR parser" do
     write_grammar <<EOF
-token one 1
+token one 1;
 Start -> E;
 E -> one E;
 E -> one;
@@ -70,8 +70,8 @@ EOF
 
   it "distinguishes between multiple identical rules with lookahead symbol" do
     write_grammar <<EOF
-token a
+token a;
-token b
+token b;
 Start -> R1 a;
 Start -> R2 b;
 R1 -> a b;
@@ -84,9 +84,9 @@ EOF
 
   it "handles reducing a rule that could be arrived at from multiple states" do
     write_grammar <<EOF
-token a
+token a;
-token b
+token b;
-drop \\s+
+drop \\s+;
 Start -> a R1;
 Start -> b R1;
 R1 -> b;