Add pattern statement

assets/parser.d.erb

@@ -163,7 +163,7 @@ class <%= @classname %>
             for (;;)
             {
                 LexedToken lt = attempt_lex_token();
-                if (lt.token != _TOKEN_DROP)
+                if ((lt.token != _TOKEN_DROP) && (lt.token != _TOKEN_NONE))
                 {
                     return lt;
                 }
@@ -236,7 +236,8 @@ class <%= @classname %>
                             attempt_match_info.delta_col++;
                         }
                         current_state = dest;
-                        if (states[current_state].token != _TOKEN_NONE)
+                        if ((states[current_state].token != _TOKEN_NONE) ||
+                            (states[current_state].code_id != 0xFFFF_FFFFu))
                         {
                             attempt_match_info.token = states[current_state].token;
                             attempt_match_info.code_id = states[current_state].code_id;
@@ -279,11 +280,7 @@ class <%= @classname %>
                         {
                             m_input_col += longest_match_info.delta_col;
                         }
-                    }
+                        lt.token = token_to_accept;
-                    if (token_to_accept != _TOKEN_NONE)
-                    {
-                        /* We have a token to accept. */
-                        lt.token = longest_match_info.token;
                         lt.length = longest_match_info.length;
                         break;
                     }

lib/propane/grammar.rb

@@ -32,6 +32,7 @@ class Propane
       elsif parse_comment_line!
       elsif parse_module_statement!
       elsif parse_class_statement!
+      elsif parse_pattern_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
@@ -125,6 +126,18 @@ class Propane
       end
     end
 
+    def parse_pattern_statement!
+      if pattern = parse_pattern!
+        consume!(/\s+/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block to follow pattern")
+        end
+        code_id = @code_id
+        @code_id += 1
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, code_id: code_id)
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""

lib/propane/lexer.rb

@@ -19,8 +19,10 @@ class Propane
             TOKEN_NONE
           elsif state.accepts.drop?
             TOKEN_DROP
-          else
+          elsif state.accepts.token
             state.accepts.token.id
+          else
+            TOKEN_NONE
           end
         code_id =
           if state.accepts && state.accepts.code_id

spec/propane_spec.rb

@@ -110,4 +110,17 @@ EOF
     compile("spec/test_user_code.d")
     run
   end
+
+  it "supports a pattern statement" do
+    write_grammar <<EOF
+token abc;
+/def/ <<
+  writeln("def!");
+>>
+Start -> abc;
+EOF
+    build_parser
+    compile("spec/test_pattern.d")
+    run
+  end
 end

spec/test_pattern.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+
+    input = "defabcdef";
+    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+}