Execute user code blocks assigned to tokens

lib/propane/grammar.rb

@@ -12,6 +12,7 @@ class Propane
       @patterns = []
       @tokens = []
       @rules = []
+      @code_id = 0
       input = input.gsub("\r\n", "\n")
       parse_grammar(input)
     end
@@ -29,7 +30,7 @@ class Propane
           @modulename = $1
         elsif sliced = input.slice!(/\Aclass\s+(\S+)\s*;/)
           @classname = $1
-        elsif sliced = input.slice!(/\Atoken\s+(\S+?)(?:\s+(.+?))?\s*(?:;|<<\n(.*?)^>>\n)/m)
+        elsif sliced = input.slice!(/\Atoken\s+(\S+?)(?:\s+([^\n]+?))?\s*(?:;|<<\n(.*?)^>>\n)/m)
           name, pattern, code = $1, $2, $3
           if pattern.nil?
             pattern = name
@@ -39,7 +40,13 @@ class Propane
           end
           token = Token.new(name: name, id: @tokens.size, line_number: line_number)
           @tokens << token
-          pattern = Pattern.new(pattern: pattern, token: token, line_number: line_number)
+          if code
+            code_id = @code_id
+            @code_id += 1
+          else
+            code_id = nil
+          end
+          pattern = Pattern.new(pattern: pattern, token: token, line_number: line_number, code: code, code_id: code_id)
           @patterns << pattern
         elsif sliced = input.slice!(/\Atokenid\s+(\S+?)\s*;/m)
           name = $1
@@ -51,7 +58,7 @@ class Propane
         elsif sliced = input.slice!(/\Adrop\s+(\S+)\s*;/)
           pattern = $1
           @patterns << Pattern.new(pattern: pattern, line_number: line_number, drop: true)
-        elsif sliced = input.slice!(/\A(\S+)\s*->\s*(.*?)(?:;|<<\n(.*?)^>>\n)/m)
+        elsif sliced = input.slice!(/\A(\S+)\s*->\s*([^\n]*?)(?:;|<<\n(.*?)^>>\n)/m)
           rule_name, components, code = $1, $2, $3
           unless rule_name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
             raise Error.new("Invalid rule name #{name.inspect}")

lib/propane/lexer.rb

@@ -22,10 +22,17 @@ class Propane
           else
             state.accepts.token.id
           end
+        code_id =
+          if state.accepts && state.accepts.code_id
+            state.accepts.code_id
+          else
+            0xFFFF_FFFF
+          end
         state_table << {
           transition_table_index: transition_table.size,
           n_transitions: state.transitions.size,
           accepts: accepts,
+          code_id: code_id,
         }
         state.transitions.each do |transition|
           transition_table << {

lib/propane/pattern.rb

@@ -2,6 +2,14 @@ class Propane
 
   class Pattern
 
+    # @return [String, nil]
+    #   Code block to execute when the pattern is matched.
+    attr_reader :code
+
+    # @option options [Integer, nil] :code_id
+    #   Code block ID.
+    attr_reader :code_id
+
     # @return [String, nil]
     #   Pattern.
     attr_reader :pattern
@@ -22,6 +30,10 @@ class Propane
     #
     # @param options [Hash]
     #   Optional parameters.
+    # @option options [String, nil] :code
+    #   Code block to execute when the pattern is matched.
+    # @option options [Integer, nil] :code_id
+    #   Code block ID.
     # @option options [Boolean] :drop
     #   Whether this is a drop pattern.
     # @option options [String, nil] :pattern
@@ -31,6 +43,8 @@ class Propane
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
     def initialize(options)
+      @code = options[:code]
+      @code_id = options[:code_id]
       @drop = options[:drop]
       @pattern = options[:pattern]
       @token = options[:token]