Add lexer modes and $mode() code expansion

lib/propane/generator.rb

@@ -26,6 +26,17 @@ class Propane
     private
 
     def process_grammar!
+      # Assign default pattern mode to patterns without a mode assigned.
+      found_default = false
+      @grammar.patterns.each do |pattern|
+        if pattern.mode.nil?
+          pattern.mode = "default"
+          found_default = true
+        end
+      end
+      unless found_default
+        raise Error.new("No patterns found for default mode")
+      end
       # Add EOF token.
       @grammar.tokens << Token.new("$EOF", nil)
       tokens_by_name = {}
@@ -152,6 +163,13 @@ class Propane
     def expand_code(code)
       code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
+      end.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
+        mode_name = $1
+        mode_id = @lexer.mode_id(mode_name)
+        unless mode_id
+          raise Error.new("Lexer mode '#{mode_name}' not found")
+        end
+        "m_mode = #{mode_id}u"
       end
     end
 

lib/propane/lexer.rb

@@ -1,51 +1,74 @@
 class Propane
   class Lexer
 
-    # @return [DFA]
-    #   Lexer DFA.
-    attr_accessor :dfa
-
     def initialize(grammar)
       @grammar = grammar
-      @dfa = DFA.new(grammar.patterns)
     end
 
     def build_tables
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
       transition_table = []
       state_table = []
-      states = @dfa.enumerate
-      states.each do |state, id|
-        token =
-          if state.accepts.nil?
-            @grammar.tokens.size
-          elsif state.accepts.drop?
-            TOKEN_DROP
-          elsif state.accepts.token
-            state.accepts.token.id
-          else
-            @grammar.tokens.size
-          end
-        code_id =
-          if state.accepts && state.accepts.code_id
-            state.accepts.code_id
-          else
-            0xFFFF_FFFF
-          end
-        state_table << {
-          transition_table_index: transition_table.size,
-          n_transitions: state.transitions.size,
-          token: token,
-          code_id: code_id,
+      mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = state_table.size
+        mode_table << {
+          state_table_offset: state_table_offset,
         }
-        state.transitions.each do |transition|
-          transition_table << {
-            first: transition.code_point_range.first,
-            last: transition.code_point_range.last,
-            destination: states[transition.destination],
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          token =
+            if state.accepts.nil?
+              @grammar.tokens.size
+            elsif state.accepts.drop?
+              TOKEN_DROP
+            elsif state.accepts.token
+              state.accepts.token.id
+            else
+              @grammar.tokens.size
+            end
+          code_id =
+            if state.accepts && state.accepts.code_id
+              state.accepts.code_id
+            else
+              0xFFFF_FFFF
+            end
+          state_table << {
+            transition_table_index: transition_table.size,
+            n_transitions: state.transitions.size,
+            token: token,
+            code_id: code_id,
           }
+          state.transitions.each do |transition|
+            transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
         end
       end
-      [transition_table, state_table]
+      [transition_table, state_table, mode_table]
+    end
+
+    # Get ID for a mode.
+    #
+    # @param mode_name [String]
+    #   Mode name.
+    #
+    # @return [Integer, nil]
+    #   Mode ID.
+    def mode_id(mode_name)
+      if mode_info = @modes[mode_name]
+        mode_info[:id]
+      end
     end
 
   end

lib/propane/pattern.rb

@@ -28,7 +28,7 @@ class Propane
 
     # @return [String, nil]
     #   Lexer mode for this pattern.
-    attr_reader :mode
+    attr_accessor :mode
 
     # Construct a Pattern.
     #