Remove _TOKEN_NONE and use _TOKEN_COUNT instead

lib/propane.rb

@@ -31,9 +31,6 @@ class Propane
   # Token ID for a "dropped" token.
   TOKEN_DROP = 0xFFFFFFFE
 
-  # Invalid token ID.
-  TOKEN_NONE = 0xFFFFFFFF
-
   class Error < RuntimeError
   end
 

lib/propane/generator.rb

@@ -81,7 +81,7 @@ class Propane
       end
       determine_possibly_empty_rulesets!(rule_sets)
       # Generate the lexer.
-      @lexer = Lexer.new(@grammar.patterns)
+      @lexer = Lexer.new(@grammar)
       # Generate the parser.
       @parser = Parser.new(@grammar, rule_sets, @log)
     end

lib/propane/lexer.rb

@@ -5,8 +5,9 @@ class Propane
     #   Lexer DFA.
     attr_accessor :dfa
 
-    def initialize(patterns)
-      @dfa = DFA.new(patterns)
+    def initialize(grammar)
+      @grammar = grammar
+      @dfa = DFA.new(grammar.patterns)
     end
 
     def build_tables
@@ -16,13 +17,13 @@ class Propane
       states.each do |state, id|
         token =
           if state.accepts.nil?
-            TOKEN_NONE
+            @grammar.tokens.size
           elsif state.accepts.drop?
             TOKEN_DROP
           elsif state.accepts.token
             state.accepts.token.id
           else
-            TOKEN_NONE
+            @grammar.tokens.size
           end
         code_id =
           if state.accepts && state.accepts.code_id

lib/propane/parser.rb

@@ -55,7 +55,7 @@ class Propane
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: TOKEN_NONE, rule_id: ra.id,
+            [{token_id: @grammar.tokens.size, rule_id: ra.id,
               rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
           when Hash
             ra.map do |token, rule|