Remove TOKEN_EOF; define EOF token and start rule in Generator

lib/propane.rb

@@ -25,9 +25,6 @@ require_relative "propane/version"
 
 class Propane
 
-  # EOF.
-  TOKEN_EOF = 0xFFFFFFFC
-
   # Decoding error.
   TOKEN_DECODE_ERROR = 0xFFFFFFFD
 

lib/propane/generator.rb

@@ -26,33 +26,41 @@ class Propane
     private
 
     def process_grammar!
+      # Add EOF token.
+      @grammar.tokens << Token.new("$EOF", nil)
       tokens_by_name = {}
-      @grammar.tokens.each do |token|
+      @grammar.tokens.each_with_index do |token, token_id|
+        # Assign token ID.
+        token.id = token_id
         # Check for token name conflicts.
         if tokens_by_name.include?(token.name)
           raise Error.new("Duplicate token name #{token.name.inspect}")
         end
         tokens_by_name[token.name] = token
       end
+      # Check for user start rule.
+      unless @grammar.rules.find {|rule| rule.name == "Start"}
+        raise Error.new("Start rule not found")
+      end
+      # Add "real" start rule.
+      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil))
       rule_sets = {}
-      @grammar.rules.each do |rule|
+      rule_set_id = @grammar.tokens.size
+      @grammar.rules.each_with_index do |rule, rule_id|
+        # Assign rule ID.
+        rule.id = rule_id
         # Check for token/rule name conflict.
         if tokens_by_name.include?(rule.name)
           raise Error.new("Rule name collides with token name #{rule.name.inspect}")
         end
         # Build rule sets of all rules with the same name.
-        @_rule_set_id ||= @grammar.tokens.size
         unless rule_sets[rule.name]
-          rule_sets[rule.name] = RuleSet.new(rule.name, @_rule_set_id)
-          @_rule_set_id += 1
+          rule_sets[rule.name] = RuleSet.new(rule.name, rule_set_id)
+          rule_set_id += 1
         end
         rule.rule_set = rule_sets[rule.name]
         rule_sets[rule.name] << rule
       end
-      # Check for start rule.
-      unless rule_sets["Start"]
-        raise Error.new("Start rule not found")
-      end
       # Generate lexer user code IDs for lexer patterns with user code blocks.
       @grammar.patterns.select do |pattern|
         pattern.code
@@ -75,7 +83,7 @@ class Propane
       # Generate the lexer.
       @lexer = Lexer.new(@grammar.patterns)
       # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, rule_sets["Start"], @log)
+      @parser = Parser.new(@grammar, rule_sets, @log)
     end
 
     # Determine which grammar rules could expand to empty sequences.

lib/propane/grammar.rb

@@ -80,7 +80,7 @@ class Propane
         unless code = parse_code_block!
           consume!(/;/, "expected pattern or `;' or code block")
         end
-        token = Token.new(name: name, id: @tokens.size, line_number: @line_number)
+        token = Token.new(name, @line_number)
         @tokens << token
         pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code)
         @patterns << pattern
@@ -93,7 +93,7 @@ class Propane
         unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
           raise Error.new("Invalid token name #{name.inspect}")
         end
-        token = Token.new(name: name, id: @tokens.size, line_number: @line_number)
+        token = Token.new(name, @line_number)
         @tokens << token
       end
     end
@@ -117,8 +117,7 @@ class Propane
           raise Error.new("Invalid rule name #{name.inspect}")
         end
         components = components.strip.split(/\s+/)
-        # Reserve rule ID 0 for the "real" start rule.
-        @rules << Rule.new(rule_name, components, code, @line_number, @rules.size + 1)
+        @rules << Rule.new(rule_name, components, code, @line_number)
       end
     end
 

lib/propane/parser.rb

@@ -2,15 +2,13 @@ class Propane
 
   class Parser
 
-    def initialize(grammar, rule_sets, start_rule_set, log)
+    def initialize(grammar, rule_sets, log)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
-      @eof_token = Token.new(name: "$", id: TOKEN_EOF)
-      @start_rule = Rule.new("$$", [start_rule_set, @eof_token], nil, nil, 0)
       @item_sets = []
       @item_sets_set = {}
-      start_item = Item.new(@start_rule, 0)
+      start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
 
       while eval_item_sets.size > 0
@@ -21,7 +19,7 @@ class Propane
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
           item_set.following_symbols.each do |following_symbol|
-            unless following_symbol == @eof_token
+            unless following_symbol.name == "$EOF"
               following_set = item_set.build_following_item_set(following_symbol)
               eval_item_sets << following_set
             end
@@ -44,7 +42,7 @@ class Propane
       @item_sets.each do |item_set|
         shift_entries = item_set.following_symbols.map do |following_symbol|
           state_id =
-            if following_symbol == @eof_token
+            if following_symbol.name == "$EOF"
               0
             else
               item_set.following_item_set[following_symbol].id
@@ -83,7 +81,7 @@ class Propane
 
     def process_item_set(item_set)
       item_set.following_symbols.each do |following_symbol|
-        unless following_symbol == @eof_token
+        unless following_symbol.name == "$EOF"
           following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
           item_set.following_item_set[following_symbol] = following_set
           following_set.in_sets << item_set
@@ -206,7 +204,7 @@ class Propane
 
     def write_log!
       @log.puts Util.banner("Parser Rules")
-      ([@start_rule] + @grammar.rules).each do |rule|
+      @grammar.rules.each do |rule|
         @log.puts
         @log.puts "Rule #{rule.id}:"
         @log.puts "  #{rule}"

lib/propane/rule.rb

@@ -12,7 +12,7 @@ class Propane
 
     # @return [Integer]
     #   Rule ID.
-    attr_reader :id
+    attr_accessor :id
 
     # @return [Integer]
     #   Line number where the rule was defined in the input grammar.
@@ -36,13 +36,10 @@ class Propane
     #   User code associated with the rule.
     # @param line_number [Integer]
     #   Line number where the rule was defined in the input grammar.
-    # @param id [Integer]
-    #   Rule ID.
-    def initialize(name, components, code, line_number, id)
+    def initialize(name, components, code, line_number)
       @name = name
       @components = components
       @code = code
-      @id = id
       @line_number = line_number
     end
 

lib/propane/token.rb

@@ -2,13 +2,25 @@ class Propane
 
   class Token
 
+    class << self
+
+      # Name of the token to use in code (special characters replaced).
+      #
+      # @return [String]
+      #   Name of the token to use in code (special characters replaced).
+      def code_name(name)
+        name.sub(/^\$/, "0")
+      end
+
+    end
+
     # @return [String, nil]
     #   Token name.
     attr_reader :name
 
     # @return [Integer, nil]
     #   Token ID.
-    attr_reader :id
+    attr_accessor :id
 
     # @return [Integer, nil]
     #   Line number where the token was defined in the input grammar.
@@ -20,14 +32,19 @@ class Propane
     #   Optional parameters.
     # @option options [String, nil] :name
     #   Token name.
-    # @option options [Integer, nil] :id
-    #   Token ID.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    def initialize(options)
-      @name = options[:name]
-      @id = options[:id]
-      @line_number = options[:line_number]
+    def initialize(name, line_number)
+      @name = name
+      @line_number = line_number
+    end
+
+    # Name of the token to use in code (special characters replaced).
+    #
+    # @return [String]
+    #   Name of the token to use in code (special characters replaced).
+    def code_name
+      self.class.code_name(@name)
     end
 
     def to_s