From f3ed678fe139b4214b23bd6a2b8e27dae558ffab Mon Sep 17 00:00:00 2001
From: Josh Holtrop <jholtrop@gmail.com>
Date: Mon, 27 Sep 2021 21:40:12 -0400
Subject: [PATCH] Store tokens in Hash by name

---
 assets/parser.d.erb       |  4 ++--
 lib/imbecile.rb           | 22 ++++++++++------------
 lib/imbecile/lexer/dfa.rb |  2 +-
 3 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/assets/parser.d.erb b/assets/parser.d.erb
index fd04ec5..43a9839 100644
--- a/assets/parser.d.erb
+++ b/assets/parser.d.erb
@@ -6,7 +6,7 @@ class <%= classname %>
 {
     enum
     {
-<% @tokens.each_with_index do |token, index| %>
+<% @tokens.each_with_index do |(name, token), index| %>
 <%   if token.name %>
         TOKEN_<%= token.c_name %> = <%= index %>,
 <%   end %>
@@ -18,7 +18,7 @@ class <%= classname %>
     }
 
     static immutable string TokenNames[] = [
-<% @tokens.each_with_index do |token, index| %>
+<% @tokens.each_with_index do |(name, token), index| %>
 <%   if token.name %>
         "<%= token.name %>",
 <%   else %>
diff --git a/lib/imbecile.rb b/lib/imbecile.rb
index 6fdf234..40c2f66 100644
--- a/lib/imbecile.rb
+++ b/lib/imbecile.rb
@@ -35,7 +35,7 @@ class Imbecile
   end
 
   def initialize(input)
-    @tokens = []
+    @tokens = {}
     @rules = {}
     input = input.gsub("\r\n", "\n")
     while !input.empty?
@@ -74,10 +74,14 @@ class Imbecile
       unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
         raise Error.new("Invalid token name #{name}")
       end
-      @tokens << Token.new(name, pattern, @tokens.size)
+      if @tokens[name]
+        raise Error.new("Duplicate token name #{name}")
+      else
+        @tokens[name] = Token.new(name, pattern, @tokens.size)
+      end
     elsif input.slice!(/\Adrop\s+(\S+)\n/)
       pattern = $1
-      @tokens << Token.new(nil, pattern, @tokens.size)
+      @tokens[name] = Token.new(nil, pattern, @tokens.size)
     elsif input.slice!(/\A(\S+)\s*:\s*\[(.*?)\] <<\n(.*?)^>>\n/m)
       rule_name, components, code = $1, $2, $3
       components = components.strip.split(/\s+/)
@@ -92,14 +96,8 @@ class Imbecile
   end
 
   def expand_rules
-    token_names = @tokens.each_with_object({}) do |token, token_names|
-      if token_names.include?(token.name)
-        raise Error.new("Duplicate token name #{token.name}")
-      end
-      token_names[token.name] = token
-    end
     @rules.each do |rule_name, rule|
-      if token_names.include?(rule_name)
+      if @tokens.include?(rule_name)
         raise Error.new("Rule name collides with token name #{rule_name}")
       end
     end
@@ -109,8 +107,8 @@ class Imbecile
     @rules.each do |rule_name, rule|
       rule.patterns.each do |rule|
         rule.components.map! do |component|
-          if token_names[component]
-            token_names[component]
+          if @tokens[component]
+            @tokens[component]
           elsif @rules[component]
             @rules[component]
           else
diff --git a/lib/imbecile/lexer/dfa.rb b/lib/imbecile/lexer/dfa.rb
index 0930fb5..137fcda 100644
--- a/lib/imbecile/lexer/dfa.rb
+++ b/lib/imbecile/lexer/dfa.rb
@@ -6,7 +6,7 @@ class Imbecile
       def initialize(tokens)
         super()
         start_nfa = Regex::NFA.new
-        tokens.each do |token|
+        tokens.each do |name, token|
           start_nfa.start_state.add_transition(nil, token.nfa.start_state)
         end
         @nfa_state_sets = {}