From ffccc659aad28ff8c97c117c98e4e94e711ca89a Mon Sep 17 00:00:00 2001
From: Josh Holtrop <jholtrop@gmail.com>
Date: Tue, 14 Mar 2023 19:38:42 -0400
Subject: [PATCH] Clean up Lexer.State a bit

---
 assets/parser.d.erb  | 26 ++++++++++++++++++--------
 lib/propane/lexer.rb | 10 +++-------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/assets/parser.d.erb b/assets/parser.d.erb
index 637ac45..6477fe6 100644
--- a/assets/parser.d.erb
+++ b/assets/parser.d.erb
@@ -223,8 +223,14 @@ class <%= @classname %>
         {
             uint transition_table_index;
             uint n_transitions;
-            uint token;
+            Token token;
             UserCodeID code_id;
+            bool drop;
+
+            bool accepts() const
+            {
+                return drop || token.is_valid() || code_id.is_valid();
+            }
         }
 
         private struct Mode
@@ -243,12 +249,17 @@ class <%= @classname %>
 <% state_table.each do |state_table_entry| %>
             State(<%= state_table_entry[:transition_table_index] %>u,
                   <%= state_table_entry[:n_transitions] %>u,
-                  <%= state_table_entry[:token] %>u,
-<%   if state_table_entry[:code_id] %>
-                  UserCodeID(<%= state_table_entry[:code_id] %>u)),
+<%   if state_table_entry[:token] %>
+                  Token(<%= state_table_entry[:token] %>u),
 <%   else %>
-                  UserCodeID.invalid()),
+                  Token.invalid(),
 <%   end %>
+<%   if state_table_entry[:code_id] %>
+                  UserCodeID(<%= state_table_entry[:code_id] %>u),
+<%   else %>
+                  UserCodeID.invalid(),
+<%   end %>
+                  <%= state_table_entry[:drop] %>),
 <% end %>
         ];
 
@@ -375,7 +386,7 @@ class <%= @classname %>
                 }
                 result.token = token_to_accept;
                 result.length = match_info.length;
-                if (result.token == _TOKEN_DROP)
+                if (match_info.accepting_state.drop)
                 {
                     result.type = Result.Type.DROP;
                 }
@@ -418,8 +429,7 @@ class <%= @classname %>
                             attempt_match_info.delta_col++;
                         }
                         current_state = dest;
-                        if ((states[current_state].token != _TOKEN_COUNT) ||
-                            (states[current_state].code_id.is_valid()))
+                        if (states[current_state].accepts())
                         {
                             attempt_match_info.accepting_state = &states[current_state];
                             *match_info = attempt_match_info;
diff --git a/lib/propane/lexer.rb b/lib/propane/lexer.rb
index 472d0f8..9c096e7 100644
--- a/lib/propane/lexer.rb
+++ b/lib/propane/lexer.rb
@@ -24,15 +24,10 @@ class Propane
         }
         states = mode_info[:dfa].enumerate
         states.each do |state, id|
+          drop = state.accepts && state.accepts.drop?
           token =
-            if state.accepts.nil?
-              @grammar.tokens.size
-            elsif state.accepts.drop?
-              TOKEN_DROP
-            elsif state.accepts.token
+            if state.accepts && state.accepts.token
               state.accepts.token.id
-            else
-              @grammar.tokens.size
             end
           code_id =
             if state.accepts && state.accepts.code_id
@@ -41,6 +36,7 @@ class Propane
           state_table << {
             transition_table_index: transition_table.size,
             n_transitions: state.transitions.size,
+            drop: drop,
             token: token,
             code_id: code_id,
           }