From f2cc5b112e15189211be876837dcfc61aaeeb9f3 Mon Sep 17 00:00:00 2001
From: Josh Holtrop <jholtrop@gmail.com>
Date: Sat, 25 Jun 2022 16:16:20 -0400
Subject: [PATCH] Handle shifting states after reducing

---
 assets/parser.d.erb   | 37 +++++++++++++++++++++++++++++++------
 lib/propane/parser.rb |  4 +---
 spec/propane_spec.rb  |  3 +++
 spec/test_d_lexer2.d  | 22 ++++++++++++++++++++++
 4 files changed, 57 insertions(+), 9 deletions(-)
 create mode 100644 spec/test_d_lexer2.d

diff --git a/assets/parser.d.erb b/assets/parser.d.erb
index 7a29ae5..67025a4 100644
--- a/assets/parser.d.erb
+++ b/assets/parser.d.erb
@@ -2,6 +2,9 @@
 module <%= @grammar.modulename %>;
 
 <% end %>
+
+import std.stdio;
+
 class <%= @classname %>
 {
     enum
@@ -16,7 +19,7 @@ class <%= @classname %>
         _TOKEN_NONE = <%= TOKEN_NONE %>,
     }
 
-    static immutable string TokenNames[] = [
+    static immutable string token_names[] = [
 <% @grammar.tokens.each_with_index do |token, index| %>
         "<%= token.name %>",
 <% end %>
@@ -303,7 +306,7 @@ class <%= @classname %>
             m_lexer = new Lexer(input, input_length);
         }
 
-        void parse()
+        bool parse()
         {
             Lexer.LexedToken lexed_token;
             uint token = _TOKEN_NONE;
@@ -331,7 +334,7 @@ class <%= @classname %>
                     if (token == _TOKEN_EOF)
                     {
                         /* Successful parse. */
-                        return;
+                        return true;
                     }
                     states ~= shift_state;
                     token = _TOKEN_NONE;
@@ -347,18 +350,39 @@ class <%= @classname %>
                 }
 
                 /* Error, unexpected token. */
-                return;
+                write("Unexpected token ");
+                if (token == _TOKEN_EOF)
+                {
+                    writeln("{EOF}");
+                }
+                else if (token < _TOKEN_COUNT)
+                {
+                    writeln(token_names[token]);
+                }
+                else
+                {
+                    writeln("{other}");
+                }
+                return false;
             }
         }
 
-        private uint check_shift(uint state, uint token)
+        private uint check_shift(uint state, uint symbol)
         {
             uint start = states[state].shift_table_index;
             uint end = start + states[state].n_shift_entries;
             for (uint i = start; i < end; i++)
             {
-                if (shifts[i].symbol == token)
+                if (shifts[i].symbol == symbol)
                 {
+                    if (symbol < _TOKEN_COUNT)
+                    {
+                        writeln("Shifting ", token_names[symbol]);
+                    }
+                    else
+                    {
+                        writeln("Shifting rule set ", symbol);
+                    }
                     return shifts[i].state;
                 }
             }
@@ -374,6 +398,7 @@ class <%= @classname %>
                 if ((reduces[i].token == token) ||
                     (reduces[i].token == _TOKEN_NONE))
                 {
+                    writeln("Reducing rule ", reduces[i].rule, ", rule set ", reduces[i].rule_set);
                     return i;
                 }
             }
diff --git a/lib/propane/parser.rb b/lib/propane/parser.rb
index a72a265..8047e94 100644
--- a/lib/propane/parser.rb
+++ b/lib/propane/parser.rb
@@ -62,9 +62,7 @@ class Propane
       state_table = []
       reduce_table = []
       @item_sets.each do |item_set|
-        shift_entries = item_set.following_symbols.select do |following_symbol|
-          following_symbol.is_a?(Token)
-        end.map do |following_symbol|
+        shift_entries = item_set.following_symbols.map do |following_symbol|
           state_id =
             if following_symbol == @eof_token
               0
diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb
index 832da56..1023ad4 100644
--- a/spec/propane_spec.rb
+++ b/spec/propane_spec.rb
@@ -74,10 +74,13 @@ EOF
     write_grammar <<EOF
 token a
 token b
+drop \\s+
 Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
     build_parser
+    compile("spec/test_d_lexer2.d")
+    run
   end
 end
diff --git a/spec/test_d_lexer2.d b/spec/test_d_lexer2.d
new file mode 100644
index 0000000..4b28ca8
--- /dev/null
+++ b/spec/test_d_lexer2.d
@@ -0,0 +1,22 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == false);
+
+    input = "a b";
+    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+
+    input = "bb";
+    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+}