From 2fbe13e07199c4f4b3cd88e357c9e3d0c7823747 Mon Sep 17 00:00:00 2001
From: Josh Holtrop <jholtrop@gmail.com>
Date: Sat, 25 Jun 2022 21:35:54 -0400
Subject: [PATCH] Do not consume lookahead token when reducing

---
 assets/parser.d.erb  | 24 +++++++++++++++++++++---
 spec/propane_spec.rb |  2 ++
 spec/test_d_lexer3.d | 18 ++++++++++++++++++
 3 files changed, 41 insertions(+), 3 deletions(-)
 create mode 100644 spec/test_d_lexer3.d

diff --git a/assets/parser.d.erb b/assets/parser.d.erb
index 67025a4..96e8e68 100644
--- a/assets/parser.d.erb
+++ b/assets/parser.d.erb
@@ -323,7 +323,6 @@ class <%= @classname %>
                 if (reduced_rule_set != 0xFFFFFFFFu)
                 {
                     shift_state = check_shift(states[$-1], reduced_rule_set);
-                    reduced_rule_set = 0xFFFFFFFFu;
                 }
                 if (shift_state == 0xFFFFFFFFu)
                 {
@@ -337,7 +336,14 @@ class <%= @classname %>
                         return true;
                     }
                     states ~= shift_state;
-                    token = _TOKEN_NONE;
+                    if (reduced_rule_set == 0xFFFFFFFFu)
+                    {
+                        token = _TOKEN_NONE;
+                    }
+                    else
+                    {
+                        reduced_rule_set = 0xFFFFFFFFu;
+                    }
                     continue;
                 }
 
@@ -398,7 +404,19 @@ class <%= @classname %>
                 if ((reduces[i].token == token) ||
                     (reduces[i].token == _TOKEN_NONE))
                 {
-                    writeln("Reducing rule ", reduces[i].rule, ", rule set ", reduces[i].rule_set);
+                    write("Reducing rule ", reduces[i].rule, ", rule set ", reduces[i].rule_set, " lookahead ");
+                    if (token < _TOKEN_COUNT)
+                    {
+                        writeln(token_names[token]);
+                    }
+                    else if (token == _TOKEN_EOF)
+                    {
+                        writeln("{EOF}");
+                    }
+                    else
+                    {
+                        writeln("{other}");
+                    }
                     return i;
                 }
             }
diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb
index 1023ad4..a7a0dcd 100644
--- a/spec/propane_spec.rb
+++ b/spec/propane_spec.rb
@@ -68,6 +68,8 @@ R1 -> a b;
 R2 -> a b;
 EOF
     build_parser
+    compile("spec/test_d_lexer3.d")
+    run
   end
 
   it "handles reducing a rule that could be arrived at from multiple states" do
diff --git a/spec/test_d_lexer3.d b/spec/test_d_lexer3.d
new file mode 100644
index 0000000..772711f
--- /dev/null
+++ b/spec/test_d_lexer3.d
@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aba";
+    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+
+    input = "abb";
+    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    assert(parser.parse() == true);
+}