Convert Decoder.decode_code_point return type to a SumType

assets/parser.d.erb

@@ -2,8 +2,8 @@
 module <%= @grammar.modulename %>;
 
 <% end %>
-
 import std.stdio;
+import std.sumtype;
 
 <% @grammar.code_blocks.each do |code| %>
 <%= code %>
@@ -22,8 +22,6 @@ class <%= @classname %>
 <%   end %>
 <% end %>
         _TOKEN_COUNT = <%= @grammar.tokens.size %>,
-        _TOKEN_DECODE_ERROR = <%= TOKEN_DECODE_ERROR %>,
-        _TOKEN_DROP = <%= TOKEN_DROP %>,
     }
 
     struct Token
@@ -58,7 +56,7 @@ class <%= @classname %>
 
     alias CodePoint = uint;
 
-    static immutable string token_names[] = [
+    static immutable string[] token_names = [
 <% @grammar.tokens.each_with_index do |token, index| %>
         "<%= token.name %>",
 <% end %>
@@ -73,54 +71,23 @@ class <%= @classname %>
 
     static class Decoder
     {
-        struct Result
+        struct Success
         {
-            enum Type
-            {
-                SUCCESS,
-                EOF,
-                DECODE_ERROR,
-            }
-            private Type type;
             CodePoint code_point;
             uint code_point_length;
-
-            static Result success(CodePoint code_point, uint code_point_length)
-            {
-                return Result(Type.SUCCESS, code_point, code_point_length);
-            }
-
-            bool is_success()
-            {
-                return type == Type.SUCCESS;
-            }
-
-            static Result eof()
-            {
-                return Result(Type.EOF);
-            }
-
-            bool is_eof()
-            {
-                return type == Type.EOF;
-            }
-
-            static Result decode_error()
-            {
-                return Result(Type.DECODE_ERROR);
-            }
-
-            bool is_decode_error()
-            {
-                return type == Type.DECODE_ERROR;
-            }
         }
+        struct EOF {}
+        struct DecodeError {}
 
-        static Result decode_code_point(string input)
+        static SumType!(
+                Success,
+                EOF,
+                DecodeError)
+            decode_code_point(string input)
         {
             if (input.length == 0u)
             {
-                return Result.eof();
+                return cast(ReturnType!decode_code_point)EOF();
             }
             char c = input[0];
             CodePoint code_point;
@@ -160,11 +127,11 @@ class <%= @classname %>
                 }
                 else
                 {
-                    return Result.decode_error();
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
                 if (input.length <= following_bytes)
                 {
-                    return Result.decode_error();
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
                 code_point_length = following_bytes + 1u;
                 for (size_t i = 0u; i < following_bytes; i++)
@@ -172,12 +139,12 @@ class <%= @classname %>
                     char b = input[i + 1u];
                     if ((b & 0xC0u) != 0x80u)
                     {
-                        return Result.decode_error();
+                        return cast(ReturnType!decode_code_point)DecodeError();
                     }
                     code_point = (code_point << 6u) | (b & 0x3Fu);
                 }
             }
-            return Result.success(code_point, code_point_length);
+            return cast(ReturnType!decode_code_point)Success(code_point, code_point_length);
         }
     }
 
@@ -239,13 +206,13 @@ class <%= @classname %>
         }
 
 <% transition_table, state_table, mode_table = @lexer.build_tables %>
-        private static immutable Transition transitions[] = [
+        private static immutable Transition[] transitions = [
 <% transition_table.each do |transition_table_entry| %>
             Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
 <% end %>
         ];
 
-        private static immutable State states[] = [
+        private static immutable State[] states = [
 <% state_table.each do |state_table_entry| %>
             State(<%= state_table_entry[:transition_table_index] %>u,
                   <%= state_table_entry[:n_transitions] %>u,
@@ -263,20 +230,12 @@ class <%= @classname %>
 <% end %>
         ];
 
-        private static immutable Mode modes[] = [
+        private static immutable Mode[] modes = [
 <% mode_table.each do |mode_table_entry| %>
             Mode(<%= mode_table_entry[:state_table_offset] %>),
 <% end %>
         ];
 
-        struct MatchInfo
-        {
-            size_t length;
-            size_t delta_row;
-            size_t delta_col;
-            const(State) * accepting_state;
-        }
-
         struct Result
         {
             enum Type
@@ -284,6 +243,7 @@ class <%= @classname %>
                 DECODE_ERROR,
                 DROP,
                 TOKEN,
+                UNEXPECTED_INPUT,
             }
 
             Type type;
@@ -325,10 +285,10 @@ class <%= @classname %>
          * @param match Matched text for this pattern.
          * @param result Result lexer result in progress.
          *
-         * @return Token ID to accept, or _TOKEN_COUNT if the user code does
+         * @return Token to accept, or invalid token if the user code does
          *   not explicitly return a token.
          */
-        private uint user_code(UserCodeID code_id, string match, Result * result)
+        private Token user_code(UserCodeID code_id, string match, Result * result)
         {
             switch (code_id)
             {
@@ -342,7 +302,7 @@ class <%= @classname %>
             default: break;
             }
 
-            return _TOKEN_COUNT;
+            return Token.invalid();
         }
 
         private Result attempt_lex_token()
@@ -351,105 +311,250 @@ class <%= @classname %>
             result.row = m_input_row;
             result.col = m_input_col;
             result.token = _TOKEN_COUNT;
-            MatchInfo match_info;
-            find_longest_match(&result, &match_info);
-            if (result.token != _TOKEN_COUNT)
+            auto match_result = find_longest_match();
+            if (match_result.is_eof())
             {
+                result.type = Result.Type.TOKEN;
+                result.token = TOKEN___EOF;
                 return result;
             }
-            if (match_info.accepting_state != null)
+            else if (match_result.is_decode_error())
             {
-                uint token_to_accept = match_info.accepting_state.token;
-                if (match_info.accepting_state.code_id.is_valid())
+                result.type = Result.Type.DECODE_ERROR;
+                return result;
+            }
+            else if (match_result.is_unexpected_input())
+            {
+                result.type = Result.Type.UNEXPECTED_INPUT;
+                return result;
+            }
+            uint token_to_accept = match_result.accepting_state.token;
+            if (match_result.accepting_state.code_id.is_valid())
+            {
+                Token user_code_token = user_code(match_result.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_result.length)], &result);
+                /* An invalid Token from user_code() means that the user
+                 * code did not explicitly return a token. So only override
+                 * the token to return if the user code does explicitly
+                 * return a token. */
+                if (user_code_token.is_valid())
                 {
-                    uint user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_info.length)], &result);
-                    /* A return of _TOKEN_COUNT from user_code() means
-                     * that the user code did not explicitly return a
-                     * token. So only override the token to return if the
-                     * user code does explicitly return a token. */
-                    if (user_code_token != _TOKEN_COUNT)
-                    {
-                        token_to_accept = user_code_token;
-                    }
+                    token_to_accept = user_code_token.token;
                 }
+            }
 
-                /* Update the input position tracking. */
-                m_input_position += match_info.length;
-                m_input_row += match_info.delta_row;
-                if (match_info.delta_row != 0u)
-                {
-                    m_input_col = match_info.delta_col;
-                }
-                else
-                {
-                    m_input_col += match_info.delta_col;
-                }
-                result.token = token_to_accept;
-                result.length = match_info.length;
-                if (match_info.accepting_state.drop)
-                {
-                    result.type = Result.Type.DROP;
-                }
-                else
-                {
-                    result.type = Result.Type.TOKEN;
-                }
+            /* Update the input position tracking. */
+            m_input_position += match_result.length;
+            m_input_row += match_result.delta_row;
+            if (match_result.delta_row != 0u)
+            {
+                m_input_col = match_result.delta_col;
+            }
+            else
+            {
+                m_input_col += match_result.delta_col;
+            }
+            result.token = token_to_accept;
+            result.length = match_result.length;
+            if (match_result.accepting_state.drop)
+            {
+                result.type = Result.Type.DROP;
+            }
+            else
+            {
+                result.type = Result.Type.TOKEN;
             }
             return result;
         }
 
-        private void find_longest_match(Result * result, MatchInfo * match_info)
+        /**
+         * Result type for find_longest_match().
+         *
+         * Alternatives:
+         * - decode_error
+         * - eof
+         * - found_match(length, delta_row, delta_col, accepting_state)
+         * - unexpected_input(unexpected_input_length)
+         */
+        struct FindLongestMatchResult
         {
-            MatchInfo attempt_match_info;
-            uint current_state = modes[m_mode].state_table_offset;
-            for (;;)
+            enum : ubyte
             {
-                auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match_info.length)..(m_input.length)]);
-                if (decoded.is_decode_error())
+                FOUND_MATCH,
+                DECODE_ERROR,
+                EOF,
+                UNEXPECTED_INPUT,
+            }
+
+            ubyte type;
+            alias type this;
+            union
+            {
+                struct
                 {
-                    result.type = Result.Type.DECODE_ERROR;
-                    result.token = _TOKEN_DECODE_ERROR;
-                    return;
-                }
-                bool lex_continue = false;
-                if (!decoded.is_eof())
-                {
-                    uint dest = transition(current_state, decoded.code_point);
-                    if (dest != cast(uint)-1)
-                    {
-                        lex_continue = true;
-                        attempt_match_info.length += decoded.code_point_length;
-                        if (decoded.code_point == '\n')
-                        {
-                            attempt_match_info.delta_row++;
-                            attempt_match_info.delta_col = 0u;
-                        }
-                        else
-                        {
-                            attempt_match_info.delta_col++;
-                        }
-                        current_state = dest;
-                        if (states[current_state].accepts())
-                        {
-                            attempt_match_info.accepting_state = &states[current_state];
-                            *match_info = attempt_match_info;
-                        }
-                    }
-                }
-                else if (attempt_match_info.length == 0u)
-                {
-                    result.token = TOKEN___EOF;
-                    result.type = Result.Type.TOKEN;
-                    return;
-                }
-                if (!lex_continue)
-                {
-                    return;
+                    size_t length;
+                    size_t delta_row;
+                    size_t delta_col;
+                    const(State) * accepting_state;
                 }
+                size_t unexpected_input_length;
+            }
+
+            this(ubyte type)
+            {
+                this.type = type;
+            }
+
+            this(ubyte type, size_t unexpected_input_length)
+            {
+                this.type = type;
+                this.unexpected_input_length = unexpected_input_length;
+            }
+
+            this(ubyte type, size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                this.type = type;
+                this.length = length;
+                this.delta_row = delta_row;
+                this.delta_col = delta_col;
+                this.accepting_state = accepting_state;
+            }
+
+            static FindLongestMatchResult found_match(size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                return FindLongestMatchResult(FOUND_MATCH, length, delta_row, delta_col, accepting_state);
+            }
+
+            static FindLongestMatchResult decode_error()
+            {
+                return FindLongestMatchResult(DECODE_ERROR);
+            }
+
+            static FindLongestMatchResult eof()
+            {
+                return FindLongestMatchResult(EOF);
+            }
+
+            static FindLongestMatchResult unexpected_input(size_t unexpected_input_length)
+            {
+                return FindLongestMatchResult(UNEXPECTED_INPUT, unexpected_input_length);
+            }
+
+            bool is_found_match()
+            {
+                return type == FOUND_MATCH;
+            }
+
+            bool is_decode_error()
+            {
+                return type == DECODE_ERROR;
+            }
+
+            bool is_eof()
+            {
+                return type == EOF;
+            }
+
+            bool is_unexpected_input()
+            {
+                return type == UNEXPECTED_INPUT;
             }
         }
 
-        private uint transition(uint current_state, uint code_point)
+        private FindLongestMatchResult find_longest_match()
+        {
+            FindLongestMatchResult longest_match = FindLongestMatchResult.found_match(0, 0, 0, null);
+            FindLongestMatchResult attempt_match = longest_match;
+            uint current_state = modes[m_mode].state_table_offset;
+            for (;;)
+            {
+                bool lex_continue = false;
+                Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]).match!(
+                        (Decoder.DecodeError de) {return FindLongestMatchResult.decode_error();},
+                        (Decoder.EOF e) {
+                            /* We hit EOF. */
+                            if (longest_match.length > 0)
+                            {
+                                /* We have a match, so use it. */
+                                return longest_match;
+                            }
+                            else if (attempt_match.length != 0)
+                            {
+                                /* There is a partial match - error! */
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length);
+                            }
+                            else
+                            {
+                                /* Valid EOF return. */
+                                return FindLongestMatchResult.eof();
+                            }
+                        },
+                        (Decoder.Success decoded) {
+                            auto transition_result = transition(current_state, decoded.code_point);
+                            if (transition_result.found())
+                            {
+                                lex_continue = true;
+                                attempt_match.length += decoded.code_point_length;
+                                if (decoded.code_point == '\n')
+                                {
+                                    attempt_match.delta_row++;
+                                    attempt_match.delta_col = 0u;
+                                }
+                                else
+                                {
+                                    attempt_match.delta_col++;
+                                }
+                                current_state = transition_result.destination();
+                                if (states[current_state].accepts())
+                                {
+                                    attempt_match.accepting_state = &states[current_state];
+                                    longest_match = attempt_match;
+                                }
+                            }
+                            else if (longest_match.length > 0)
+                            {
+                                return longest_match;
+                            }
+                            else
+                            {
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length);
+                            }
+                        });
+            }
+        }
+
+        /**
+         * Result type for transition().
+         *
+         * Alternatives:
+         * - found(destination)
+         * - not_found
+         */
+        struct TransitionResult
+        {
+            private uint m_destination;
+
+            static TransitionResult found(uint destination)
+            {
+                return TransitionResult(destination);
+            }
+
+            static TransitionResult not_found()
+            {
+                return TransitionResult(cast(uint)-1);
+            }
+
+            bool found()
+            {
+                return m_destination != cast(uint)-1;
+            }
+
+            @property uint destination()
+            {
+                return m_destination;
+            }
+        }
+
+        private TransitionResult transition(uint current_state, uint code_point)
         {
             uint transition_table_index = states[current_state].transition_table_index;
             for (uint i = 0u; i < states[current_state].n_transitions; i++)
@@ -457,10 +562,11 @@ class <%= @classname %>
                 if ((transitions[transition_table_index + i].first <= code_point) &&
                     (code_point <= transitions[transition_table_index + i].last))
                 {
-                    return transitions[transition_table_index + i].destination;
+                    uint destination = transitions[transition_table_index + i].destination;
+                    return TransitionResult.found(destination);
                 }
             }
-            return cast(uint)-1;
+            return TransitionResult.not_found();
         }
     }
 
@@ -500,19 +606,19 @@ class <%= @classname %>
         }
 
 <% state_table, shift_table, reduce_table = @parser.build_tables %>
-        private static immutable Shift shifts[] = [
+        private static immutable Shift[] shifts = [
 <%   shift_table.each do |shift| %>
             Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u),
 <%   end %>
         ];
 
-        private static immutable Reduce reduces[] = [
+        private static immutable Reduce[] reduces = [
 <%   reduce_table.each do |reduce| %>
             Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
 <%   end %>
         ];
 
-        private static immutable State states[] = [
+        private static immutable State[] states = [
 <%   state_table.each do |state| %>
             State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
 <%   end %>

lib/propane.rb

@@ -25,12 +25,6 @@ require_relative "propane/version"
 
 class Propane
 
-  # Decoding error.
-  TOKEN_DECODE_ERROR = 0xFFFFFFFD
-
-  # Token ID for a "dropped" token.
-  TOKEN_DROP = 0xFFFFFFFE
-
   class Error < RuntimeError
   end
 

lib/propane/generator.rb

@@ -183,7 +183,7 @@ class Propane
     #   Expanded user code block.
     def expand_code(code, parser, rule, pattern)
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
-        "TOKEN_#{Token.code_name($1)}"
+        "Token(TOKEN_#{Token.code_name($1)})"
       end
       if parser
         code = code.gsub(/\$\$/) do |match|

spec/propane_spec.rb

@@ -20,7 +20,7 @@ describe Propane do
   end
 
   def compile(*test_files)
-    result = system(*%w[gdc -funittest -o spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
+    result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
     expect(result).to be_truthy
   end
 
@@ -28,6 +28,7 @@ describe Propane do
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
+    stderr.sub!(/^.*modules passed unittests\n/, "")
     Results.new(stdout, stderr, status)
   end