Convert Decoder.decode_code_point return type to a SumType

.gitignore

@@ -2,6 +2,7 @@
 /.yardoc
 /_yardoc/
 /coverage/
+/dist/
 /gen/
 /pkg/
 /spec/reports/

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2022 Josh Holtrop
+Copyright (c) 2010-2023 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Rakefile

@@ -1,5 +1,9 @@
 require "rspec/core/rake_task"
 
+task :build_dist do
+  sh "ruby rb/build_dist.rb"
+end
+
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]

assets/parser.d.erb

@@ -2,8 +2,8 @@
 module <%= @grammar.modulename %>;
 
 <% end %>
-
 import std.stdio;
+import std.sumtype;
 
 <% @grammar.code_blocks.each do |code| %>
 <%= code %>
@@ -11,17 +11,52 @@ import std.stdio;
 
 class <%= @classname %>
 {
-    enum
+    alias TokenID = uint;
+
+    enum : TokenID
     {
 <% @grammar.tokens.each_with_index do |token, index| %>
         TOKEN_<%= token.code_name %> = <%= index %>,
+<%   unless token.id == index %>
+<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
+<%   end %>
 <% end %>
         _TOKEN_COUNT = <%= @grammar.tokens.size %>,
-        _TOKEN_DECODE_ERROR = <%= TOKEN_DECODE_ERROR %>,
-        _TOKEN_DROP = <%= TOKEN_DROP %>,
     }
 
-    static immutable string token_names[] = [
+    struct Token
+    {
+        /* Number of tokens in this parser. */
+        enum count = <%= @grammar.tokens.size %>;
+        TokenID token;
+        alias token this;
+
+        @disable this();
+
+        this(TokenID token)
+        {
+            this.token = token;
+        }
+
+        static Token invalid()
+        {
+            return Token(count);
+        }
+
+        bool is_valid() const
+        {
+            return token < count;
+        }
+
+        bool is_invalid() const
+        {
+            return !is_valid();
+        }
+    }
+
+    alias CodePoint = uint;
+
+    static immutable string[] token_names = [
 <% @grammar.tokens.each_with_index do |token, index| %>
         "<%= token.name %>",
 <% end %>
@@ -36,26 +71,26 @@ class <%= @classname %>
 
     static class Decoder
     {
-        enum
+        struct Success
         {
-            CODE_POINT_INVALID = 0xFFFFFFFE,
-            CODE_POINT_EOF = 0xFFFFFFFF,
-        }
-
-        struct DecodedCodePoint
-        {
-            uint code_point;
+            CodePoint code_point;
             uint code_point_length;
         }
+        struct EOF {}
+        struct DecodeError {}
 
-        static DecodedCodePoint decode_code_point(string input)
+        static SumType!(
+                Success,
+                EOF,
+                DecodeError)
+            decode_code_point(string input)
         {
             if (input.length == 0u)
             {
-                return DecodedCodePoint(CODE_POINT_EOF, 0u);
+                return cast(ReturnType!decode_code_point)EOF();
             }
             char c = input[0];
-            uint code_point;
+            CodePoint code_point;
             uint code_point_length;
             if ((c & 0x80u) == 0u)
             {
@@ -92,11 +127,11 @@ class <%= @classname %>
                 }
                 else
                 {
-                    return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
                 if (input.length <= following_bytes)
                 {
-                    return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
                 code_point_length = following_bytes + 1u;
                 for (size_t i = 0u; i < following_bytes; i++)
@@ -104,12 +139,12 @@ class <%= @classname %>
                     char b = input[i + 1u];
                     if ((b & 0xC0u) != 0x80u)
                     {
-                        return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                        return cast(ReturnType!decode_code_point)DecodeError();
                     }
                     code_point = (code_point << 6u) | (b & 0x3Fu);
                 }
             }
-            return DecodedCodePoint(code_point, code_point_length);
+            return cast(ReturnType!decode_code_point)Success(code_point, code_point_length);
         }
     }
 
@@ -117,17 +152,52 @@ class <%= @classname %>
     {
         private struct Transition
         {
-            uint first;
-            uint last;
+            CodePoint first;
+            CodePoint last;
             uint destination;
         }
 
+        private struct UserCodeID
+        {
+            enum count = <%= (@grammar.patterns.map(&:code_id).compact.max || -1) + 1%>;
+            uint user_code_id;
+            alias user_code_id this;
+
+            @disable this();
+
+            this(uint user_code_id)
+            {
+                this.user_code_id = user_code_id;
+            }
+
+            static UserCodeID invalid()
+            {
+                return UserCodeID(count);
+            }
+
+            bool is_valid() const
+            {
+                return user_code_id < count;
+            }
+
+            bool is_invalid() const
+            {
+                return !is_valid();
+            }
+        }
+
         private struct State
         {
             uint transition_table_index;
             uint n_transitions;
-            uint token;
-            uint code_id;
+            Token token;
+            UserCodeID code_id;
+            bool drop;
+
+            bool accepts() const
+            {
+                return drop || token.is_valid() || code_id.is_valid();
+            }
         }
 
         private struct Mode
@@ -136,29 +206,47 @@ class <%= @classname %>
         }
 
 <% transition_table, state_table, mode_table = @lexer.build_tables %>
-        private static immutable Transition transitions[] = [
+        private static immutable Transition[] transitions = [
 <% transition_table.each do |transition_table_entry| %>
             Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
 <% end %>
         ];
 
-        private static immutable State states[] = [
+        private static immutable State[] states = [
 <% state_table.each do |state_table_entry| %>
             State(<%= state_table_entry[:transition_table_index] %>u,
                   <%= state_table_entry[:n_transitions] %>u,
-                  <%= state_table_entry[:token] %>u,
-                  <%= state_table_entry[:code_id] %>u),
+<%   if state_table_entry[:token] %>
+                  Token(<%= state_table_entry[:token] %>u),
+<%   else %>
+                  Token.invalid(),
+<%   end %>
+<%   if state_table_entry[:code_id] %>
+                  UserCodeID(<%= state_table_entry[:code_id] %>u),
+<%   else %>
+                  UserCodeID.invalid(),
+<%   end %>
+                  <%= state_table_entry[:drop] %>),
 <% end %>
         ];
 
-        private static immutable Mode modes[] = [
+        private static immutable Mode[] modes = [
 <% mode_table.each do |mode_table_entry| %>
             Mode(<%= mode_table_entry[:state_table_offset] %>),
 <% end %>
         ];
 
-        struct LexedToken
+        struct Result
         {
+            enum Type
+            {
+                DECODE_ERROR,
+                DROP,
+                TOKEN,
+                UNEXPECTED_INPUT,
+            }
+
+            Type type;
             size_t row;
             size_t col;
             size_t length;
@@ -178,14 +266,14 @@ class <%= @classname %>
             m_mode = <%= @lexer.mode_id("default") %>;
         }
 
-        LexedToken lex_token()
+        Result lex_token()
         {
             for (;;)
             {
-                LexedToken lt = attempt_lex_token();
-                if (lt.token < _TOKEN_COUNT)
+                Result result = attempt_lex_token();
+                if (result.token < _TOKEN_COUNT)
                 {
-                    return lt;
+                    return result;
                 }
             }
         }
@@ -195,12 +283,12 @@ class <%= @classname %>
          *
          * @param code_id The ID of the user code block to execute.
          * @param match Matched text for this pattern.
-         * @param lt LexedToken lexer result in progress.
+         * @param result Result lexer result in progress.
          *
-         * @return Token ID to accept, or _TOKEN_COUNT if the user code does
+         * @return Token to accept, or invalid token if the user code does
          *   not explicitly return a token.
          */
-        private uint user_code(uint code_id, string match, LexedToken * lt)
+        private Token user_code(UserCodeID code_id, string match, Result * result)
         {
             switch (code_id)
             {
@@ -214,107 +302,259 @@ class <%= @classname %>
             default: break;
             }
 
-            return _TOKEN_COUNT;
+            return Token.invalid();
         }
 
-        private LexedToken attempt_lex_token()
+        private Result attempt_lex_token()
         {
-            LexedToken lt;
-            lt.row = m_input_row;
-            lt.col = m_input_col;
-            lt.token = _TOKEN_COUNT;
-            struct MatchInfo
+            Result result;
+            result.row = m_input_row;
+            result.col = m_input_col;
+            result.token = _TOKEN_COUNT;
+            auto match_result = find_longest_match();
+            if (match_result.is_eof())
             {
-                size_t length;
-                size_t delta_row;
-                size_t delta_col;
-                uint token;
-                uint code_id;
+                result.type = Result.Type.TOKEN;
+                result.token = TOKEN___EOF;
+                return result;
             }
-            MatchInfo longest_match_info;
-            longest_match_info.token = _TOKEN_COUNT;
-            MatchInfo attempt_match_info;
+            else if (match_result.is_decode_error())
+            {
+                result.type = Result.Type.DECODE_ERROR;
+                return result;
+            }
+            else if (match_result.is_unexpected_input())
+            {
+                result.type = Result.Type.UNEXPECTED_INPUT;
+                return result;
+            }
+            uint token_to_accept = match_result.accepting_state.token;
+            if (match_result.accepting_state.code_id.is_valid())
+            {
+                Token user_code_token = user_code(match_result.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_result.length)], &result);
+                /* An invalid Token from user_code() means that the user
+                 * code did not explicitly return a token. So only override
+                 * the token to return if the user code does explicitly
+                 * return a token. */
+                if (user_code_token.is_valid())
+                {
+                    token_to_accept = user_code_token.token;
+                }
+            }
+
+            /* Update the input position tracking. */
+            m_input_position += match_result.length;
+            m_input_row += match_result.delta_row;
+            if (match_result.delta_row != 0u)
+            {
+                m_input_col = match_result.delta_col;
+            }
+            else
+            {
+                m_input_col += match_result.delta_col;
+            }
+            result.token = token_to_accept;
+            result.length = match_result.length;
+            if (match_result.accepting_state.drop)
+            {
+                result.type = Result.Type.DROP;
+            }
+            else
+            {
+                result.type = Result.Type.TOKEN;
+            }
+            return result;
+        }
+
+        /**
+         * Result type for find_longest_match().
+         *
+         * Alternatives:
+         * - decode_error
+         * - eof
+         * - found_match(length, delta_row, delta_col, accepting_state)
+         * - unexpected_input(unexpected_input_length)
+         */
+        struct FindLongestMatchResult
+        {
+            enum : ubyte
+            {
+                FOUND_MATCH,
+                DECODE_ERROR,
+                EOF,
+                UNEXPECTED_INPUT,
+            }
+
+            ubyte type;
+            alias type this;
+            union
+            {
+                struct
+                {
+                    size_t length;
+                    size_t delta_row;
+                    size_t delta_col;
+                    const(State) * accepting_state;
+                }
+                size_t unexpected_input_length;
+            }
+
+            this(ubyte type)
+            {
+                this.type = type;
+            }
+
+            this(ubyte type, size_t unexpected_input_length)
+            {
+                this.type = type;
+                this.unexpected_input_length = unexpected_input_length;
+            }
+
+            this(ubyte type, size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                this.type = type;
+                this.length = length;
+                this.delta_row = delta_row;
+                this.delta_col = delta_col;
+                this.accepting_state = accepting_state;
+            }
+
+            static FindLongestMatchResult found_match(size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                return FindLongestMatchResult(FOUND_MATCH, length, delta_row, delta_col, accepting_state);
+            }
+
+            static FindLongestMatchResult decode_error()
+            {
+                return FindLongestMatchResult(DECODE_ERROR);
+            }
+
+            static FindLongestMatchResult eof()
+            {
+                return FindLongestMatchResult(EOF);
+            }
+
+            static FindLongestMatchResult unexpected_input(size_t unexpected_input_length)
+            {
+                return FindLongestMatchResult(UNEXPECTED_INPUT, unexpected_input_length);
+            }
+
+            bool is_found_match()
+            {
+                return type == FOUND_MATCH;
+            }
+
+            bool is_decode_error()
+            {
+                return type == DECODE_ERROR;
+            }
+
+            bool is_eof()
+            {
+                return type == EOF;
+            }
+
+            bool is_unexpected_input()
+            {
+                return type == UNEXPECTED_INPUT;
+            }
+        }
+
+        private FindLongestMatchResult find_longest_match()
+        {
+            FindLongestMatchResult longest_match = FindLongestMatchResult.found_match(0, 0, 0, null);
+            FindLongestMatchResult attempt_match = longest_match;
             uint current_state = modes[m_mode].state_table_offset;
             for (;;)
             {
-                auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match_info.length)..(m_input.length)]);
-                if (decoded.code_point == Decoder.CODE_POINT_INVALID)
-                {
-                    lt.token = _TOKEN_DECODE_ERROR;
-                    return lt;
-                }
                 bool lex_continue = false;
-                if (decoded.code_point != Decoder.CODE_POINT_EOF)
-                {
-                    uint dest = transition(current_state, decoded.code_point);
-                    if (dest != cast(uint)-1)
-                    {
-                        lex_continue = true;
-                        attempt_match_info.length += decoded.code_point_length;
-                        if (decoded.code_point == '\n')
-                        {
-                            attempt_match_info.delta_row++;
-                            attempt_match_info.delta_col = 0u;
-                        }
-                        else
-                        {
-                            attempt_match_info.delta_col++;
-                        }
-                        current_state = dest;
-                        if ((states[current_state].token != _TOKEN_COUNT) ||
-                            (states[current_state].code_id != 0xFFFF_FFFFu))
-                        {
-                            attempt_match_info.token = states[current_state].token;
-                            attempt_match_info.code_id = states[current_state].code_id;
-                            longest_match_info = attempt_match_info;
-                        }
-                    }
-                }
-                else if (attempt_match_info.length == 0u)
-                {
-                    lt.token = TOKEN_0EOF;
-                    break;
-                }
-                if (!lex_continue)
-                {
-                    bool pattern_accepted = false;
-                    uint token_to_accept = longest_match_info.token;
-                    if (longest_match_info.code_id != 0xFFFF_FFFFu)
-                    {
-                        uint user_code_token = user_code(longest_match_info.code_id, m_input[m_input_position..(m_input_position + longest_match_info.length)], &lt);
-                        /* A return of _TOKEN_COUNT from user_code() means
-                         * that the user code did not explicitly return a
-                         * token. So only override the token to return if the
-                         * user code does explicitly return a token. */
-                        if (user_code_token != _TOKEN_COUNT)
-                        {
-                            token_to_accept = user_code_token;
-                        }
-                        pattern_accepted = true;
-                    }
-                    if (pattern_accepted || (token_to_accept != _TOKEN_COUNT))
-                    {
-                        /* Update the input position tracking. */
-                        m_input_position += longest_match_info.length;
-                        m_input_row += longest_match_info.delta_row;
-                        if (longest_match_info.delta_row != 0u)
-                        {
-                            m_input_col = longest_match_info.delta_col;
-                        }
-                        else
-                        {
-                            m_input_col += longest_match_info.delta_col;
-                        }
-                        lt.token = token_to_accept;
-                        lt.length = longest_match_info.length;
-                        break;
-                    }
-                }
+                Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]).match!(
+                        (Decoder.DecodeError de) {return FindLongestMatchResult.decode_error();},
+                        (Decoder.EOF e) {
+                            /* We hit EOF. */
+                            if (longest_match.length > 0)
+                            {
+                                /* We have a match, so use it. */
+                                return longest_match;
+                            }
+                            else if (attempt_match.length != 0)
+                            {
+                                /* There is a partial match - error! */
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length);
+                            }
+                            else
+                            {
+                                /* Valid EOF return. */
+                                return FindLongestMatchResult.eof();
+                            }
+                        },
+                        (Decoder.Success decoded) {
+                            auto transition_result = transition(current_state, decoded.code_point);
+                            if (transition_result.found())
+                            {
+                                lex_continue = true;
+                                attempt_match.length += decoded.code_point_length;
+                                if (decoded.code_point == '\n')
+                                {
+                                    attempt_match.delta_row++;
+                                    attempt_match.delta_col = 0u;
+                                }
+                                else
+                                {
+                                    attempt_match.delta_col++;
+                                }
+                                current_state = transition_result.destination();
+                                if (states[current_state].accepts())
+                                {
+                                    attempt_match.accepting_state = &states[current_state];
+                                    longest_match = attempt_match;
+                                }
+                            }
+                            else if (longest_match.length > 0)
+                            {
+                                return longest_match;
+                            }
+                            else
+                            {
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length);
+                            }
+                        });
             }
-            return lt;
         }
 
-        private uint transition(uint current_state, uint code_point)
+        /**
+         * Result type for transition().
+         *
+         * Alternatives:
+         * - found(destination)
+         * - not_found
+         */
+        struct TransitionResult
+        {
+            private uint m_destination;
+
+            static TransitionResult found(uint destination)
+            {
+                return TransitionResult(destination);
+            }
+
+            static TransitionResult not_found()
+            {
+                return TransitionResult(cast(uint)-1);
+            }
+
+            bool found()
+            {
+                return m_destination != cast(uint)-1;
+            }
+
+            @property uint destination()
+            {
+                return m_destination;
+            }
+        }
+
+        private TransitionResult transition(uint current_state, uint code_point)
         {
             uint transition_table_index = states[current_state].transition_table_index;
             for (uint i = 0u; i < states[current_state].n_transitions; i++)
@@ -322,10 +562,11 @@ class <%= @classname %>
                 if ((transitions[transition_table_index + i].first <= code_point) &&
                     (code_point <= transitions[transition_table_index + i].last))
                 {
-                    return transitions[transition_table_index + i].destination;
+                    uint destination = transitions[transition_table_index + i].destination;
+                    return TransitionResult.found(destination);
                 }
             }
-            return cast(uint)-1;
+            return TransitionResult.not_found();
         }
     }
 
@@ -365,19 +606,19 @@ class <%= @classname %>
         }
 
 <% state_table, shift_table, reduce_table = @parser.build_tables %>
-        private static immutable Shift shifts[] = [
+        private static immutable Shift[] shifts = [
 <%   shift_table.each do |shift| %>
             Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u),
 <%   end %>
         ];
 
-        private static immutable Reduce reduces[] = [
+        private static immutable Reduce[] reduces = [
 <%   reduce_table.each do |reduce| %>
             Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
 <%   end %>
         ];
 
-        private static immutable State states[] = [
+        private static immutable State[] states = [
 <%   state_table.each do |state| %>
             State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
 <%   end %>
@@ -394,7 +635,7 @@ class <%= @classname %>
 
         bool parse()
         {
-            Lexer.LexedToken lexed_token;
+            Lexer.Result lexed_token;
             uint token = _TOKEN_COUNT;
             StateValue[] statevalues = new StateValue[](1);
             uint reduced_rule_set = 0xFFFFFFFFu;
@@ -414,7 +655,7 @@ class <%= @classname %>
                 if (shift_state == 0xFFFFFFFFu)
                 {
                     shift_state = check_shift(statevalues[$-1].state, token);
-                    if ((shift_state != 0xFFFFFFFFu) && (token == TOKEN_0EOF))
+                    if ((shift_state != 0xFFFFFFFFu) && (token == TOKEN___EOF))
                     {
                         /* Successful parse. */
                         parse_result = statevalues[$-1].pvalue;

doc/user_guide.md

@@ -1,3 +1,10 @@
+${remove}
+WARNING: This user guide is meant to be preprocessed and rendered by a custom
+script.
+The markdown source file is not intended to be viewed directly and will not
+include all intended content.
+${/remove}
+
 #> Overview
 
 Propane is an LR Parser Generator (LPG) which:
@@ -8,12 +15,62 @@ Propane is an LR Parser Generator (LPG) which:
   * generates a table-driven parser to parse input in linear time
   * is MIT-licensed
   * is distributable as a standalone Ruby script
+  * supports D language
 
-${remove}
-WARNING: This user guide is meant to be preprocessed and rendered by a custom
-script.
-The markdown source file is not intended to be viewed directly and will not
-include all intended content.
-Visit [https://holtrop.github.io/rscons/index.html](https://holtrop.github.io/rscons/index.html)
-to view the properly rendered version.
-${/remove}
+#> Installation
+
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
+
+#> Command Line Usage
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+#> Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+#> License
+
+Propane is licensed under the terms of the MIT License:
+
+```
+${include LICENSE.txt}
+```
+
+#> Contributing
+
+Propane is developed on [github](https://github.com/holtrop/propane).
+
+Issues may be submitted to [https://github.com/holtrop/propane/issues](https://github.com/holtrop/propane/issues).
+
+Pull requests may be submitted as well:
+
+  1. Fork it
+  2. Create your feature branch (`git checkout -b my-new-feature`)
+  3. Commit your changes (`git commit -am 'Add some feature'`)
+  4. Push to the branch (`git push origin my-new-feature`)
+  5. Create new Pull Request
+
+#> Change Log
+
+${changelog}

lib/propane.rb

@@ -25,12 +25,6 @@ require_relative "propane/version"
 
 class Propane
 
-  # Decoding error.
-  TOKEN_DECODE_ERROR = 0xFFFFFFFD
-
-  # Token ID for a "dropped" token.
-  TOKEN_DROP = 0xFFFFFFFE
-
   class Error < RuntimeError
   end
 

lib/propane/generator.rb

@@ -183,7 +183,7 @@ class Propane
     #   Expanded user code block.
     def expand_code(code, parser, rule, pattern)
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
-        "TOKEN_#{Token.code_name($1)}"
+        "Token(TOKEN_#{Token.code_name($1)})"
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
@@ -195,7 +195,7 @@ class Propane
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          "lt.pvalue.v_#{pattern.ptypename}"
+          "result.pvalue.v_#{pattern.ptypename}"
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1

lib/propane/grammar.rb

@@ -2,7 +2,8 @@ class Propane
 
   class Grammar
 
-    IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z_0-9]*/
+    # Reserve identifiers beginning with a double-underscore for internal use.
+    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :classname
     attr_reader :modulename

lib/propane/lexer.rb

@@ -24,25 +24,19 @@ class Propane
         }
         states = mode_info[:dfa].enumerate
         states.each do |state, id|
+          drop = state.accepts && state.accepts.drop?
           token =
-            if state.accepts.nil?
-              @grammar.tokens.size
-            elsif state.accepts.drop?
-              TOKEN_DROP
-            elsif state.accepts.token
+            if state.accepts && state.accepts.token
               state.accepts.token.id
-            else
-              @grammar.tokens.size
             end
           code_id =
             if state.accepts && state.accepts.code_id
               state.accepts.code_id
-            else
-              0xFFFF_FFFF
             end
           state_table << {
             transition_table_index: transition_table.size,
             n_transitions: state.transitions.size,
+            drop: drop,
             token: token,
             code_id: code_id,
           }

lib/propane/token.rb

@@ -9,7 +9,7 @@ class Propane
       # @return [String]
       #   Name of the token to use in code (special characters replaced).
       def code_name(name)
-        name.sub(/^\$/, "0")
+        name.sub(/^\$/, "__")
       end
 
     end

rb/build_dist.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require "fileutils"
+require "digest/md5"
+
+if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
+  VERSION = $1
+else
+  raise "Could not determine version."
+end
+PROG_NAME = "propane"
+START_FILE = "bin/#{PROG_NAME}"
+LIB_DIR = "lib"
+DIST = "dist"
+
+files_processed = {}
+combined_file = []
+
+combine_files = lambda do |file|
+  File.read(file, mode: "rb").each_line do |line|
+    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
+      require_name = $1
+      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
+        path = "#{LIB_DIR}/#{require_name}.rb"
+        if File.exist?(path)
+          unless files_processed[path]
+            files_processed[path] = true
+            combine_files[path]
+          end
+        else
+          raise "require path #{path.inspect} not found"
+        end
+      else
+        combined_file << line
+      end
+    else
+      combined_file << line
+    end
+  end
+end
+
+combine_files[START_FILE]
+
+# Strip Ruby comment lines and empty lines to save some space, but do not
+# remove lines that are in heredoc sections. This isn't terribly robust to be
+# used in the wild, but works for the heredoc instances for this project.
+stripped = []
+heredoc_end = nil
+combined_file.each do |line|
+  if line =~ /<<-?([A-Z]+)/
+    heredoc_end = $1
+  end
+  if heredoc_end and line =~ /^\s*#{heredoc_end}/
+    heredoc_end = nil
+  end
+  if line !~ /#\sspecs/
+    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
+      stripped << line
+    end
+  end
+end
+
+license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
+  if line.size > 0
+    "# #{line}"
+  else
+    "#"
+  end
+end
+
+require "zlib"
+require "base64"
+compressed_script = Zlib::Deflate.deflate(stripped.join)
+encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
+hash = Digest::MD5.hexdigest(encoded_compressed_script)
+
+FileUtils.rm_rf(DIST)
+FileUtils.mkdir_p(DIST)
+File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
+  fh.write(<<EOF)
+#!/usr/bin/env ruby
+
+#{license}
+
+script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
+unless File.exist?(script)
+  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
+    require "zlib"
+    require "base64"
+    encoded_compressed = $1
+    unescaped_compressed = Base64.decode64(encoded_compressed)
+    inflated = Zlib::Inflate.inflate(unescaped_compressed)
+    File.open(script, "wb") do |fh|
+      fh.write(inflated)
+    end
+  else
+    raise "Could not decompress."
+  end
+end
+load script
+if __FILE__ == $0
+  Propane::CLI.new.run(ARGV)
+end
+#==>#{encoded_compressed_script}
+EOF
+end

spec/json_parser.propane

@@ -0,0 +1,179 @@
+<<
+import std.math;
+import json_types;
+private string string_value;
+>>
+
+ptype JSONValue;
+ptype array = JSONValue[];
+ptype dict = JSONValue[string];
+ptype string = string;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n;
+    bool negative;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = new JSONNumber(n);
+>>
+token true <<
+  $$ = new JSONTrue();
+>>
+token false <<
+  $$ = new JSONFalse();
+>>
+token null <<
+  $$ = new JSONNull();
+>>
+/"/ <<
+  $mode(string);
+  string_value = "";
+>>
+string: token string (string) /"/ <<
+  $$ = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  string_value ~= "\"";
+>>
+string: /\\\\/ <<
+  string_value ~= "\\";
+>>
+string: /\\\// <<
+  string_value ~= "/";
+>>
+string: /\\b/ <<
+  string_value ~= "\b";
+>>
+string: /\\f/ <<
+  string_value ~= "\f";
+>>
+string: /\\n/ <<
+  string_value ~= "\n";
+>>
+string: /\\r/ <<
+  string_value ~= "\r";
+>>
+string: /\\t/ <<
+  string_value ~= "\t";
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  string_value ~= "{" ~ match[2..6] ~ "}";
+>>
+string: /[^\\]/ <<
+  string_value ~= match;
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = new JSONString($1);
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = new JSONObject();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = new JSONObject($2);
+>>
+KeyValues (dict) -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  foreach (key, value; $3)
+  {
+    $1[key] = value;
+  }
+  $$ = $1;
+>>
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+Array -> lbracket rbracket <<
+  $$ = new JSONArray();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = new JSONArray($2);
+>>
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>

spec/propane_spec.rb

@@ -20,7 +20,7 @@ describe Propane do
   end
 
   def compile(*test_files)
-    result = system(*%w[gdc -funittest -o spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
+    result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
     expect(result).to be_truthy
   end
 
@@ -28,6 +28,7 @@ describe Propane do
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
+    stderr.sub!(/^.*modules passed unittests\n/, "")
     Results.new(stdout, stderr, status)
   end
 
@@ -336,187 +337,7 @@ EOF
   end
 
   it "allows creating a JSON parser" do
-    write_grammar <<EOF
-<<
-    import std.math;
-    import json_types;
-    string string_value;
->>
-
-ptype JSONValue;
-ptype array = JSONValue[];
-ptype dict = JSONValue[string];
-ptype string = string;
-
-drop /\\s+/;
-token lbrace /\\{/;
-token rbrace /\\}/;
-token lbracket /\\[/;
-token rbracket /\\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n;
-    bool negative;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = new JSONNumber(n);
->>
-token true <<
-  $$ = new JSONTrue();
->>
-token false <<
-  $$ = new JSONFalse();
->>
-token null <<
-  $$ = new JSONNull();
->>
-/"/ <<
-  $mode(string);
-  string_value = "";
->>
-string: token string (string) /"/ <<
-  $$ = string_value;
-  $mode(default);
->>
-string: /\\\\"/ <<
-  string_value ~= "\\"";
->>
-string: /\\\\\\\\/ <<
-  string_value ~= "\\\\";
->>
-string: /\\\\\\// <<
-  string_value ~= "/";
->>
-string: /\\\\b/ <<
-  string_value ~= "\\b";
->>
-string: /\\\\f/ <<
-  string_value ~= "\\f";
->>
-string: /\\\\n/ <<
-  string_value ~= "\\n";
->>
-string: /\\\\r/ <<
-  string_value ~= "\\r";
->>
-string: /\\\\t/ <<
-  string_value ~= "\\t";
->>
-string: /\\\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
->>
-string: /[^\\\\]/ <<
-  string_value ~= match;
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = new JSONString($1);
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = new JSONObject();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = new JSONObject($2);
->>
-KeyValues (dict) -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  foreach (key, value; $3)
-  {
-    $1[key] = value;
-  }
-  $$ = $1;
->>
-KeyValue (dict) -> string colon Value <<
-  $$ = [$1: $3];
->>
-Array -> lbracket rbracket <<
-  $$ = new JSONArray();
->>
-Array -> lbracket Values rbracket <<
-  $$ = new JSONArray($2);
->>
-Values (array) -> Value <<
-  $$ = [$1];
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>
-EOF
+    write_grammar(File.read("spec/json_parser.propane"))
     build_parser
     compile("spec/test_parsing_json.d", "spec/json_types.d")
   end

spec/test_d_lexer.d

@@ -8,45 +8,45 @@ int main()
 
 unittest
 {
-    alias DCP = Testparser.Decoder.DecodedCodePoint;
-    DCP dcp;
+    alias Result = Testparser.Decoder.Result;
+    Result result;
 
-    dcp = Testparser.Decoder.decode_code_point("5");
-    assert(dcp == DCP('5', 1u));
+    result = Testparser.Decoder.decode_code_point("5");
+    assert(result == Result.success('5', 1u));
 
-    dcp = Testparser.Decoder.decode_code_point("");
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
+    result = Testparser.Decoder.decode_code_point("");
+    assert(result == Result.eof());
 
-    dcp = Testparser.Decoder.decode_code_point("\xC2\xA9");
-    assert(dcp == DCP(0xA9u, 2u));
+    result = Testparser.Decoder.decode_code_point("\xC2\xA9");
+    assert(result == Result.success(0xA9u, 2u));
 
-    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
-    assert(dcp == DCP(0x1F9E1, 4u));
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
+    assert(result == Result.success(0x1F9E1, 4u));
 
-    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
+    assert(result == Result.decode_error());
 
-    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
+    assert(result == Result.decode_error());
 
-    dcp = Testparser.Decoder.decode_code_point("\xfe");
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("\xfe");
+    assert(result == Result.decode_error());
 }
 
 unittest
 {
-    alias LT = Testparser.Lexer.LexedToken;
+    alias Result = Testparser.Lexer.Result;
     string input = "5 + 4 * \n677 + 567";
     Testparser.Lexer lexer = new Testparser.Lexer(input);
-    assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_plus));
-    assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_times));
-    assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_plus));
-    assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_0EOF));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 2, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 4, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 6, 1, Testparser.TOKEN_times));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 0, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 4, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 6, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 9, 0, Testparser.TOKEN___EOF));
 
     lexer = new Testparser.Lexer("");
-    assert(lexer.lex_token() == LT(0, 0, 0, Testparser.TOKEN_0EOF));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 0, Testparser.TOKEN___EOF));
 }