Convert Decoder.decode_code_point return type to a SumType

Also fix a couple UTF-8 decoder bugs!

.gitignore

@@ -2,7 +2,8 @@
 /.yardoc
 /_yardoc/
 /coverage/
-/doc/
+/dist/
+/gen/
 /pkg/
 /spec/reports/
 /tmp/

CHANGELOG.md

@@ -0,0 +1,3 @@
+## v1.0.0
+
+- Initial release

Gemfile

@@ -2,3 +2,6 @@ source "https://rubygems.org"
 
 gem "rake"
 gem "rspec"
+gem "rdoc"
+gem "redcarpet"
+gem "syntax"

Gemfile.lock

@@ -2,7 +2,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
+    psych (5.0.1)
+      stringio
     rake (13.0.6)
+    rdoc (6.5.0)
+      psych (>= 4.0.0)
+    redcarpet (3.5.1)
     rspec (3.11.0)
       rspec-core (~> 3.11.0)
       rspec-expectations (~> 3.11.0)
@@ -16,13 +21,18 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.11.0)
     rspec-support (3.11.0)
+    stringio (3.0.4)
+    syntax (1.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   rake
+  rdoc
+  redcarpet
   rspec
+  syntax
 
 BUNDLED WITH
-   2.4.0.dev
+   2.3.7

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2022 Josh Holtrop
+Copyright (c) 2010-2023 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Rakefile

@@ -1,5 +1,9 @@
 require "rspec/core/rake_task"
 
+task :build_dist do
+  sh "ruby rb/build_dist.rb"
+end
+
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
@@ -7,3 +11,8 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
 end
 
 task :default => :spec
+
+desc "Build user guide"
+task :user_guide do
+  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
+end

assets/parser.d.erb

@@ -2,51 +2,95 @@
 module <%= @grammar.modulename %>;
 
 <% end %>
-
 import std.stdio;
+import std.sumtype;
+
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 class <%= @classname %>
 {
-    enum
+    alias TokenID = uint;
+
+    enum : TokenID
     {
 <% @grammar.tokens.each_with_index do |token, index| %>
-        TOKEN_<%= token.name %> = <%= index %>,
+        TOKEN_<%= token.code_name %> = <%= index %>,
+<%   unless token.id == index %>
+<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
+<%   end %>
 <% end %>
         _TOKEN_COUNT = <%= @grammar.tokens.size %>,
-        _TOKEN_EOF = <%= TOKEN_EOF %>,
-        _TOKEN_DECODE_ERROR = <%= TOKEN_DECODE_ERROR %>,
-        _TOKEN_DROP = <%= TOKEN_DROP %>,
-        _TOKEN_NONE = <%= TOKEN_NONE %>,
     }
 
-    static immutable string token_names[] = [
+    struct Token
+    {
+        /* Number of tokens in this parser. */
+        enum count = <%= @grammar.tokens.size %>;
+        TokenID token;
+        alias token this;
+
+        @disable this();
+
+        this(TokenID token)
+        {
+            this.token = token;
+        }
+
+        static Token invalid()
+        {
+            return Token(count);
+        }
+
+        bool is_valid() const
+        {
+            return token < count;
+        }
+
+        bool is_invalid() const
+        {
+            return !is_valid();
+        }
+    }
+
+    alias CodePoint = uint;
+
+    static immutable string[] token_names = [
 <% @grammar.tokens.each_with_index do |token, index| %>
         "<%= token.name %>",
 <% end %>
     ];
 
+    static union ParserValue
+    {
+<% @grammar.ptypes.each do |name, typestring| %>
+        <%= typestring %> v_<%= name %>;
+<% end %>
+    }
+
     static class Decoder
     {
-        enum
+        struct Success
         {
-            CODE_POINT_INVALID = 0xFFFFFFFE,
-            CODE_POINT_EOF = 0xFFFFFFFF,
-        }
-
-        struct DecodedCodePoint
-        {
-            uint code_point;
+            CodePoint code_point;
             uint code_point_length;
         }
+        struct EOF {}
+        struct DecodeError {}
 
-        static DecodedCodePoint decode_code_point(const(ubyte) * input, size_t input_length)
+        static SumType!(
+                Success,
+                EOF,
+                DecodeError)
+            decode_code_point(string input)
         {
-            if (input_length == 0u)
+            if (input.length == 0u)
             {
-                return DecodedCodePoint(CODE_POINT_EOF, 0u);
+                return cast(ReturnType!decode_code_point)EOF();
             }
-            ubyte c = *input;
-            uint code_point;
+            char c = input[0];
+            CodePoint code_point;
             uint code_point_length;
             if ((c & 0x80u) == 0u)
             {
@@ -83,25 +127,24 @@ class <%= @classname %>
                 }
                 else
                 {
-                    return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
-                if (input_length <= following_bytes)
+                if (input.length <= following_bytes)
                 {
-                    return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                    return cast(ReturnType!decode_code_point)DecodeError();
                 }
                 code_point_length = following_bytes + 1u;
-                while (following_bytes-- > 0u)
+                for (size_t i = 0u; i < following_bytes; i++)
                 {
-                    input++;
-                    ubyte b = *input;
-                    if ((b & 0xC0u) != 0u)
+                    char b = input[i + 1u];
+                    if ((b & 0xC0u) != 0x80u)
                     {
-                        return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                        return cast(ReturnType!decode_code_point)DecodeError();
                     }
-                    code_point = (code_point << 6u) | b;
+                    code_point = (code_point << 6u) | (b & 0x3Fu);
                 }
             }
-            return DecodedCodePoint(code_point, code_point_length);
+            return cast(ReturnType!decode_code_point)Success(code_point, code_point_length);
         }
     }
 
@@ -109,63 +152,128 @@ class <%= @classname %>
     {
         private struct Transition
         {
-            uint first;
-            uint last;
+            CodePoint first;
+            CodePoint last;
             uint destination;
         }
 
+        private struct UserCodeID
+        {
+            enum count = <%= (@grammar.patterns.map(&:code_id).compact.max || -1) + 1%>;
+            uint user_code_id;
+            alias user_code_id this;
+
+            @disable this();
+
+            this(uint user_code_id)
+            {
+                this.user_code_id = user_code_id;
+            }
+
+            static UserCodeID invalid()
+            {
+                return UserCodeID(count);
+            }
+
+            bool is_valid() const
+            {
+                return user_code_id < count;
+            }
+
+            bool is_invalid() const
+            {
+                return !is_valid();
+            }
+        }
+
         private struct State
         {
             uint transition_table_index;
             uint n_transitions;
-            uint token;
-            uint code_id;
+            Token token;
+            UserCodeID code_id;
+            bool drop;
+
+            bool accepts() const
+            {
+                return drop || token.is_valid() || code_id.is_valid();
+            }
         }
 
-<% transition_table, state_table = @lexer.build_tables %>
-        private static immutable Transition transitions[] = [
+        private struct Mode
+        {
+            uint state_table_offset;
+        }
+
+<% transition_table, state_table, mode_table = @lexer.build_tables %>
+        private static immutable Transition[] transitions = [
 <% transition_table.each do |transition_table_entry| %>
             Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
 <% end %>
         ];
 
-        private static const State states[] = [
+        private static immutable State[] states = [
 <% state_table.each do |state_table_entry| %>
             State(<%= state_table_entry[:transition_table_index] %>u,
                   <%= state_table_entry[:n_transitions] %>u,
-                  <%= state_table_entry[:token] %>u,
-                  <%= state_table_entry[:code_id] %>u),
+<%   if state_table_entry[:token] %>
+                  Token(<%= state_table_entry[:token] %>u),
+<%   else %>
+                  Token.invalid(),
+<%   end %>
+<%   if state_table_entry[:code_id] %>
+                  UserCodeID(<%= state_table_entry[:code_id] %>u),
+<%   else %>
+                  UserCodeID.invalid(),
+<%   end %>
+                  <%= state_table_entry[:drop] %>),
 <% end %>
         ];
 
-        struct LexedToken
+        private static immutable Mode[] modes = [
+<% mode_table.each do |mode_table_entry| %>
+            Mode(<%= mode_table_entry[:state_table_offset] %>),
+<% end %>
+        ];
+
+        struct Result
         {
+            enum Type
+            {
+                DECODE_ERROR,
+                DROP,
+                TOKEN,
+                UNEXPECTED_INPUT,
+            }
+
+            Type type;
             size_t row;
             size_t col;
             size_t length;
             uint token;
+            ParserValue pvalue;
         }
 
-        private const(ubyte) * m_input;
-        private size_t m_input_length;
+        private string m_input;
         private size_t m_input_position;
         private size_t m_input_row;
         private size_t m_input_col;
+        private size_t m_mode;
 
-        this(const(ubyte) * input, size_t input_length)
+        this(string input)
         {
             m_input = input;
-            m_input_length = input_length;
+            m_mode = <%= @lexer.mode_id("default") %>;
         }
 
-        LexedToken lex_token()
+        Result lex_token()
         {
             for (;;)
             {
-                LexedToken lt = attempt_lex_token();
-                if (lt.token != _TOKEN_DROP)
+                Result result = attempt_lex_token();
+                if (result.token < _TOKEN_COUNT)
                 {
-                    return lt;
+                    return result;
                 }
             }
         }
@@ -174,125 +282,279 @@ class <%= @classname %>
          * Execute user code associated with a lexer pattern.
          *
          * @param code_id The ID of the user code block to execute.
+         * @param match Matched text for this pattern.
+         * @param result Result lexer result in progress.
          *
-         * @return Token ID to accept, or _TOKEN_NONE if the user code does
+         * @return Token to accept, or invalid token if the user code does
          *   not explicitly return a token.
          */
-        private uint user_code(uint code_id)
+        private Token user_code(UserCodeID code_id, string match, Result * result)
         {
             switch (code_id)
             {
 <% @grammar.patterns.each do |pattern| %>
 <%   if pattern.code_id %>
             case <%= pattern.code_id %>u: {
-<%= pattern.code %>
+<%= expand_code(pattern.code, false, nil, pattern) %>
             } break;
 <%   end %>
 <% end %>
             default: break;
             }
 
-            return _TOKEN_NONE;
+            return Token.invalid();
         }
 
-        private LexedToken attempt_lex_token()
+        private Result attempt_lex_token()
         {
-            LexedToken lt = LexedToken(m_input_row, m_input_col, 0, _TOKEN_NONE);
-            struct MatchInfo
+            Result result;
+            result.row = m_input_row;
+            result.col = m_input_col;
+            result.token = _TOKEN_COUNT;
+            auto match_result = find_longest_match();
+            if (match_result.is_eof())
             {
-                size_t length;
-                size_t delta_row;
-                size_t delta_col;
-                uint token;
-                uint code_id;
+                result.type = Result.Type.TOKEN;
+                result.token = TOKEN___EOF;
+                return result;
             }
-            MatchInfo longest_match_info;
-            longest_match_info.token = _TOKEN_NONE;
-            MatchInfo attempt_match_info;
-            uint current_state;
+            else if (match_result.is_decode_error())
+            {
+                result.type = Result.Type.DECODE_ERROR;
+                return result;
+            }
+            else if (match_result.is_unexpected_input())
+            {
+                result.type = Result.Type.UNEXPECTED_INPUT;
+                return result;
+            }
+            uint token_to_accept = match_result.accepting_state.token;
+            if (match_result.accepting_state.code_id.is_valid())
+            {
+                Token user_code_token = user_code(match_result.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_result.length)], &result);
+                /* An invalid Token from user_code() means that the user
+                 * code did not explicitly return a token. So only override
+                 * the token to return if the user code does explicitly
+                 * return a token. */
+                if (user_code_token.is_valid())
+                {
+                    token_to_accept = user_code_token.token;
+                }
+            }
+
+            /* Update the input position tracking. */
+            m_input_position += match_result.length;
+            m_input_row += match_result.delta_row;
+            if (match_result.delta_row != 0u)
+            {
+                m_input_col = match_result.delta_col;
+            }
+            else
+            {
+                m_input_col += match_result.delta_col;
+            }
+            result.token = token_to_accept;
+            result.length = match_result.length;
+            if (match_result.accepting_state.drop)
+            {
+                result.type = Result.Type.DROP;
+            }
+            else
+            {
+                result.type = Result.Type.TOKEN;
+            }
+            return result;
+        }
+
+        /**
+         * Result type for find_longest_match().
+         *
+         * Alternatives:
+         * - decode_error
+         * - eof
+         * - found_match(length, delta_row, delta_col, accepting_state)
+         * - unexpected_input(unexpected_input_length)
+         */
+        struct FindLongestMatchResult
+        {
+            enum : ubyte
+            {
+                FOUND_MATCH,
+                DECODE_ERROR,
+                EOF,
+                UNEXPECTED_INPUT,
+            }
+
+            ubyte type;
+            alias type this;
+            union
+            {
+                struct
+                {
+                    size_t length;
+                    size_t delta_row;
+                    size_t delta_col;
+                    const(State) * accepting_state;
+                }
+                size_t unexpected_input_length;
+            }
+
+            this(ubyte type)
+            {
+                this.type = type;
+            }
+
+            this(ubyte type, size_t unexpected_input_length)
+            {
+                this.type = type;
+                this.unexpected_input_length = unexpected_input_length;
+            }
+
+            this(ubyte type, size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                this.type = type;
+                this.length = length;
+                this.delta_row = delta_row;
+                this.delta_col = delta_col;
+                this.accepting_state = accepting_state;
+            }
+
+            static FindLongestMatchResult found_match(size_t length, size_t delta_row, size_t delta_col, const(State) * accepting_state)
+            {
+                return FindLongestMatchResult(FOUND_MATCH, length, delta_row, delta_col, accepting_state);
+            }
+
+            static FindLongestMatchResult decode_error()
+            {
+                return FindLongestMatchResult(DECODE_ERROR);
+            }
+
+            static FindLongestMatchResult eof()
+            {
+                return FindLongestMatchResult(EOF);
+            }
+
+            static FindLongestMatchResult unexpected_input(size_t unexpected_input_length)
+            {
+                return FindLongestMatchResult(UNEXPECTED_INPUT, unexpected_input_length);
+            }
+
+            bool is_found_match()
+            {
+                return type == FOUND_MATCH;
+            }
+
+            bool is_decode_error()
+            {
+                return type == DECODE_ERROR;
+            }
+
+            bool is_eof()
+            {
+                return type == EOF;
+            }
+
+            bool is_unexpected_input()
+            {
+                return type == UNEXPECTED_INPUT;
+            }
+        }
+
+        private FindLongestMatchResult find_longest_match()
+        {
+            FindLongestMatchResult longest_match = FindLongestMatchResult.found_match(0, 0, 0, null);
+            FindLongestMatchResult attempt_match = longest_match;
+            uint current_state = modes[m_mode].state_table_offset;
             for (;;)
             {
-                auto decoded = Decoder.decode_code_point(&m_input[m_input_position + attempt_match_info.length], m_input_length - m_input_position - attempt_match_info.length);
-                if (decoded.code_point == Decoder.CODE_POINT_INVALID)
-                {
-                    lt.token = _TOKEN_DECODE_ERROR;
-                    return lt;
-                }
                 bool lex_continue = false;
-                if (decoded.code_point != Decoder.CODE_POINT_EOF)
-                {
-                    uint dest = transition(current_state, decoded.code_point);
-                    if (dest != cast(uint)-1)
-                    {
-                        lex_continue = true;
-                        attempt_match_info.length += decoded.code_point_length;
-                        if (decoded.code_point == '\n')
-                        {
-                            attempt_match_info.delta_row++;
-                            attempt_match_info.delta_col = 0u;
-                        }
-                        else
-                        {
-                            attempt_match_info.delta_col++;
-                        }
-                        current_state = dest;
-                        if (states[current_state].token != _TOKEN_NONE)
-                        {
-                            attempt_match_info.token = states[current_state].token;
-                            attempt_match_info.code_id = states[current_state].code_id;
-                            longest_match_info = attempt_match_info;
-                        }
-                    }
-                }
-                else if (attempt_match_info.length == 0u)
-                {
-                    lt.token = _TOKEN_EOF;
-                    break;
-                }
-                if (!lex_continue)
-                {
-                    bool pattern_accepted = false;
-                    uint token_to_accept = longest_match_info.token;
-                    if (longest_match_info.code_id != 0xFFFF_FFFFu)
-                    {
-                        uint user_code_token = user_code(longest_match_info.code_id);
-                        /* A return of _TOKEN_NONE from user_code() means
-                         * that the user code did not explicitly return a
-                         * token. So only override the token to return if the
-                         * user code does explicitly return a token. */
-                        if (user_code_token != _TOKEN_NONE)
-                        {
-                            token_to_accept = user_code_token;
-                        }
-                        pattern_accepted = true;
-                    }
-                    if (pattern_accepted || (token_to_accept != _TOKEN_NONE))
-                    {
-                        /* Update the input position tracking. */
-                        m_input_position += longest_match_info.length;
-                        m_input_row += longest_match_info.delta_row;
-                        if (longest_match_info.delta_row != 0u)
-                        {
-                            m_input_col = longest_match_info.delta_col;
-                        }
-                        else
-                        {
-                            m_input_col += longest_match_info.delta_col;
-                        }
-                    }
-                    if (token_to_accept != _TOKEN_NONE)
-                    {
-                        /* We have a token to accept. */
-                        lt.token = longest_match_info.token;
-                        lt.length = longest_match_info.length;
-                        break;
-                    }
-                }
+                Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]).match!(
+                        (Decoder.DecodeError de) {return FindLongestMatchResult.decode_error();},
+                        (Decoder.EOF e) {
+                            /* We hit EOF. */
+                            if (longest_match.length > 0)
+                            {
+                                /* We have a match, so use it. */
+                                return longest_match;
+                            }
+                            else if (attempt_match.length != 0)
+                            {
+                                /* There is a partial match - error! */
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length);
+                            }
+                            else
+                            {
+                                /* Valid EOF return. */
+                                return FindLongestMatchResult.eof();
+                            }
+                        },
+                        (Decoder.Success decoded) {
+                            auto transition_result = transition(current_state, decoded.code_point);
+                            if (transition_result.found())
+                            {
+                                lex_continue = true;
+                                attempt_match.length += decoded.code_point_length;
+                                if (decoded.code_point == '\n')
+                                {
+                                    attempt_match.delta_row++;
+                                    attempt_match.delta_col = 0u;
+                                }
+                                else
+                                {
+                                    attempt_match.delta_col++;
+                                }
+                                current_state = transition_result.destination();
+                                if (states[current_state].accepts())
+                                {
+                                    attempt_match.accepting_state = &states[current_state];
+                                    longest_match = attempt_match;
+                                }
+                            }
+                            else if (longest_match.length > 0)
+                            {
+                                return longest_match;
+                            }
+                            else
+                            {
+                                return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length);
+                            }
+                        });
             }
-            return lt;
         }
 
-        private uint transition(uint current_state, uint code_point)
+        /**
+         * Result type for transition().
+         *
+         * Alternatives:
+         * - found(destination)
+         * - not_found
+         */
+        struct TransitionResult
+        {
+            private uint m_destination;
+
+            static TransitionResult found(uint destination)
+            {
+                return TransitionResult(destination);
+            }
+
+            static TransitionResult not_found()
+            {
+                return TransitionResult(cast(uint)-1);
+            }
+
+            bool found()
+            {
+                return m_destination != cast(uint)-1;
+            }
+
+            @property uint destination()
+            {
+                return m_destination;
+            }
+        }
+
+        private TransitionResult transition(uint current_state, uint code_point)
         {
             uint transition_table_index = states[current_state].transition_table_index;
             for (uint i = 0u; i < states[current_state].n_transitions; i++)
@@ -300,10 +562,11 @@ class <%= @classname %>
                 if ((transitions[transition_table_index + i].first <= code_point) &&
                     (code_point <= transitions[transition_table_index + i].last))
                 {
-                    return transitions[transition_table_index + i].destination;
+                    uint destination = transitions[transition_table_index + i].destination;
+                    return TransitionResult.found(destination);
                 }
             }
-            return cast(uint)-1;
+            return TransitionResult.not_found();
         }
     }
 
@@ -331,20 +594,31 @@ class <%= @classname %>
             uint n_reduce_entries;
         }
 
+        private struct StateValue
+        {
+            uint state;
+            ParserValue pvalue;
+
+            this(uint state)
+            {
+                this.state = state;
+            }
+        }
+
 <% state_table, shift_table, reduce_table = @parser.build_tables %>
-        private static immutable Shift shifts[] = [
+        private static immutable Shift[] shifts = [
 <%   shift_table.each do |shift| %>
             Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u),
 <%   end %>
         ];
 
-        private static immutable Reduce reduces[] = [
+        private static immutable Reduce[] reduces = [
 <%   reduce_table.each do |reduce| %>
             Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
 <%   end %>
         ];
 
-        private static immutable State states[] = [
+        private static immutable State[] states = [
 <%   state_table.each do |state| %>
             State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
 <%   end %>
@@ -352,20 +626,23 @@ class <%= @classname %>
 
         private Lexer m_lexer;
 
-        this(const(ubyte) * input, size_t input_length)
+        private ParserValue parse_result;
+
+        this(string input)
         {
-            m_lexer = new Lexer(input, input_length);
+            m_lexer = new Lexer(input);
         }
 
         bool parse()
         {
-            Lexer.LexedToken lexed_token;
-            uint token = _TOKEN_NONE;
-            uint[] states = new uint[](1);
+            Lexer.Result lexed_token;
+            uint token = _TOKEN_COUNT;
+            StateValue[] statevalues = new StateValue[](1);
             uint reduced_rule_set = 0xFFFFFFFFu;
+            ParserValue reduced_parser_value;
             for (;;)
             {
-                if (token == _TOKEN_NONE)
+                if (token == _TOKEN_COUNT)
                 {
                     lexed_token = m_lexer.lex_token();
                     token = lexed_token.token;
@@ -373,46 +650,52 @@ class <%= @classname %>
                 uint shift_state = 0xFFFFFFFFu;
                 if (reduced_rule_set != 0xFFFFFFFFu)
                 {
-                    shift_state = check_shift(states[$-1], reduced_rule_set);
+                    shift_state = check_shift(statevalues[$-1].state, reduced_rule_set);
                 }
                 if (shift_state == 0xFFFFFFFFu)
                 {
-                    shift_state = check_shift(states[$-1], token);
+                    shift_state = check_shift(statevalues[$-1].state, token);
+                    if ((shift_state != 0xFFFFFFFFu) && (token == TOKEN___EOF))
+                    {
+                        /* Successful parse. */
+                        parse_result = statevalues[$-1].pvalue;
+                        return true;
+                    }
                 }
                 if (shift_state != 0xFFFFFFFFu)
                 {
-                    if (token == _TOKEN_EOF)
-                    {
-                        /* Successful parse. */
-                        return true;
-                    }
-                    states ~= shift_state;
+                    /* We have something to shift. */
+                    statevalues ~= StateValue(shift_state);
                     if (reduced_rule_set == 0xFFFFFFFFu)
                     {
-                        token = _TOKEN_NONE;
+                        /* We shifted a token, mark it consumed. */
+                        token = _TOKEN_COUNT;
+                        statevalues[$-1].pvalue = lexed_token.pvalue;
                     }
                     else
                     {
+                        /* We shifted a RuleSet. */
+                        statevalues[$-1].pvalue = reduced_parser_value;
+                        ParserValue new_parse_result;
+                        reduced_parser_value = new_parse_result;
                         reduced_rule_set = 0xFFFFFFFFu;
                     }
                     continue;
                 }
 
-                uint reduce_index = check_reduce(states[$-1], token);
+                uint reduce_index = check_reduce(statevalues[$-1].state, token);
                 if (reduce_index != 0xFFFFFFFFu)
                 {
+                    /* We have something to reduce. */
+                    reduced_parser_value = user_code(reduces[reduce_index].rule, statevalues, reduces[reduce_index].n_states);
                     reduced_rule_set = reduces[reduce_index].rule_set;
-                    states.length -= reduces[reduce_index].n_states;
+                    statevalues.length -= reduces[reduce_index].n_states;
                     continue;
                 }
 
                 /* Error, unexpected token. */
                 write("Unexpected token ");
-                if (token == _TOKEN_EOF)
-                {
-                    writeln("{EOF}");
-                }
-                else if (token < _TOKEN_COUNT)
+                if (token < _TOKEN_COUNT)
                 {
                     writeln(token_names[token]);
                 }
@@ -424,6 +707,11 @@ class <%= @classname %>
             }
         }
 
+        @property <%= start_rule_type[1] %> result()
+        {
+            return parse_result.v_<%= start_rule_type[0] %>;
+        }
+
         private uint check_shift(uint state, uint symbol)
         {
             uint start = states[state].shift_table_index;
@@ -453,17 +741,13 @@ class <%= @classname %>
             for (uint i = start; i < end; i++)
             {
                 if ((reduces[i].token == token) ||
-                    (reduces[i].token == _TOKEN_NONE))
+                    (reduces[i].token == _TOKEN_COUNT))
                 {
 //                    write("Reducing rule ", reduces[i].rule, ", rule set ", reduces[i].rule_set, " lookahead ");
 //                    if (token < _TOKEN_COUNT)
 //                    {
 //                        writeln(token_names[token]);
 //                    }
-//                    else if (token == _TOKEN_EOF)
-//                    {
-//                        writeln("{EOF}");
-//                    }
 //                    else
 //                    {
 //                        writeln("{other}");
@@ -473,5 +757,31 @@ class <%= @classname %>
             }
             return 0xFFFFFFFFu;
         }
+
+        /**
+         * Execute user code associated with a parser rule.
+         *
+         * @param rule The ID of the rule.
+         *
+         * @return Parse value.
+         */
+        private ParserValue user_code(uint rule, StateValue[] statevalues, uint n_states)
+        {
+            ParserValue _pvalue;
+
+            switch (rule)
+            {
+<% @grammar.rules.each do |rule| %>
+<%   if rule.code %>
+            case <%= rule.id %>u: {
+<%= expand_code(rule.code, true, rule, nil) %>
+            } break;
+<%   end %>
+<% end %>
+            default: break;
+            }
+
+            return _pvalue;
+        }
     }
 }

doc/user_guide.md

@@ -0,0 +1,76 @@
+${remove}
+WARNING: This user guide is meant to be preprocessed and rendered by a custom
+script.
+The markdown source file is not intended to be viewed directly and will not
+include all intended content.
+${/remove}
+
+#> Overview
+
+Propane is an LR Parser Generator (LPG) which:
+
+  * accepts LR(0), SLR, and LALR grammars
+  * generates a built-in lexer to tokenize input
+  * supports UTF-8 lexer inputs
+  * generates a table-driven parser to parse input in linear time
+  * is MIT-licensed
+  * is distributable as a standalone Ruby script
+  * supports D language
+
+#> Installation
+
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
+
+#> Command Line Usage
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+#> Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+#> License
+
+Propane is licensed under the terms of the MIT License:
+
+```
+${include LICENSE.txt}
+```
+
+#> Contributing
+
+Propane is developed on [github](https://github.com/holtrop/propane).
+
+Issues may be submitted to [https://github.com/holtrop/propane/issues](https://github.com/holtrop/propane/issues).
+
+Pull requests may be submitted as well:
+
+  1. Fork it
+  2. Create your feature branch (`git checkout -b my-new-feature`)
+  3. Commit your changes (`git commit -am 'Add some feature'`)
+  4. Push to the branch (`git push origin my-new-feature`)
+  5. Create new Pull Request
+
+#> Change Log
+
+${changelog}

lib/propane.rb

@@ -25,18 +25,6 @@ require_relative "propane/version"
 
 class Propane
 
-  # EOF.
-  TOKEN_EOF = 0xFFFFFFFC
-
-  # Decoding error.
-  TOKEN_DECODE_ERROR = 0xFFFFFFFD
-
-  # Token ID for a "dropped" token.
-  TOKEN_DROP = 0xFFFFFFFE
-
-  # Invalid token ID.
-  TOKEN_NONE = 0xFFFFFFFF
-
   class Error < RuntimeError
   end
 

lib/propane/generator.rb

@@ -26,29 +26,74 @@ class Propane
     private
 
     def process_grammar!
+      # Assign default pattern mode to patterns without a mode assigned.
+      found_default = false
+      @grammar.patterns.each do |pattern|
+        if pattern.mode.nil?
+          pattern.mode = "default"
+          found_default = true
+        end
+        pattern.ptypename ||= "default"
+      end
+      unless found_default
+        raise Error.new("No patterns found for default mode")
+      end
+      # Add EOF token.
+      @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
-      @grammar.tokens.each do |token|
+      @grammar.tokens.each_with_index do |token, token_id|
+        # Assign token ID.
+        token.id = token_id
+        # Set default ptypename if none given.
+        token.ptypename ||= "default"
+        # Check for token name conflicts.
         if tokens_by_name.include?(token.name)
           raise Error.new("Duplicate token name #{token.name.inspect}")
         end
         tokens_by_name[token.name] = token
       end
+      # Check for user start rule.
+      unless @grammar.rules.find {|rule| rule.name == "Start"}
+        raise Error.new("Start rule not found")
+      end
+      # Add "real" start rule.
+      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
       rule_sets = {}
-      @grammar.rules.each do |rule|
+      rule_set_id = @grammar.tokens.size
+      @grammar.rules.each_with_index do |rule, rule_id|
+        # Assign rule ID.
+        rule.id = rule_id
+        # Check for token/rule name conflict.
         if tokens_by_name.include?(rule.name)
           raise Error.new("Rule name collides with token name #{rule.name.inspect}")
         end
-        @_rule_set_id ||= @grammar.tokens.size
+        # Build rule sets of all rules with the same name.
         unless rule_sets[rule.name]
-          rule_sets[rule.name] = RuleSet.new(rule.name, @_rule_set_id)
-          @_rule_set_id += 1
+          rule_sets[rule.name] = RuleSet.new(rule.name, rule_set_id)
+          rule_set_id += 1
         end
-        rule.rule_set = rule_sets[rule.name]
-        rule_sets[rule.name] << rule
+        rule_set = rule_sets[rule.name]
+        if rule_set.ptypename && rule.ptypename && rule_set.ptypename != rule.ptypename
+          raise Error.new("Conflicting ptypes for rule #{rule.name}")
+        end
+        rule_set.ptypename ||= rule.ptypename
+        rule.rule_set = rule_set
+        rule_set << rule
       end
-      unless rule_sets["Start"]
-        raise Error.new("Start rule not found")
+      rule_sets.each do |name, rule_set|
+        rule_set.ptypename ||= "default"
+        # Assign rule set ptypenames back to rules.
+        rule_set.rules.each do |rule|
+          rule.ptypename = rule_set.ptypename
+        end
       end
+      # Generate lexer user code IDs for lexer patterns with user code blocks.
+      @grammar.patterns.select do |pattern|
+        pattern.code
+      end.each_with_index do |pattern, code_id|
+        pattern.code_id = code_id
+      end
+      # Map rule components from names to Token/RuleSet objects.
       @grammar.rules.each do |rule|
         rule.components.map! do |component|
           if tokens_by_name[component]
@@ -61,8 +106,10 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
-      @lexer = Lexer.new(@grammar.patterns)
-      @parser = Parser.new(@grammar, rule_sets, rule_sets["Start"], @log)
+      # Generate the lexer.
+      @lexer = Lexer.new(@grammar)
+      # Generate the parser.
+      @parser = Parser.new(@grammar, rule_sets, @log)
     end
 
     # Determine which grammar rules could expand to empty sequences.
@@ -121,6 +168,58 @@ class Propane
       end
     end
 
+    # Expand expansions in user code block.
+    #
+    # @param code [String]
+    #   User code block.
+    # @param parser [Boolean]
+    #   Whether the user code is for the parser or lexer.
+    # @param rule [Rule, nil]
+    #   The Rule associated with the user code if user code is for the parser.
+    # @param pattern [Pattern, nil]
+    #   The Pattern associated with the user code if user code is for the lexer.
+    #
+    # @return [String]
+    #   Expanded user code block.
+    def expand_code(code, parser, rule, pattern)
+      code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
+        "Token(TOKEN_#{Token.code_name($1)})"
+      end
+      if parser
+        code = code.gsub(/\$\$/) do |match|
+          "_pvalue.v_#{rule.ptypename}"
+        end
+        code = code.gsub(/\$(\d+)/) do |match|
+          index = $1.to_i
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
+        end
+      else
+        code = code.gsub(/\$\$/) do |match|
+          "result.pvalue.v_#{pattern.ptypename}"
+        end
+        code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
+          mode_name = $1
+          mode_id = @lexer.mode_id(mode_name)
+          unless mode_id
+            raise Error.new("Lexer mode '#{mode_name}' not found")
+          end
+          "m_mode = #{mode_id}u"
+        end
+      end
+      code
+    end
+
+    # Get the parser value type for the start rule.
+    #
+    # @return [Array<String>]
+    #   Start rule parser value type name and type string.
+    def start_rule_type
+      start_rule = @grammar.rules.find do |rule|
+        rule.name == "Start"
+      end
+      [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
+    end
+
   end
 
 end

lib/propane/grammar.rb

@@ -2,22 +2,34 @@ class Propane
 
   class Grammar
 
+    # Reserve identifiers beginning with a double-underscore for internal use.
+    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
+
     attr_reader :classname
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
     attr_reader :tokens
+    attr_reader :code_blocks
+    attr_reader :ptypes
 
     def initialize(input)
       @patterns = []
       @tokens = []
       @rules = []
-      @code_id = 0
+      @code_blocks = []
       @line_number = 1
+      @next_line_number = @line_number
+      @mode = nil
       @input = input.gsub("\r\n", "\n")
+      @ptypes = {"default" => "void *"}
       parse_grammar!
     end
 
+    def ptype
+      @ptypes["default"]
+    end
+
     private
 
     def parse_grammar!
@@ -27,112 +39,208 @@ class Propane
     end
 
     def parse_statement!
-      @next_line_number = @line_number
       if parse_white_space!
       elsif parse_comment_line!
+      elsif @mode.nil? && parse_mode_label!
       elsif parse_module_statement!
       elsif parse_class_statement!
+      elsif parse_ptype_statement!
+      elsif parse_pattern_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
       elsif parse_rule_statement!
+      elsif parse_code_block_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
         end
         raise Error.new("Unexpected grammar input at line #{@line_number}: #{@input.chomp}")
       end
-      @line_number = @next_line_number
+    end
+
+    def parse_mode_label!
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
+        @mode = md[1]
+      end
     end
 
     def parse_white_space!
-      consume!(/\A\s+/)
+      consume!(/\s+/)
     end
 
     def parse_comment_line!
-      consume!(/\A#.*\n/)
+      consume!(/#.*\n/)
     end
 
     def parse_module_statement!
-      if md = consume!(/\Amodule\s+(\S+)\s*;/)
+      if consume!(/module\s+/)
+        md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
+        consume!(/;/, "expected `;'")
+        @mode = nil
+        true
       end
     end
 
     def parse_class_statement!
-      if md = consume!(/\Aclass\s+(\S+)\s*;/)
+      if consume!(/class\s+/)
+        md = consume!(/([\w.]+)\s*/, "expected class name")
         @classname = md[1]
+        consume!(/;/, "expected `;'")
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_ptype_statement!
+      if consume!(/ptype\s+/)
+        name = "default"
+        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          name = md[1]
+        end
+        md = consume!(/([^;]+);/, "expected parser result type expression")
+        @ptypes[name] = md[1].strip
       end
     end
 
     def parse_token_statement!
-      if md = consume!(/\Atoken\s+(\S+?)(?:\s+([^\n]+?))?\s*(?:;|<<\n(.*?)^>>\n)/m)
-        name, pattern, code = *md[1, 3]
-        if pattern.nil?
-          pattern = name
+      if consume!(/token\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
+        name = md[1]
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
         end
-        unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-          raise Error.new("Invalid token name #{name.inspect}")
+        pattern = parse_pattern! || name
+        consume!(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected pattern or `;' or code block")
         end
-        token = Token.new(name: name, id: @tokens.size, line_number: @line_number)
+        token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        if code
-          code_id = @code_id
-          @code_id += 1
-        else
-          code_id = nil
-        end
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, code_id: code_id)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
         @patterns << pattern
+        @mode = nil
+        true
       end
     end
 
     def parse_tokenid_statement!
-      if md = consume!(/\Atokenid\s+(\S+?)\s*;/m)
+      if md = consume!(/tokenid\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
-        unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-          raise Error.new("Invalid token name #{name.inspect}")
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
         end
-        token = Token.new(name: name, id: @tokens.size, line_number: @line_number)
+        consume!(/;/, "expected `;'");
+        token = Token.new(name, ptypename, @line_number)
         @tokens << token
+        @mode = nil
+        true
       end
     end
 
     def parse_drop_statement!
-      if md = consume!(/\Adrop\s+(\S+)\s*;/)
-        pattern = md[1]
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, drop: true)
+      if md = consume!(/drop\s+/)
+        pattern = parse_pattern!
+        unless pattern
+          raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
+        end
+        consume!(/\s+/)
+        consume!(/;/, "expected `;'")
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, drop: true, mode: @mode)
+        @mode = nil
+        true
       end
     end
 
     def parse_rule_statement!
-      if md = consume!(/\A(\S+)\s*->\s*([^\n]*?)(?:;|<<\n(.*?)^>>\n)/m)
-        rule_name, components, code = *md[1, 3]
-        unless rule_name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-          raise Error.new("Invalid rule name #{name.inspect}")
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
+        rule_name, ptypename = *md[1, 2]
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        components = md[1].strip.split(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected pattern or `;' or code block")
         end
-        components = components.strip.split(/\s+/)
-        # Reserve rule ID 0 for the "real" start rule.
-        @rules << Rule.new(rule_name, components, code, @line_number, @rules.size + 1)
+        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_pattern_statement!
+      if pattern = parse_pattern!
+        consume!(/\s+/)
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
+        end
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block to follow pattern")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_code_block_statement!
+      if code = parse_code_block!
+        @code_blocks << code
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_pattern!
+      if md = consume!(%r{/})
+        pattern = ""
+        while !consume!(%r{/})
+          if consume!(%r{\\})
+            pattern += "\\"
+            if md = consume!(%r{(.)})
+              pattern += md[1]
+            else
+              raise Error.new("Line #{@line_number}: unterminated escape sequence")
+            end
+          elsif md = consume!(%r{(.)})
+            pattern += md[1]
+          end
+        end
+        pattern
+      end
+    end
+
+    def parse_code_block!
+      if md = consume!(/<<\n(.*?)^>>\n/m)
+        md[1]
       end
     end
 
     # Check if the input string matches the given regex.
     #
     # If so, remove the match from the input string, and update the line
-    # number.
+    # number. If the regex is not matched and an error message is provided,
+    # the error is raised.
     #
     # @param regex [Regexp]
     #   Regex to attempt to match.
+    # @param error_message [String, nil]
+    #   Error message to display if the regex is not matched. If nil and the
+    #   regex is not matched, an error is not raised.
     #
     # @return [MatchData, nil]
     #   MatchData for the given regex if it was matched and removed from the
     #   input.
-    def consume!(regex)
-      if md = @input.match(regex)
+    def consume!(regex, error_message = nil)
+      @line_number = @next_line_number
+      if md = @input.match(/\A#{regex}/)
         @input.slice!(0, md[0].size)
         @next_line_number += md[0].count("\n")
         md
+      elsif error_message
+        raise Error.new("Line #{@line_number}: Error: #{error_message}")
+      else
+        false
       end
     end
 

lib/propane/lexer.rb

@@ -1,48 +1,68 @@
 class Propane
   class Lexer
 
-    # @return [DFA]
-    #   Lexer DFA.
-    attr_accessor :dfa
-
-    def initialize(patterns)
-      @dfa = DFA.new(patterns)
+    def initialize(grammar)
+      @grammar = grammar
     end
 
     def build_tables
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
       transition_table = []
       state_table = []
-      states = @dfa.enumerate
-      states.each do |state, id|
-        token =
-          if state.accepts.nil?
-            TOKEN_NONE
-          elsif state.accepts.drop?
-            TOKEN_DROP
-          else
-            state.accepts.token.id
-          end
-        code_id =
-          if state.accepts && state.accepts.code_id
-            state.accepts.code_id
-          else
-            0xFFFF_FFFF
-          end
-        state_table << {
-          transition_table_index: transition_table.size,
-          n_transitions: state.transitions.size,
-          token: token,
-          code_id: code_id,
+      mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = state_table.size
+        mode_table << {
+          state_table_offset: state_table_offset,
         }
-        state.transitions.each do |transition|
-          transition_table << {
-            first: transition.code_point_range.first,
-            last: transition.code_point_range.last,
-            destination: states[transition.destination],
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          drop = state.accepts && state.accepts.drop?
+          token =
+            if state.accepts && state.accepts.token
+              state.accepts.token.id
+            end
+          code_id =
+            if state.accepts && state.accepts.code_id
+              state.accepts.code_id
+            end
+          state_table << {
+            transition_table_index: transition_table.size,
+            n_transitions: state.transitions.size,
+            drop: drop,
+            token: token,
+            code_id: code_id,
           }
+          state.transitions.each do |transition|
+            transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
         end
       end
-      [transition_table, state_table]
+      [transition_table, state_table, mode_table]
+    end
+
+    # Get ID for a mode.
+    #
+    # @param mode_name [String]
+    #   Mode name.
+    #
+    # @return [Integer, nil]
+    #   Mode ID.
+    def mode_id(mode_name)
+      if mode_info = @modes[mode_name]
+        mode_info[:id]
+      end
     end
 
   end

lib/propane/parser.rb

@@ -2,15 +2,13 @@ class Propane
 
   class Parser
 
-    def initialize(grammar, rule_sets, start_rule_set, log)
+    def initialize(grammar, rule_sets, log)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
-      @eof_token = Token.new(name: "$", id: TOKEN_EOF)
-      @start_rule = Rule.new("$$", [start_rule_set, @eof_token], nil, nil, 0)
       @item_sets = []
       @item_sets_set = {}
-      start_item = Item.new(@start_rule, 0)
+      start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
 
       while eval_item_sets.size > 0
@@ -21,7 +19,7 @@ class Propane
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
           item_set.following_symbols.each do |following_symbol|
-            unless following_symbol == @eof_token
+            unless following_symbol.name == "$EOF"
               following_set = item_set.build_following_item_set(following_symbol)
               eval_item_sets << following_set
             end
@@ -44,7 +42,7 @@ class Propane
       @item_sets.each do |item_set|
         shift_entries = item_set.following_symbols.map do |following_symbol|
           state_id =
-            if following_symbol == @eof_token
+            if following_symbol.name == "$EOF"
               0
             else
               item_set.following_item_set[following_symbol].id
@@ -57,7 +55,7 @@ class Propane
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: TOKEN_NONE, rule_id: ra.id,
+            [{token_id: @grammar.tokens.size, rule_id: ra.id,
               rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
           when Hash
             ra.map do |token, rule|
@@ -83,7 +81,7 @@ class Propane
 
     def process_item_set(item_set)
       item_set.following_symbols.each do |following_symbol|
-        unless following_symbol == @eof_token
+        unless following_symbol.name == "$EOF"
           following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
           item_set.following_item_set[following_symbol] = following_set
           following_set.in_sets << item_set
@@ -206,7 +204,7 @@ class Propane
 
     def write_log!
       @log.puts Util.banner("Parser Rules")
-      ([@start_rule] + @grammar.rules).each do |rule|
+      @grammar.rules.each do |rule|
         @log.puts
         @log.puts "Rule #{rule.id}:"
         @log.puts "  #{rule}"

lib/propane/pattern.rb

@@ -8,7 +8,7 @@ class Propane
 
     # @option options [Integer, nil] :code_id
     #   Code block ID.
-    attr_reader :code_id
+    attr_accessor :code_id
 
     # @return [String, nil]
     #   Pattern.
@@ -26,14 +26,20 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
+    # @return [String, nil]
+    #   Lexer mode for this pattern.
+    attr_accessor :mode
+
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
+
     # Construct a Pattern.
     #
     # @param options [Hash]
     #   Optional parameters.
     # @option options [String, nil] :code
     #   Code block to execute when the pattern is matched.
-    # @option options [Integer, nil] :code_id
-    #   Code block ID.
     # @option options [Boolean] :drop
     #   Whether this is a drop pattern.
     # @option options [String, nil] :pattern
@@ -42,13 +48,16 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
+    # @option options [String, nil] :mode
+    #   Lexer mode for this pattern.
     def initialize(options)
       @code = options[:code]
-      @code_id = options[:code_id]
       @drop = options[:drop]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
+      @mode = options[:mode]
+      @ptypename = options[:ptypename]
       regex = Regex.new(@pattern)
       regex.nfa.end_state.accepts = self
       @nfa = regex.nfa

lib/propane/rule.rb

@@ -12,7 +12,11 @@ class Propane
 
     # @return [Integer]
     #   Rule ID.
-    attr_reader :id
+    attr_accessor :id
+
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
 
     # @return [Integer]
     #   Line number where the rule was defined in the input grammar.
@@ -34,15 +38,15 @@ class Propane
     #   Rule components.
     # @param code [String]
     #   User code associated with the rule.
+    # @param ptypename [String, nil]
+    #   Parser type name.
     # @param line_number [Integer]
     #   Line number where the rule was defined in the input grammar.
-    # @param id [Integer]
-    #   Rule ID.
-    def initialize(name, components, code, line_number, id)
+    def initialize(name, components, code, ptypename, line_number)
       @name = name
       @components = components
       @code = code
-      @id = id
+      @ptypename = ptypename
       @line_number = line_number
     end
 

lib/propane/rule_set.rb

@@ -10,6 +10,10 @@ class Propane
     #   Name of the RuleSet.
     attr_reader :name
 
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
+
     # @return [Array<Rule>]
     #   Rules in the RuleSet.
     attr_reader :rules

lib/propane/token.rb

@@ -2,13 +2,29 @@ class Propane
 
   class Token
 
+    class << self
+
+      # Name of the token to use in code (special characters replaced).
+      #
+      # @return [String]
+      #   Name of the token to use in code (special characters replaced).
+      def code_name(name)
+        name.sub(/^\$/, "__")
+      end
+
+    end
+
     # @return [String, nil]
     #   Token name.
     attr_reader :name
 
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
+
     # @return [Integer, nil]
     #   Token ID.
-    attr_reader :id
+    attr_accessor :id
 
     # @return [Integer, nil]
     #   Line number where the token was defined in the input grammar.
@@ -16,18 +32,24 @@ class Propane
 
     # Construct a Token.
     #
-    # @param options [Hash]
-    #   Optional parameters.
-    # @option options [String, nil] :name
+    # @param name [String, nil]
     #   Token name.
-    # @option options [Integer, nil] :id
-    #   Token ID.
-    # @option options [Integer, nil] :line_number
+    # @param ptypename [String, nil]
+    #   Parser value type for this token.
+    # @param line_number [Integer, nil]
     #   Line number where the token was defined in the input grammar.
-    def initialize(options)
-      @name = options[:name]
-      @id = options[:id]
-      @line_number = options[:line_number]
+    def initialize(name, ptypename, line_number)
+      @name = name
+      @ptypename = ptypename
+      @line_number = line_number
+    end
+
+    # Name of the token to use in code (special characters replaced).
+    #
+    # @return [String]
+    #   Name of the token to use in code (special characters replaced).
+    def code_name
+      self.class.code_name(@name)
     end
 
     def to_s

rb/assets/user_guide.html.erb

@@ -0,0 +1,68 @@
+<html>
+  <head>
+    <title>Propane User Guide<%= subpage_title %> - Version <%= Propane::VERSION %></title>
+    <style>
+      body {
+        background-color: #CCC;
+        margin: 0px;
+        padding: 0px;
+      }
+      #body-content {
+        margin-left: auto;
+        margin-right: auto;
+        margin-top: 1em;
+        margin-bottom: 1em;
+        border: 1px solid black;
+        background-color: #FFF;
+        width: 120ex;
+        padding: 2ex;
+      }
+      a {
+        text-decoration: none;
+      }
+      .separator {
+        height: 2em;
+      }
+      .img_block_center {
+        display: block;
+        margin-left: auto;
+        margin-right: auto;
+      }
+      .page_nav {
+        width: 100%;
+        table-layout: fixed;
+      }
+      .page_nav_toc {
+        text-align: center;
+      }
+      .page_nav_next {
+        text-align: right;
+      }
+      .code {
+        padding-left: 2ex;
+        width: 116ex;
+        overflow-x: auto;
+      }
+      .ruby_code .normal {}
+      .ruby_code .comment { color: #005; font-style: italic; }
+      .ruby_code .keyword { color: #A00; font-weight: bold; }
+      .ruby_code .method { color: #077; }
+      .ruby_code .class { color: #074; }
+      .ruby_code .module { color: #050; }
+      .ruby_code .punct { color: #447; font-weight: bold; }
+      .ruby_code .symbol { color: #099; }
+      .ruby_code .string { color: #090; }
+      .ruby_code .char { color: #F07; }
+      .ruby_code .ident { color: #004; }
+      .ruby_code .constant { color: #07F; }
+      .ruby_code .regex { color: #B66; }
+      .ruby_code .number { color: #D55; }
+      .ruby_code .attribute { color: #377; }
+      .ruby_code .global { color: #3B7; }
+      .ruby_code .expr { color: #227; }
+    </style>
+  </head>
+  <body>
+    <div id="body-content"><%= content %></div>
+  </body>
+</html>

rb/build_dist.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require "fileutils"
+require "digest/md5"
+
+if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
+  VERSION = $1
+else
+  raise "Could not determine version."
+end
+PROG_NAME = "propane"
+START_FILE = "bin/#{PROG_NAME}"
+LIB_DIR = "lib"
+DIST = "dist"
+
+files_processed = {}
+combined_file = []
+
+combine_files = lambda do |file|
+  File.read(file, mode: "rb").each_line do |line|
+    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
+      require_name = $1
+      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
+        path = "#{LIB_DIR}/#{require_name}.rb"
+        if File.exist?(path)
+          unless files_processed[path]
+            files_processed[path] = true
+            combine_files[path]
+          end
+        else
+          raise "require path #{path.inspect} not found"
+        end
+      else
+        combined_file << line
+      end
+    else
+      combined_file << line
+    end
+  end
+end
+
+combine_files[START_FILE]
+
+# Strip Ruby comment lines and empty lines to save some space, but do not
+# remove lines that are in heredoc sections. This isn't terribly robust to be
+# used in the wild, but works for the heredoc instances for this project.
+stripped = []
+heredoc_end = nil
+combined_file.each do |line|
+  if line =~ /<<-?([A-Z]+)/
+    heredoc_end = $1
+  end
+  if heredoc_end and line =~ /^\s*#{heredoc_end}/
+    heredoc_end = nil
+  end
+  if line !~ /#\sspecs/
+    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
+      stripped << line
+    end
+  end
+end
+
+license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
+  if line.size > 0
+    "# #{line}"
+  else
+    "#"
+  end
+end
+
+require "zlib"
+require "base64"
+compressed_script = Zlib::Deflate.deflate(stripped.join)
+encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
+hash = Digest::MD5.hexdigest(encoded_compressed_script)
+
+FileUtils.rm_rf(DIST)
+FileUtils.mkdir_p(DIST)
+File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
+  fh.write(<<EOF)
+#!/usr/bin/env ruby
+
+#{license}
+
+script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
+unless File.exist?(script)
+  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
+    require "zlib"
+    require "base64"
+    encoded_compressed = $1
+    unescaped_compressed = Base64.decode64(encoded_compressed)
+    inflated = Zlib::Inflate.inflate(unescaped_compressed)
+    File.open(script, "wb") do |fh|
+      fh.write(inflated)
+    end
+  else
+    raise "Could not decompress."
+  end
+end
+load script
+if __FILE__ == $0
+  Propane::CLI.new.run(ARGV)
+end
+#==>#{encoded_compressed_script}
+EOF
+end

rb/gen_user_guide.rb

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "fileutils"
+require "redcarpet"
+require "syntax"
+require "syntax/convertors/html"
+require "propane/version"
+
+def load_file(file_name)
+  contents = File.read(file_name)
+  contents.gsub(/\$\{include (.*?)\}/) do |match|
+    include_file_name = $1
+    File.read(include_file_name)
+  end
+end
+
+class Generator
+  class Section
+    attr_reader :number
+    attr_reader :title
+    attr_reader :page
+    attr_reader :contents
+    attr_reader :anchor
+    def initialize(number, title, page, anchor)
+      @number = number
+      @title = title.gsub(/[`]/, "")
+      @page = page
+      @anchor = anchor
+      @contents = ""
+    end
+    def append(contents)
+      @contents += contents
+    end
+  end
+
+  class Page
+    attr_reader :name
+    attr_reader :title
+    attr_accessor :contents
+    def initialize(name, title, contents = "")
+      @name = name
+      @title = title
+      @contents = contents
+    end
+  end
+
+  def initialize(input, output_file, multi_page)
+    current_page =
+      if multi_page
+        nil
+      else
+        File.basename(output_file).sub(/\.html$/, "")
+      end
+    @sections = []
+    @current_section_number = [0]
+    @lines = input.lines
+    while @lines.size > 0
+      line = @lines.slice!(0)
+      if line =~ /^```(.*)$/
+        @sections.last.append(gather_code_section($1))
+      elsif line.chomp == "${remove}"
+        remove_section
+      elsif line =~ /^(#+)(>)?\s*(.*)$/
+        level_text, new_page_text, title_text = $1, $2, $3
+        level = $1.size
+        new_page = !new_page_text.nil?
+        section_number = get_next_section_number(level)
+        anchor = make_anchor(section_number, title_text)
+        if multi_page and (new_page or current_page.nil?)
+          current_page = anchor
+        end
+        @sections << Section.new(section_number, title_text, current_page, anchor)
+        @sections.last.append("#{level_text} #{section_number} #{title_text}")
+      elsif @sections.size > 0
+        @sections.last.append(line)
+      end
+    end
+
+    renderer = Redcarpet::Render::HTML.new
+    @markdown_renderer = Redcarpet::Markdown.new(renderer)
+    changelog = @markdown_renderer.render(File.read("CHANGELOG.md"))
+
+    @pages = [Page.new("toc", "Table of Contents", render_toc)]
+    @sections.each do |section|
+      unless @pages.last.name == section.page
+        @pages << Page.new(section.page, "#{section.number} #{section.title}")
+      end
+      @pages.last.contents += render_section(section)
+    end
+
+    @pages.each do |page|
+      page.contents.gsub!("${changelog}", changelog)
+      page.contents.gsub!(%r{\$\{#(.+?)\}}) do |match|
+        section_name = $1
+        href = get_link_to_section(section_name)
+        %[<a href="#{href}">#{section_name}</a>]
+      end
+    end
+
+    template = File.read("rb/assets/user_guide.html.erb")
+    erb = ERB.new(template, trim_mode: "<>")
+
+    if multi_page
+      @pages.each_with_index do |page, page_index|
+        subpage_title = " - #{page.title}"
+        page_nav_bar = render_page_nav_bar(page_index)
+        content = page_nav_bar + separator + page.contents + separator + page_nav_bar
+        html_result = erb.result(binding.clone)
+        File.open(File.join(output_file, "#{page.name}.html"), "w") do |fh|
+          fh.write(html_result)
+        end
+      end
+    else
+      subpage_title = ""
+      content = @pages.reduce("") do |result, page|
+        result + page.contents
+      end
+      html_result = erb.result(binding.clone)
+      File.open(output_file, "w") do |fh|
+        fh.write(html_result)
+      end
+    end
+  end
+
+  def separator
+    %[<div class="separator"></div>]
+  end
+
+  def render_page_nav_bar(page_index)
+    page_nav_prev =
+      if page_index > 1
+        %[<a href="#{@pages[page_index - 1].name}.html">&laquo; Prev<br/>#{@pages[page_index - 1].title}</a>]
+      else
+        ""
+      end
+    page_nav_toc =
+      if page_index > 0
+        %[<a href="toc.html">Table of Contents</a>]
+      else
+        ""
+      end
+    page_nav_next =
+      if page_index < @pages.size - 1
+        %[<a href="#{@pages[page_index + 1].name}.html">Next &raquo<br/>#{@pages[page_index + 1].title}</a>]
+      else
+        ""
+      end
+    %[<table class="page_nav"><tr>] + \
+    %[<td class="page_nav_prev">#{page_nav_prev}</td>] + \
+    %[<td class="page_nav_toc">#{page_nav_toc}</td>] + \
+    %[<td class="page_nav_next">#{page_nav_next}</td>] + \
+    %[</tr></table>]
+  end
+
+  def render_toc
+    toc_content = %[<h1>Table of Contents</h1>\n]
+    @sections.each do |section|
+      indent = section.number.split(".").size - 1
+      toc_content += %[<span style="padding-left: #{4 * indent}ex;">]
+      toc_content += %[<a href="#{section.page}.html##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
+      toc_content += %[</span>]
+    end
+    toc_content
+  end
+
+  def render_section(section)
+    %[<a name="#{section.anchor}" />] + \
+      @markdown_renderer.render(section.contents)
+  end
+
+  def make_anchor(section_number, section_title)
+    "s" + ("#{section_number} #{section_title}").gsub(/[^a-zA-Z0-9]/, "_")
+  end
+
+  def get_link_to_section(section_name)
+    section = @sections.find do |section|
+      section.title == section_name
+    end
+    raise "Could not find section #{section_name}" unless section
+    "#{section.page}.html##{section.anchor}"
+  end
+
+  def gather_code_section(syntax)
+    code = ""
+    loop do
+      line = @lines.slice!(0)
+      if line =~ /^```/
+        break
+      end
+      code += line
+    end
+    if syntax != ""
+      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
+      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
+    else
+      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
+    end
+  end
+
+  def remove_section
+    loop do
+      line = @lines.slice!(0)
+      if line.chomp == "${/remove}"
+        break
+      end
+    end
+  end
+
+  def get_next_section_number(level)
+    if @current_section_number.size == level - 1
+      @current_section_number << 1
+    elsif @current_section_number.size >= level
+      @current_section_number[level - 1] += 1
+      @current_section_number.slice!(level, @current_section_number.size)
+    else
+      raise "Section level change from #{@current_section_number.size} to #{level}"
+    end
+    @current_section_number.join(".")
+  end
+end
+
+input = load_file("doc/user_guide.md")
+FileUtils.rm_rf("gen/user_guide")
+FileUtils.mkdir_p("gen/user_guide")
+Generator.new(input, "gen/user_guide/user_guide.html", false)
+Generator.new(input, "gen/user_guide", true)

spec/json_parser.propane

@@ -0,0 +1,179 @@
+<<
+import std.math;
+import json_types;
+private string string_value;
+>>
+
+ptype JSONValue;
+ptype array = JSONValue[];
+ptype dict = JSONValue[string];
+ptype string = string;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n;
+    bool negative;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = new JSONNumber(n);
+>>
+token true <<
+  $$ = new JSONTrue();
+>>
+token false <<
+  $$ = new JSONFalse();
+>>
+token null <<
+  $$ = new JSONNull();
+>>
+/"/ <<
+  $mode(string);
+  string_value = "";
+>>
+string: token string (string) /"/ <<
+  $$ = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  string_value ~= "\"";
+>>
+string: /\\\\/ <<
+  string_value ~= "\\";
+>>
+string: /\\\// <<
+  string_value ~= "/";
+>>
+string: /\\b/ <<
+  string_value ~= "\b";
+>>
+string: /\\f/ <<
+  string_value ~= "\f";
+>>
+string: /\\n/ <<
+  string_value ~= "\n";
+>>
+string: /\\r/ <<
+  string_value ~= "\r";
+>>
+string: /\\t/ <<
+  string_value ~= "\t";
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  string_value ~= "{" ~ match[2..6] ~ "}";
+>>
+string: /[^\\]/ <<
+  string_value ~= match;
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = new JSONString($1);
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = new JSONObject();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = new JSONObject($2);
+>>
+KeyValues (dict) -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  foreach (key, value; $3)
+  {
+    $1[key] = value;
+  }
+  $$ = $1;
+>>
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+Array -> lbracket rbracket <<
+  $$ = new JSONArray();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = new JSONArray($2);
+>>
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>

spec/json_types.d

@@ -0,0 +1,63 @@
+class JSONValue
+{
+}
+
+class JSONObject : JSONValue
+{
+    JSONValue[string] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[string] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONArray : JSONValue
+{
+    JSONValue[] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONNumber : JSONValue
+{
+    double value;
+
+    this(double value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONString : JSONValue
+{
+    string value;
+
+    this(string value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONTrue : JSONValue
+{
+}
+
+class JSONFalse : JSONValue
+{
+}
+
+class JSONNull : JSONValue
+{
+}

spec/propane/grammar_spec.rb

@@ -6,11 +6,12 @@ class Propane
 
 module a.b;
 class Foobar;
+ptype   XYZ  *  ;
 
 token while;
 
 token id
-  [a-zA-Z_][a-zA-Z_0-9]*;
+  /[a-zA-Z_][a-zA-Z_0-9]*/;
 
 token token_with_code <<
 Code for the token
@@ -18,7 +19,7 @@ Code for the token
 
 tokenid token_with_no_pattern;
 
-drop \\s+;
+drop /\\s+/;
 
 A -> B <<
   a = 42;
@@ -31,55 +32,50 @@ EOF
       grammar = Grammar.new(input)
       expect(grammar.classname).to eq "Foobar"
       expect(grammar.modulename).to eq "a.b"
+      expect(grammar.ptype).to eq "XYZ  *"
+      expect(grammar.ptypes).to eq("default" => "XYZ  *")
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 6
-      expect(o.id).to eq 0
+      expect(o.line_number).to eq 7
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "while"
-      expect(o.line_number).to eq 6
-      expect(o.code_id).to be_nil
+      expect(o.line_number).to eq 7
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "id"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 8
-      expect(o.id).to eq 1
+      expect(o.line_number).to eq 10
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
-      expect(o.line_number).to eq 8
-      expect(o.code_id).to be_nil
+      expect(o.line_number).to eq 10
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "token_with_code"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 11
-      expect(o.id).to eq 2
+      expect(o.line_number).to eq 12
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
-      expect(o.line_number).to eq 11
-      expect(o.code_id).to eq 0
+      expect(o.line_number).to eq 12
       expect(o.code).to eq "Code for the token\n"
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 15
+      expect(o.line_number).to eq 16
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 17
+      expect(o.line_number).to eq 18
       expect(o.token).to be_nil
-      expect(o.code_id).to be_nil
       expect(o.code).to be_nil
 
       expect(grammar.rules.size).to eq 3
@@ -87,22 +83,19 @@ EOF
       o = grammar.rules[0]
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
-      expect(o.id).to eq 1
-      expect(o.line_number).to eq 19
+      expect(o.line_number).to eq 20
       expect(o.code).to eq "  a = 42;\n"
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
       expect(o.components).to eq %w[C while id]
-      expect(o.id).to eq 2
-      expect(o.line_number).to eq 22
+      expect(o.line_number).to eq 23
       expect(o.code).to be_nil
 
       o = grammar.rules[2]
       expect(o.name).to eq "B"
       expect(o.components).to eq []
-      expect(o.id).to eq 3
-      expect(o.line_number).to eq 23
+      expect(o.line_number).to eq 24
       expect(o.code).to eq "  b = 0;\n"
     end
 
@@ -123,23 +116,114 @@ EOF
 
       o = grammar.tokens.find {|token| token.name == "code1"}
       expect(o).to_not be_nil
-      expect(o.id).to eq 0
       expect(o.line_number).to eq 1
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code_id).to eq 0
       expect(o.code).to eq "  a = b;\n  return c;\n"
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
-      expect(o.id).to eq 1
       expect(o.line_number).to eq 6
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code_id).to eq 1
       expect(o.code).to eq %[  writeln("Hello there");\n]
     end
+
+    it "supports mode labels" do
+      input = <<EOF
+token a;
+m1: token b;
+/foo/ <<
+>>
+m2: /bar/ <<
+>>
+drop /q/;
+m3: drop /r/;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "a"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "b"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m1"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m2"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m3"
+    end
+
+    it "allows assigning ptypes to tokens and rules" do
+      input = <<EOF
+ptype Subnode *;
+ptype string = char *;
+ptype integer = int;
+ptype node = Node *;
+
+token abc(string);
+token bar;
+tokenid int(integer);
+
+/xyz/ (string) <<
+>>
+
+/z28/ <<
+>>
+
+Start (node) -> R;
+R -> abc int;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "abc"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.tokens.find {|token| token.name == "bar"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "int"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "integer"
+
+      o = grammar.rules.find {|rule| rule.name == "Start"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "node"
+
+      o = grammar.rules.find {|rule| rule.name == "R"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "xyz"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "z28"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+    end
   end
 end

spec/propane/lexer/dfa_spec.rb

@@ -82,15 +82,15 @@ EOF
     expect(run(<<EOF, "foobar")).to eq expected
 token foo;
 token bar;
-token identifier [a-z]+;
+token identifier /[a-z]+/;
 EOF
     expected = [
       ["plusplus", "++"],
       ["plus", "+"],
     ]
     expect(run(<<EOF, "+++")).to eq expected
-token plus \\+;
-token plusplus \\+\\+;
+token plus /\\+/;
+token plusplus /\\+\\+/;
 EOF
   end
 
@@ -103,7 +103,7 @@ EOF
     expect(run(<<EOF, "foo \tbar")).to eq expected
 token foo;
 token bar;
-token WS \\s+;
+token WS /\\s+/;
 EOF
   end
 
@@ -116,7 +116,7 @@ EOF
     expect(run(<<EOF, "foo \tbar")).to eq expected
 token foo;
 token bar;
-drop \\s+;
+drop /\\s+/;
 EOF
   end
 
@@ -125,7 +125,7 @@ EOF
       ["semicolon", ";"],
     ]
     expect(run(<<EOF, ";")).to eq expected
-token semicolon \;;
+token semicolon /;/;
 EOF
   end
 end

spec/propane_spec.rb

@@ -1,23 +1,60 @@
 require "fileutils"
+require "open3"
+
+Results = Struct.new(:stdout, :stderr, :status)
 
 describe Propane do
   def write_grammar(grammar)
     File.write("spec/run/testparser.propane", grammar)
   end
 
-  def build_parser
-    result = system(*%w[./propane.sh spec/run/testparser.propane spec/run/testparser.d --log spec/run/testparser.log])
-    expect(result).to be_truthy
+  def build_parser(options = {})
+    command = %w[./propane.sh spec/run/testparser.propane spec/run/testparser.d --log spec/run/testparser.log]
+    if (options[:capture])
+      stdout, stderr, status = Open3.capture3(*command)
+      Results.new(stdout, stderr, status)
+    else
+      result = system(*command)
+      expect(result).to be_truthy
+    end
   end
 
-  def compile(test_file)
-    result = system(*%w[gdc -funittest -o spec/run/testparser spec/run/testparser.d], test_file)
+  def compile(*test_files)
+    result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
     expect(result).to be_truthy
   end
 
   def run
-    result = system("spec/run/testparser")
-    expect(result).to be_truthy
+    stdout, stderr, status = Open3.capture3("spec/run/testparser")
+    File.binwrite("spec/run/.stderr", stderr)
+    File.binwrite("spec/run/.stdout", stdout)
+    stderr.sub!(/^.*modules passed unittests\n/, "")
+    Results.new(stdout, stderr, status)
+  end
+
+  def lines(str)
+    str.lines.map(&:chomp)
+  end
+
+  def verify_lines(lines, patterns)
+    if lines.is_a?(String)
+      lines = lines.lines.map(&:chomp)
+    end
+    patterns.each_with_index do |pattern, i|
+      found_index =
+        if pattern.is_a?(Regexp)
+          lines.find_index {|line| line =~ pattern}
+        else
+          lines.find_index do |line|
+            line.chomp == pattern.chomp
+          end
+        end
+      unless found_index
+        $stderr.puts "Lines:"
+        $stderr.puts lines
+        raise "A line matching #{pattern.inspect} (index #{i}) was not found."
+      end
+    end
   end
 
   before(:each) do
@@ -27,10 +64,10 @@ describe Propane do
 
   it "generates a D lexer" do
     write_grammar <<EOF
-token int \\d+;
-token plus \\+;
-token times \\*;
-drop \\s+;
+token int /\\d+/;
+token plus /\\+/;
+token times /\\*/;
+drop /\\s+/;
 Start -> Foo;
 Foo -> int <<
 >>
@@ -39,15 +76,17 @@ Foo -> plus <<
 EOF
     build_parser
     compile("spec/test_d_lexer.d")
-    run
+    results = run
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
   end
 
   it "generates a parser" do
     write_grammar <<EOF
-token plus \\+;
-token times \\*;
-token zero 0;
-token one 1;
+token plus /\\+/;
+token times /\\*/;
+token zero /0/;
+token one /1/;
 Start -> E;
 E -> E times B;
 E -> E plus B;
@@ -60,7 +99,7 @@ EOF
 
   it "generates an SLR parser" do
     write_grammar <<EOF
-token one 1;
+token one /1/;
 Start -> E;
 E -> one E;
 E -> one;
@@ -78,22 +117,24 @@ R1 -> a b;
 R2 -> a b;
 EOF
     build_parser
-    compile("spec/test_d_lexer3.d")
-    run
+    compile("spec/test_d_parser_identical_rules_lookahead.d")
+    results = run
+    expect(results.status).to eq 0
   end
 
   it "handles reducing a rule that could be arrived at from multiple states" do
     write_grammar <<EOF
 token a;
 token b;
-drop \\s+;
+drop /\\s+/;
 Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
     build_parser
-    compile("spec/test_d_lexer2.d")
-    run
+    compile("spec/test_d_parser_rule_from_multiple_states.d")
+    results = run
+    expect(results.status).to eq 0
   end
 
   it "executes user code when matching lexer token" do
@@ -108,6 +149,196 @@ Abcs -> abc Abcs;
 EOF
     build_parser
     compile("spec/test_user_code.d")
-    run
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "abc!",
+      "pass1",
+      "abc!",
+      "abc!",
+      "pass2",
+    ])
+  end
+
+  it "supports a pattern statement" do
+    write_grammar <<EOF
+token abc;
+/def/ <<
+  writeln("def!");
+>>
+Start -> abc;
+EOF
+    build_parser
+    compile("spec/test_pattern.d")
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "def!",
+      "pass1",
+      "def!",
+      "def!",
+      "pass2",
+    ])
+  end
+
+  it "supports returning tokens from pattern code blocks" do
+    write_grammar <<EOF
+token abc;
+/def/ <<
+  writeln("def!");
+>>
+/ghi/ <<
+  writeln("ghi!");
+  return $token(abc);
+>>
+Start -> abc;
+EOF
+    build_parser
+    compile("spec/test_return_token_from_pattern.d")
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "def!",
+      "ghi!",
+      "def!",
+    ])
+  end
+
+  it "supports lexer modes" do
+    write_grammar <<EOF
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  writeln("begin string mode");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  writeln("captured string");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
+EOF
+    build_parser
+    compile("spec/test_lexer_modes.d")
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "begin string mode",
+      "captured string",
+      "pass1",
+      "begin string mode",
+      "captured string",
+      "pass2",
+    ])
+  end
+
+  it "executes user code associated with a parser rule" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> A B <<
+  writeln("Start!");
+>>
+A -> a <<
+  writeln("A!");
+>>
+B -> b <<
+  writeln("B!");
+>>
+EOF
+    build_parser
+    compile("spec/test_parser_rule_user_code.d")
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "A!",
+      "B!",
+      "Start!",
+    ])
+  end
+
+  it "parses lists" do
+    write_grammar <<EOF
+ptype uint;
+token a;
+Start -> As <<
+  $$ = $1;
+>>
+As -> <<
+  $$ = 0u;
+>>
+As -> As a <<
+  $$ = $1 + 1u;
+>>
+EOF
+    build_parser
+    compile("spec/test_parsing_lists.d")
+    results = run
+    expect(results.status).to eq 0
+    expect(results.stderr).to eq ""
+  end
+
+  it "fails to generate a parser for a LR(1) grammar that is not LALR" do
+    write_grammar <<EOF
+token a;
+token b;
+token c;
+token d;
+token e;
+Start -> a E c;
+Start -> a F d;
+Start -> b F c;
+Start -> b E d;
+E -> e;
+F -> e;
+EOF
+    results = build_parser(capture: true)
+    expect(results.status).to_not eq 0
+    expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+  end
+
+  it "provides matched text to user code blocks" do
+    write_grammar <<EOF
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  writeln("Matched token is ", match);
+>>
+Start -> id;
+EOF
+    build_parser
+    compile("spec/test_lexer_match_text.d")
+    results = run
+    expect(results.status).to eq 0
+    verify_lines(results.stdout, [
+      "Matched token is identifier_123",
+      "pass1",
+    ])
+  end
+
+  it "allows storing a result value for the lexer" do
+    write_grammar <<EOF
+ptype ulong;
+token word /[a-z]+/ <<
+  $$ = match.length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+EOF
+    build_parser
+    compile("spec/test_lexer_result_value.d")
+    results = run
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "allows creating a JSON parser" do
+    write_grammar(File.read("spec/json_parser.propane"))
+    build_parser
+    compile("spec/test_parsing_json.d", "spec/json_types.d")
   end
 end

spec/test_d_lexer.d

@@ -8,77 +8,45 @@ int main()
 
 unittest
 {
-    alias DCP = Testparser.Decoder.DecodedCodePoint;
-    string inputstring = "5+\n 66";
-    const(ubyte) * input = cast(const(ubyte) *)inputstring.ptr;
-    size_t input_length = inputstring.length;
-    DCP dcp;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('5', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('+', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('\n', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(' ', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('6', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('6', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
+    alias Result = Testparser.Decoder.Result;
+    Result result;
 
-    inputstring = "\xf0\x1f\x27\x21";
-    input = cast(const(ubyte) *)inputstring.ptr;
-    input_length = inputstring.length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(0x1F9E1, 4u));
+    result = Testparser.Decoder.decode_code_point("5");
+    assert(result == Result.success('5', 1u));
 
-    inputstring = "\xf0\x1f\x27";
-    input = cast(const(ubyte) *)inputstring.ptr;
-    input_length = inputstring.length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("");
+    assert(result == Result.eof());
 
-    inputstring = "\xf0\x1f\x27\xFF";
-    input = cast(const(ubyte) *)inputstring.ptr;
-    input_length = inputstring.length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("\xC2\xA9");
+    assert(result == Result.success(0xA9u, 2u));
 
-    inputstring = "\xfe";
-    input = cast(const(ubyte) *)inputstring.ptr;
-    input_length = inputstring.length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
+    assert(result == Result.success(0x1F9E1, 4u));
+
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
+    assert(result == Result.decode_error());
+
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
+    assert(result == Result.decode_error());
+
+    result = Testparser.Decoder.decode_code_point("\xfe");
+    assert(result == Result.decode_error());
 }
 
 unittest
 {
-    alias LT = Testparser.Lexer.LexedToken;
+    alias Result = Testparser.Lexer.Result;
     string input = "5 + 4 * \n677 + 567";
-    Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
-    assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_plus));
-    assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_times));
-    assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_plus));
-    assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_int));
-    assert(lexer.lex_token() == LT(1, 9, 0, Testparser._TOKEN_EOF));
+    Testparser.Lexer lexer = new Testparser.Lexer(input);
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 2, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 4, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 6, 1, Testparser.TOKEN_times));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 0, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 4, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 6, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 9, 0, Testparser.TOKEN___EOF));
 
-    lexer = new Testparser.Lexer(null, 0u);
-    assert(lexer.lex_token() == LT(0, 0, 0, Testparser._TOKEN_EOF));
+    lexer = new Testparser.Lexer("");
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 0, Testparser.TOKEN___EOF));
 }

spec/test_d_parser_identical_rules_lookahead.d

@@ -9,10 +9,10 @@ int main()
 unittest
 {
     string input = "aba";
-    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    auto parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
 
     input = "abb";
-    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
 }

spec/test_d_parser_rule_from_multiple_states.d

@@ -9,14 +9,14 @@ int main()
 unittest
 {
     string input = "a";
-    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    auto parser = new Testparser.Parser(input);
     assert(parser.parse() == false);
 
     input = "a b";
-    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
 
     input = "bb";
-    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
 }

spec/test_lexer_match_text.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `identifier_123`;
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    writeln("pass1");
+}

spec/test_lexer_modes.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc "a string" def`;
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    writeln("pass1");
+
+    input = `abc "abc def" def`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    writeln("pass2");
+}

spec/test_lexer_result_value.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `x`;
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 1u);
+
+    input = `fabulous`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 8u);
+}

spec/test_parser_rule_user_code.d

@@ -0,0 +1,14 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_parsing_json.d

@@ -0,0 +1,55 @@
+import testparser;
+import std.stdio;
+import json_types;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = ``;
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse());
+
+    input = `{}`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
+
+    input = `[]`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONArray)parser.result);
+
+    input = `-45.6`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONNumber)parser.result);
+    assert((cast(JSONNumber)parser.result).value == -45.6);
+
+    input = `2E-2`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONNumber)parser.result);
+    assert((cast(JSONNumber)parser.result).value == 0.02);
+
+    input = `{"hi":true}`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
+    JSONObject o = cast(JSONObject)parser.result;
+    assert(o.value["hi"]);
+    assert(cast(JSONTrue)o.value["hi"]);
+
+    input = `{"ff": false, "nn": null}`;
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
+    o = cast(JSONObject)parser.result;
+    assert(o.value["ff"]);
+    assert(cast(JSONFalse)o.value["ff"]);
+    assert(o.value["nn"]);
+    assert(cast(JSONNull)o.value["nn"]);
+}

spec/test_parsing_lists.d

@@ -0,0 +1,25 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 1u);
+
+    input = "";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 16u);
+}

spec/test_pattern.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    writeln("pass1");
+
+    input = "defabcdef";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    writeln("pass2");
+}

spec/test_return_token_from_pattern.d

@@ -0,0 +1,14 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "defghidef";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_user_code.d

@@ -9,10 +9,12 @@ int main()
 unittest
 {
     string input = "abcdef";
-    auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    auto parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
+    writeln("pass1");
 
     input = "abcabcdef";
-    parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
+    parser = new Testparser.Parser(input);
     assert(parser.parse() == true);
+    writeln("pass2");
 }