Convert Decoder.decode_code_point return type to a SumType

Gemfile.lock

@@ -2,26 +2,26 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
-    psych (5.1.0)
+    psych (5.0.1)
       stringio
     rake (13.0.6)
     rdoc (6.5.0)
       psych (>= 4.0.0)
-    redcarpet (3.6.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    redcarpet (3.5.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
-    stringio (3.0.7)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    stringio (3.0.4)
     syntax (1.2.2)
 
 PLATFORMS

doc/user_guide.md

@@ -49,60 +49,6 @@ detailed information about the parser states and transitions.
 A Propane grammar file provides Propane with the patterns, tokens, grammar
 rules, and user code blocks from which to build the generated lexer and parser.
 
-Example grammar file:
-
-```
-<<
-import std.math;
->>
-
-ptype ulong;
-
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-```
-
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane/generator.rb

@@ -183,7 +183,7 @@ class Propane
     #   Expanded user code block.
     def expand_code(code, parser, rule, pattern)
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
-        "TOKEN_#{Token.code_name($1)}"
+        "Token(TOKEN_#{Token.code_name($1)})"
       end
       if parser
         code = code.gsub(/\$\$/) do |match|
@@ -195,7 +195,7 @@ class Propane
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          "out_token_info.pvalue.v_#{pattern.ptypename}"
+          "result.pvalue.v_#{pattern.ptypename}"
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
@@ -203,7 +203,7 @@ class Propane
           unless mode_id
             raise Error.new("Lexer mode '#{mode_name}' not found")
           end
-          "context.mode = #{mode_id}u"
+          "m_mode = #{mode_id}u"
         end
       end
       code
@@ -220,23 +220,6 @@ class Propane
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end
 
-    # Get an unsigned integer type that can hold the given maximum value.
-    #
-    # @param max [Integer]
-    #   Maximum value to store.
-    #
-    # @return [String]
-    #   Type.
-    def get_type_for(max)
-      if max <= 0xFF
-        "ubyte"
-      elsif max <= 0xFFFF
-        "ushort"
-      else
-        "uint"
-      end
-    end
-
   end
 
 end

lib/propane/grammar.rb

@@ -12,7 +12,6 @@ class Propane
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
-    attr_reader :prefix
 
     def initialize(input)
       @patterns = []
@@ -24,7 +23,6 @@ class Propane
       @mode = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
-      @prefix = "p_"
       parse_grammar!
     end
 
@@ -32,10 +30,6 @@ class Propane
       @ptypes["default"]
     end
 
-    def invalid_token_id
-      @tokens.size
-    end
-
     private
 
     def parse_grammar!
@@ -57,7 +51,6 @@ class Propane
       elsif parse_drop_statement!
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
-      elsif parse_prefix_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -155,7 +148,7 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, drop: true, mode: @mode)
         @mode = nil
         true
       end
@@ -198,13 +191,6 @@ class Propane
       end
     end
 
-    def parse_prefix_statement!
-      if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
-        @prefix = md[1]
-        true
-      end
-    end
-
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""

lib/propane/lexer.rb

@@ -1,13 +1,55 @@
 class Propane
   class Lexer
 
-    attr_reader :state_table
-    attr_reader :transition_table
-    attr_reader :mode_table
-
     def initialize(grammar)
       @grammar = grammar
-      build_tables!
+    end
+
+    def build_tables
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
+      transition_table = []
+      state_table = []
+      mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = state_table.size
+        mode_table << {
+          state_table_offset: state_table_offset,
+        }
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          drop = state.accepts && state.accepts.drop?
+          token =
+            if state.accepts && state.accepts.token
+              state.accepts.token.id
+            end
+          code_id =
+            if state.accepts && state.accepts.code_id
+              state.accepts.code_id
+            end
+          state_table << {
+            transition_table_index: transition_table.size,
+            n_transitions: state.transitions.size,
+            drop: drop,
+            token: token,
+            code_id: code_id,
+          }
+          state.transitions.each do |transition|
+            transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
+        end
+      end
+      [transition_table, state_table, mode_table]
     end
 
     # Get ID for a mode.
@@ -23,46 +65,5 @@ class Propane
       end
     end
 
-    private
-
-    def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
-        pattern.mode
-      end.transform_values do |patterns|
-        {dfa: DFA.new(patterns)}
-      end
-      @modes.each_with_index do |(mode_name, mode_info), index|
-        mode_info[:id] = index
-      end
-      @state_table = []
-      @transition_table = []
-      @mode_table = []
-      @modes.each do |mode_name, mode_info|
-        state_table_offset = @state_table.size
-        @mode_table << {
-          state_table_offset: state_table_offset,
-        }
-        states = mode_info[:dfa].enumerate
-        states.each do |state, id|
-          token = state.accepts && state.accepts.token && state.accepts.token.id
-          code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
-          @state_table << {
-            transition_table_index: @transition_table.size,
-            n_transitions: state.transitions.size,
-            accepts: !!state.accepts,
-            token: token,
-            code_id: code_id,
-          }
-          state.transitions.each do |transition|
-            @transition_table << {
-              first: transition.code_point_range.first,
-              last: transition.code_point_range.last,
-              destination: states[transition.destination] + state_table_offset,
-            }
-          end
-        end
-      end
-    end
-
   end
 end

lib/propane/parser.rb

@@ -2,11 +2,6 @@ class Propane
 
   class Parser
 
-    attr_reader :state_table
-    attr_reader :shift_table
-    attr_reader :reduce_table
-    attr_reader :rule_sets
-
     def initialize(grammar, rule_sets, log)
       @grammar = grammar
       @rule_sets = rule_sets
@@ -38,15 +33,12 @@ class Propane
 
       build_reduce_actions!
       write_log!
-      build_tables!
     end
 
-    private
-
-    def build_tables!
-      @state_table = []
-      @shift_table = []
-      @reduce_table = []
+    def build_tables
+      shift_table = []
+      state_table = []
+      reduce_table = []
       @item_sets.each do |item_set|
         shift_entries = item_set.following_symbols.map do |following_symbol|
           state_id =
@@ -56,14 +48,14 @@ class Propane
               item_set.following_item_set[following_symbol].id
             end
           {
-            symbol_id: following_symbol.id,
+            token_id: following_symbol.id,
             state_id: state_id,
           }
         end
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
+            [{token_id: @grammar.tokens.size, rule_id: ra.id,
               rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
           when Hash
             ra.map do |token, rule|
@@ -73,17 +65,20 @@ class Propane
           else
             []
           end
-        @state_table << {
-          shift_index: @shift_table.size,
+        state_table << {
+          shift_index: shift_table.size,
           n_shifts: shift_entries.size,
-          reduce_index: @reduce_table.size,
+          reduce_index: reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += shift_entries
-        @reduce_table += reduce_entries
+        shift_table += shift_entries
+        reduce_table += reduce_entries
       end
+      [state_table, shift_table, reduce_table]
     end
 
+    private
+
     def process_item_set(item_set)
       item_set.following_symbols.each do |following_symbol|
         unless following_symbol.name == "$EOF"
@@ -247,7 +242,7 @@ class Propane
         @log.puts "  Reduce actions:"
         case item_set.reduce_actions
         when Rule
-          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
+          @log.puts "    * => #{item_set.reduce_actions.id} (#{item_set.reduce_actions.name})"
         when Hash
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"

lib/propane/pattern.rb

@@ -40,6 +40,8 @@ class Propane
     #   Optional parameters.
     # @option options [String, nil] :code
     #   Code block to execute when the pattern is matched.
+    # @option options [Boolean] :drop
+    #   Whether this is a drop pattern.
     # @option options [String, nil] :pattern
     #   Pattern.
     # @option options [Token, nil] :token
@@ -50,6 +52,7 @@ class Propane
     #   Lexer mode for this pattern.
     def initialize(options)
       @code = options[:code]
+      @drop = options[:drop]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
@@ -60,6 +63,14 @@ class Propane
       @nfa = regex.nfa
     end
 
+    # Whether the pattern is a drop pattern.
+    #
+    # @return [Boolean]
+    #   Whether the pattern is a drop pattern.
+    def drop?
+      @drop
+    end
+
   end
 
 end

lib/propane/rule_set.rb

@@ -4,7 +4,7 @@ class Propane
 
     # @return [Integer]
     #   ID of the RuleSet.
-    attr_reader :id
+    attr_accessor :id
 
     # @return [String]
     #   Name of the RuleSet.

spec/propane/grammar_spec.rb

@@ -34,7 +34,6 @@ EOF
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
-      expect(grammar.prefix).to eq "p_"
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
@@ -112,11 +111,8 @@ token code2 <<
 >>
 
 tokenid token_with_no_pattern;
-
-prefix myparser_;
 EOF
       grammar = Grammar.new(input)
-      expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
       expect(o).to_not be_nil

spec/propane_spec.rb

@@ -4,14 +4,12 @@ require "open3"
 Results = Struct.new(:stdout, :stderr, :status)
 
 describe Propane do
-  def write_grammar(grammar, options = {})
-    options[:name] ||= ""
-    File.write("spec/run/testparser#{options[:name]}.propane", grammar)
+  def write_grammar(grammar)
+    File.write("spec/run/testparser.propane", grammar)
   end
 
   def build_parser(options = {})
-    options[:name] ||= ""
-    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.d --log spec/run/testparser#{options[:name]}.log]
+    command = %w[./propane.sh spec/run/testparser.propane spec/run/testparser.d --log spec/run/testparser.log]
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -21,13 +19,8 @@ describe Propane do
     end
   end
 
-  def compile(test_files, options = {})
-    test_files = Array(test_files)
-    options[:parsers] ||= [""]
-    parsers = options[:parsers].map do |name|
-      "spec/run/testparser#{name}.d"
-    end
-    result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+  def compile(*test_files)
+    result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
     expect(result).to be_truthy
   end
 
@@ -69,7 +62,7 @@ describe Propane do
     FileUtils.mkdir_p("spec/run")
   end
 
-  it "generates a lexer" do
+  it "generates a D lexer" do
     write_grammar <<EOF
 token int /\\d+/;
 token plus /\\+/;
@@ -82,30 +75,7 @@ Foo -> plus <<
 >>
 EOF
     build_parser
-    compile("spec/test_lexer.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
-
-  it "detects a lexer error when an unknown character is seen" do
-    write_grammar <<EOF
-ptype int;
-token int /\\d+/ <<
-  int v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-Start -> int <<
-  $$ = $1;
->>
-EOF
-    build_parser
-    compile("spec/test_lexer_unknown_character.d")
+    compile("spec/test_d_lexer.d")
     results = run
     expect(results.stderr).to eq ""
     expect(results.status).to eq 0
@@ -127,65 +97,6 @@ EOF
     build_parser
   end
 
-  it "generates a parser that does basic math - user guide example" do
-    write_grammar <<EOF
-<<
-import std.math;
->>
-
-ptype ulong;
-
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-EOF
-    build_parser
-    compile("spec/test_basic_math_grammar.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
-
   it "generates an SLR parser" do
     write_grammar <<EOF
 token one /1/;
@@ -206,7 +117,7 @@ R1 -> a b;
 R2 -> a b;
 EOF
     build_parser
-    compile("spec/test_parser_identical_rules_lookahead.d")
+    compile("spec/test_d_parser_identical_rules_lookahead.d")
     results = run
     expect(results.status).to eq 0
   end
@@ -221,16 +132,13 @@ Start -> b R1;
 R1 -> b;
 EOF
     build_parser
-    compile("spec/test_parser_rule_from_multiple_states.d")
+    compile("spec/test_d_parser_rule_from_multiple_states.d")
     results = run
     expect(results.status).to eq 0
   end
 
   it "executes user code when matching lexer token" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc <<
   writeln("abc!");
 >>
@@ -254,9 +162,6 @@ EOF
 
   it "supports a pattern statement" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 /def/ <<
   writeln("def!");
@@ -278,9 +183,6 @@ EOF
 
   it "supports returning tokens from pattern code blocks" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 /def/ <<
   writeln("def!");
@@ -304,9 +206,6 @@ EOF
 
   it "supports lexer modes" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 token def;
 tokenid string;
@@ -340,9 +239,6 @@ EOF
 
   it "executes user code associated with a parser rule" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token a;
 token b;
 Start -> A B <<
@@ -408,9 +304,6 @@ EOF
 
   it "provides matched text to user code blocks" do
     write_grammar <<EOF
-<<
-import std.stdio;
->>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   writeln("Matched token is ", match);
 >>
@@ -443,46 +336,9 @@ EOF
     expect(results.status).to eq 0
   end
 
-  it "tracks position of parser errors" do
-    write_grammar <<EOF
-token a;
-token num /\\d+/;
-drop /\\s+/;
-Start -> a num Start;
-Start -> a num;
-EOF
-    build_parser
-    compile("spec/test_error_positions.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
-
   it "allows creating a JSON parser" do
     write_grammar(File.read("spec/json_parser.propane"))
     build_parser
-    compile(["spec/test_parsing_json.d", "spec/json_types.d"])
-  end
-
-  it "allows generating multiple parsers in the same program" do
-    write_grammar(<<EOF, name: "myp1")
-prefix myp1_;
-token a;
-token num /\\d+/;
-drop /\\s+/;
-Start -> a num;
-EOF
-    build_parser(name: "myp1")
-    write_grammar(<<EOF, name: "myp2")
-prefix myp2_;
-token b;
-token c;
-Start -> b c b;
-EOF
-    build_parser(name: "myp2")
-    compile("spec/test_multiple_parsers.d", parsers: %w[myp1 myp2])
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
+    compile("spec/test_parsing_json.d", "spec/json_types.d")
   end
 end

spec/test_basic_math_grammar.d

@@ -1,32 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
-
-    input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
-
-    input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
-
-    input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
-}

spec/test_d_lexer.d

@@ -0,0 +1,52 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    alias Result = Testparser.Decoder.Result;
+    Result result;
+
+    result = Testparser.Decoder.decode_code_point("5");
+    assert(result == Result.success('5', 1u));
+
+    result = Testparser.Decoder.decode_code_point("");
+    assert(result == Result.eof());
+
+    result = Testparser.Decoder.decode_code_point("\xC2\xA9");
+    assert(result == Result.success(0xA9u, 2u));
+
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
+    assert(result == Result.success(0x1F9E1, 4u));
+
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
+    assert(result == Result.decode_error());
+
+    result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
+    assert(result == Result.decode_error());
+
+    result = Testparser.Decoder.decode_code_point("\xfe");
+    assert(result == Result.decode_error());
+}
+
+unittest
+{
+    alias Result = Testparser.Lexer.Result;
+    string input = "5 + 4 * \n677 + 567";
+    Testparser.Lexer lexer = new Testparser.Lexer(input);
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 2, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 4, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 6, 1, Testparser.TOKEN_times));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 0, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 4, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 6, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 9, 0, Testparser.TOKEN___EOF));
+
+    lexer = new Testparser.Lexer("");
+    assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 0, Testparser.TOKEN___EOF));
+}

spec/test_d_parser_identical_rules_lookahead.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aba";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+
+    input = "abb";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_d_parser_rule_from_multiple_states.d

@@ -0,0 +1,22 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == false);
+
+    input = "a b";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+
+    input = "bb";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_error_positions.d

@@ -1,38 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
-
-    input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
-
-    input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
-
-    input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    writeln(p_position(&context));
-    assert(p_position(&context) == p_position_t(5, 4));
-}

spec/test_lexer.d

@@ -1,69 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    size_t result;
-    p_code_point_t code_point;
-    ubyte code_point_length;
-
-    result = p_decode_code_point("5", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == '5');
-    assert(code_point_length == 1u);
-
-    result = p_decode_code_point("", &code_point, &code_point_length);
-    assert(result == P_EOF);
-
-    result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0xA9u);
-    assert(code_point_length == 2u);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0x1F9E1u);
-    assert(code_point_length == 4u);
-
-    result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xfe", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-}
-
-unittest
-{
-    p_token_info_t token_info;
-    string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
-
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
-}

spec/test_lexer_match_text.d

@@ -9,8 +9,7 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass1");
 }

spec/test_lexer_modes.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/test_lexer_result_value.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 1u);
 
     input = `fabulous`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 8u);
 }

spec/test_lexer_unknown_character.d

@@ -1,20 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-
-    input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
-}

spec/test_multiple_parsers.d

@@ -1,21 +0,0 @@
-import testparsermyp1;
-import testparsermyp2;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
-
-    string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
-}

spec/test_parser_identical_rules_lookahead.d

@@ -1,19 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_from_multiple_states.d

@@ -1,25 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
-    assert(context.token == TOKEN___EOF);
-
-    input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_user_code.d

@@ -9,7 +9,6 @@ int main()
 unittest
 {
     string input = "ab";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
 }

spec/test_parsing_json.d

@@ -10,45 +10,44 @@ int main()
 unittest
 {
     string input = ``;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse());
 
     input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
 
     input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONArray)parser.result);
 
     input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONNumber)parser.result);
+    assert((cast(JSONNumber)parser.result).value == -45.6);
 
     input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONNumber)parser.result);
+    assert((cast(JSONNumber)parser.result).value == 0.02);
 
     input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
+    JSONObject o = cast(JSONObject)parser.result;
     assert(o.value["hi"]);
     assert(cast(JSONTrue)o.value["hi"]);
 
     input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse());
+    assert(cast(JSONObject)parser.result);
+    o = cast(JSONObject)parser.result;
     assert(o.value["ff"]);
     assert(cast(JSONFalse)o.value["ff"]);
     assert(o.value["nn"]);

spec/test_parsing_lists.d

@@ -9,18 +9,17 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 1u);
 
     input = "";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 0u);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+    assert(parser.result == 16u);
 }

spec/test_pattern.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass1");
 
     input = "defabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/test_return_token_from_pattern.d

@@ -9,7 +9,6 @@ int main()
 unittest
 {
     string input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
 }

spec/test_user_code.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass1");
 
     input = "abcabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/testutils.d

@@ -1,10 +0,0 @@
-import std.stdio;
-
-void assert_eq(T)(T expected, T actual, string file = __FILE__, size_t line = __LINE__)
-{
-    if (expected != actual)
-    {
-        stderr.writeln(file, ":", line, ": expected ", expected, ", got ", actual);
-        assert(false);
-    }
-}