propane/spec/propane_spec.rb

require "fileutils"
require "open3"

Results = Struct.new(:stdout, :stderr, :status)

describe Propane do
  def write_grammar(grammar, options = {})
    options[:name] ||= ""
    File.write("spec/run/testparser#{options[:name]}.propane", grammar)
  end

  def build_parser(options = {})
    options[:name] ||= ""
    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.d --log spec/run/testparser#{options[:name]}.log]
    if (options[:capture])
      stdout, stderr, status = Open3.capture3(*command)
      Results.new(stdout, stderr, status)
    else
      result = system(*command)
      expect(result).to be_truthy
    end
  end

  def compile(test_files, options = {})
    test_files = Array(test_files)
    options[:parsers] ||= [""]
    parsers = options[:parsers].map do |name|
      "spec/run/testparser#{name}.d"
    end
    result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files)
    expect(result).to be_truthy
  end

  def run
    stdout, stderr, status = Open3.capture3("spec/run/testparser")
    File.binwrite("spec/run/.stderr", stderr)
    File.binwrite("spec/run/.stdout", stdout)
    stderr.sub!(/^.*modules passed unittests\n/, "")
    Results.new(stdout, stderr, status)
  end

  def lines(str)
    str.lines.map(&:chomp)
  end

  def verify_lines(lines, patterns)
    if lines.is_a?(String)
      lines = lines.lines.map(&:chomp)
    end
    patterns.each_with_index do |pattern, i|
      found_index =
        if pattern.is_a?(Regexp)
          lines.find_index {|line| line =~ pattern}
        else
          lines.find_index do |line|
            line.chomp == pattern.chomp
          end
        end
      unless found_index
        $stderr.puts "Lines:"
        $stderr.puts lines
        raise "A line matching #{pattern.inspect} (index #{i}) was not found."
      end
    end
  end

  before(:each) do
    FileUtils.rm_rf("spec/run")
    FileUtils.mkdir_p("spec/run")
  end

  it "generates a D lexer" do
    write_grammar <<EOF
token int /\\d+/;
token plus /\\+/;
token times /\\*/;
drop /\\s+/;
Start -> Foo;
Foo -> int <<
>>
Foo -> plus <<
>>
EOF
    build_parser
    compile("spec/test_d_lexer.d")
    results = run
    expect(results.stderr).to eq ""
    expect(results.status).to eq 0
  end

  it "detects a lexer error when an unknown character is seen" do
    write_grammar <<EOF
ptype int;
token int /\\d+/ <<
  int v;
  foreach (c; match)
  {
    v *= 10;
    v += (c - '0');
  }
  $$ = v;
>>
Start -> int <<
  $$ = $1;
>>
EOF
    build_parser
    compile("spec/test_lexer_unknown_character.d")
    results = run
    expect(results.stderr).to eq ""
    expect(results.status).to eq 0
  end

  it "generates a parser" do
    write_grammar <<EOF
token plus /\\+/;
token times /\\*/;
token zero /0/;
token one /1/;
Start -> E;
E -> E times B;
E -> E plus B;
E -> B;
B -> zero;
B -> one;
EOF
    build_parser
  end

  it "generates an SLR parser" do
    write_grammar <<EOF
token one /1/;
Start -> E;
E -> one E;
E -> one;
EOF
    build_parser
  end

  it "distinguishes between multiple identical rules with lookahead symbol" do
    write_grammar <<EOF
token a;
token b;
Start -> R1 a;
Start -> R2 b;
R1 -> a b;
R2 -> a b;
EOF
    build_parser
    compile("spec/test_d_parser_identical_rules_lookahead.d")
    results = run
    expect(results.status).to eq 0
  end

  it "handles reducing a rule that could be arrived at from multiple states" do
    write_grammar <<EOF
token a;
token b;
drop /\\s+/;
Start -> a R1;
Start -> b R1;
R1 -> b;
EOF
    build_parser
    compile("spec/test_d_parser_rule_from_multiple_states.d")
    results = run
    expect(results.status).to eq 0
  end

  it "executes user code when matching lexer token" do
    write_grammar <<EOF
token abc <<
  writeln("abc!");
>>
token def;
Start -> Abcs def;
Abcs -> ;
Abcs -> abc Abcs;
EOF
    build_parser
    compile("spec/test_user_code.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "abc!",
      "pass1",
      "abc!",
      "abc!",
      "pass2",
    ])
  end

  it "supports a pattern statement" do
    write_grammar <<EOF
token abc;
/def/ <<
  writeln("def!");
>>
Start -> abc;
EOF
    build_parser
    compile("spec/test_pattern.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "def!",
      "pass1",
      "def!",
      "def!",
      "pass2",
    ])
  end

  it "supports returning tokens from pattern code blocks" do
    write_grammar <<EOF
token abc;
/def/ <<
  writeln("def!");
>>
/ghi/ <<
  writeln("ghi!");
  return $token(abc);
>>
Start -> abc;
EOF
    build_parser
    compile("spec/test_return_token_from_pattern.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "def!",
      "ghi!",
      "def!",
    ])
  end

  it "supports lexer modes" do
    write_grammar <<EOF
token abc;
token def;
tokenid string;
drop /\\s+/;
/"/ <<
  writeln("begin string mode");
  $mode(string);
>>
string: /[^"]+/ <<
  writeln("captured string");
>>
string: /"/ <<
  $mode(default);
  return $token(string);
>>
Start -> abc string def;
EOF
    build_parser
    compile("spec/test_lexer_modes.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "begin string mode",
      "captured string",
      "pass1",
      "begin string mode",
      "captured string",
      "pass2",
    ])
  end

  it "executes user code associated with a parser rule" do
    write_grammar <<EOF
token a;
token b;
Start -> A B <<
  writeln("Start!");
>>
A -> a <<
  writeln("A!");
>>
B -> b <<
  writeln("B!");
>>
EOF
    build_parser
    compile("spec/test_parser_rule_user_code.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "A!",
      "B!",
      "Start!",
    ])
  end

  it "parses lists" do
    write_grammar <<EOF
ptype uint;
token a;
Start -> As <<
  $$ = $1;
>>
As -> <<
  $$ = 0u;
>>
As -> As a <<
  $$ = $1 + 1u;
>>
EOF
    build_parser
    compile("spec/test_parsing_lists.d")
    results = run
    expect(results.status).to eq 0
    expect(results.stderr).to eq ""
  end

  it "fails to generate a parser for a LR(1) grammar that is not LALR" do
    write_grammar <<EOF
token a;
token b;
token c;
token d;
token e;
Start -> a E c;
Start -> a F d;
Start -> b F c;
Start -> b E d;
E -> e;
F -> e;
EOF
    results = build_parser(capture: true)
    expect(results.status).to_not eq 0
    expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
  end

  it "provides matched text to user code blocks" do
    write_grammar <<EOF
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
  writeln("Matched token is ", match);
>>
Start -> id;
EOF
    build_parser
    compile("spec/test_lexer_match_text.d")
    results = run
    expect(results.status).to eq 0
    verify_lines(results.stdout, [
      "Matched token is identifier_123",
      "pass1",
    ])
  end

  it "allows storing a result value for the lexer" do
    write_grammar <<EOF
ptype ulong;
token word /[a-z]+/ <<
  $$ = match.length;
>>
Start -> word <<
  $$ = $1;
>>
EOF
    build_parser
    compile("spec/test_lexer_result_value.d")
    results = run
    expect(results.stderr).to eq ""
    expect(results.status).to eq 0
  end

  it "tracks position of parser errors" do
    write_grammar <<EOF
token a;
token num /\\d+/;
drop /\\s+/;
Start -> a num Start;
Start -> a num;
EOF
    build_parser
    compile("spec/test_error_positions.d")
    results = run
    expect(results.stderr).to eq ""
    expect(results.status).to eq 0
  end

  it "allows creating a JSON parser" do
    write_grammar(File.read("spec/json_parser.propane"))
    build_parser
    compile(["spec/test_parsing_json.d", "spec/json_types.d"])
  end

  it "allows generating multiple parsers in the same program" do
    write_grammar(<<EOF, name: "myp1")
prefix myp1_;
token a;
token num /\\d+/;
drop /\\s+/;
Start -> a num;
EOF
    build_parser(name: "myp1")
    write_grammar(<<EOF, name: "myp2")
prefix myp2_;
token b;
token c;
Start -> b c b;
EOF
    build_parser(name: "myp2")
    compile("spec/test_multiple_parsers.d", parsers: %w[myp1 myp2])
    results = run
    expect(results.stderr).to eq ""
    expect(results.status).to eq 0
  end
end