wip

assets/parser.h.erb

@@ -4,8 +4,7 @@
  * This file is generated by Propane.
  */
 
-#ifndef PROPANE_PARSER_H
+#pragma once
-#define PROPANE_PARSER_H
 
 #include <stdint.h>
 #include <stddef.h>
@@ -121,5 +120,3 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
-
-#endif

lib/propane/generator.rb

@@ -206,15 +206,20 @@ class Propane
           index = $1.to_i
           case @language
           when "c"
-            "state_value_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
+            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
           when "d"
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
       else
         code = code.gsub(/\$\$/) do |match|
+          case @language
+          when "c"
+            "out_token_info->pvalue.v_#{pattern.ptypename}"
+          when "d"
             "out_token_info.pvalue.v_#{pattern.ptypename}"
           end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)

spec/propane_spec.rb

@@ -29,7 +29,7 @@ describe Propane do
     end
     case options[:language]
     when "c"
-      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files)
+      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
     when "d"
       result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
     end
@@ -98,6 +98,24 @@ EOF
       end
 
       it "detects a lexer error when an unknown character is seen" do
+        case language
+        when "c"
+          write_grammar <<EOF
+ptype int;
+token int /\\d+/ <<
+  int v = 0;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+Start -> int <<
+  $$ = $1;
+>>
+EOF
+        when "d"
           write_grammar <<EOF
 ptype int;
 token int /\\d+/ <<
@@ -113,6 +131,7 @@ Start -> int <<
   $$ = $1;
 >>
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_lexer_unknown_character.#{language}", language: language)
         results = run
@@ -137,6 +156,60 @@ EOF
       end
 
       it "generates a parser that does basic math - user guide example" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <math.h>
+>>
+
+ptype size_t;
+
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  size_t v = 0u;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = (size_t)pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.math;
@@ -188,6 +261,7 @@ E4 -> lparen E1 rparen <<
   $$ = $2;
 >>
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_basic_math_grammar.#{language}", language: language)
         results = run
@@ -236,6 +310,21 @@ EOF
       end
 
       it "executes user code when matching lexer token" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc <<
+  printf("abc!\\n");
+>>
+token def;
+Start -> Abcs def;
+Abcs -> ;
+Abcs -> abc Abcs;
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -248,6 +337,7 @@ Start -> Abcs def;
 Abcs -> ;
 Abcs -> abc Abcs;
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_user_code.#{language}", language: language)
         results = run
@@ -262,6 +352,19 @@ EOF
       end
 
       it "supports a pattern statement" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+/def/ <<
+  printf("def!\\n");
+>>
+Start -> abc;
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -272,6 +375,7 @@ token abc;
 >>
 Start -> abc;
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_pattern.#{language}", language: language)
         results = run
@@ -286,6 +390,23 @@ EOF
       end
 
       it "supports returning tokens from pattern code blocks" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+/def/ <<
+  printf("def!\\n");
+>>
+/ghi/ <<
+  printf("ghi!\\n");
+  return $token(abc);
+>>
+Start -> abc;
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -300,6 +421,7 @@ token abc;
 >>
 Start -> abc;
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_return_token_from_pattern.#{language}", language: language)
         results = run
@@ -312,6 +434,30 @@ EOF
       end
 
       it "supports lexer modes" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  printf("begin string mode\\n");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  printf("captured string\\n");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -333,6 +479,7 @@ string: /"/ <<
 >>
 Start -> abc string def;
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_lexer_modes.#{language}", language: language)
         results = run
@@ -348,6 +495,25 @@ EOF
       end
 
       it "executes user code associated with a parser rule" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token a;
+token b;
+Start -> A B <<
+  printf("Start!\\n");
+>>
+A -> a <<
+  printf("A!\\n");
+>>
+B -> b <<
+  printf("B!\\n");
+>>
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -364,6 +530,7 @@ B -> b <<
   writeln("B!");
 >>
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_parser_rule_user_code.#{language}", language: language)
         results = run
@@ -377,7 +544,7 @@ EOF
 
       it "parses lists" do
         write_grammar <<EOF
-ptype uint;
+ptype #{language == "c" ? "uint32_t" : "uint"};
 token a;
 Start -> As <<
   $$ = $1;
@@ -416,6 +583,22 @@ EOF
       end
 
       it "provides matched text to user code blocks" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <stdlib.h>
+>>
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * t = malloc(match_length + 1);
+  strncpy(t, (char *)match, match_length);
+  printf("Matched token is %s\\n", t);
+  free(t);
+>>
+Start -> id;
+EOF
+        when "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -425,6 +608,7 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
 >>
 Start -> id;
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_lexer_match_text.#{language}", language: language)
         results = run
@@ -436,6 +620,18 @@ EOF
       end
 
       it "allows storing a result value for the lexer" do
+        case language
+        when "c"
+          write_grammar <<EOF
+ptype uint64_t;
+token word /[a-z]+/ <<
+  $$ = match_length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+EOF
+        when "d"
           write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
@@ -445,6 +641,7 @@ Start -> word <<
   $$ = $1;
 >>
 EOF
+        end
         build_parser(language: language)
         compile("spec/test_lexer_result_value.#{language}", language: language)
         results = run

spec/test_basic_math_grammar.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "1 + 2 * 3 + 4";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(11, p_result(&context));
+
+    input = "1 * 2 ** 4 * 3";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(48, p_result(&context));
+
+    input = "(1 + 2) * 3 + 4";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(13, p_result(&context));
+
+    input = "(2 * 2) ** 3 + 4 + 5";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(73, p_result(&context));
+
+    return 0;
+}

spec/test_error_positions.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a 42";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "a\n123\na  a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 3);
+    assert(context.token == TOKEN_a);
+
+    input = "12";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 0);
+    assert(context.token == TOKEN_num);
+
+    input = "a 12\n\nab";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 1);
+
+    input = "a 12\n\na\n\n77\na   \xAA";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_DECODE_ERROR);
+    assert(p_position(&context).row == 5);
+    assert(p_position(&context).col == 4);
+
+    return 0;
+}

spec/test_error_positions.d

@@ -33,6 +33,5 @@ unittest
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
-    writeln(p_position(&context));
     assert(p_position(&context) == p_position_t(5, 4));
 }

spec/test_lexer_match_text.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "identifier_123";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    return 0;
+}

spec/test_lexer_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc \"a string\" def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc \"abc def\" def";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_result_value.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "fabulous";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 8u);
+
+    return 0;
+}

spec/test_lexer_unknown_character.c

@@ -0,0 +1,18 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+
+    input = "123";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 123u);
+
+    return 0;
+}

spec/test_multiple_parsers.c

@@ -0,0 +1,19 @@
+#include "testparsermyp1.h"
+#include "testparsermyp2.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input1 = "a\n1";
+    myp1_context_t context1;
+    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+
+    char const * input2 = "bcb";
+    myp2_context_t context2;
+    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+
+    return 0;
+}

spec/test_parser_identical_rules_lookahead.c

@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "aba";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parser_rule_from_multiple_states.c

@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 1);
+    assert(context.token == TOKEN___EOF);
+
+    input = "a b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "bb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parser_rule_user_code.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parsing_lists.c

@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 16u);
+
+    return 0;
+}

spec/test_pattern.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "defabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_return_token_from_pattern.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "defghidef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_user_code.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abcabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/testutils.c

@@ -0,0 +1,12 @@
+#include <stdio.h>
+#include <assert.h>
+#include <stdbool.h>
+
+void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
+{
+    if (expected != actual)
+    {
+        fprintf(stderr, "%s:%lu: expected %lu, got %lu\n", file, line, expected, actual);
+        assert(false);
+    }
+}

spec/testutils.h

@@ -0,0 +1,7 @@
+#pragma once
+
+void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
+
+#define assert_eq(expected, actual) \
+    assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
+