wip

assets/parser.c.erb

@@ -1,4 +1,4 @@
-#include "<TBD>.h"
+#include "<%= File.basename(output_file).sub(%r{\.[a-z]+$}, "") %>.h"
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
@@ -326,15 +326,15 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
     lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
 {
-    lexer_match_info_t longest_match;
-    lexer_match_info_t attempt_match;
+    lexer_match_info_t longest_match = {0};
+    lexer_match_info_t attempt_match = {0};
     *out_match_info = longest_match;
-    uint32_t current_state = lexer_mode_table[context.mode].state_table_offset;
+    uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
     for (;;)
     {
-        size_t const input_index = context.input_index + attempt_match.length;
-        uint8_t const * input = &context.input[input_index];
-        size_t input_length = context.input_length - input_index;
+        size_t const input_index = context->input_index + attempt_match.length;
+        uint8_t const * input = &context->input[input_index];
+        size_t input_length = context->input_length - input_index;
         <%= @grammar.prefix %>code_point_t code_point;
         uint8_t code_point_length;
         size_t result = <%= @grammar.prefix %>decode_code_point(input, input_length, &code_point, &code_point_length);
@@ -427,8 +427,8 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
-    <%= @grammar.prefix %>token_info_t token_info;
-    token_info.position = context.text_position;
+    <%= @grammar.prefix %>token_info_t token_info = {0};
+    token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
     *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
@@ -437,12 +437,12 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     switch (result)
     {
     case P_SUCCESS:
-        <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
-        if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
+        <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match = &context.input[context.input_index];
+            uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state.code_id, match, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match, match_info.length, &token_info);
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -454,15 +454,15 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
 
         /* Update the input position tracking. */
-        context.input_index += match_info.length;
-        context.text_position.row += match_info.delta_position.row;
+        context->input_index += match_info.length;
+        context->text_position.row += match_info.delta_position.row;
         if (match_info.delta_position.row != 0u)
         {
-            context.text_position.col = match_info.delta_position.col;
+            context->text_position.col = match_info.delta_position.col;
         }
         else
         {
-            context.text_position.col += match_info.delta_position.col;
+            context->text_position.col += match_info.delta_position.col;
         }
 
         if (token_to_accept == INVALID_TOKEN_ID)
@@ -481,15 +481,15 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
 
     case P_DECODE_ERROR:
         /* Update the input position tracking. */
-        context.input_index += match_info.length;
-        context.text_position.row += match_info.delta_position.row;
+        context->input_index += match_info.length;
+        context->text_position.row += match_info.delta_position.row;
         if (match_info.delta_position.row != 0u)
         {
-            context.text_position.col = match_info.delta_position.col;
+            context->text_position.col = match_info.delta_position.col;
         }
         else
         {
-            context.text_position.col += match_info.delta_position.col;
+            context->text_position.col += match_info.delta_position.col;
         }
         return result;
 
@@ -531,7 +531,7 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  *************************************************************************/
 
 /** Reduce ID type. */
-alias reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
+typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
 
 /**
  * A symbol ID can hold either a token ID or a rule set ID.
@@ -539,16 +539,16 @@ alias reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
  * Token IDs and rule set IDs share the same namespace, with rule set IDs
  * beginning after token IDs end.
  */
-alias symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
+typedef <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %> symbol_id_t;
 
 /** Parser state ID type. */
-alias parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
+typedef <%= get_type_for(@parser.state_table.size) %> parser_state_id_t;
 
 /** Parser rule ID type. */
-alias rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
+typedef <%= get_type_for(@grammar.rules.size) %> rule_id_t;
 
 /** Parser shift ID type. */
-alias shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
+typedef <%= get_type_for(@parser.shift_table.size) %> shift_id_t;
 
 /** Shift table entry. */
 typedef struct
@@ -622,25 +622,25 @@ typedef struct
 } state_value_t;
 
 /** Parser shift table. */
-static immutable shift_t[] parser_shift_table = [
+static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
-    shift_t(<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u),
+    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
 <%   end %>
-];
+};
 
 /** Parser reduce table. */
-static immutable reduce_t[] parser_reduce_table = [
+static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
+    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
 <%   end %>
-];
+};
 
 /** Parser state table. */
-static immutable parser_state_t[] parser_state_table = [
+static const parser_state_t parser_state_table[] = {
 <%   @parser.state_table.each do |state| %>
-    parser_state_t(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
+    {<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u},
 <%   end %>
-];
+};
 
 /* state_values stack functionality */
 
@@ -658,7 +658,7 @@ typedef struct
  * @param stack
  *   state_values stack structure.
  */
-void state_values_stack_init(state_values_stack_t * stack)
+static void state_values_stack_init(state_values_stack_t * stack)
 {
     const size_t initial_capacity = 10u;
     stack->length = 0u;
@@ -666,13 +666,35 @@ void state_values_stack_init(state_values_stack_t * stack)
     stack->entries = (state_value_t *)malloc(initial_capacity * sizeof(state_value_t));
 }
 
+/**
+ * Index a state_values stack.
+ *
+ * @param stack
+ *   state_values stack structure.
+ * @param index
+ *   Index to the stack.
+ *
+ * @return Pointer to the state value structure at the given index.
+ */
+static state_value_t * state_values_stack_index(state_values_stack_t * stack, int index)
+{
+    if (index >= 0)
+    {
+        return &stack->entries[index];
+    }
+    else
+    {
+        return &stack->entries[stack->length - (size_t)(unsigned int)(-index)];
+    }
+}
+
 /**
  * Push a new state_value to the state_values stack.
  *
  * @param stack
  *   state_values stack structure.
  */
-void state_values_stack_push(state_values_stack_t * stack)
+static void state_values_stack_push(state_values_stack_t * stack)
 {
     size_t const current_capacity = stack->capacity;
     size_t const current_length = stack->length;
@@ -680,7 +702,7 @@ void state_values_stack_push(state_values_stack_t * stack)
     {
         size_t const new_capacity = current_capacity * 2u;
         state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
-        memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t);
+        memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
         stack->entries = new_entries;
@@ -689,6 +711,30 @@ void state_values_stack_push(state_values_stack_t * stack)
     stack->length = current_length + 1u;
 }
 
+/**
+ * Pop entries from a state_values stack.
+ *
+ * @param stack
+ *   state_values stack structure.
+ * @param n
+ *   Number of states to pop.
+ */
+static void state_values_stack_pop(state_values_stack_t * stack, size_t n)
+{
+    stack->length -= n;
+}
+
+/**
+ * Free memory for a state_values stack structure.
+ *
+ * @param stack
+ *   state_values stack structure.
+ */
+static void state_values_stack_free(state_values_stack_t * stack)
+{
+     free(stack->entries);
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -696,9 +742,9 @@ void state_values_stack_push(state_values_stack_t * stack)
  *
  * @return Parse value.
  */
-static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_value_t[] statevalues, uint32_t n_states)
+static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
 {
-    <%= @grammar.prefix %>value_t _pvalue;
+    <%= @grammar.prefix %>value_t _pvalue = {0};
 
     switch (rule)
     {
@@ -775,7 +821,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context.token holds the unexpected token.
+ *   The value context->token holds the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -785,9 +831,12 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
-    state_value_t[] statevalues = new state_value_t[](1);
+    state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
     <%= @grammar.prefix %>value_t reduced_parser_value;
+    state_values_stack_init(&statevalues);
+    state_values_stack_push(&statevalues);
+    size_t result;
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
@@ -795,53 +844,56 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
-                return lexer_result;
+                result = lexer_result;
+                break;
             }
             token = token_info.token;
         }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
-            shift_state = check_shift(statevalues[$-1].state_id, reduced_rule_set);
+            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, reduced_rule_set);
         }
         if (shift_state == INVALID_ID)
         {
-            shift_state = check_shift(statevalues[$-1].state_id, token);
+            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, token);
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
-                context.parse_result = statevalues[$-1].pvalue;
-                return P_SUCCESS;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+                result = P_SUCCESS;
+                break;
             }
         }
         if (shift_state != INVALID_ID)
         {
             /* We have something to shift. */
-            statevalues ~= state_value_t(shift_state);
+            state_values_stack_push(&statevalues);
+            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
                 token = INVALID_TOKEN_ID;
-                statevalues[$-1].pvalue = token_info.pvalue;
+                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
             }
             else
             {
                 /* We shifted a RuleSet. */
-                statevalues[$-1].pvalue = reduced_parser_value;
-                <%= @grammar.prefix %>value_t new_parse_result;
+                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                <%= @grammar.prefix %>value_t new_parse_result = {0};
                 reduced_parser_value = new_parse_result;
                 reduced_rule_set = INVALID_ID;
             }
             continue;
         }
 
-        size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states);
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
-            statevalues.length -= parser_reduce_table[reduce_index].n_states;
+            state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
         }
 
@@ -850,10 +902,13 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
          * the context text position to point to the token rather than the text
          * after it, so that if the caller wants to report the error position,
          * it will point to the correct position of the unexpected token. */
-        context.text_position = token_info.position;
-        context.token = token;
-        return P_UNEXPECTED_TOKEN;
+        context->text_position = token_info.position;
+        context->token = token;
+        result = P_UNEXPECTED_TOKEN;
+        break;
     }
+    state_values_stack_free(&statevalues);
+    return result;
 }
 
 /**
@@ -866,7 +921,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  */
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
-    return context.parse_result.v_<%= start_rule_type[0] %>;
+    return context->parse_result.v_<%= start_rule_type[0] %>;
 }
 
 /**
@@ -879,5 +934,5 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  */
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
 {
-    return context.text_position;
+    return context->text_position;
 }

assets/parser.h.erb

@@ -8,6 +8,7 @@
 #define PROPANE_PARSER_H
 
 #include <stdint.h>
+#include <stddef.h>
 
 /**************************************************************************
  * Public types
@@ -87,6 +88,9 @@ typedef struct
     /** Input text. */
     uint8_t const * input;
 
+    /** Input text length. */
+    size_t input_length;
+
     /** Input text index (byte offset). */
     size_t input_index;
 
@@ -105,4 +109,17 @@ typedef struct
     <%= @grammar.prefix %>token_t token;
 } <%= @grammar.prefix %>context_t;
 
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+
+size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
+        <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
+
+size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
+
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
+
+<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
+
 #endif

lib/propane/generator.rb

@@ -11,14 +11,27 @@ class Propane
         @log = StringIO.new
       end
       @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
+      @language =
+        if output_file =~ /\.([a-z]+)$/
+          $1
+        else
+          "d"
+        end
       process_grammar!
     end
 
     def generate
-      erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.d.erb")), trim_mode: "<>")
-      result = erb.result(binding.clone)
-      File.open(@output_file, "wb") do |fh|
-        fh.write(result)
+      extensions = [@language]
+      if @language == "c"
+        extensions += %w[h]
+      end
+      extensions.each do |extension|
+        erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.#{extension}.erb")), trim_mode: "<>")
+        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        result = erb.result(binding.clone)
+        File.open(output_file, "wb") do |fh|
+          fh.write(result)
+        end
       end
       @log.close
     end
@@ -191,7 +204,12 @@ class Propane
         end
         code = code.gsub(/\$(\d+)/) do |match|
           index = $1.to_i
-          "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
+          case @language
+          when "c"
+            "state_value_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
+          when "d"
+            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
+          end
         end
       else
         code = code.gsub(/\$\$/) do |match|
@@ -203,7 +221,12 @@ class Propane
           unless mode_id
             raise Error.new("Lexer mode '#{mode_name}' not found")
           end
-          "context.mode = #{mode_id}u"
+          case @language
+          when "c"
+            "context->mode = #{mode_id}u"
+          when "d"
+            "context.mode = #{mode_id}u"
+          end
         end
       end
       code
@@ -229,11 +252,26 @@ class Propane
     #   Type.
     def get_type_for(max)
       if max <= 0xFF
-        "ubyte"
+        case @language
+        when "c"
+          "uint8_t"
+        when "d"
+          "ubyte"
+        end
       elsif max <= 0xFFFF
-        "ushort"
+        case @language
+        when "c"
+          "uint16_t"
+        when "d"
+          "ushort"
+        end
       else
-        "uint"
+        case @language
+        when "c"
+          "uint32_t"
+        else
+          "uint"
+        end
       end
     end
 

spec/propane_spec.rb

@@ -11,7 +11,7 @@ describe Propane do
 
   def build_parser(options = {})
     options[:name] ||= ""
-    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.d --log spec/run/testparser#{options[:name]}.log]
+    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -25,9 +25,14 @@ describe Propane do
     test_files = Array(test_files)
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
-      "spec/run/testparser#{name}.d"
+      "spec/run/testparser#{name}.#{options[:language]}"
+    end
+    case options[:language]
+    when "c"
+      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files)
+    when "d"
+      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
     end
-    result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
     expect(result).to be_truthy
   end
 
@@ -69,8 +74,12 @@ describe Propane do
     FileUtils.mkdir_p("spec/run")
   end
 
-  it "generates a lexer" do
-    write_grammar <<EOF
+  %w[d c].each do |language|
+
+    context "#{language.upcase} language" do
+
+      it "generates a lexer" do
+        write_grammar <<EOF
 token int /\\d+/;
 token plus /\\+/;
 token times /\\*/;
@@ -81,15 +90,15 @@ Foo -> int <<
 Foo -> plus <<
 >>
 EOF
-    build_parser
-    compile("spec/test_lexer.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_lexer.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
 
-  it "detects a lexer error when an unknown character is seen" do
-    write_grammar <<EOF
+      it "detects a lexer error when an unknown character is seen" do
+        write_grammar <<EOF
 ptype int;
 token int /\\d+/ <<
   int v;
@@ -104,15 +113,15 @@ Start -> int <<
   $$ = $1;
 >>
 EOF
-    build_parser
-    compile("spec/test_lexer_unknown_character.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_lexer_unknown_character.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
 
-  it "generates a parser" do
-    write_grammar <<EOF
+      it "generates a parser" do
+        write_grammar <<EOF
 token plus /\\+/;
 token times /\\*/;
 token zero /0/;
@@ -124,11 +133,11 @@ E -> B;
 B -> zero;
 B -> one;
 EOF
-    build_parser
-  end
+        build_parser(language: language)
+      end
 
-  it "generates a parser that does basic math - user guide example" do
-    write_grammar <<EOF
+      it "generates a parser that does basic math - user guide example" do
+        write_grammar <<EOF
 <<
 import std.math;
 >>
@@ -179,25 +188,25 @@ E4 -> lparen E1 rparen <<
   $$ = $2;
 >>
 EOF
-    build_parser
-    compile("spec/test_basic_math_grammar.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_basic_math_grammar.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
 
-  it "generates an SLR parser" do
-    write_grammar <<EOF
+      it "generates an SLR parser" do
+        write_grammar <<EOF
 token one /1/;
 Start -> E;
 E -> one E;
 E -> one;
 EOF
-    build_parser
-  end
+        build_parser(language: language)
+      end
 
-  it "distinguishes between multiple identical rules with lookahead symbol" do
-    write_grammar <<EOF
+      it "distinguishes between multiple identical rules with lookahead symbol" do
+        write_grammar <<EOF
 token a;
 token b;
 Start -> R1 a;
@@ -205,14 +214,14 @@ Start -> R2 b;
 R1 -> a b;
 R2 -> a b;
 EOF
-    build_parser
-    compile("spec/test_parser_identical_rules_lookahead.d")
-    results = run
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+      end
 
-  it "handles reducing a rule that could be arrived at from multiple states" do
-    write_grammar <<EOF
+      it "handles reducing a rule that could be arrived at from multiple states" do
+        write_grammar <<EOF
 token a;
 token b;
 drop /\\s+/;
@@ -220,14 +229,14 @@ Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
-    build_parser
-    compile("spec/test_parser_rule_from_multiple_states.d")
-    results = run
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+      end
 
-  it "executes user code when matching lexer token" do
-    write_grammar <<EOF
+      it "executes user code when matching lexer token" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -239,21 +248,21 @@ Start -> Abcs def;
 Abcs -> ;
 Abcs -> abc Abcs;
 EOF
-    build_parser
-    compile("spec/test_user_code.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "abc!",
-      "pass1",
-      "abc!",
-      "abc!",
-      "pass2",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_user_code.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "abc!",
+          "pass1",
+          "abc!",
+          "abc!",
+          "pass2",
+        ])
+      end
 
-  it "supports a pattern statement" do
-    write_grammar <<EOF
+      it "supports a pattern statement" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -263,21 +272,21 @@ token abc;
 >>
 Start -> abc;
 EOF
-    build_parser
-    compile("spec/test_pattern.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "def!",
-      "pass1",
-      "def!",
-      "def!",
-      "pass2",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_pattern.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "def!",
+          "pass1",
+          "def!",
+          "def!",
+          "pass2",
+        ])
+      end
 
-  it "supports returning tokens from pattern code blocks" do
-    write_grammar <<EOF
+      it "supports returning tokens from pattern code blocks" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -291,19 +300,19 @@ token abc;
 >>
 Start -> abc;
 EOF
-    build_parser
-    compile("spec/test_return_token_from_pattern.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "def!",
-      "ghi!",
-      "def!",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_return_token_from_pattern.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "def!",
+          "ghi!",
+          "def!",
+        ])
+      end
 
-  it "supports lexer modes" do
-    write_grammar <<EOF
+      it "supports lexer modes" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -324,22 +333,22 @@ string: /"/ <<
 >>
 Start -> abc string def;
 EOF
-    build_parser
-    compile("spec/test_lexer_modes.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "begin string mode",
-      "captured string",
-      "pass1",
-      "begin string mode",
-      "captured string",
-      "pass2",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_lexer_modes.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "begin string mode",
+          "captured string",
+          "pass1",
+          "begin string mode",
+          "captured string",
+          "pass2",
+        ])
+      end
 
-  it "executes user code associated with a parser rule" do
-    write_grammar <<EOF
+      it "executes user code associated with a parser rule" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -355,19 +364,19 @@ B -> b <<
   writeln("B!");
 >>
 EOF
-    build_parser
-    compile("spec/test_parser_rule_user_code.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "A!",
-      "B!",
-      "Start!",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_parser_rule_user_code.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "A!",
+          "B!",
+          "Start!",
+        ])
+      end
 
-  it "parses lists" do
-    write_grammar <<EOF
+      it "parses lists" do
+        write_grammar <<EOF
 ptype uint;
 token a;
 Start -> As <<
@@ -380,15 +389,15 @@ As -> As a <<
   $$ = $1 + 1u;
 >>
 EOF
-    build_parser
-    compile("spec/test_parsing_lists.d")
-    results = run
-    expect(results.status).to eq 0
-    expect(results.stderr).to eq ""
-  end
+        build_parser(language: language)
+        compile("spec/test_parsing_lists.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        expect(results.stderr).to eq ""
+      end
 
-  it "fails to generate a parser for a LR(1) grammar that is not LALR" do
-    write_grammar <<EOF
+      it "fails to generate a parser for a LR(1) grammar that is not LALR" do
+        write_grammar <<EOF
 token a;
 token b;
 token c;
@@ -401,13 +410,13 @@ Start -> b E d;
 E -> e;
 F -> e;
 EOF
-    results = build_parser(capture: true)
-    expect(results.status).to_not eq 0
-    expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
-  end
+        results = build_parser(capture: true, language: language)
+        expect(results.status).to_not eq 0
+        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+      end
 
-  it "provides matched text to user code blocks" do
-    write_grammar <<EOF
+      it "provides matched text to user code blocks" do
+        write_grammar <<EOF
 <<
 import std.stdio;
 >>
@@ -416,18 +425,18 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
 >>
 Start -> id;
 EOF
-    build_parser
-    compile("spec/test_lexer_match_text.d")
-    results = run
-    expect(results.status).to eq 0
-    verify_lines(results.stdout, [
-      "Matched token is identifier_123",
-      "pass1",
-    ])
-  end
+        build_parser(language: language)
+        compile("spec/test_lexer_match_text.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "Matched token is identifier_123",
+          "pass1",
+        ])
+      end
 
-  it "allows storing a result value for the lexer" do
-    write_grammar <<EOF
+      it "allows storing a result value for the lexer" do
+        write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
   $$ = match.length;
@@ -436,53 +445,55 @@ Start -> word <<
   $$ = $1;
 >>
 EOF
-    build_parser
-    compile("spec/test_lexer_result_value.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_lexer_result_value.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
 
-  it "tracks position of parser errors" do
-    write_grammar <<EOF
+      it "tracks position of parser errors" do
+        write_grammar <<EOF
 token a;
 token num /\\d+/;
 drop /\\s+/;
 Start -> a num Start;
 Start -> a num;
 EOF
-    build_parser
-    compile("spec/test_error_positions.d")
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
-  end
+        build_parser(language: language)
+        compile("spec/test_error_positions.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
 
-  it "allows creating a JSON parser" do
-    write_grammar(File.read("spec/json_parser.propane"))
-    build_parser
-    compile(["spec/test_parsing_json.d", "spec/json_types.d"])
-  end
+      it "allows creating a JSON parser" do
+        write_grammar(File.read("spec/json_parser.propane"))
+        build_parser(language: language)
+        compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
+      end
 
-  it "allows generating multiple parsers in the same program" do
-    write_grammar(<<EOF, name: "myp1")
+      it "allows generating multiple parsers in the same program" do
+        write_grammar(<<EOF, name: "myp1")
 prefix myp1_;
 token a;
 token num /\\d+/;
 drop /\\s+/;
 Start -> a num;
 EOF
-    build_parser(name: "myp1")
-    write_grammar(<<EOF, name: "myp2")
+        build_parser(name: "myp1", language: language)
+        write_grammar(<<EOF, name: "myp2")
 prefix myp2_;
 token b;
 token c;
 Start -> b c b;
 EOF
-    build_parser(name: "myp2")
-    compile("spec/test_multiple_parsers.d", parsers: %w[myp1 myp2])
-    results = run
-    expect(results.stderr).to eq ""
-    expect(results.status).to eq 0
+        build_parser(name: "myp2", language: language)
+        compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+    end
   end
 end

spec/test_lexer.c

@@ -0,0 +1,92 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    size_t result;
+    p_code_point_t code_point;
+    uint8_t code_point_length;
+
+    result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == '5');
+    assert(code_point_length == 1u);
+
+    result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
+    assert(result == P_EOF);
+
+    result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0xA9u);
+    assert(code_point_length == 2u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0x1F9E1u);
+    assert(code_point_length == 4u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+
+    p_token_info_t token_info;
+    char const * input = "5 + 4 * \n677 + 567";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 2u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_times);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 9u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    p_context_init(&context, (uint8_t const *)"", 0u);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    return 0;
+}