JSON parser checkpoint

.gitignore

@@ -2,8 +2,7 @@
 /.yardoc
 /_yardoc/
 /coverage/
-/dist/
+/doc/
-/gen/
 /pkg/
 /spec/reports/
 /tmp/

CHANGELOG.md

@@ -1,3 +0,0 @@
-## v1.0.0
-
-- Initial release

Gemfile

@@ -2,6 +2,3 @@ source "https://rubygems.org"
 
 gem "rake"
 gem "rspec"
-gem "rdoc"
-gem "redcarpet"
-gem "syntax"

Gemfile.lock

@@ -2,37 +2,27 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
-    psych (5.1.0)
-      stringio
     rake (13.0.6)
-    rdoc (6.5.0)
+    rspec (3.11.0)
-      psych (>= 4.0.0)
+      rspec-core (~> 3.11.0)
-    redcarpet (3.6.0)
+      rspec-expectations (~> 3.11.0)
-    rspec (3.12.0)
+      rspec-mocks (~> 3.11.0)
-      rspec-core (~> 3.12.0)
+    rspec-core (3.11.0)
-      rspec-expectations (~> 3.12.0)
+      rspec-support (~> 3.11.0)
-      rspec-mocks (~> 3.12.0)
+    rspec-expectations (3.11.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
+      rspec-support (~> 3.11.0)
-    rspec-mocks (3.12.6)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
+      rspec-support (~> 3.11.0)
-    rspec-support (3.12.1)
+    rspec-support (3.11.0)
-    stringio (3.0.7)
-    syntax (1.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   rake
-  rdoc
-  redcarpet
   rspec
-  syntax
 
 BUNDLED WITH
-   2.3.7
+   2.4.0.dev

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2023 Josh Holtrop
+Copyright (c) 2010-2022 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Rakefile

@@ -1,9 +1,5 @@
 require "rspec/core/rake_task"
 
-task :build_dist do
-  sh "ruby rb/build_dist.rb"
-end
-
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
@@ -11,8 +7,3 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
 end
 
 task :default => :spec
-
-desc "Build user guide"
-task :user_guide do
-  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
-end

assets/parser.c.erb

@@ -1,938 +0,0 @@
-#include "<%= File.basename(output_file).sub(%r{\.[a-z]+$}, "") %>.h"
-#include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-
-/**************************************************************************
- * User code blocks
- *************************************************************************/
-
-<% @grammar.code_blocks.each do |code| %>
-<%= code %>
-<% end %>
-
-/**************************************************************************
- * Private types
- *************************************************************************/
-
-<% if @grammar.prefix.upcase != "P_" %>
-/* Result codes. */
-#define P_SUCCESS 0u
-#define P_DECODE_ERROR 1u
-#define P_UNEXPECTED_INPUT 2u
-#define P_UNEXPECTED_TOKEN 3u
-#define P_DROP 4u
-#define P_EOF 5u
-<% end %>
-
-/* An invalid ID value. */
-#define INVALID_ID ((size_t)-1)
-
-/**************************************************************************
- * State initialization
- *************************************************************************/
-
-/**
- * Initialize lexer/parser context structure.
- *
- * @param[out] context
- *   Lexer/parser context structure.
- * @param input
- *   Text input.
- * @param input_length
- *   Text input length.
- */
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
-{
-    /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext = {0};
-
-    /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.input_length = input_length;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
-
-    /* Copy to the user's context structure. */
-    *context = newcontext;
-}
-
-/**************************************************************************
- * Decoder
- *************************************************************************/
-
-/**
- * Decode a UTF-8 code point.
- *
- * @param input
- *   Text input to decode.
- * @param input_length
- *   Input text length.
- * @param[out] out_code_point
- *   The decoded code point is stored here if the return value is P_SUCCESS.
- * @param[out] out_code_point_length
- *   The number of bytes the code point used is stored here if the return value
- *   is P_SUCCESS.
- *
- * @retval P_SUCCESS on a successful code point decode
- * @retval P_DECODE_ERROR when an encoding error is observed
- * @retval P_EOF when the end of the text input is reached
- */
-size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
-    <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length)
-{
-    if (input_length == 0u)
-    {
-        return P_EOF;
-    }
-    char c = input[0];
-    <%= @grammar.prefix %>code_point_t code_point;
-    uint8_t code_point_length;
-    if ((c & 0x80u) == 0u)
-    {
-        code_point = c;
-        code_point_length = 1u;
-    }
-    else
-    {
-        uint8_t following_bytes;
-        if ((c & 0xE0u) == 0xC0u)
-        {
-            code_point = c & 0x1Fu;
-            following_bytes = 1u;
-        }
-        else if ((c & 0xF0u) == 0xE0u)
-        {
-            code_point = c & 0x0Fu;
-            following_bytes = 2u;
-        }
-        else if ((c & 0xF8u) == 0xF0u)
-        {
-            code_point = c & 0x07u;
-            following_bytes = 3u;
-        }
-        else if ((c & 0xFCu) == 0xF8u)
-        {
-            code_point = c & 0x03u;
-            following_bytes = 4u;
-        }
-        else if ((c & 0xFEu) == 0xFCu)
-        {
-            code_point = c & 0x01u;
-            following_bytes = 5u;
-        }
-        else
-        {
-            return P_DECODE_ERROR;
-        }
-        if (input_length <= following_bytes)
-        {
-            return P_DECODE_ERROR;
-        }
-        code_point_length = (uint8_t)(following_bytes + 1u);
-        for (size_t i = 0u; i < following_bytes; i++)
-        {
-            char b = input[i + 1u];
-            if ((b & 0xC0u) != 0x80u)
-            {
-                return P_DECODE_ERROR;
-            }
-            code_point = (code_point << 6u) | (b & 0x3Fu);
-        }
-    }
-    *out_code_point = code_point;
-    *out_code_point_length = code_point_length;
-    return P_SUCCESS;
-}
-
-/**************************************************************************
- * Lexer
- *************************************************************************/
-
-/** Lexer state ID type. */
-typedef <%= get_type_for(@lexer.state_table.size) %> lexer_state_id_t;
-
-/** Invalid lexer state ID. */
-#define INVALID_LEXER_STATE_ID <%= @lexer.state_table.size %>u
-
-/** Lexer user code ID type. */
-<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
-typedef <%= get_type_for(user_code_id_count) %> lexer_user_code_id_t;
-
-/** Invalid lexer user code ID. */
-#define INVALID_USER_CODE_ID <%= user_code_id_count %>u
-
-/**
- * Lexer transition table entry.
- *
- * An incoming code point matching the range for a transition entry will cause
- * the lexer to progress to the destination state.
- */
-typedef struct
-{
-    /** First code point in the range for this transition. */
-    <%= @grammar.prefix %>code_point_t first;
-
-    /** Last code point in the range for this transition. */
-    <%= @grammar.prefix %>code_point_t last;
-
-    /** Destination lexer state ID for this transition. */
-    lexer_state_id_t destination_state;
-} lexer_transition_t;
-
-/** Lexer state table entry. */
-typedef struct
-{
-    /** Index to the transition table for this state. */
-    <%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
-
-    /** Number of transition table entries for this state. */
-    <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
-
-    /** Lexer token formed at this state. */
-    <%= @grammar.prefix %>token_t token;
-
-    /** Lexer user code ID to execute at this state. */
-    lexer_user_code_id_t code_id;
-
-    /** Whether this state matches a lexer pattern. */
-    bool accepts;
-} lexer_state_t;
-
-/** Lexer mode table entry. */
-typedef struct
-{
-    /** Offset in the state table to be used for this mode. */
-    uint32_t state_table_offset;
-} lexer_mode_t;
-
-/**
- * Lexer match info structure.
- *
- * This structure holds output values from the lexer upon a successful pattern
- * match.
- */
-typedef struct
-{
-    /** Number of bytes of input text used to match. */
-    size_t length;
-
-    /** Input text position delta. */
-    <%= @grammar.prefix %>position_t delta_position;
-
-    /** Accepting lexer state from the match. */
-    lexer_state_t const * accepting_state;
-} lexer_match_info_t;
-
-/** Lexer transition table. */
-static lexer_transition_t lexer_transition_table[] = {
-<% @lexer.transition_table.each do |transition_table_entry| %>
-    {<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u},
-<% end %>
-};
-
-/** Lexer state table. */
-static lexer_state_t lexer_state_table[] = {
-<% @lexer.state_table.each do |state_table_entry| %>
-    {<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>},
-<% end %>
-};
-
-/** Lexer mode table. */
-static lexer_mode_t lexer_mode_table[] = {
-<% @lexer.mode_table.each do |mode_table_entry| %>
-    {<%= mode_table_entry[:state_table_offset] %>},
-<% end %>
-};
-
-/**
- * Execute user code associated with a lexer pattern.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param code_id
- *   The ID of the user code block to execute.
- * @param match
- *   Matched text for this pattern.
- * @param match_length
- *   Matched text length.
- * @param out_token_info
- *   Lexer token info in progress.
- *
- * @return Token to accept, or invalid token if the user code does
- *   not explicitly return a token.
- */
-static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match,
-    size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
-{
-    switch (code_id)
-    {
-<% @grammar.patterns.each do |pattern| %>
-<%   if pattern.code_id %>
-    case <%= pattern.code_id %>u: {
-<%= expand_code(pattern.code, false, nil, pattern) %>
-    } break;
-<%   end %>
-<% end %>
-    default: break;
-    }
-
-    return INVALID_TOKEN_ID;
-}
-
-/**
- * Check if there is a transition from the current lexer state to another
- * based on the given input code point.
- *
- * @param current_state
- *   Current lexer state.
- * @param code_point
- *   Input code point.
- *
- * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
- */
-static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t code_point)
-{
-    uint32_t transition_table_index = lexer_state_table[current_state].transition_table_index;
-    for (uint32_t i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
-    {
-        if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
-            (code_point <= lexer_transition_table[transition_table_index + i].last))
-        {
-            return lexer_transition_table[transition_table_index + i].destination_state;
-        }
-    }
-    return INVALID_LEXER_STATE_ID;
-}
-
-/**
- * Find the longest lexer pattern match at the current position.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
- *
- * @reval P_SUCCESS
- *   A token was successfully lexed.
- * @reval P_DECODE_ERROR
- *   The decoder encountered invalid text encoding.
- * @reval P_UNEXPECTED_INPUT
- *   Input text does not match any lexer pattern.
- * @retval P_EOF
- *   The end of the text input was reached.
- */
-static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
-    lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
-{
-    lexer_match_info_t longest_match = {0};
-    lexer_match_info_t attempt_match = {0};
-    *out_match_info = longest_match;
-    uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
-    for (;;)
-    {
-        size_t const input_index = context->input_index + attempt_match.length;
-        uint8_t const * input = &context->input[input_index];
-        size_t input_length = context->input_length - input_index;
-        <%= @grammar.prefix %>code_point_t code_point;
-        uint8_t code_point_length;
-        size_t result = <%= @grammar.prefix %>decode_code_point(input, input_length, &code_point, &code_point_length);
-        switch (result)
-        {
-        case P_SUCCESS:
-            lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
-            if (transition_state != INVALID_LEXER_STATE_ID)
-            {
-                attempt_match.length += code_point_length;
-                if (code_point == '\n')
-                {
-                    attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
-                }
-                else
-                {
-                    attempt_match.delta_position.col++;
-                }
-                current_state = transition_state;
-                if (lexer_state_table[current_state].accepts)
-                {
-                    attempt_match.accepting_state = &lexer_state_table[current_state];
-                    longest_match = attempt_match;
-                }
-            }
-            else if (longest_match.length > 0)
-            {
-                *out_match_info = longest_match;
-                return P_SUCCESS;
-            }
-            else
-            {
-                *out_unexpected_input_length = attempt_match.length + code_point_length;
-                return P_UNEXPECTED_INPUT;
-            }
-            break;
-
-        case P_EOF:
-            /* We hit EOF. */
-            if (longest_match.length > 0)
-            {
-                /* We have a match, so use it. */
-                *out_match_info = longest_match;
-                return P_SUCCESS;
-            }
-            else if (attempt_match.length != 0)
-            {
-                /* There is a partial match - error! */
-                *out_unexpected_input_length = attempt_match.length;
-                return P_UNEXPECTED_INPUT;
-            }
-            else
-            {
-                /* Valid EOF return. */
-                return P_EOF;
-            }
-            break;
-
-        case P_DECODE_ERROR:
-            /* If we see a decode error, we may be partially in the middle of
-             * matching a pattern, so return the attempted match info so that
-             * the input text position can be updated. */
-            *out_match_info = attempt_match;
-            return result;
-
-        default:
-            return result;
-        }
-    }
-}
-
-/**
- * Attempt to lex the next token in the input stream.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
- *
- * @reval P_SUCCESS
- *   A token was successfully lexed.
- * @reval P_DECODE_ERROR
- *   The decoder encountered invalid text encoding.
- * @reval P_UNEXPECTED_INPUT
- *   Input text does not match any lexer pattern.
- * @retval P_DROP
- *   A drop pattern was matched so the lexer should continue.
- */
-static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
-{
-    <%= @grammar.prefix %>token_info_t token_info = {0};
-    token_info.position = context->text_position;
-    token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
-    lexer_match_info_t match_info;
-    size_t unexpected_input_length;
-    size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
-    switch (result)
-    {
-    case P_SUCCESS:
-        <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
-        if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
-        {
-            uint8_t const * match = &context->input[context->input_index];
-            <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match, match_info.length, &token_info);
-            /* An invalid token returned from lexer_user_code() means that the
-             * user code did not explicitly return a token. So only override
-             * the token to return if the user code does explicitly return a
-             * token. */
-            if (user_code_token != INVALID_TOKEN_ID)
-            {
-                token_to_accept = user_code_token;
-            }
-        }
-
-        /* Update the input position tracking. */
-        context->input_index += match_info.length;
-        context->text_position.row += match_info.delta_position.row;
-        if (match_info.delta_position.row != 0u)
-        {
-            context->text_position.col = match_info.delta_position.col;
-        }
-        else
-        {
-            context->text_position.col += match_info.delta_position.col;
-        }
-
-        if (token_to_accept == INVALID_TOKEN_ID)
-        {
-            return P_DROP;
-        }
-        token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        *out_token_info = token_info;
-        return P_SUCCESS;
-
-    case P_EOF:
-        token_info.token = TOKEN___EOF;
-        *out_token_info = token_info;
-        return P_SUCCESS;
-
-    case P_DECODE_ERROR:
-        /* Update the input position tracking. */
-        context->input_index += match_info.length;
-        context->text_position.row += match_info.delta_position.row;
-        if (match_info.delta_position.row != 0u)
-        {
-            context->text_position.col = match_info.delta_position.col;
-        }
-        else
-        {
-            context->text_position.col += match_info.delta_position.col;
-        }
-        return result;
-
-    default:
-        return result;
-    }
-}
-
-/**
- * Lex the next token in the input stream.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
- *
- * @reval P_SUCCESS
- *   A token was successfully lexed.
- * @reval P_DECODE_ERROR
- *   The decoder encountered invalid text encoding.
- * @reval P_UNEXPECTED_INPUT
- *   Input text does not match any lexer pattern.
- */
-size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
-{
-    for (;;)
-    {
-        size_t result = attempt_lex_token(context, out_token_info);
-        if (result != P_DROP)
-        {
-            return result;
-        }
-    }
-}
-
-/**************************************************************************
- * Parser
- *************************************************************************/
-
-/** Reduce ID type. */
-typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
-
-/**
- * A symbol ID can hold either a token ID or a rule set ID.
- *
- * Token IDs and rule set IDs share the same namespace, with rule set IDs
- * beginning after token IDs end.
- */
-typedef <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %> symbol_id_t;
-
-/** Parser state ID type. */
-typedef <%= get_type_for(@parser.state_table.size) %> parser_state_id_t;
-
-/** Parser rule ID type. */
-typedef <%= get_type_for(@grammar.rules.size) %> rule_id_t;
-
-/** Parser shift ID type. */
-typedef <%= get_type_for(@parser.shift_table.size) %> shift_id_t;
-
-/** Shift table entry. */
-typedef struct
-{
-    /** Token or rule set ID. */
-    symbol_id_t symbol_id;
-
-    /** Parser state to shift to. */
-    parser_state_id_t state_id;
-} shift_t;
-
-/** Reduce table entry. */
-typedef struct
-{
-    /** Lookahead token. */
-    <%= @grammar.prefix %>token_t token;
-
-    /**
-     * Rule ID.
-     *
-     * This is used to execute the parser user code block associated with a
-     * grammar rule.
-     */
-    rule_id_t rule;
-
-    /**
-     * Rule set ID.
-     *
-     * This is used as the new top symbol ID of the parse stack after this
-     * reduce action.
-     */
-    symbol_id_t rule_set;
-
-    /**
-     * Number of states leading to this reduce action.
-     *
-     * This is the number of entries popped from the parse stack after this
-     * reduce action.
-     */
-    parser_state_id_t n_states;
-} reduce_t;
-
-/** Parser state entry. */
-typedef struct
-{
-    /** First shift table entry for this parser state. */
-    shift_id_t shift_table_index;
-
-    /** Number of shift table entries for this parser state. */
-    shift_id_t n_shift_entries;
-
-    /** First reduce table entry for this parser state. */
-    reduce_id_t reduce_table_index;
-
-    /** Number of reduce table entries for this parser state. */
-    reduce_id_t n_reduce_entries;
-} parser_state_t;
-
-/**
- * Structure to hold a state ID and value pair.
- *
- * A stack of these structures makes up the parse stack.
- */
-typedef struct
-{
-    /** Parser state ID. */
-    size_t state_id;
-
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-} state_value_t;
-
-/** Parser shift table. */
-static const shift_t parser_shift_table[] = {
-<%   @parser.shift_table.each do |shift| %>
-    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
-<%   end %>
-};
-
-/** Parser reduce table. */
-static const reduce_t parser_reduce_table[] = {
-<%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
-<%   end %>
-};
-
-/** Parser state table. */
-static const parser_state_t parser_state_table[] = {
-<%   @parser.state_table.each do |state| %>
-    {<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u},
-<%   end %>
-};
-
-/* state_values stack functionality */
-
-/** state_values stack type. */
-typedef struct
-{
-    size_t length;
-    size_t capacity;
-    state_value_t * entries;
-} state_values_stack_t;
-
-/**
- * Initialize state_values stack structure.
- *
- * @param stack
- *   state_values stack structure.
- */
-static void state_values_stack_init(state_values_stack_t * stack)
-{
-    const size_t initial_capacity = 10u;
-    stack->length = 0u;
-    stack->capacity = initial_capacity;
-    stack->entries = (state_value_t *)malloc(initial_capacity * sizeof(state_value_t));
-}
-
-/**
- * Index a state_values stack.
- *
- * @param stack
- *   state_values stack structure.
- * @param index
- *   Index to the stack.
- *
- * @return Pointer to the state value structure at the given index.
- */
-static state_value_t * state_values_stack_index(state_values_stack_t * stack, int index)
-{
-    if (index >= 0)
-    {
-        return &stack->entries[index];
-    }
-    else
-    {
-        return &stack->entries[stack->length - (size_t)(unsigned int)(-index)];
-    }
-}
-
-/**
- * Push a new state_value to the state_values stack.
- *
- * @param stack
- *   state_values stack structure.
- */
-static void state_values_stack_push(state_values_stack_t * stack)
-{
-    size_t const current_capacity = stack->capacity;
-    size_t const current_length = stack->length;
-    if (current_length >= current_capacity)
-    {
-        size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
-        memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
-        free(stack->entries);
-        stack->capacity = new_capacity;
-        stack->entries = new_entries;
-    }
-    memset(&stack->entries[current_length], 0, sizeof(state_value_t));
-    stack->length = current_length + 1u;
-}
-
-/**
- * Pop entries from a state_values stack.
- *
- * @param stack
- *   state_values stack structure.
- * @param n
- *   Number of states to pop.
- */
-static void state_values_stack_pop(state_values_stack_t * stack, size_t n)
-{
-    stack->length -= n;
-}
-
-/**
- * Free memory for a state_values stack structure.
- *
- * @param stack
- *   state_values stack structure.
- */
-static void state_values_stack_free(state_values_stack_t * stack)
-{
-     free(stack->entries);
-}
-
-/**
- * Execute user code associated with a parser rule.
- *
- * @param rule The ID of the rule.
- *
- * @return Parse value.
- */
-static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
-{
-    <%= @grammar.prefix %>value_t _pvalue = {0};
-
-    switch (rule)
-    {
-<% @grammar.rules.each do |rule| %>
-<%   if rule.code %>
-    case <%= rule.id %>u: {
-<%= expand_code(rule.code, true, rule, nil) %>
-    } break;
-<%   end %>
-<% end %>
-    default: break;
-    }
-
-    return _pvalue;
-}
-
-/**
- * Check if the parser should shift to a new state.
- *
- * @param state_id
- *   Parser state ID.
- * @param symbol_id
- *   Incoming token/rule set ID.
- *
- * @return State to shift to, or INVALID_ID if none.
- */
-static size_t check_shift(size_t state_id, size_t symbol_id)
-{
-    uint32_t start = parser_state_table[state_id].shift_table_index;
-    uint32_t end = start + parser_state_table[state_id].n_shift_entries;
-    for (uint32_t i = start; i < end; i++)
-    {
-        if (parser_shift_table[i].symbol_id == symbol_id)
-        {
-            return parser_shift_table[i].state_id;
-        }
-    }
-    return INVALID_ID;
-}
-
-/**
- * Check if the parser should reduce to a new state.
- *
- * @param state_id
- *   Parser state ID.
- * @param token
- *   Incoming token.
- *
- * @return State to reduce to, or INVALID_ID if none.
- */
-static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
-{
-    size_t start = parser_state_table[state_id].reduce_table_index;
-    size_t end = start + parser_state_table[state_id].n_reduce_entries;
-    for (size_t i = start; i < end; i++)
-    {
-        if ((parser_reduce_table[i].token == token) ||
-            (parser_reduce_table[i].token == INVALID_TOKEN_ID))
-        {
-            return i;
-        }
-    }
-    return INVALID_ID;
-}
-
-/**
- * Run the parser.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @retval P_SUCCESS
- *   The parser successfully matched the input text. The parse result value
- *   can be accessed with <%= @grammar.prefix %>result().
- * @retval P_UNEXPECTED_TOKEN
- *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context->token holds the unexpected token.
- * @reval P_DECODE_ERROR
- *   The decoder encountered invalid text encoding.
- * @reval P_UNEXPECTED_INPUT
- *   Input text does not match any lexer pattern.
- */
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
-{
-    <%= @grammar.prefix %>token_info_t token_info;
-    <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
-    state_values_stack_t statevalues;
-    size_t reduced_rule_set = INVALID_ID;
-    <%= @grammar.prefix %>value_t reduced_parser_value;
-    state_values_stack_init(&statevalues);
-    state_values_stack_push(&statevalues);
-    size_t result;
-    for (;;)
-    {
-        if (token == INVALID_TOKEN_ID)
-        {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
-            if (lexer_result != P_SUCCESS)
-            {
-                result = lexer_result;
-                break;
-            }
-            token = token_info.token;
-        }
-        size_t shift_state = INVALID_ID;
-        if (reduced_rule_set != INVALID_ID)
-        {
-            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, reduced_rule_set);
-        }
-        if (shift_state == INVALID_ID)
-        {
-            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, token);
-            if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
-            {
-                /* Successful parse. */
-                context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
-                result = P_SUCCESS;
-                break;
-            }
-        }
-        if (shift_state != INVALID_ID)
-        {
-            /* We have something to shift. */
-            state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
-            if (reduced_rule_set == INVALID_ID)
-            {
-                /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
-            }
-            else
-            {
-                /* We shifted a RuleSet. */
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
-                <%= @grammar.prefix %>value_t new_parse_result = {0};
-                reduced_parser_value = new_parse_result;
-                reduced_rule_set = INVALID_ID;
-            }
-            continue;
-        }
-
-        size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
-        if (reduce_index != INVALID_ID)
-        {
-            /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
-            reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
-            state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
-            continue;
-        }
-
-        /* A token was successfully lexed, so the input text position was
-         * advanced. However, this is an unexpected token, so we want to reset
-         * the context text position to point to the token rather than the text
-         * after it, so that if the caller wants to report the error position,
-         * it will point to the correct position of the unexpected token. */
-        context->text_position = token_info.position;
-        context->token = token;
-        result = P_UNEXPECTED_TOKEN;
-        break;
-    }
-    state_values_stack_free(&statevalues);
-    return result;
-}
-
-/**
- * Get the parse result value.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return Parse result value.
- */
-<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
-{
-    return context->parse_result.v_<%= start_rule_type[0] %>;
-}
-
-/**
- * Get the current text input position.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return Current text position.
- */
-<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
-{
-    return context->text_position;
-}

assets/parser.h.erb

@@ -1,122 +0,0 @@
-/**
- * @file
- *
- * This file is generated by Propane.
- */
-
-#pragma once
-
-#include <stdint.h>
-#include <stddef.h>
-
-/**************************************************************************
- * Public types
- *************************************************************************/
-
-/* Result codes. */
-#define <%= @grammar.prefix.upcase %>SUCCESS 0u
-#define <%= @grammar.prefix.upcase %>DECODE_ERROR 1u
-#define <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT 2u
-#define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
-#define <%= @grammar.prefix.upcase %>DROP 4u
-#define <%= @grammar.prefix.upcase %>EOF 5u
-
-/** Token type. */
-typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
-
-/** Token IDs. */
-<% @grammar.tokens.each_with_index do |token, index| %>
-#define TOKEN_<%= token.code_name %> <%= index %>u
-<%   unless token.id == index %>
-<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
-<%   end %>
-<% end %>
-#define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
-
-/** Code point type. */
-typedef uint32_t <%= @grammar.prefix %>code_point_t;
-
-/** Parser values type(s). */
-typedef union
-{
-<% @grammar.ptypes.each do |name, typestring| %>
-    <%= typestring %> v_<%= name %>;
-<% end %>
-} <%= @grammar.prefix %>value_t;
-
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
-{
-    /** Input text row (0-based). */
-    uint32_t row;
-
-    /** Input text column (0-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
-
-/** Lexed token information. */
-typedef struct
-{
-    /** Text position where the token was found. */
-    <%= @grammar.prefix %>position_t position;
-
-    /** Number of input bytes used by the token. */
-    size_t length;
-
-    /** Token that was lexed. */
-    <%= @grammar.prefix %>token_t token;
-
-    /** Parser value associated with the token. */
-    <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.prefix %>token_info_t;
-
-/**
- * Lexer and parser context.
- *
- * The user must allocate an instance of this structure and pass it to any
- * public API function.
- */
-typedef struct
-{
-    /* Lexer context data. */
-
-    /** Input text. */
-    uint8_t const * input;
-
-    /** Input text length. */
-    size_t input_length;
-
-    /** Input text index (byte offset). */
-    size_t input_index;
-
-    /** Input text position (row/column). */
-    <%= @grammar.prefix %>position_t text_position;
-
-    /** Current lexer mode. */
-    size_t mode;
-
-    /* Parser context data. */
-
-    /** Parse result value. */
-    <%= @grammar.prefix %>value_t parse_result;
-
-    /** Unexpected token received. */
-    <%= @grammar.prefix %>token_t token;
-} <%= @grammar.prefix %>context_t;
-
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
-
-size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
-        <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
-
-size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
-
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
-
-<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
-
-<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -1,130 +0,0 @@
-${remove}
-WARNING: This user guide is meant to be preprocessed and rendered by a custom
-script.
-The markdown source file is not intended to be viewed directly and will not
-include all intended content.
-${/remove}
-
-#> Overview
-
-Propane is an LR Parser Generator (LPG) which:
-
-  * accepts LR(0), SLR, and LALR grammars
-  * generates a built-in lexer to tokenize input
-  * supports UTF-8 lexer inputs
-  * generates a table-driven parser to parse input in linear time
-  * is MIT-licensed
-  * is distributable as a standalone Ruby script
-  * supports D language
-
-#> Installation
-
-Propane is designed to be distributed as a stand-alone single file script that
-can be copied into and versioned in a project's source tree.
-The only requirement to run Propane is that the system has a Ruby interpreter
-installed.
-The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
-Simply copy the `propane` executable script into the desired location within
-the project to be built (typically the root of the repository) and mark it
-executable.
-
-#> Command Line Usage
-
-Propane is typically invoked from the command-line as `./propane`.
-
-    Usage: ./propane [options] <input-file> <output-file>
-    Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
-
-The user must specify the path to a Propane input grammar file and a path to an
-output file.
-The generated source code will be written to the output file.
-If a log file path is specified, Propane will write a log file containing
-detailed information about the parser states and transitions.
-
-#> Propane Grammar File
-
-A Propane grammar file provides Propane with the patterns, tokens, grammar
-rules, and user code blocks from which to build the generated lexer and parser.
-
-Example grammar file:
-
-```
-<<
-import std.math;
->>
-
-ptype ulong;
-
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-```
-
-#> License
-
-Propane is licensed under the terms of the MIT License:
-
-```
-${include LICENSE.txt}
-```
-
-#> Contributing
-
-Propane is developed on [github](https://github.com/holtrop/propane).
-
-Issues may be submitted to [https://github.com/holtrop/propane/issues](https://github.com/holtrop/propane/issues).
-
-Pull requests may be submitted as well:
-
-  1. Fork it
-  2. Create your feature branch (`git checkout -b my-new-feature`)
-  3. Commit your changes (`git commit -am 'Add some feature'`)
-  4. Push to the branch (`git push origin my-new-feature`)
-  5. Create new Pull Request
-
-#> Change Log
-
-${changelog}

lib/propane.rb

@@ -25,6 +25,12 @@ require_relative "propane/version"
 
 class Propane
 
+  # Decoding error.
+  TOKEN_DECODE_ERROR = 0xFFFFFFFD
+
+  # Token ID for a "dropped" token.
+  TOKEN_DROP = 0xFFFFFFFE
+
   class Error < RuntimeError
   end
 

lib/propane/generator.rb

@@ -11,27 +11,14 @@ class Propane
         @log = StringIO.new
       end
       @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
-      @language =
-        if output_file =~ /\.([a-z]+)$/
-          $1
-        else
-          "d"
-        end
       process_grammar!
     end
 
     def generate
-      extensions = [@language]
+      erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.d.erb")), trim_mode: "<>")
-      if @language == "c"
+      result = erb.result(binding.clone)
-        extensions += %w[h]
+      File.open(@output_file, "wb") do |fh|
-      end
+        fh.write(result)
-      extensions.each do |extension|
-        erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.#{extension}.erb")), trim_mode: "<>")
-        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
-        result = erb.result(binding.clone)
-        File.open(output_file, "wb") do |fh|
-          fh.write(result)
-        end
       end
       @log.close
     end
@@ -204,21 +191,11 @@ class Propane
         end
         code = code.gsub(/\$(\d+)/) do |match|
           index = $1.to_i
-          case @language
+          "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.ptypename}"
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-          end
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          case @language
+          "lt.pvalue.v_#{pattern.ptypename}"
-          when "c"
-            "out_token_info->pvalue.v_#{pattern.ptypename}"
-          when "d"
-            "out_token_info.pvalue.v_#{pattern.ptypename}"
-          end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
@@ -226,12 +203,7 @@ class Propane
           unless mode_id
             raise Error.new("Lexer mode '#{mode_name}' not found")
           end
-          case @language
+          "m_mode = #{mode_id}u"
-          when "c"
-            "context->mode = #{mode_id}u"
-          when "d"
-            "context.mode = #{mode_id}u"
-          end
         end
       end
       code
@@ -248,38 +220,6 @@ class Propane
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end
 
-    # Get an unsigned integer type that can hold the given maximum value.
-    #
-    # @param max [Integer]
-    #   Maximum value to store.
-    #
-    # @return [String]
-    #   Type.
-    def get_type_for(max)
-      if max <= 0xFF
-        case @language
-        when "c"
-          "uint8_t"
-        when "d"
-          "ubyte"
-        end
-      elsif max <= 0xFFFF
-        case @language
-        when "c"
-          "uint16_t"
-        when "d"
-          "ushort"
-        end
-      else
-        case @language
-        when "c"
-          "uint32_t"
-        else
-          "uint"
-        end
-      end
-    end
-
   end
 
 end

lib/propane/grammar.rb

@@ -2,8 +2,7 @@ class Propane
 
   class Grammar
 
-    # Reserve identifiers beginning with a double-underscore for internal use.
+    IDENTIFIER_REGEX = /[a-zA-Z_][a-zA-Z_0-9]*/
-    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :classname
     attr_reader :modulename
@@ -12,7 +11,6 @@ class Propane
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
-    attr_reader :prefix
 
     def initialize(input)
       @patterns = []
@@ -24,7 +22,6 @@ class Propane
       @mode = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
-      @prefix = "p_"
       parse_grammar!
     end
 
@@ -32,10 +29,6 @@ class Propane
       @ptypes["default"]
     end
 
-    def invalid_token_id
-      @tokens.size
-    end
-
     private
 
     def parse_grammar!
@@ -57,7 +50,6 @@ class Propane
       elsif parse_drop_statement!
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
-      elsif parse_prefix_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -155,7 +147,7 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, drop: true, mode: @mode)
         @mode = nil
         true
       end
@@ -198,13 +190,6 @@ class Propane
       end
     end
 
-    def parse_prefix_statement!
-      if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
-        @prefix = md[1]
-        true
-      end
-    end
-
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""

lib/propane/lexer.rb

@@ -1,13 +1,61 @@
 class Propane
   class Lexer
 
-    attr_reader :state_table
-    attr_reader :transition_table
-    attr_reader :mode_table
-
     def initialize(grammar)
       @grammar = grammar
-      build_tables!
+    end
+
+    def build_tables
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
+      transition_table = []
+      state_table = []
+      mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = state_table.size
+        mode_table << {
+          state_table_offset: state_table_offset,
+        }
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          token =
+            if state.accepts.nil?
+              @grammar.tokens.size
+            elsif state.accepts.drop?
+              TOKEN_DROP
+            elsif state.accepts.token
+              state.accepts.token.id
+            else
+              @grammar.tokens.size
+            end
+          code_id =
+            if state.accepts && state.accepts.code_id
+              state.accepts.code_id
+            else
+              0xFFFF_FFFF
+            end
+          state_table << {
+            transition_table_index: transition_table.size,
+            n_transitions: state.transitions.size,
+            token: token,
+            code_id: code_id,
+          }
+          state.transitions.each do |transition|
+            transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
+        end
+      end
+      [transition_table, state_table, mode_table]
     end
 
     # Get ID for a mode.
@@ -23,46 +71,5 @@ class Propane
       end
     end
 
-    private
-
-    def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
-        pattern.mode
-      end.transform_values do |patterns|
-        {dfa: DFA.new(patterns)}
-      end
-      @modes.each_with_index do |(mode_name, mode_info), index|
-        mode_info[:id] = index
-      end
-      @state_table = []
-      @transition_table = []
-      @mode_table = []
-      @modes.each do |mode_name, mode_info|
-        state_table_offset = @state_table.size
-        @mode_table << {
-          state_table_offset: state_table_offset,
-        }
-        states = mode_info[:dfa].enumerate
-        states.each do |state, id|
-          token = state.accepts && state.accepts.token && state.accepts.token.id
-          code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
-          @state_table << {
-            transition_table_index: @transition_table.size,
-            n_transitions: state.transitions.size,
-            accepts: !!state.accepts,
-            token: token,
-            code_id: code_id,
-          }
-          state.transitions.each do |transition|
-            @transition_table << {
-              first: transition.code_point_range.first,
-              last: transition.code_point_range.last,
-              destination: states[transition.destination] + state_table_offset,
-            }
-          end
-        end
-      end
-    end
-
   end
 end

lib/propane/parser.rb

@@ -2,11 +2,6 @@ class Propane
 
   class Parser
 
-    attr_reader :state_table
-    attr_reader :shift_table
-    attr_reader :reduce_table
-    attr_reader :rule_sets
-
     def initialize(grammar, rule_sets, log)
       @grammar = grammar
       @rule_sets = rule_sets
@@ -38,15 +33,12 @@ class Propane
 
       build_reduce_actions!
       write_log!
-      build_tables!
     end
 
-    private
+    def build_tables
-
+      shift_table = []
-    def build_tables!
+      state_table = []
-      @state_table = []
+      reduce_table = []
-      @shift_table = []
-      @reduce_table = []
       @item_sets.each do |item_set|
         shift_entries = item_set.following_symbols.map do |following_symbol|
           state_id =
@@ -56,14 +48,14 @@ class Propane
               item_set.following_item_set[following_symbol].id
             end
           {
-            symbol_id: following_symbol.id,
+            token_id: following_symbol.id,
             state_id: state_id,
           }
         end
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
+            [{token_id: @grammar.tokens.size, rule_id: ra.id,
               rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
           when Hash
             ra.map do |token, rule|
@@ -73,17 +65,20 @@ class Propane
           else
             []
           end
-        @state_table << {
+        state_table << {
-          shift_index: @shift_table.size,
+          shift_index: shift_table.size,
           n_shifts: shift_entries.size,
-          reduce_index: @reduce_table.size,
+          reduce_index: reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += shift_entries
+        shift_table += shift_entries
-        @reduce_table += reduce_entries
+        reduce_table += reduce_entries
       end
+      [state_table, shift_table, reduce_table]
     end
 
+    private
+
     def process_item_set(item_set)
       item_set.following_symbols.each do |following_symbol|
         unless following_symbol.name == "$EOF"
@@ -247,7 +242,7 @@ class Propane
         @log.puts "  Reduce actions:"
         case item_set.reduce_actions
         when Rule
-          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
+          @log.puts "    * => #{item_set.reduce_actions.id} (#{item_set.reduce_actions.name})"
         when Hash
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"

lib/propane/pattern.rb

@@ -40,6 +40,8 @@ class Propane
     #   Optional parameters.
     # @option options [String, nil] :code
     #   Code block to execute when the pattern is matched.
+    # @option options [Boolean] :drop
+    #   Whether this is a drop pattern.
     # @option options [String, nil] :pattern
     #   Pattern.
     # @option options [Token, nil] :token
@@ -50,6 +52,7 @@ class Propane
     #   Lexer mode for this pattern.
     def initialize(options)
       @code = options[:code]
+      @drop = options[:drop]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
@@ -60,6 +63,14 @@ class Propane
       @nfa = regex.nfa
     end
 
+    # Whether the pattern is a drop pattern.
+    #
+    # @return [Boolean]
+    #   Whether the pattern is a drop pattern.
+    def drop?
+      @drop
+    end
+
   end
 
 end

lib/propane/rule_set.rb

@@ -4,7 +4,7 @@ class Propane
 
     # @return [Integer]
     #   ID of the RuleSet.
-    attr_reader :id
+    attr_accessor :id
 
     # @return [String]
     #   Name of the RuleSet.

lib/propane/token.rb

@@ -9,7 +9,7 @@ class Propane
       # @return [String]
       #   Name of the token to use in code (special characters replaced).
       def code_name(name)
-        name.sub(/^\$/, "__")
+        name.sub(/^\$/, "0")
       end
 
     end

rb/assets/user_guide.html.erb

@@ -1,68 +0,0 @@
-<html>
-  <head>
-    <title>Propane User Guide<%= subpage_title %> - Version <%= Propane::VERSION %></title>
-    <style>
-      body {
-        background-color: #CCC;
-        margin: 0px;
-        padding: 0px;
-      }
-      #body-content {
-        margin-left: auto;
-        margin-right: auto;
-        margin-top: 1em;
-        margin-bottom: 1em;
-        border: 1px solid black;
-        background-color: #FFF;
-        width: 120ex;
-        padding: 2ex;
-      }
-      a {
-        text-decoration: none;
-      }
-      .separator {
-        height: 2em;
-      }
-      .img_block_center {
-        display: block;
-        margin-left: auto;
-        margin-right: auto;
-      }
-      .page_nav {
-        width: 100%;
-        table-layout: fixed;
-      }
-      .page_nav_toc {
-        text-align: center;
-      }
-      .page_nav_next {
-        text-align: right;
-      }
-      .code {
-        padding-left: 2ex;
-        width: 116ex;
-        overflow-x: auto;
-      }
-      .ruby_code .normal {}
-      .ruby_code .comment { color: #005; font-style: italic; }
-      .ruby_code .keyword { color: #A00; font-weight: bold; }
-      .ruby_code .method { color: #077; }
-      .ruby_code .class { color: #074; }
-      .ruby_code .module { color: #050; }
-      .ruby_code .punct { color: #447; font-weight: bold; }
-      .ruby_code .symbol { color: #099; }
-      .ruby_code .string { color: #090; }
-      .ruby_code .char { color: #F07; }
-      .ruby_code .ident { color: #004; }
-      .ruby_code .constant { color: #07F; }
-      .ruby_code .regex { color: #B66; }
-      .ruby_code .number { color: #D55; }
-      .ruby_code .attribute { color: #377; }
-      .ruby_code .global { color: #3B7; }
-      .ruby_code .expr { color: #227; }
-    </style>
-  </head>
-  <body>
-    <div id="body-content"><%= content %></div>
-  </body>
-</html>

rb/build_dist.rb

@@ -1,106 +0,0 @@
-#!/usr/bin/env ruby
-
-require "fileutils"
-require "digest/md5"
-
-if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
-  VERSION = $1
-else
-  raise "Could not determine version."
-end
-PROG_NAME = "propane"
-START_FILE = "bin/#{PROG_NAME}"
-LIB_DIR = "lib"
-DIST = "dist"
-
-files_processed = {}
-combined_file = []
-
-combine_files = lambda do |file|
-  File.read(file, mode: "rb").each_line do |line|
-    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
-      require_name = $1
-      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
-        path = "#{LIB_DIR}/#{require_name}.rb"
-        if File.exist?(path)
-          unless files_processed[path]
-            files_processed[path] = true
-            combine_files[path]
-          end
-        else
-          raise "require path #{path.inspect} not found"
-        end
-      else
-        combined_file << line
-      end
-    else
-      combined_file << line
-    end
-  end
-end
-
-combine_files[START_FILE]
-
-# Strip Ruby comment lines and empty lines to save some space, but do not
-# remove lines that are in heredoc sections. This isn't terribly robust to be
-# used in the wild, but works for the heredoc instances for this project.
-stripped = []
-heredoc_end = nil
-combined_file.each do |line|
-  if line =~ /<<-?([A-Z]+)/
-    heredoc_end = $1
-  end
-  if heredoc_end and line =~ /^\s*#{heredoc_end}/
-    heredoc_end = nil
-  end
-  if line !~ /#\sspecs/
-    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
-      stripped << line
-    end
-  end
-end
-
-license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
-  if line.size > 0
-    "# #{line}"
-  else
-    "#"
-  end
-end
-
-require "zlib"
-require "base64"
-compressed_script = Zlib::Deflate.deflate(stripped.join)
-encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
-hash = Digest::MD5.hexdigest(encoded_compressed_script)
-
-FileUtils.rm_rf(DIST)
-FileUtils.mkdir_p(DIST)
-File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
-  fh.write(<<EOF)
-#!/usr/bin/env ruby
-
-#{license}
-
-script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
-unless File.exist?(script)
-  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
-    require "zlib"
-    require "base64"
-    encoded_compressed = $1
-    unescaped_compressed = Base64.decode64(encoded_compressed)
-    inflated = Zlib::Inflate.inflate(unescaped_compressed)
-    File.open(script, "wb") do |fh|
-      fh.write(inflated)
-    end
-  else
-    raise "Could not decompress."
-  end
-end
-load script
-if __FILE__ == $0
-  Propane::CLI.new.run(ARGV)
-end
-#==>#{encoded_compressed_script}
-EOF
-end

rb/gen_user_guide.rb

@@ -1,227 +0,0 @@
-#!/usr/bin/env ruby
-
-require "erb"
-require "fileutils"
-require "redcarpet"
-require "syntax"
-require "syntax/convertors/html"
-require "propane/version"
-
-def load_file(file_name)
-  contents = File.read(file_name)
-  contents.gsub(/\$\{include (.*?)\}/) do |match|
-    include_file_name = $1
-    File.read(include_file_name)
-  end
-end
-
-class Generator
-  class Section
-    attr_reader :number
-    attr_reader :title
-    attr_reader :page
-    attr_reader :contents
-    attr_reader :anchor
-    def initialize(number, title, page, anchor)
-      @number = number
-      @title = title.gsub(/[`]/, "")
-      @page = page
-      @anchor = anchor
-      @contents = ""
-    end
-    def append(contents)
-      @contents += contents
-    end
-  end
-
-  class Page
-    attr_reader :name
-    attr_reader :title
-    attr_accessor :contents
-    def initialize(name, title, contents = "")
-      @name = name
-      @title = title
-      @contents = contents
-    end
-  end
-
-  def initialize(input, output_file, multi_page)
-    current_page =
-      if multi_page
-        nil
-      else
-        File.basename(output_file).sub(/\.html$/, "")
-      end
-    @sections = []
-    @current_section_number = [0]
-    @lines = input.lines
-    while @lines.size > 0
-      line = @lines.slice!(0)
-      if line =~ /^```(.*)$/
-        @sections.last.append(gather_code_section($1))
-      elsif line.chomp == "${remove}"
-        remove_section
-      elsif line =~ /^(#+)(>)?\s*(.*)$/
-        level_text, new_page_text, title_text = $1, $2, $3
-        level = $1.size
-        new_page = !new_page_text.nil?
-        section_number = get_next_section_number(level)
-        anchor = make_anchor(section_number, title_text)
-        if multi_page and (new_page or current_page.nil?)
-          current_page = anchor
-        end
-        @sections << Section.new(section_number, title_text, current_page, anchor)
-        @sections.last.append("#{level_text} #{section_number} #{title_text}")
-      elsif @sections.size > 0
-        @sections.last.append(line)
-      end
-    end
-
-    renderer = Redcarpet::Render::HTML.new
-    @markdown_renderer = Redcarpet::Markdown.new(renderer)
-    changelog = @markdown_renderer.render(File.read("CHANGELOG.md"))
-
-    @pages = [Page.new("toc", "Table of Contents", render_toc)]
-    @sections.each do |section|
-      unless @pages.last.name == section.page
-        @pages << Page.new(section.page, "#{section.number} #{section.title}")
-      end
-      @pages.last.contents += render_section(section)
-    end
-
-    @pages.each do |page|
-      page.contents.gsub!("${changelog}", changelog)
-      page.contents.gsub!(%r{\$\{#(.+?)\}}) do |match|
-        section_name = $1
-        href = get_link_to_section(section_name)
-        %[<a href="#{href}">#{section_name}</a>]
-      end
-    end
-
-    template = File.read("rb/assets/user_guide.html.erb")
-    erb = ERB.new(template, trim_mode: "<>")
-
-    if multi_page
-      @pages.each_with_index do |page, page_index|
-        subpage_title = " - #{page.title}"
-        page_nav_bar = render_page_nav_bar(page_index)
-        content = page_nav_bar + separator + page.contents + separator + page_nav_bar
-        html_result = erb.result(binding.clone)
-        File.open(File.join(output_file, "#{page.name}.html"), "w") do |fh|
-          fh.write(html_result)
-        end
-      end
-    else
-      subpage_title = ""
-      content = @pages.reduce("") do |result, page|
-        result + page.contents
-      end
-      html_result = erb.result(binding.clone)
-      File.open(output_file, "w") do |fh|
-        fh.write(html_result)
-      end
-    end
-  end
-
-  def separator
-    %[<div class="separator"></div>]
-  end
-
-  def render_page_nav_bar(page_index)
-    page_nav_prev =
-      if page_index > 1
-        %[<a href="#{@pages[page_index - 1].name}.html">&laquo; Prev<br/>#{@pages[page_index - 1].title}</a>]
-      else
-        ""
-      end
-    page_nav_toc =
-      if page_index > 0
-        %[<a href="toc.html">Table of Contents</a>]
-      else
-        ""
-      end
-    page_nav_next =
-      if page_index < @pages.size - 1
-        %[<a href="#{@pages[page_index + 1].name}.html">Next &raquo<br/>#{@pages[page_index + 1].title}</a>]
-      else
-        ""
-      end
-    %[<table class="page_nav"><tr>] + \
-    %[<td class="page_nav_prev">#{page_nav_prev}</td>] + \
-    %[<td class="page_nav_toc">#{page_nav_toc}</td>] + \
-    %[<td class="page_nav_next">#{page_nav_next}</td>] + \
-    %[</tr></table>]
-  end
-
-  def render_toc
-    toc_content = %[<h1>Table of Contents</h1>\n]
-    @sections.each do |section|
-      indent = section.number.split(".").size - 1
-      toc_content += %[<span style="padding-left: #{4 * indent}ex;">]
-      toc_content += %[<a href="#{section.page}.html##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
-      toc_content += %[</span>]
-    end
-    toc_content
-  end
-
-  def render_section(section)
-    %[<a name="#{section.anchor}" />] + \
-      @markdown_renderer.render(section.contents)
-  end
-
-  def make_anchor(section_number, section_title)
-    "s" + ("#{section_number} #{section_title}").gsub(/[^a-zA-Z0-9]/, "_")
-  end
-
-  def get_link_to_section(section_name)
-    section = @sections.find do |section|
-      section.title == section_name
-    end
-    raise "Could not find section #{section_name}" unless section
-    "#{section.page}.html##{section.anchor}"
-  end
-
-  def gather_code_section(syntax)
-    code = ""
-    loop do
-      line = @lines.slice!(0)
-      if line =~ /^```/
-        break
-      end
-      code += line
-    end
-    if syntax != ""
-      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
-      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
-    else
-      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
-    end
-  end
-
-  def remove_section
-    loop do
-      line = @lines.slice!(0)
-      if line.chomp == "${/remove}"
-        break
-      end
-    end
-  end
-
-  def get_next_section_number(level)
-    if @current_section_number.size == level - 1
-      @current_section_number << 1
-    elsif @current_section_number.size >= level
-      @current_section_number[level - 1] += 1
-      @current_section_number.slice!(level, @current_section_number.size)
-    else
-      raise "Section level change from #{@current_section_number.size} to #{level}"
-    end
-    @current_section_number.join(".")
-  end
-end
-
-input = load_file("doc/user_guide.md")
-FileUtils.rm_rf("gen/user_guide")
-FileUtils.mkdir_p("gen/user_guide")
-Generator.new(input, "gen/user_guide/user_guide.html", false)
-Generator.new(input, "gen/user_guide", true)

spec/json_parser.propane

@@ -1,179 +0,0 @@
-<<
-import std.math;
-import json_types;
-private string string_value;
->>
-
-ptype JSONValue;
-ptype array = JSONValue[];
-ptype dict = JSONValue[string];
-ptype string = string;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n;
-    bool negative;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = new JSONNumber(n);
->>
-token true <<
-  $$ = new JSONTrue();
->>
-token false <<
-  $$ = new JSONFalse();
->>
-token null <<
-  $$ = new JSONNull();
->>
-/"/ <<
-  $mode(string);
-  string_value = "";
->>
-string: token string (string) /"/ <<
-  $$ = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  string_value ~= "\"";
->>
-string: /\\\\/ <<
-  string_value ~= "\\";
->>
-string: /\\\// <<
-  string_value ~= "/";
->>
-string: /\\b/ <<
-  string_value ~= "\b";
->>
-string: /\\f/ <<
-  string_value ~= "\f";
->>
-string: /\\n/ <<
-  string_value ~= "\n";
->>
-string: /\\r/ <<
-  string_value ~= "\r";
->>
-string: /\\t/ <<
-  string_value ~= "\t";
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
->>
-string: /[^\\]/ <<
-  string_value ~= match;
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = new JSONString($1);
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = new JSONObject();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = new JSONObject($2);
->>
-KeyValues (dict) -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  foreach (key, value; $3)
-  {
-    $1[key] = value;
-  }
-  $$ = $1;
->>
-KeyValue (dict) -> string colon Value <<
-  $$ = [$1: $3];
->>
-Array -> lbracket rbracket <<
-  $$ = new JSONArray();
->>
-Array -> lbracket Values rbracket <<
-  $$ = new JSONArray($2);
->>
-Values (array) -> Value <<
-  $$ = [$1];
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>

spec/json_types.d

@@ -2,45 +2,22 @@ class JSONValue
 {
 }
 
-class JSONObject : JSONValue
+class JSONObject < JSONValue
 {
     JSONValue[string] value;
-
-    this()
-    {
-    }
-
-    this(JSONValue[string] value)
-    {
-        this.value = value;
-    }
 }
 
-class JSONArray : JSONValue
+class JSONArray < JSONValue
 {
     JSONValue[] value;
-
-    this()
-    {
-    }
-
-    this(JSONValue[] value)
-    {
-        this.value = value;
-    }
 }
 
-class JSONNumber : JSONValue
+class JSONNumber < JSONValue
 {
     double value;
-
-    this(double value)
-    {
-        this.value = value;
-    }
 }
 
-class JSONString : JSONValue
+class JSONString < JSONValue
 {
     string value;
 
@@ -50,14 +27,14 @@ class JSONString : JSONValue
     }
 }
 
-class JSONTrue : JSONValue
+class JSONTrue < JSONValue
 {
 }
 
-class JSONFalse : JSONValue
+class JSONFalse < JSONValue
 {
 }
 
-class JSONNull : JSONValue
+class JSONNull < JSONValue
 {
 }

spec/propane/grammar_spec.rb

@@ -34,7 +34,6 @@ EOF
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
-      expect(grammar.prefix).to eq "p_"
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
@@ -112,11 +111,8 @@ token code2 <<
 >>
 
 tokenid token_with_no_pattern;
-
-prefix myparser_;
 EOF
       grammar = Grammar.new(input)
-      expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
       expect(o).to_not be_nil

spec/propane_spec.rb

@@ -4,14 +4,12 @@ require "open3"
 Results = Struct.new(:stdout, :stderr, :status)
 
 describe Propane do
-  def write_grammar(grammar, options = {})
+  def write_grammar(grammar)
-    options[:name] ||= ""
+    File.write("spec/run/testparser.propane", grammar)
-    File.write("spec/run/testparser#{options[:name]}.propane", grammar)
   end
 
   def build_parser(options = {})
-    options[:name] ||= ""
+    command = %w[./propane.sh spec/run/testparser.propane spec/run/testparser.d --log spec/run/testparser.log]
-    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -21,18 +19,8 @@ describe Propane do
     end
   end
 
-  def compile(test_files, options = {})
+  def compile(test_file)
-    test_files = Array(test_files)
+    result = system(*%w[gdc -funittest -o spec/run/testparser spec/run/testparser.d], test_file)
-    options[:parsers] ||= [""]
-    parsers = options[:parsers].map do |name|
-      "spec/run/testparser#{name}.#{options[:language]}"
-    end
-    case options[:language]
-    when "c"
-      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
-    when "d"
-      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
-    end
     expect(result).to be_truthy
   end
 
@@ -40,7 +28,6 @@ describe Propane do
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
-    stderr.sub!(/^.*modules passed unittests\n/, "")
     Results.new(stdout, stderr, status)
   end
 
@@ -74,12 +61,8 @@ describe Propane do
     FileUtils.mkdir_p("spec/run")
   end
 
-  %w[d c].each do |language|
+  it "generates a D lexer" do
-
+    write_grammar <<EOF
-    context "#{language.upcase} language" do
-
-      it "generates a lexer" do
-        write_grammar <<EOF
 token int /\\d+/;
 token plus /\\+/;
 token times /\\*/;
@@ -90,57 +73,15 @@ Foo -> int <<
 Foo -> plus <<
 >>
 EOF
-        build_parser(language: language)
+    build_parser
-        compile("spec/test_lexer.#{language}", language: language)
+    compile("spec/test_d_lexer.d")
-        results = run
+    results = run
-        expect(results.stderr).to eq ""
+    expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
+    expect(results.status).to eq 0
-      end
+  end
 
-      it "detects a lexer error when an unknown character is seen" do
+  it "generates a parser" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-ptype int;
-token int /\\d+/ <<
-  int v = 0;
-  for (size_t i = 0u; i < match_length; i++)
-  {
-    v *= 10;
-    v += (match[i] - '0');
-  }
-  $$ = v;
->>
-Start -> int <<
-  $$ = $1;
->>
-EOF
-        when "d"
-          write_grammar <<EOF
-ptype int;
-token int /\\d+/ <<
-  int v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-Start -> int <<
-  $$ = $1;
->>
-EOF
-        end
-        build_parser(language: language)
-        compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
-      it "generates a parser" do
-        write_grammar <<EOF
 token plus /\\+/;
 token times /\\*/;
 token zero /0/;
@@ -152,135 +93,21 @@ E -> B;
 B -> zero;
 B -> one;
 EOF
-        build_parser(language: language)
+    build_parser
-      end
+  end
 
-      it "generates a parser that does basic math - user guide example" do
+  it "generates an SLR parser" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <math.h>
->>
-
-ptype size_t;
-
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  size_t v = 0u;
-  for (size_t i = 0u; i < match_length; i++)
-  {
-    v *= 10;
-    v += (match[i] - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = (size_t)pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.math;
->>
-
-ptype ulong;
-
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-EOF
-        end
-        build_parser(language: language)
-        compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
-      it "generates an SLR parser" do
-        write_grammar <<EOF
 token one /1/;
 Start -> E;
 E -> one E;
 E -> one;
 EOF
-        build_parser(language: language)
+    build_parser
-      end
+  end
 
-      it "distinguishes between multiple identical rules with lookahead symbol" do
+  it "distinguishes between multiple identical rules with lookahead symbol" do
-        write_grammar <<EOF
+    write_grammar <<EOF
 token a;
 token b;
 Start -> R1 a;
@@ -288,14 +115,14 @@ Start -> R2 b;
 R1 -> a b;
 R2 -> a b;
 EOF
-        build_parser(language: language)
+    build_parser
-        compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
+    compile("spec/test_d_parser_identical_rules_lookahead.d")
-        results = run
+    results = run
-        expect(results.status).to eq 0
+    expect(results.status).to eq 0
-      end
+  end
 
-      it "handles reducing a rule that could be arrived at from multiple states" do
+  it "handles reducing a rule that could be arrived at from multiple states" do
-        write_grammar <<EOF
+    write_grammar <<EOF
 token a;
 token b;
 drop /\\s+/;
@@ -303,32 +130,14 @@ Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
-        build_parser(language: language)
+    build_parser
-        compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
+    compile("spec/test_d_parser_rule_from_multiple_states.d")
-        results = run
+    results = run
-        expect(results.status).to eq 0
+    expect(results.status).to eq 0
-      end
+  end
 
-      it "executes user code when matching lexer token" do
+  it "executes user code when matching lexer token" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
->>
-token abc <<
-  printf("abc!\\n");
->>
-token def;
-Start -> Abcs def;
-Abcs -> ;
-Abcs -> abc Abcs;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc <<
   writeln("abc!");
 >>
@@ -337,80 +146,42 @@ Start -> Abcs def;
 Abcs -> ;
 Abcs -> abc Abcs;
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_user_code.d")
-        compile("spec/test_user_code.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "abc!",
-          "abc!",
+      "pass1",
-          "pass1",
+      "abc!",
-          "abc!",
+      "abc!",
-          "abc!",
+      "pass2",
-          "pass2",
+    ])
-        ])
+  end
-      end
 
-      it "supports a pattern statement" do
+  it "supports a pattern statement" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
->>
-token abc;
-/def/ <<
-  printf("def!\\n");
->>
-Start -> abc;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 /def/ <<
   writeln("def!");
 >>
 Start -> abc;
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_pattern.d")
-        compile("spec/test_pattern.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "def!",
-          "def!",
+      "pass1",
-          "pass1",
+      "def!",
-          "def!",
+      "def!",
-          "def!",
+      "pass2",
-          "pass2",
+    ])
-        ])
+  end
-      end
 
-      it "supports returning tokens from pattern code blocks" do
+  it "supports returning tokens from pattern code blocks" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
->>
-token abc;
-/def/ <<
-  printf("def!\\n");
->>
-/ghi/ <<
-  printf("ghi!\\n");
-  return $token(abc);
->>
-Start -> abc;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 /def/ <<
   writeln("def!");
@@ -421,47 +192,19 @@ token abc;
 >>
 Start -> abc;
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_return_token_from_pattern.d")
-        compile("spec/test_return_token_from_pattern.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "def!",
-          "def!",
+      "ghi!",
-          "ghi!",
+      "def!",
-          "def!",
+    ])
-        ])
+  end
-      end
 
-      it "supports lexer modes" do
+  it "supports lexer modes" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
->>
-token abc;
-token def;
-tokenid string;
-drop /\\s+/;
-/"/ <<
-  printf("begin string mode\\n");
-  $mode(string);
->>
-string: /[^"]+/ <<
-  printf("captured string\\n");
->>
-string: /"/ <<
-  $mode(default);
-  return $token(string);
->>
-Start -> abc string def;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token abc;
 token def;
 tokenid string;
@@ -479,45 +222,22 @@ string: /"/ <<
 >>
 Start -> abc string def;
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_lexer_modes.d")
-        compile("spec/test_lexer_modes.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "begin string mode",
-          "begin string mode",
+      "captured string",
-          "captured string",
+      "pass1",
-          "pass1",
+      "begin string mode",
-          "begin string mode",
+      "captured string",
-          "captured string",
+      "pass2",
-          "pass2",
+    ])
-        ])
+  end
-      end
 
-      it "executes user code associated with a parser rule" do
+  it "executes user code associated with a parser rule" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
->>
-token a;
-token b;
-Start -> A B <<
-  printf("Start!\\n");
->>
-A -> a <<
-  printf("A!\\n");
->>
-B -> b <<
-  printf("B!\\n");
->>
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token a;
 token b;
 Start -> A B <<
@@ -530,21 +250,20 @@ B -> b <<
   writeln("B!");
 >>
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_parser_rule_user_code.d")
-        compile("spec/test_parser_rule_user_code.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "A!",
-          "A!",
+      "B!",
-          "B!",
+      "Start!",
-          "Start!",
+    ])
-        ])
+  end
-      end
 
-      it "parses lists" do
+  it "parses lists" do
-        write_grammar <<EOF
+    write_grammar <<EOF
-ptype #{language == "c" ? "uint32_t" : "uint"};
+ptype uint;
 token a;
 Start -> As <<
   $$ = $1;
@@ -556,15 +275,15 @@ As -> As a <<
   $$ = $1 + 1u;
 >>
 EOF
-        build_parser(language: language)
+    build_parser
-        compile("spec/test_parsing_lists.#{language}", language: language)
+    compile("spec/test_parsing_lists.d")
-        results = run
+    results = run
-        expect(results.status).to eq 0
+    expect(results.status).to eq 0
-        expect(results.stderr).to eq ""
+    expect(results.stderr).to eq ""
-      end
+  end
 
-      it "fails to generate a parser for a LR(1) grammar that is not LALR" do
+  it "fails to generate a parser for a LR(1) grammar that is not LALR" do
-        write_grammar <<EOF
+    write_grammar <<EOF
 token a;
 token b;
 token c;
@@ -577,62 +296,30 @@ Start -> b E d;
 E -> e;
 F -> e;
 EOF
-        results = build_parser(capture: true, language: language)
+    results = build_parser(capture: true)
-        expect(results.status).to_not eq 0
+    expect(results.status).to_not eq 0
-        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+    expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
-      end
+  end
 
-      it "provides matched text to user code blocks" do
+  it "provides matched text to user code blocks" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-<<
-#include <stdio.h>
-#include <stdlib.h>
->>
-token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * t = malloc(match_length + 1);
-  strncpy(t, (char *)match, match_length);
-  printf("Matched token is %s\\n", t);
-  free(t);
->>
-Start -> id;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-import std.stdio;
->>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   writeln("Matched token is ", match);
 >>
 Start -> id;
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_lexer_match_text.d")
-        compile("spec/test_lexer_match_text.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+    verify_lines(results.stdout, [
-        verify_lines(results.stdout, [
+      "Matched token is identifier_123",
-          "Matched token is identifier_123",
+      "pass1",
-          "pass1",
+    ])
-        ])
+  end
-      end
 
-      it "allows storing a result value for the lexer" do
+  it "allows storing a result value for the lexer" do
-        case language
+    write_grammar <<EOF
-        when "c"
-          write_grammar <<EOF
-ptype uint64_t;
-token word /[a-z]+/ <<
-  $$ = match_length;
->>
-Start -> word <<
-  $$ = $1;
->>
-EOF
-        when "d"
-          write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
   $$ = match.length;
@@ -641,56 +328,137 @@ Start -> word <<
   $$ = $1;
 >>
 EOF
-        end
+    build_parser
-        build_parser(language: language)
+    compile("spec/test_lexer_result_value.d")
-        compile("spec/test_lexer_result_value.#{language}", language: language)
+    results = run
-        results = run
+    expect(results.stderr).to eq ""
-        expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
-        expect(results.status).to eq 0
+  end
-      end
 
-      it "tracks position of parser errors" do
+  it "allows creating a JSON parser" do
-        write_grammar <<EOF
+    write_grammar <<EOF
-token a;
+<<
-token num /\\d+/;
+  import json_types;
+  string string_value;
+>>
 drop /\\s+/;
-Start -> a num Start;
+token lbrace /\\{/;
-Start -> a num;
+token rbrace /\\}/RuRfaw2CHgpvCWu;
+token lbracket /\\[/;
+token rbracket /\\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+  bool negative;
+  bool in_fraction;
+  bool in_exponent;
+  double exponent;
+  foreach (c; match)
+  {
+    if (c == '-')
+    {
+      negative = true;
+    }
+  }
+  $$ = new JSONNumber();
+>>
+token true <<
+  $$ = new JSONTrue();
+>>
+token false <<
+  $$ = new JSONFalse();
+>>
+token null <<
+  $$ = new JSONNull();
+>>
+/"/ <<
+  $mode(string);
+  string_value = "";
+>>
+string: token string /"/ <<
+  $$ = new JSONString(string_value);
+  $mode(default);
+>>
+string: /\\\\"/ <<
+  string_value ~= "\\"";
+>>
+string: /\\\\\\\\/ <<
+  string_value ~= "\\\\";
+>>
+string: /\\\\\\// <<
+  string_value ~= "/";
+>>
+string: /\\\\b/ <<
+  string_value ~= "\\b";
+>>
+string: /\\\\f/ <<
+  string_value ~= "\\f";
+>>
+string: /\\\\n/ <<
+  string_value ~= "\\n";
+>>
+string: /\\\\r/ <<
+  string_value ~= "\\r";
+>>
+string: /\\\\t/ <<
+  string_value ~= "\\t";
+>>
+string: /\\\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  string_value ~= "{" ~ match[2..6] ~ "}";
+>>
+string: /[^\\\\]/ <<
+  string_value ~= match;
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = new JSONObject();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = new JSONObject($2);
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  foreach (key, value; $3)
+  {
+    $1[key] = value;
+  }
+  $$ = $1;
+>>
+KeyValue -> string colon Value <<
+  $$ = 
+>>
+Array -> lbracket rbracket;
+Array -> lbracket Values rbracket;
+Values -> Value;
+Values -> Values comma Value;
 EOF
-        build_parser(language: language)
+    build_parser
-        compile("spec/test_error_positions.#{language}", language: language)
+    compile("spec/test_parsing_json.d")
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
-      it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.propane"))
-        build_parser(language: language)
-        compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
-      end
-
-      it "allows generating multiple parsers in the same program" do
-        write_grammar(<<EOF, name: "myp1")
-prefix myp1_;
-token a;
-token num /\\d+/;
-drop /\\s+/;
-Start -> a num;
-EOF
-        build_parser(name: "myp1", language: language)
-        write_grammar(<<EOF, name: "myp2")
-prefix myp2_;
-token b;
-token c;
-Start -> b c b;
-EOF
-        build_parser(name: "myp2", language: language)
-        compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-    end
   end
 end

spec/test_basic_math_grammar.c

@@ -1,29 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-
-int main()
-{
-    char const * input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
-
-    input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
-
-    input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
-
-    input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
-
-    return 0;
-}

spec/test_basic_math_grammar.d

@@ -1,32 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
-
-    input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
-
-    input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
-
-    input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
-}

spec/test_d_lexer.d

@@ -0,0 +1,52 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    alias DCP = Testparser.Decoder.DecodedCodePoint;
+    DCP dcp;
+
+    dcp = Testparser.Decoder.decode_code_point("5");
+    assert(dcp == DCP('5', 1u));
+
+    dcp = Testparser.Decoder.decode_code_point("");
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
+
+    dcp = Testparser.Decoder.decode_code_point("\xC2\xA9");
+    assert(dcp == DCP(0xA9u, 2u));
+
+    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
+    assert(dcp == DCP(0x1F9E1, 4u));
+
+    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    dcp = Testparser.Decoder.decode_code_point("\xfe");
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+}
+
+unittest
+{
+    alias LT = Testparser.Lexer.LexedToken;
+    string input = "5 + 4 * \n677 + 567";
+    Testparser.Lexer lexer = new Testparser.Lexer(input);
+    assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_times));
+    assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_plus));
+    assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_int));
+    assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_0EOF));
+
+    lexer = new Testparser.Lexer("");
+    assert(lexer.lex_token() == LT(0, 0, 0, Testparser.TOKEN_0EOF));
+}

spec/test_d_parser_identical_rules_lookahead.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aba";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+
+    input = "abb";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_d_parser_rule_from_multiple_states.d

@@ -0,0 +1,22 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    auto parser = new Testparser.Parser(input);
+    assert(parser.parse() == false);
+
+    input = "a b";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+
+    input = "bb";
+    parser = new Testparser.Parser(input);
+    assert(parser.parse() == true);
+}

spec/test_error_positions.c

@@ -1,39 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a 42";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "a\n123\na  a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 3);
-    assert(context.token == TOKEN_a);
-
-    input = "12";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 0);
-    assert(context.token == TOKEN_num);
-
-    input = "a 12\n\nab";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 1);
-
-    input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 5);
-    assert(p_position(&context).col == 4);
-
-    return 0;
-}

spec/test_error_positions.d

@@ -1,37 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
-
-    input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
-
-    input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
-
-    input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(5, 4));
-}

spec/test_lexer.c

@@ -1,92 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    size_t result;
-    p_code_point_t code_point;
-    uint8_t code_point_length;
-
-    result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == '5');
-    assert(code_point_length == 1u);
-
-    result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
-    assert(result == P_EOF);
-
-    result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0xA9u);
-    assert(code_point_length == 2u);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0x1F9E1u);
-    assert(code_point_length == 4u);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-
-    p_token_info_t token_info;
-    char const * input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 2u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_times);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.length == 3u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.length == 3u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 9u);
-    assert(token_info.length == 0u);
-    assert(token_info.token == TOKEN___EOF);
-
-    p_context_init(&context, (uint8_t const *)"", 0u);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.length == 0u);
-    assert(token_info.token == TOKEN___EOF);
-
-    return 0;
-}

spec/test_lexer.d

@@ -1,69 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    size_t result;
-    p_code_point_t code_point;
-    ubyte code_point_length;
-
-    result = p_decode_code_point("5", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == '5');
-    assert(code_point_length == 1u);
-
-    result = p_decode_code_point("", &code_point, &code_point_length);
-    assert(result == P_EOF);
-
-    result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0xA9u);
-    assert(code_point_length == 2u);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0x1F9E1u);
-    assert(code_point_length == 4u);
-
-    result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xfe", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-}
-
-unittest
-{
-    p_token_info_t token_info;
-    string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
-
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
-}

spec/test_lexer_match_text.c

@@ -1,15 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "identifier_123";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    return 0;
-}

spec/test_lexer_match_text.d

@@ -9,8 +9,7 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 }

spec/test_lexer_modes.c

@@ -1,20 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "abc \"a string\" def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    input = "abc \"abc def\" def";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass2\n");
-
-    return 0;
-}

spec/test_lexer_modes.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/test_lexer_result_value.c

@@ -1,19 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
-
-    input = "fabulous";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
-
-    return 0;
-}

spec/test_lexer_result_value.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.result == 1u);
-    assert(p_result(&context) == 1u);
 
     input = `fabulous`;
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
-    assert(p_result(&context) == 8u);
+    assert(parser.result == 8u);
 }

spec/test_lexer_unknown_character.c

@@ -1,18 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-
-    input = "123";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
-
-    return 0;
-}

spec/test_lexer_unknown_character.d

@@ -1,20 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-
-    input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
-}

spec/test_multiple_parsers.c

@@ -1,19 +0,0 @@
-#include "testparsermyp1.h"
-#include "testparsermyp2.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
-
-    char const * input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
-
-    return 0;
-}

spec/test_multiple_parsers.d

@@ -1,21 +0,0 @@
-import testparsermyp1;
-import testparsermyp2;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
-
-    string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
-}

spec/test_parser_identical_rules_lookahead.c

@@ -1,17 +0,0 @@
-#include "testparser.h"
-#include <string.h>
-#include <assert.h>
-
-int main()
-{
-    char const * input = "aba";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_identical_rules_lookahead.d

@@ -1,19 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_from_multiple_states.c

@@ -1,24 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 1);
-    assert(context.token == TOKEN___EOF);
-
-    input = "a b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "bb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_rule_from_multiple_states.d

@@ -1,25 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
-    assert(context.token == TOKEN___EOF);
-
-    input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_user_code.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_rule_user_code.d

@@ -9,7 +9,6 @@ int main()
 unittest
 {
     string input = "ab";
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
 }

spec/test_parsing_json.d

@@ -1,6 +1,5 @@
 import testparser;
 import std.stdio;
-import json_types;
 
 int main()
 {
@@ -10,47 +9,6 @@ int main()
 unittest
 {
     string input = ``;
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-
-    input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
-
-    input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
-
-    input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
-
-    input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
-    assert(o.value["hi"]);
-    assert(cast(JSONTrue)o.value["hi"]);
-
-    input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
-    assert(o.value["ff"]);
-    assert(cast(JSONFalse)o.value["ff"]);
-    assert(o.value["nn"]);
-    assert(cast(JSONNull)o.value["nn"]);
 }

spec/test_parsing_lists.c

@@ -1,24 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
-
-    input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
-
-    input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
-
-    return 0;
-}

spec/test_parsing_lists.d

@@ -9,18 +9,17 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.result == 1u);
-    assert(p_result(&context) == 1u);
 
     input = "";
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
-    assert(p_result(&context) == 0u);
+    assert(parser.result == 0u);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
-    assert(p_result(&context) == 16u);
+    assert(parser.result == 16u);
 }

spec/test_pattern.c

@@ -1,20 +0,0 @@
-#include "testparser.h"
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    input = "defabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass2\n");
-
-    return 0;
-}

spec/test_pattern.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 
     input = "defabcdef";
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/test_return_token_from_pattern.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_return_token_from_pattern.d

@@ -9,7 +9,6 @@ int main()
 unittest
 {
     string input = "defghidef";
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
 }

spec/test_user_code.c

@@ -1,20 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    input = "abcabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass2\n");
-
-    return 0;
-}

spec/test_user_code.d

@@ -9,13 +9,12 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
+    auto parser = new Testparser.Parser(input);
-    p_context_init(&context, input);
+    assert(parser.parse() == true);
-    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 
     input = "abcabcdef";
-    p_context_init(&context, input);
+    parser = new Testparser.Parser(input);
-    assert(p_parse(&context) == P_SUCCESS);
+    assert(parser.parse() == true);
     writeln("pass2");
 }

spec/testutils.c

@@ -1,12 +0,0 @@
-#include <stdio.h>
-#include <assert.h>
-#include <stdbool.h>
-
-void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
-{
-    if (expected != actual)
-    {
-        fprintf(stderr, "%s:%lu: expected %lu, got %lu\n", file, line, expected, actual);
-        assert(false);
-    }
-}

spec/testutils.d

@@ -1,10 +0,0 @@
-import std.stdio;
-
-void assert_eq(T)(T expected, T actual, string file = __FILE__, size_t line = __LINE__)
-{
-    if (expected != actual)
-    {
-        stderr.writeln(file, ":", line, ": expected ", expected, ", got ", actual);
-        assert(false);
-    }
-}

spec/testutils.h

@@ -1,7 +0,0 @@
-#pragma once
-
-void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
-
-#define assert_eq(expected, actual) \
-    assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
-