wip

CHANGELOG.md

@@ -1,25 +1,3 @@
-## v1.2.0
-
-### New Features
-
-- Allow one line user code blocks (#21)
-- Add backslash escape codes (#19)
-- Add API to access unexpected token found (#18)
-- Add token_names API (#17)
-- Add D example to user guide for p_context_init() (#16)
-- Allow user termination from lexer code blocks (#15)
-
-### Fixes
-
-- Fix generator hang when state transition cycle is present (#20)
-
-## v1.1.0
-
-### New Features
-
-- Add user parser terminations (#13)
-- Document generated parser API in user guide (#14)
-
 ## v1.0.0
 
 - Initial release

Gemfile

@@ -5,4 +5,3 @@ gem "rspec"
 gem "rdoc"
 gem "redcarpet"
 gem "syntax"
-gem "simplecov"

Gemfile.lock

@@ -2,7 +2,6 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
-    docile (1.4.0)
     psych (5.1.0)
       stringio
     rake (13.0.6)
@@ -22,12 +21,6 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.1)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.4)
     stringio (3.0.7)
     syntax (1.2.2)
 
@@ -39,7 +32,6 @@ DEPENDENCIES
   rdoc
   redcarpet
   rspec
-  simplecov
   syntax
 
 BUNDLED WITH

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2023 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,113 +1,21 @@
 # The Propane Parser Generator
 
-Propane is a LALR Parser Generator (LPG) which:
+Propane is an LR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven shift/reduce parser to parse input in linear time
+  * generates a table-driven parser to parse input in linear time
-  * target C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
 ## Installation
 
-Propane is designed to be distributed as a stand-alone single file script that
+TODO
-can be copied into and versioned in a project's source tree.
-The only requirement to run Propane is that the system has a Ruby interpreter
-installed.
-The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
-Simply copy the `propane` executable script into the desired location within
-the project to be built (typically the root of the repository) and mark it
-executable.
 
 ## Usage
 
-### Command Line Interface
+TODO: Write usage instructions here
-
-Propane is typically invoked from the command-line as `./propane`.
-
-    Usage: ./propane [options] <input-file> <output-file>
-    Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
-
-The user must specify the path to a Propane input grammar file and a path to an
-output file.
-The generated source code will be written to the output file.
-If a log file path is specified, Propane will write a log file containing
-detailed information about the parser states and transitions.
-
-### Propane Grammar File
-
-A Propane grammar file provides Propane with the patterns, tokens, grammar
-rules, and user code blocks from which to build the generated lexer and parser.
-
-Example grammar file:
-
-```
-<<
-import std.math;
->>
-
-# Parser values are unsigned integers.
-ptype ulong;
-
-# A few basic arithmetic operators.
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\\(/;
-token rparen /\\)/;
-# Drop whitespace.
-drop /\\s+/;
-
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
-```
-
-Grammar files can contain comment lines beginning with `#` which are ignored.
-White space in the grammar file is also ignored.
-
-It is convention to use the extension `.propane` for the Propane grammar file,
-however any file name is accepted by Propane.
-
-See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
 
 ## Development
 

Rakefile

@@ -1,8 +1,5 @@
-require "rake/clean"
 require "rspec/core/rake_task"
 
-CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
-
 task :build_dist do
   sh "ruby rb/build_dist.rb"
 end
@@ -13,20 +10,9 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   end
 end
 
-# dspec task is useful to test the distributable release script, but is not
-# useful for coverage information.
-desc "Dist Specs"
-task :dspec, [:example_string] => :build_dist do |task, args|
-  ENV["dist_specs"] = "1"
-  Rake::Task["spec"].execute(args)
-  ENV.delete("dist_specs")
-end
-
 task :default => :spec
 
 desc "Build user guide"
 task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
-
-task :all => [:spec, :dspec, :user_guide]

assets/parser.c.erb

@@ -3,22 +3,13 @@
 #include <stdlib.h>
 #include <string.h>
 
-/**************************************************************************
- * Public data
- *************************************************************************/
-
-/** Token names. */
-const char * <%= @grammar.prefix %>token_names[] = {
-<% @grammar.tokens.each_with_index do |token, index| %>
-    "<%= token.name %>",
-<% end %>
-};
-
 /**************************************************************************
  * User code blocks
  *************************************************************************/
 
-<%= @grammar.code_blocks.fetch("", "") %>
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 /**************************************************************************
  * Private types
@@ -32,7 +23,6 @@ const char * <%= @grammar.prefix %>token_names[] = {
 #define P_UNEXPECTED_TOKEN 3u
 #define P_DROP 4u
 #define P_EOF 5u
-#define P_USER_TERMINATED 6u
 <% end %>
 
 /* An invalid ID value. */
@@ -402,6 +392,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
+            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -433,8 +424,6 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -454,12 +443,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state->code_id, match, match_info.length, &token_info);
-            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
-             * that the user code is requesting to terminate the lexer. */
-            if (user_code_token == TERMINATE_TOKEN_ID)
-            {
-                return P_USER_TERMINATED;
-            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -530,8 +513,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -759,13 +740,12 @@ static void state_values_stack_free(state_values_stack_t * stack)
  *
  * @param rule The ID of the rule.
  *
- * @retval P_SUCCESS
+ * @return Parse value.
- *   Continue parsing.
- * @retval P_USER_TERMINATED
- *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
 {
+    <%= @grammar.prefix %>value_t _pvalue = {0};
+
     switch (rule)
     {
 <% @grammar.rules.each do |rule| %>
@@ -778,7 +758,7 @@ static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t
     default: break;
     }
 
-    return P_SUCCESS;
+    return _pvalue;
 }
 
 /**
@@ -841,7 +821,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The function p_token(&context) can be used to get the unexpected token.
+ *   The value context->token holds the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -911,12 +891,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
-            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
-            {
-                return P_USER_TERMINATED;
-            }
-            reduced_parser_value = reduced_parser_value2;
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
@@ -961,26 +936,3 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
     return context->text_position;
 }
-
-/**
- * Get the user terminate code.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return User terminate code.
- */
-size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
-{
-    return context->user_terminate_code;
-}
-
-/**
- * Get the parse token.
- *
- * @return Parse token.
- */
-<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
-{
-    return context->token;
-}

assets/parser.d.erb

@@ -12,7 +12,9 @@ module <%= @grammar.modulename %>;
  * User code blocks
  *************************************************************************/
 
-<%= @grammar.code_blocks.fetch("", "") %>
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 /**************************************************************************
  * Public types
@@ -27,11 +29,10 @@ public enum : size_t
     <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN,
     <%= @grammar.prefix.upcase %>DROP,
     <%= @grammar.prefix.upcase %>EOF,
-    <%= @grammar.prefix.upcase %>USER_TERMINATED,
 }
 
 /** Token type. */
-public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
 
 /** Token IDs. */
 public enum : <%= @grammar.prefix %>token_t
@@ -43,7 +44,6 @@ public enum : <%= @grammar.prefix %>token_t
 <%   end %>
 <% end %>
     INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
-    TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
 }
 
 /** Code point type. */
@@ -116,9 +116,6 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
-
-    /** User terminate code. */
-    size_t user_terminate_code;
 }
 
 /**************************************************************************
@@ -146,7 +143,6 @@ private enum : size_t
     P_UNEXPECTED_TOKEN,
     P_DROP,
     P_EOF,
-    P_USER_TERMINATED,
 }
 <% end %>
 
@@ -508,6 +504,7 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
+            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -539,8 +536,6 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -560,12 +555,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state.code_id, match, &token_info);
-            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
-             * that the user code is requesting to terminate the lexer. */
-            if (user_code_token == TERMINATE_TOKEN_ID)
-            {
-                return P_USER_TERMINATED;
-            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -636,8 +625,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -777,13 +764,12 @@ private immutable parser_state_t[] parser_state_table = [
  *
  * @param rule The ID of the rule.
  *
- * @retval P_SUCCESS
+ * @return Parse value.
- *   Continue parsing.
- * @retval P_USER_TERMINATED
- *   User requested to terminate parsing.
  */
-private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
+private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[] statevalues, uint n_states)
 {
+    <%= @grammar.prefix %>value_t _pvalue;
+
     switch (rule)
     {
 <% @grammar.rules.each do |rule| %>
@@ -796,7 +782,7 @@ private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint ru
     default: break;
     }
 
-    return P_SUCCESS;
+    return _pvalue;
 }
 
 /**
@@ -859,7 +845,7 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The function p_token(&context) can be used to get the unexpected token.
+ *   The value context.token holds the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -923,12 +909,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states);
-            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
-            {
-                return P_USER_TERMINATED;
-            }
-            reduced_parser_value = reduced_parser_value2;
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
             continue;
@@ -970,26 +951,3 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
 {
     return context.text_position;
 }
-
-/**
- * Get the user terminate code.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return User terminate code.
- */
-public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
-{
-    return context.user_terminate_code;
-}
-
-/**
- * Get the parse token.
- *
- * @return Parse token.
- */
-public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
-{
-    return context.token;
-}

assets/parser.h.erb

@@ -20,10 +20,9 @@
 #define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
 #define <%= @grammar.prefix.upcase %>DROP 4u
 #define <%= @grammar.prefix.upcase %>EOF 5u
-#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
 
 /** Token type. */
-typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
 
 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -33,14 +32,10 @@ typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>t
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
-#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
 
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
-/** User header code blocks. */
-<%= @grammar.code_blocks.fetch("header", "") %>
-
 /** Parser values type(s). */
 typedef union
 {
@@ -111,18 +106,8 @@ typedef struct
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
-
-    /** User terminate code. */
-    size_t user_terminate_code;
 } <%= @grammar.prefix %>context_t;
 
-/**************************************************************************
- * Public data
- *************************************************************************/
-
-/** Token names. */
-extern const char * <%= @grammar.prefix %>token_names[];
-
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
@@ -135,7 +120,3 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
-
-size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
-
-<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -7,15 +7,15 @@ ${/remove}
 
 #> Overview
 
-Propane is a LALR Parser Generator (LPG) which:
+Propane is an LR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven shift/reduce parser to parse input in linear time
+  * generates a table-driven parser to parse input in linear time
-  * targets C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
+  * supports D language
 
 #> Installation
 
@@ -56,10 +56,8 @@ Example grammar file:
 import std.math;
 >>
 
-# Parser values are unsigned integers.
 ptype ulong;
 
-# A few basic arithmetic operators.
 token plus /\\+/;
 token times /\\*/;
 token power /\\*\\*/;
@@ -74,643 +72,35 @@ token integer /\\d+/ <<
 >>
 token lparen /\\(/;
 token rparen /\\)/;
-# Drop whitespace.
 drop /\\s+/;
 
-Start -> E1 << $$ = $1; >>
+Start -> E1 <<
-E1 -> E2 << $$ = $1; >>
+  $$ = $1;
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E2 -> E3 << $$ = $1; >>
-E2 -> E2 times E3 << $$ = $1 * $3; >>
-E3 -> E4 << $$ = $1; >>
-E3 -> E3 power E4 << $$ = pow($1, $3); >>
-E4 -> integer << $$ = $1; >>
-E4 -> lparen E1 rparen << $$ = $2; >>
-```
-
-Grammar files can contain comment lines beginning with `#` which are ignored.
-White space in the grammar file is also ignored.
-
-It is convention to use the extension `.propane` for the Propane grammar file,
-however any file name is accepted by Propane.
-
-This user guide follows the convention of beginning a token name with a
-lowercase character and beginning a rule name with an uppercase character.
-
-##> User Code Blocks
-
-User code blocks begin following a "<<" token and end with a ">>" token found
-at the end of a line.
-All text lines in the code block are copied verbatim into the output file.
-
-### Standalone Code Blocks
-
-C example:
-
-```
-<<
-#include &lt;stdio.h>
 >>
-```
+E1 -> E2 <<
-
+  $$ = $1;
-D example:
-
-```
-<<
-import std.stdio;
 >>
-```
+E1 -> E1 plus E2 <<
-
+  $$ = $1 + $3;
-Standalone code blocks are emitted early in the output file as top-level code
-outside the context of any function.
-Standalone code blocks are a good place to include/import any other necessary
-supporting code modules.
-They can also define helper functions that can be reused by lexer or parser
-user code blocks.
-They are emitted in the order they are defined in the grammar file.
-
-For a C target, the word "header" may immediately follow the "<<" token to
-cause Propane to emit the code block in the generated header file rather than
-the generated implementation file.
-This allows including another header that may be necessary to define any types
-needed by a `ptype` directive, for example:
-
-```
-<&lt;header
-#include "mytypes.h"
 >>
-```
+E2 -> E3 <<
-
+  $$ = $1;
-### Lexer pattern code blocks
-
-Example:
-
-```
-ptype ulong;
-
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
 >>
-```
+E2 -> E2 times E3 <<
-
+  $$ = $1 * $3;
-Lexer code blocks appear following a `token` or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
-### Parser rule code blocks
-
-Example:
-
-```
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
-```
-
-Parser rule code blocks appear following a rule expression.
-User code in a parser rule code block will be executed when the parser reduces
-the given rule.
-Assignment to the `$$` symbol will associate a parser value with the reduced
-rule.
-Parser values for the rules or tokens in the rule pattern can be accessed
-positionally with tokens `$1`, `$2`, `$3`, etc...
-
-##> Specifying tokens - the `token` statement
-
-The `token` statement allows defining a lexer token and a pattern to match that
-token.
-The name of the token must be specified immediately following the `token`
-keyword.
-A regular expression pattern may optionally follow the token name.
-If a regular expression pattern is not specified, the name of the token is
-taken to be the pattern.
-See also: ${#Regular expression syntax}.
-
-Example:
-
-```
-token for;
-```
-
-In this example, the token name is `for` and the pattern to match it is
-`/for/`.
-
-Example:
-
-```
-token lbrace /\{/;
-```
-
-In this example, the token name is `lbrace` and a single left curly brace will
-match it.
-
-The `token` statement can also include a user code block.
-The user code block will be executed whenever the token is matched by the
-lexer.
-
-Example:
-
-```
-token if << writeln("'if' keyword lexed"); >>
-```
-
-The `token` statement is actually a shortcut statement for a combination of a
-`tokenid` statement and a pattern statement.
-To define a lexer token without an associated pattern to match it, use a
-`tokenid` statement.
-To define a lexer pattern that may or may not result in a matched token, use
-a pattern statement.
-
-##> Defining tokens without a matching pattern - the `tokenid` statement
-
-The `tokenid` statement can be used to define a token without associating it
-with a lexer pattern that matches it.
-
-Example:
-
-```
-tokenid string;
-```
-
-The `tokenid` statement can be useful when defining a token that may optionally
-be returned by user code associated with a pattern.
-
-It is also useful when lexer modes and multiple lexer patterns are required to
-build up a full token.
-A common example is parsing a string.
-See the ${#Lexer modes} chapter for more information.
-
-##> Specifying a lexer pattern - the pattern statement
-
-A pattern statement is used to define a lexer pattern that can execute user
-code but may not result in a matched token.
-
-Example:
-
-```
-/foo+/ << writeln("saw a foo pattern"); >>
-```
-
-This can be especially useful with ${#Lexer modes}.
-
-See also ${#Regular expression syntax}.
-
-##> Ignoring input sections - the `drop` statement
-
-A `drop` statement can be used to specify a lexer pattern that when matched
-should result in the matched input being dropped and lexing continuing after
-the matched input.
-
-A common use for a `drop` statement would be to ignore whitespace sequences in
-the user input.
-
-Example:
-
-```
-drop /\s+/;
-```
-
-See also ${#Regular expression syntax}.
-
-##> Regular expression syntax
-
-A regular expression ("regex") is used to define lexer patterns in `token`,
-pattern, and `drop` statements.
-A regular expression begins and ends with a `/` character.
-
-Example:
-
-```
-/#.*$/
-```
-
-Regular expressions can include many special characters:
-
-  * The `.` character matches any input character other than a newline.
-  * The `*` character matches any number of the previous regex element.
-  * The `+` character matches one or more of the previous regex element.
-  * The `?` character matches 0 or 1 of the previous regex element.
-  * The `[` character begins a character class.
-  * The `(` character begins a matching group.
-  * The `{` character begins a count qualifier.
-  * The `\` character escapes the following character and changes its meaning:
-    * The `\a` sequence matches an ASCII bell character (0x07).
-    * The `\b` sequence matches an ASCII backspace character (0x08).
-    * The `\d` sequence matches any character `0` through `9`.
-    * The `\f` sequence matches an ASCII form feed character (0x0C).
-    * The `\n` sequence matches an ASCII new line character (0x0A).
-    * The `\r` sequence matches an ASCII carriage return character (0x0D).
-    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
-    `\r`, a form feed `\f`, or a vertical tab `\v` character.
-    * The `\t` sequence matches an ASCII tab character (0x09).
-    * The `\v` sequence matches an ASCII vertical tab character (0x0B).
-    * Any other character matches itself.
-  * The `|` character creates an alternate match.
-
-Any other character just matches itself in the input stream.
-
-A character class consists of a list of character alternates or character
-ranges that can be matched by the character class.
-For example `[a-zA-Z_]` matches any lowercase character between `a` and `z` or
-any uppercase character between `A` and `Z` or the underscore `_` character.
-Character classes can also be negative character classes if the first character
-after the `[` is a `^` character.
-In this case, the set of characters matched by the character class is the
-inverse of what it otherwise would have been.
-For example, `[^0-9]` matches any character other than 0 through 9.
-
-A matching group can be used to override the pattern sequence that multiplicity
-specifiers apply to.
-For example, the pattern `/foo+/` matches "foo" or "foooo", while the pattern
-`/(foo)+/` matches "foo" or "foofoofoo", but not "foooo".
-
-A count qualifier in curly braces can be used to restrict the number of matches
-of the preceding atom to an explicit minimum and maximum range.
-For example, the pattern `\d{3}` matches exactly 3 digits 0-9.
-Both a minimum and maximum multiplicity count can be specified and separated by
-a comma.
-For example, `/a{1,5}/` matches between 1 and 5 `a` characters.
-Either the minimum or maximum count can be omitted to omit the corresponding
-restriction in the number of matches allowed.
-
-An alternate match is created with the `|` character.
-For example, the pattern `/foo|bar/` matches either the sequence "foo" or the
-sequence "bar".
-
-##> Lexer modes
-
-Lexer modes can be used to change the set of patterns that are matched by the
-lexer.
-A common use for lexer modes is to match strings.
-
-Example:
-
-```
-<<
-string mystringvalue;
 >>
-
+E3 -> E4 <<
-tokenid str;
+  $$ = $1;
-
-# String processing
-/"/ <<
-  mystringvalue = "";
-  $mode(string);
 >>
-string: /[^"]+/ << mystringvalue += match; >>
+E3 -> E3 power E4 <<
-string: /"/ <<
+  $$ = pow($1, $3);
-  $mode(default);
+>>
-  return $token(str);
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
 >>
-```
-
-A lexer mode is defined by placing the name before a colon (`:`) character that
-precedes a token or pattern statement.
-The token or pattern statement is restricted to only applying if the named mode
-is active.
-
-By default, the active lexer mode is named `default`.
-A `$mode()` call within a lexer code block can be used to change lexer modes.
-
-In the above example, when the lexer in the default mode sees a doublequote
-(`"`) character, the lexer code block will clear the `mystringvalue` variable
-and will set the lexer mode to `string`.
-When the lexer begins looking for patterns to match against the input, it will
-now look only for patterns tagged for the `string` lexer mode.
-Any non-`"` character will be appended to the `mystringvalue` string.
-A `"` character will end the `string` lexer mode and return to the `default`
-lexer mode.
-It also returns the `str` token now that the token is complete.
-
-Note that the token name `str` above could have been `string` instead - the
-namespace for token names is distinct from the namespace for lexer modes.
-
-##> Specifying parser value types - the `ptype` statement
-
-The `ptype` statement is used to define parser value type(s).
-Example:
-
-```
-ptype void *;
-```
-
-This defines the default parser value type to be `void *` (this is, in fact,
-the default parser value type if the grammar file does not specify otherwise).
-
-Each defined lexer token type and parser rule has an associated parser value
-type.
-When the lexer runs, each lexed token has a parser value associated with it.
-When the parser runs, each instance of a reduced rule has a parser value
-associated with it.
-Propane supports using different parser value types for different rules and
-token types.
-The example `ptype` statement above defines the default parser value type.
-A parser value type name can optionally be specified following the `ptype`
-keyword.
-For example:
-
-```
-ptype Value;
-ptype array = Value[];
-ptype dict = Value[string];
-
-Object -> lbrace rbrace << $$ = new Value(); >>
-
-Values (array) -> Value << $$ = [$1]; >>
-Values -> Values comma Value << $$ = $1 ~ [$3]; >>
-
-KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
-```
-
-In this example, the default parser value type is `Value`.
-A parser value type named `array` is defined to mean `Value[]`.
-A parser value type named `dict` is defined to mean `Value[string]`.
-Any defined tokens or rules that do not specify a parser value type will have
-the default parser value type associated with them.
-To associate a different parser value type with a token or rule, write the
-parser value type name in parentheses following the name of the token or rule.
-In this example:
-
-  * a reduced `Object`'s parser value has a type of `Value`.
-  * a reduced `Values`'s parser value has a type of `Value[]`.
-  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
-
-##> Specifying a parser rule - the rule statement
-
-Rule statements create parser rules which define the grammar that will be
-parsed by the generated parser.
-
-Multiple rules with the same name can be specified.
-Rules with the same name define a rule set for that name and act as
-alternatives that the parser can accept when attempting to match a reference to
-that rule.
-
-The grammar file must define a rule with the name `Start` which will be used as
-the top-level starting rule that the parser attempts to reduce.
-
-Example:
-
-```
-ptype ulong;
-token word /[a-z]+/ << $$ = match.length; >>
-Start -> word << $$ = $1; >>
-```
-
-In the above example the `Start` rule is defined to match a single `word`
-token.
-
-Example:
-
-```
-Start -> E1 << $$ = $1; >>
-E1 -> E2 << $$ = $1; >>
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E2 -> E3 << $$ = $1; >>
-E2 -> E2 times E3 << $$ = $1 * $3; >>
-E3 -> E4 << $$ = $1; >>
-E3 -> E3 power E4 << $$ = pow($1, $3); >>
-E4 -> integer << $$ = $1; >>
-E4 -> lparen E1 rparen << $$ = $2; >>
-```
-
-A parser rule has zero or more terms on the right side of its definition.
-Each of these terms is either a token name or a rule name.
-
-In a parser rule code block, parser values for the right side terms are
-accessible as `$1` for the first term's parser value, `$2` for the second
-term's parser value, etc...
-The `$$` symbol accesses the output parser value for this rule.
-The above examples demonstrate how the parser values for the rule components
-can be used to produce the parser value for the accepted rule.
-
-##> Specifying the parser module name - the `module` statement
-
-The `module` statement can be used to specify the module name for a generated
-D module.
-
-```
-module proj.parser;
-```
-
-If a module statement is not present, then the generated D module will not
-contain a module statement and the default module name will be used.
-
-##> Specifying the generated API prefix - the `prefix` statement
-
-By default the public API (types, constants, and functions) of the generated
-lexer and parser uses a prefix of `p_`.
-
-This prefix can be changed with the `prefix` statement.
-
-Example:
-
-```
-prefix myparser_;
-```
-
-With a parser generated with this `prefix` statement, instead of calling
-`p_context_init()` you would call `myparser_context_init()`.
-
-The `prefix` statement can be optionally used if you would like to change the
-prefix used by your generated lexer and parser to something other than the
-default.
-
-It can also be used when generating multiple lexers/parsers to be used in the
-same program to avoid symbol collisions.
-
-##> User termination of the lexer or parser
-
-Propane supports allowing lexer or parser user code blocks to terminate
-execution of the parser.
-Some example uses of this functionality could be to:
-
-  * Detect integer overflow when lexing an integer literal constant.
-  * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
-  * Determine whether parsing should stop and instead be performed using a different parser version.
-
-To terminate parsing from a lexer or parser user code block, use the
-`$terminate(code)` function, passing an integer expression argument.
-For example:
-
-```
-NewExpression -> new Expression << $terminate(42); >>
-```
-
-The value passed to the `$terminate()` function is known as the "user terminate
-code".
-If the parser returns a `P_USER_TERMINATED` result code, then the user
-terminate code can be accessed using the `p_user_terminate_code()` API
-function.
-
-#> Propane generated API
-
-By default, Propane uses a prefix of `p_` when generating a lexer/parser.
-This prefix is used for all publicly declared types and functions.
-The uppercase version of the prefix is used for all constant values.
-
-This section documents the generated API using the default `p_` or `P_` names.
-
-##> Constants
-
-Propane generates the following result code constants:
-
-* `P_SUCCESS`: A successful decode/lex/parse operation has taken place.
-* `P_DECODE_ERROR`: An error occurred when decoding UTF-8 input.
-* `P_UNEXPECTED_INPUT`: Input was received by the lexer that does not match any lexer pattern.
-* `P_UNEXPECTED_TOKEN`: A token was seen in a location that does not match any parser rule.
-* `P_DROP`: The lexer matched a drop pattern.
-* `P_EOF`: The lexer reached the end of the input string.
-* `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
-
-Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
-
-##> Types
-
-### `p_context_t`
-
-Propane defines a `p_context_t` structure type.
-The structure is intended to be used opaquely and stores information related to
-the state of the lexer and parser.
-Integrating code must define an instance of the `p_context_t` structure.
-A pointer to this instance is passed to the generated functions.
-
-### `p_position_t`
-
-The `p_position_t` structure contains two fields `row` and `col`.
-These fields contain the 0-based row and column describing a parser position.
-
-##> Functions
-
-### `p_context_init`
-
-The `p_context_init()` function must be called to initialize the context
-structure.
-The input to be used for lexing/parsing is passed in when initializing the
-context structure.
-
-C example:
-
-```
-p_context_t context;
-p_context_init(&context, input, input_length);
-```
-
-D example:
-
-```
-p_context_t context;
-p_context_init(&context, input);
-```
-
-### `p_parse`
-
-The `p_parse()` function is the main entry point to the parser.
-It must be passed a pointer to an initialized context structure.
-
-Example:
-
-```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-```
-
-### `p_result`
-
-The `p_result()` function can be used to retrieve the final parse value after
-`p_parse()` returns a `P_SUCCESS` value.
-
-Example:
-
-```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_SUCCESS)
-{
-    result = p_result(&context);
-}
-```
-
-### `p_position`
-
-The `p_position()` function can be used to retrieve the parser position where
-an error occurred.
-
-Example:
-
-```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
-{
-    p_position_t error_position = p_position(&context);
-    fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
-}
-```
-
-### `p_user_terminate_code`
-
-The `p_user_terminate_code()` function can be used to retrieve the user
-terminate code after `p_parse()` returns a `P_USER_TERMINATED` value.
-User terminate codes are arbitrary values that can be defined by the user to
-be returned when the user requests to terminate parsing.
-They have no particular meaning to Propane.
-
-Example:
-
-```
-if (p_parse(&context) == P_USER_TERMINATED)
-{
-    size_t user_terminate_code = p_user_terminate_code(&context);
-}
-```
-
-### `p_token`
-
-The `p_token()` function can be used to retrieve the current parse token.
-This is useful after `p_parse()` returns a `P_UNEXPECTED_TOKEN` value.
-terminate code after `p_parse()` returns a `P_USER_TERMINATED` value to
-indicate what token the parser was not expecting.
-
-Example:
-
-```
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
-{
-    p_token_t unexpected_token = p_token(&context);
-}
-```
-
-##> Data
-
-### `p_token_names`
-
-The `p_token_names` array contains the grammar-specified token names.
-It is indexed by the token ID.
-
-C example:
-
-```
-p_context_t context;
-p_context_init(&context, input, input_length);
-size_t result = p_parse(&context);
-if (p_parse(&context) == P_UNEXPECTED_TOKEN)
-{
-    p_position_t error_position = p_position(&context);
-    fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
-        p_token_names[context.token],
-        error_position.row + 1, error_position.col + 1);
-}
 ```
 
 #> License

lib/propane.rb

@@ -1,7 +1,6 @@
 require "erb"
 require "set"
 require "stringio"
-require_relative "propane/assets"
 require_relative "propane/cli"
 require_relative "propane/code_point_range"
 require_relative "propane/fa"

lib/propane/assets.rb

@@ -1,10 +0,0 @@
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
-        File.binread(path)
-      end
-    end
-  end
-end

lib/propane/generator.rb

@@ -10,6 +10,7 @@ class Propane
       else
         @log = StringIO.new
       end
+      @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
       @language =
         if output_file =~ /\.([a-z]+)$/
           $1
@@ -25,8 +26,7 @@ class Propane
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension}.erb")
+        erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.#{extension}.erb")), trim_mode: "<>")
-        erb = ERB.new(template, trim_mode: "<>")
         output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         result = erb.result(binding.clone)
         File.open(output_file, "wb") do |fh|
@@ -198,25 +198,10 @@ class Propane
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
       end
-      code = code.gsub(/\$terminate\((.*)\);/) do |match|
-        user_terminate_code = $1
-        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
-        case @language
-        when "c"
-          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        when "d"
-          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        end
-      end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "_pvalue->v_#{rule.ptypename}"
-          when "d"
           "_pvalue.v_#{rule.ptypename}"
         end
-        end
         code = code.gsub(/\$(\d+)/) do |match|
           index = $1.to_i
           case @language

lib/propane/grammar.rb

@@ -5,6 +5,7 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :classname
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
@@ -17,7 +18,7 @@ class Propane
       @patterns = []
       @tokens = []
       @rules = []
-      @code_blocks = {}
+      @code_blocks = []
       @line_number = 1
       @next_line_number = @line_number
       @mode = nil
@@ -35,10 +36,6 @@ class Propane
       @tokens.size
     end
 
-    def terminate_token_id
-      @tokens.size + 1
-    end
-
     private
 
     def parse_grammar!
@@ -52,6 +49,7 @@ class Propane
       elsif parse_comment_line!
       elsif @mode.nil? && parse_mode_label!
       elsif parse_module_statement!
+      elsif parse_class_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
       elsif parse_token_statement!
@@ -92,6 +90,16 @@ class Propane
       end
     end
 
+    def parse_class_statement!
+      if consume!(/class\s+/)
+        md = consume!(/([\w.]+)\s*/, "expected class name")
+        @classname = md[1]
+        consume!(/;/, "expected `;'")
+        @mode = nil
+        true
+      end
+    end
+
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
         name = "default"
@@ -183,15 +191,8 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
+      if code = parse_code_block!
-        name, code = md[1..2]
+        @code_blocks << code
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
-        if @code_blocks[name]
-          @code_blocks[name] += code
-        else
-          @code_blocks[name] = code
-        end
         @mode = nil
         true
       end
@@ -224,11 +225,8 @@ class Propane
     end
 
     def parse_code_block!
-      if md = consume!(/<<(.*?)>>\n/m)
+      if md = consume!(/<<\n(.*?)^>>\n/m)
-        code = md[1]
+        md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
-        code
       end
     end
 

lib/propane/parser.rb

@@ -134,7 +134,7 @@ class Propane
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
-      item_sets = Set[item_set] + item_set.leading_item_sets
+      item_sets = item_set.leading_item_sets
       reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|

lib/propane/parser/item_set.rb

@@ -87,25 +87,15 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
+      # This set includes this ItemSet.
+      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        result = Set.new
+        @in_sets.reduce(Set[self]) do |result, item_set|
-        eval_sets = Set[self]
+          result + item_set.leading_item_sets
-        evaled = Set.new
-        while eval_sets.size > 0
-          eval_set = eval_sets.first
-          eval_sets.delete(eval_set)
-          evaled << eval_set
-          eval_set.in_sets.each do |in_set|
-            result << in_set
-            unless evaled.include?(in_set)
-              eval_sets << in_set
         end
       end
-        end
-        result
-      end
 
       # Represent the ItemSet as a String.
       #

lib/propane/regex.rb

@@ -134,18 +134,8 @@ class Propane
       else
         c = @pattern.slice!(0)
         case c
-        when "a"
-          CharacterRangeUnit.new("\a", "\a")
-        when "b"
-          CharacterRangeUnit.new("\b", "\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
-        when "f"
-          CharacterRangeUnit.new("\f", "\f")
-        when "n"
-          CharacterRangeUnit.new("\n", "\n")
-        when "r"
-          CharacterRangeUnit.new("\r", "\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -155,10 +145,6 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
-        when "t"
-          CharacterRangeUnit.new("\t", "\t")
-        when "v"
-          CharacterRangeUnit.new("\v", "\v")
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.2.0"
+  VERSION = "0.1.0"
 end

propane.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec bundle exec ruby -Ilib bin/propane "$@"

rb/build_dist.rb

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require "erb"
 require "fileutils"
 require "digest/md5"
 
@@ -14,24 +13,6 @@ START_FILE = "bin/#{PROG_NAME}"
 LIB_DIR = "lib"
 DIST = "dist"
 
-ASSETS_TEMPLATE = <<EOF
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        case name
-<% Dir.glob("assets/*").each  do |asset_file| %>
-        when <%= File.basename(asset_file).inspect %>
-          <%= File.binread(asset_file).inspect %>
-<% end %>
-        end
-      end
-    end
-  end
-end
-EOF
-
-assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
 files_processed = {}
 combined_file = []
 
@@ -44,12 +25,8 @@ combine_files = lambda do |file|
         if File.exist?(path)
           unless files_processed[path]
             files_processed[path] = true
-            if require_name == "propane/assets"
-              combined_file << assets_module
-            else
             combine_files[path]
           end
-          end
         else
           raise "require path #{path.inspect} not found"
         end

spec/json_parser.c.propane

@@ -1,183 +0,0 @@
-<<header
-#include "json_types.h"
-#include "testutils.h"
->>
-<<
-#include "math.h"
-#include <stdbool.h>
-static str_t string_value;
->>
-
-ptype JSONValue *;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n = 0.0;
-    bool negative = false;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative = false;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = JSONValue_new(JSON_NUMBER);
-    $$->number = n;
->>
-token true <<
-  $$ = JSONValue_new(JSON_TRUE);
->>
-token false <<
-  $$ = JSONValue_new(JSON_FALSE);
->>
-token null <<
-  $$ = JSONValue_new(JSON_NULL);
->>
-/"/ <<
-  $mode(string);
-  str_init(&string_value, "");
->>
-string: token string /"/ <<
-  $$ = JSONValue_new(JSON_STRING);
-  $$->string = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  str_append(&string_value, "\"");
->>
-string: /\\\\/ <<
-  str_append(&string_value, "\\");
->>
-string: /\\\// <<
-  str_append(&string_value, "/");
->>
-string: /\\b/ <<
-  str_append(&string_value, "\b");
->>
-string: /\\f/ <<
-  str_append(&string_value, "\f");
->>
-string: /\\n/ <<
-  str_append(&string_value, "\n");
->>
-string: /\\r/ <<
-  str_append(&string_value, "\r");
->>
-string: /\\t/ <<
-  str_append(&string_value, "\t");
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
-  str_append(&string_value, s);
->>
-string: /[^\\]/ <<
-  char s[] = {match[0], 0};
-  str_append(&string_value, s);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONObject_new();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
-  $$ = $1;
->>
-KeyValue -> string colon Value <<
-  $$ = JSONObject_new();
-  JSONObject_append($$, str_cstr(&$1->string), $3);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONArray_new();
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  JSONArray_append($1, $3);
-  $$ = $1;
->>

spec/json_types.c

@@ -1,64 +0,0 @@
-#include "json_types.h"
-#include <string.h>
-#include <stdlib.h>
-#include "testutils.h"
-
-JSONValue * JSONValue_new(size_t id)
-{
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
-    jv->id = id;
-    return jv;
-}
-
-JSONValue * JSONObject_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_OBJECT);
-    jv->object.size = 0u;
-    return jv;
-}
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
-{
-    size_t const size = object->object.size;
-    for (size_t i = 0u; i < size; i++)
-    {
-        if (strcmp(name, object->object.entries[i].name) == 0)
-        {
-            object->object.entries[i].value = value;
-            return;
-        }
-    }
-    size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
-    if (size > 0)
-    {
-        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
-        free(object->object.entries);
-    }
-    object->object.entries = new_entries;
-    object->object.entries[size].name = name;
-    object->object.entries[size].value = value;
-    object->object.size = new_size;
-}
-
-JSONValue * JSONArray_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_ARRAY);
-    jv->array.size = 0u;
-    return jv;
-}
-
-void JSONArray_append(JSONValue * array, JSONValue * value)
-{
-    size_t const size = array->array.size;
-    size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
-    if (array->array.size > 0)
-    {
-        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
-        free(array->array.entries);
-    }
-    array->array.entries = new_entries;
-    array->array.entries[size] = value;
-    array->array.size = new_size;
-}

spec/json_types.h

@@ -1,46 +0,0 @@
-#pragma once
-
-#include <stddef.h>
-#include "testutils.h"
-
-#define JSON_OBJECT 0u
-#define JSON_ARRAY 1u
-#define JSON_NUMBER 2u
-#define JSON_STRING 3u
-#define JSON_TRUE 4u
-#define JSON_FALSE 5u
-#define JSON_NULL 6u
-
-typedef struct JSONValue_s
-{
-    size_t id;
-    union
-    {
-        struct
-        {
-            size_t size;
-            struct
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
-        } object;
-        struct
-        {
-            size_t size;
-            struct JSONValue_s ** entries;
-        } array;
-        double number;
-        str_t string;
-    };
-} JSONValue;
-
-JSONValue * JSONValue_new(size_t id);
-
-JSONValue * JSONObject_new(void);
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
-
-JSONValue * JSONArray_new(void);
-
-void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane/grammar_spec.rb

@@ -5,6 +5,7 @@ class Propane
 # Comment line
 
 module a.b;
+class Foobar;
 ptype   XYZ  *  ;
 
 token while;
@@ -29,6 +30,7 @@ B -> <<
 >>
 EOF
       grammar = Grammar.new(input)
+      expect(grammar.classname).to eq "Foobar"
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -36,44 +38,44 @@ EOF
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 6
+      expect(o.line_number).to eq 7
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "while"
-      expect(o.line_number).to eq 6
+      expect(o.line_number).to eq 7
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "id"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 9
+      expect(o.line_number).to eq 10
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
-      expect(o.line_number).to eq 9
+      expect(o.line_number).to eq 10
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "token_with_code"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 11
+      expect(o.line_number).to eq 12
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
-      expect(o.line_number).to eq 11
+      expect(o.line_number).to eq 12
       expect(o.code).to eq "Code for the token\n"
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 15
+      expect(o.line_number).to eq 16
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 17
+      expect(o.line_number).to eq 18
       expect(o.token).to be_nil
       expect(o.code).to be_nil
 
@@ -82,19 +84,19 @@ EOF
       o = grammar.rules[0]
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
-      expect(o.line_number).to eq 19
+      expect(o.line_number).to eq 20
       expect(o.code).to eq "  a = 42;\n"
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
       expect(o.components).to eq %w[C while id]
-      expect(o.line_number).to eq 22
+      expect(o.line_number).to eq 23
       expect(o.code).to be_nil
 
       o = grammar.rules[2]
       expect(o.name).to eq "B"
       expect(o.components).to eq []
-      expect(o.line_number).to eq 23
+      expect(o.line_number).to eq 24
       expect(o.code).to eq "  b = 0;\n"
     end
 

spec/propane_spec.rb

@@ -4,52 +4,14 @@ require "open3"
 Results = Struct.new(:stdout, :stderr, :status)
 
 describe Propane do
-  before(:all) do
-    @statics = {}
-  end
-
   def write_grammar(grammar, options = {})
     options[:name] ||= ""
     File.write("spec/run/testparser#{options[:name]}.propane", grammar)
   end
 
   def build_parser(options = {})
-    @statics[:build_test_id] ||= 0
+    options[:name] ||= ""
-    @statics[:build_test_id] += 1
+    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
-    if ENV["dist_specs"]
-      command = %W[dist/propane]
-    else
-      command = %W[ruby -I spec/run -r _simplecov_setup -I lib bin/propane]
-      command_prefix =
-        if ENV["partial_specs"]
-          "p"
-        else
-          "b"
-        end
-      command_name = "#{command_prefix}#{@statics[:build_test_id]}"
-      File.open("spec/run/_simplecov_setup.rb", "w") do |fh|
-        fh.puts <<EOF
-require "bundler"
-Bundler.setup
-require "simplecov"
-class MyFormatter
-  def format(*args)
-  end
-end
-SimpleCov.start do
-  command_name(#{command_name.inspect})
-  filters.clear
-  add_filter do |src|
-    !(src.filename[SimpleCov.root])
-  end
-  formatter(MyFormatter)
-end
-# force color off
-ENV["TERM"] = nil
-EOF
-      end
-    end
-    command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -123,8 +85,10 @@ token plus /\\+/;
 token times /\\*/;
 drop /\\s+/;
 Start -> Foo;
-Foo -> int <<>>
+Foo -> int <<
-Foo -> plus <<>>
+>>
+Foo -> plus <<
+>>
 EOF
         build_parser(language: language)
         compile("spec/test_lexer.#{language}", language: language)
@@ -147,7 +111,9 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int << $$ = $1; >>
+Start -> int <<
+  $$ = $1;
+>>
 EOF
         when "d"
           write_grammar <<EOF
@@ -161,7 +127,9 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int << $$ = $1; >>
+Start -> int <<
+  $$ = $1;
+>>
 EOF
         end
         build_parser(language: language)
@@ -213,15 +181,33 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 << $$ = $1; >>
+Start -> E1 <<
-E1 -> E2 << $$ = $1; >>
+  $$ = $1;
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
+>>
-E2 -> E3 << $$ = $1; >>
+E1 -> E2 <<
-E2 -> E2 times E3 << $$ = $1 * $3; >>
+  $$ = $1;
-E3 -> E4 << $$ = $1; >>
+>>
-E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
+E1 -> E1 plus E2 <<
-E4 -> integer << $$ = $1; >>
+  $$ = $1 + $3;
-E4 -> lparen E1 rparen << $$ = $2; >>
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = (size_t)pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
 EOF
         when "d"
           write_grammar <<EOF
@@ -247,15 +233,33 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 << $$ = $1; >>
+Start -> E1 <<
-E1 -> E2 << $$ = $1; >>
+  $$ = $1;
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
+>>
-E2 -> E3 << $$ = $1; >>
+E1 -> E2 <<
-E2 -> E2 times E3 << $$ = $1 * $3; >>
+  $$ = $1;
-E3 -> E4 << $$ = $1; >>
+>>
-E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E1 -> E1 plus E2 <<
-E4 -> integer << $$ = $1; >>
+  $$ = $1 + $3;
-E4 -> lparen E1 rparen << $$ = $2; >>
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
 EOF
         end
         build_parser(language: language)
@@ -366,7 +370,9 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ << writeln("def!"); >>
+/def/ <<
+  writeln("def!");
+>>
 Start -> abc;
 EOF
         end
@@ -391,7 +397,9 @@ EOF
 #include <stdio.h>
 >>
 token abc;
-/def/ << printf("def!\\n"); >>
+/def/ <<
+  printf("def!\\n");
+>>
 /ghi/ <<
   printf("ghi!\\n");
   return $token(abc);
@@ -404,7 +412,9 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ << writeln("def!"); >>
+/def/ <<
+  writeln("def!");
+>>
 /ghi/ <<
   writeln("ghi!");
   return $token(abc);
@@ -493,9 +503,15 @@ EOF
 >>
 token a;
 token b;
-Start -> A B << printf("Start!\\n"); >>
+Start -> A B <<
-A -> a << printf("A!\\n"); >>
+  printf("Start!\\n");
-B -> b << printf("B!\\n"); >>
+>>
+A -> a <<
+  printf("A!\\n");
+>>
+B -> b <<
+  printf("B!\\n");
+>>
 EOF
         when "d"
           write_grammar <<EOF
@@ -504,9 +520,15 @@ import std.stdio;
 >>
 token a;
 token b;
-Start -> A B << writeln("Start!"); >>
+Start -> A B <<
-A -> a << writeln("A!"); >>
+  writeln("Start!");
-B -> b << writeln("B!"); >>
+>>
+A -> a <<
+  writeln("A!");
+>>
+B -> b <<
+  writeln("B!");
+>>
 EOF
         end
         build_parser(language: language)
@@ -524,9 +546,15 @@ EOF
         write_grammar <<EOF
 ptype #{language == "c" ? "uint32_t" : "uint"};
 token a;
-Start -> As << $$ = $1; >>
+Start -> As <<
-As -> << $$ = 0u; >>
+  $$ = $1;
-As -> As a << $$ = $1 + 1u; >>
+>>
+As -> <<
+  $$ = 0u;
+>>
+As -> As a <<
+  $$ = $1 + 1u;
+>>
 EOF
         build_parser(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
@@ -637,7 +665,7 @@ EOF
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        write_grammar(File.read("spec/json_parser.propane"))
         build_parser(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end
@@ -663,110 +691,6 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
-
-      it "allows the user to terminate the lexer" do
-        write_grammar <<EOF
-token a;
-token b <<
-  $terminate(8675309);
->>
-token c;
-Start -> Any;
-Any -> a;
-Any -> b;
-Any -> c;
-EOF
-        build_parser(language: language)
-        compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
-      it "allows the user to terminate the parser" do
-        write_grammar <<EOF
-token a;
-token b;
-token c;
-Start -> Any;
-Any -> a Any;
-Any -> b Any << $terminate(4200); >>
-Any -> c Any;
-Any -> ;
-EOF
-        build_parser(language: language)
-        compile("spec/test_user_terminate.#{language}", language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
-      it "matches backslash escape sequences" do
-        case language
-        when "c"
-          write_grammar <<EOF
-<<
-  #include <stdio.h>
->>
-tokenid t;
-/\\a/ << printf("A\\n"); >>
-/\\b/ << printf("B\\n"); >>
-/\\t/ << printf("T\\n"); >>
-/\\n/ << printf("N\\n"); >>
-/\\v/ << printf("V\\n"); >>
-/\\f/ << printf("F\\n"); >>
-/\\r/ << printf("R\\n"); >>
-/t/ << return $token(t); >>
-Start -> t;
-EOF
-        when "d"
-          write_grammar <<EOF
-<<
-  import std.stdio;
->>
-tokenid t;
-/\\a/ << writeln("A"); >>
-/\\b/ << writeln("B"); >>
-/\\t/ << writeln("T"); >>
-/\\n/ << writeln("N"); >>
-/\\v/ << writeln("V"); >>
-/\\f/ << writeln("F"); >>
-/\\r/ << writeln("R"); >>
-/t/ <<
-  return $token(t);
->>
-Start -> t;
-EOF
-        end
-        build_parser(language: language)
-        compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-        verify_lines(results.stdout, [
-          "A",
-          "B",
-          "T",
-          "N",
-          "V",
-          "F",
-          "R",
-        ])
-      end
-
-      it "handles when an item set leads to itself" do
-        write_grammar <<EOF
-token one;
-token two;
-
-Start -> Opt one Start;
-Start -> ;
-
-Opt -> two;
-Opt -> ;
-EOF
-        build_parser(language: language)
-      end
     end
   end
 end

spec/spec_helper.rb

@@ -1,18 +1,5 @@
-unless ENV["dist_specs"]
 require "bundler/setup"
-  require "simplecov"
+require "propane"
-
-  SimpleCov.start do
-    add_filter "/spec/"
-    add_filter "/.bundle/"
-    if ENV["partial_specs"]
-      command_name "RSpec-partial"
-    else
-      command_name "RSpec"
-    end
-    project_name "Propane"
-    merge_timeout 3600
-  end
 
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
@@ -22,6 +9,3 @@ unless ENV["dist_specs"]
     c.syntax = :expect
   end
 end
-end
-
-require "propane"

spec/test_error_positions.c

@@ -14,14 +14,14 @@ int main()
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 2);
     assert(p_position(&context).col == 3);
-    assert(p_token(&context) == TOKEN_a);
+    assert(context.token == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 0);
     assert(p_position(&context).col == 0);
-    assert(p_token(&context) == TOKEN_num);
+    assert(context.token == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
@@ -35,8 +35,5 @@ int main()
     assert(p_position(&context).row == 5);
     assert(p_position(&context).col == 4);
 
-    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
-    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
-
     return 0;
 }

spec/test_error_positions.d

@@ -17,13 +17,13 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(2, 3));
-    assert(p_token(&context) == TOKEN_a);
+    assert(context.token == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(0, 0));
-    assert(p_token(&context) == TOKEN_num);
+    assert(context.token == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
@@ -34,7 +34,4 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
     assert(p_position(&context) == p_position_t(5, 4));
-
-    assert(p_token_names[TOKEN_a] == "a");
-    assert(p_token_names[TOKEN_num] == "num");
 }

spec/test_match_backslashes.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_match_backslashes.d

@@ -1,15 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parsing_json.c

@@ -1,56 +0,0 @@
-#include "testparser.h"
-#include "json_types.h"
-#include <string.h>
-#include <assert.h>
-
-int main()
-{
-    char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
-
-    input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
-
-    input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
-
-    input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
-
-    input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(1, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "hi") == 0);
-    assert(o->object.entries[0].value->id == JSON_TRUE);
-
-    input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(2, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "ff") == 0);
-    assert(o->object.entries[0].value->id == JSON_FALSE);
-    assert(strcmp(o->object.entries[1].name, "nn") == 0);
-    assert(o->object.entries[1].value->id == JSON_NULL);
-
-    return 0;
-}

spec/test_user_terminate.c

@@ -1,19 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "aacc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
-
-    return 0;
-}

spec/test_user_terminate.d

@@ -1,20 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "aacc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 4200);
-}

spec/test_user_terminate_lexer.c

@@ -1,19 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 8675309);
-
-    return 0;
-}

spec/test_user_terminate_lexer.d

@@ -1,20 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(&context) == 8675309);
-}

spec/testutils.c

@@ -1,9 +1,6 @@
 #include <stdio.h>
 #include <assert.h>
 #include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-#include "testutils.h"
 
 void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
 {
@@ -13,26 +10,3 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
         assert(false);
     }
 }
-
-void str_init(str_t * str, char const * cs)
-{
-    size_t length = strlen(cs);
-    str->cs = malloc(length + 1u);
-    strcpy(str->cs, cs);
-}
-
-void str_append(str_t * str, char const * cs)
-{
-    size_t length = strlen(str->cs);
-    size_t length2 = strlen(cs);
-    char * new_cs = malloc(length + length2 + 1u);
-    memcpy(new_cs, str->cs, length);
-    strcpy(&new_cs[length], cs);
-    free(str->cs);
-    str->cs = new_cs;
-}
-
-void str_free(str_t * str)
-{
-    free(str->cs);
-}

spec/testutils.h

@@ -5,15 +5,3 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 #define assert_eq(expected, actual) \
     assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
 
-typedef struct
-{
-    char * cs;
-} str_t;
-
-void str_init(str_t * str, char const * cs);
-void str_append(str_t * str, char const * cs);
-void str_free(str_t * str);
-static inline char * str_cstr(str_t * str)
-{
-    return str->cs;
-}