Bump version to 1.0.0

Gemfile

@@ -5,3 +5,4 @@ gem "rspec"
 gem "rdoc"
 gem "redcarpet"
 gem "syntax"
+gem "simplecov"

Gemfile.lock

@@ -2,6 +2,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
+    docile (1.4.0)
     psych (5.1.0)
       stringio
     rake (13.0.6)
@@ -21,6 +22,12 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
     stringio (3.0.7)
     syntax (1.2.2)
 
@@ -32,6 +39,7 @@ DEPENDENCIES
   rdoc
   redcarpet
   rspec
+  simplecov
   syntax
 
 BUNDLED WITH

README.md

@@ -1,21 +1,113 @@
 # The Propane Parser Generator
 
-Propane is an LR Parser Generator (LPG) which:
+Propane is a LALR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven parser to parse input in linear time
+  * generates a table-driven shift/reduce parser to parse input in linear time
+  * target C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
 ## Installation
 
-TODO
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
 
 ## Usage
 
-TODO: Write usage instructions here
+### Command Line Interface
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+### Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+Example grammar file:
+
+```
+<<
+import std.math;
+>>
+
+# Parser values are unsigned integers.
+ptype ulong;
+
+# A few basic arithmetic operators.
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+# Drop whitespace.
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+```
+
+Grammar files can contain comment lines beginning with `#` which are ignored.
+White space in the grammar file is also ignored.
+
+It is convention to use the extension `.propane` for the Propane grammar file,
+however any file name is accepted by Propane.
+
+See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
 
 ## Development
 

Rakefile

@@ -1,5 +1,8 @@
+require "rake/clean"
 require "rspec/core/rake_task"
 
+CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
+
 task :build_dist do
   sh "ruby rb/build_dist.rb"
 end
@@ -10,9 +13,20 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   end
 end
 
+# dspec task is useful to test the distributable release script, but is not
+# useful for coverage information.
+desc "Dist Specs"
+task :dspec, [:example_string] => :build_dist do |task, args|
+  ENV["dist_specs"] = "1"
+  Rake::Task["spec"].execute(args)
+  ENV.delete("dist_specs")
+end
+
 task :default => :spec
 
 desc "Build user guide"
 task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
+
+task :all => [:spec, :dspec, :user_guide]

assets/parser.c.erb

@@ -7,9 +7,7 @@
  * User code blocks
  *************************************************************************/
 
-<% @grammar.code_blocks.each do |code| %>
+<%= @grammar.code_blocks.fetch("", "") %>
-<%= code %>
-<% end %>
 
 /**************************************************************************
  * Private types

assets/parser.d.erb

@@ -12,9 +12,7 @@ module <%= @grammar.modulename %>;
  * User code blocks
  *************************************************************************/
 
-<% @grammar.code_blocks.each do |code| %>
+<%= @grammar.code_blocks.fetch("", "") %>
-<%= code %>
-<% end %>
 
 /**************************************************************************
  * Public types

assets/parser.h.erb

@@ -36,6 +36,9 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
+/** User header code blocks. */
+<%= @grammar.code_blocks.fetch("header", "") %>
+
 /** Parser values type(s). */
 typedef union
 {

doc/user_guide.md

@@ -7,15 +7,15 @@ ${/remove}
 
 #> Overview
 
-Propane is an LR Parser Generator (LPG) which:
+Propane is a LALR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven parser to parse input in linear time
+  * generates a table-driven shift/reduce parser to parse input in linear time
+  * target C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
-  * supports D language
 
 #> Installation
 
@@ -56,8 +56,10 @@ Example grammar file:
 import std.math;
 >>
 
+# Parser values are unsigned integers.
 ptype ulong;
 
+# A few basic arithmetic operators.
 token plus /\\+/;
 token times /\\*/;
 token power /\\*\\*/;
@@ -72,6 +74,7 @@ token integer /\\d+/ <<
 >>
 token lparen /\\(/;
 token rparen /\\)/;
+# Drop whitespace.
 drop /\\s+/;
 
 Start -> E1 <<
@@ -103,6 +106,474 @@ E4 -> lparen E1 rparen <<
 >>
 ```
 
+Grammar files can contain comment lines beginning with `#` which are ignored.
+White space in the grammar file is also ignored.
+
+It is convention to use the extension `.propane` for the Propane grammar file,
+however any file name is accepted by Propane.
+
+This user guide follows the convention of beginning a token name with a
+lowercase character and beginning a rule name with an uppercase character.
+
+##> User Code Blocks
+
+User code blocks begin with the line following a "<<" token and end with the
+line preceding a grammar line consisting of solely the ">>" token.
+All text lines in the code block are copied verbatim into the output file.
+
+### Standalone Code Blocks
+
+C example:
+
+```
+<<
+#include &lt;stdio.h>
+>>
+```
+
+D example:
+
+```
+<<
+import std.stdio;
+>>
+```
+
+Standalone code blocks are emitted early in the output file as top-level code
+outside the context of any function.
+Standalone code blocks are a good place to include/import any other necessary
+supporting code modules.
+They can also define helper functions that can be reused by lexer or parser
+user code blocks.
+They are emitted in the order they are defined in the grammar file.
+
+For a C target, the word "header" may immediately follow the "<<" token to
+cause Propane to emit the code block in the generated header file rather than
+the generated implementation file.
+This allows including another header that may be necessary to define any types
+needed by a `ptype` directive, for example:
+
+```
+<&lt;header
+#include "mytypes.h"
+>>
+```
+
+### Lexer pattern code blocks
+
+Example:
+
+```
+ptype ulong;
+
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+```
+
+Lexer code blocks appear following a `token` or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+### Parser rule code blocks
+
+Example:
+
+```
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+```
+
+Parser rule code blocks appear following a rule expression.
+User code in a parser rule code block will be executed when the parser reduces
+the given rule.
+Assignment to the `$$` symbol will associate a parser value with the reduced
+rule.
+Parser values for the rules or tokens in the rule pattern can be accessed
+positionally with tokens `$1`, `$2`, `$3`, etc...
+
+##> Specifying tokens - the `token` statement
+
+The `token` statement allows defining a lexer token and a pattern to match that
+token.
+The name of the token must be specified immediately following the `token`
+keyword.
+A regular expression pattern may optionally follow the token name.
+If a regular expression pattern is not specified, the name of the token is
+taken to be the pattern.
+See also: ${#Regular expression syntax}.
+
+Example:
+
+```
+token for;
+```
+
+In this example, the token name is `for` and the pattern to match it is
+`/for/`.
+
+Example:
+
+```
+token lbrace /\{/;
+```
+
+In this example, the token name is `lbrace` and a single left curly brace will
+match it.
+
+The `token` statement can also include a user code block.
+The user code block will be executed whenever the token is matched by the
+lexer.
+
+Example:
+
+```
+token if <<
+  writeln("'if' keyword lexed");
+>>
+```
+
+The `token` statement is actually a shortcut statement for a combination of a
+`tokenid` statement and a pattern statement.
+To define a lexer token without an associated pattern to match it, use a
+`tokenid` statement.
+To define a lexer pattern that may or may not result in a matched token, use
+a pattern statement.
+
+##> Defining tokens without a matching pattern - the `tokenid` statement
+
+The `tokenid` statement can be used to define a token without associating it
+with a lexer pattern that matches it.
+
+Example:
+
+```
+tokenid string;
+```
+
+The `tokenid` statement can be useful when defining a token that may optionally
+be returned by user code associated with a pattern.
+
+It is also useful when lexer modes and multiple lexer patterns are required to
+build up a full token.
+A common example is parsing a string.
+See the ${#Lexer modes} chapter for more information.
+
+##> Specifying a lexer pattern - the pattern statement
+
+A pattern statement is used to define a lexer pattern that can execute user
+code but may not result in a matched token.
+
+Example:
+
+```
+/foo+/ <<
+  writeln("saw a foo pattern");
+>>
+```
+
+This can be especially useful with ${#Lexer modes}.
+
+See also ${#Regular expression syntax}.
+
+##> Ignoring input sections - the `drop` statement
+
+A `drop` statement can be used to specify a lexer pattern that when matched
+should result in the matched input being dropped and lexing continuing after
+the matched input.
+
+A common use for a `drop` statement would be to ignore whitespace sequences in
+the user input.
+
+Example:
+
+```
+drop /\s+/;
+```
+
+See also ${#Regular expression syntax}.
+
+##> Regular expression syntax
+
+A regular expression ("regex") is used to define lexer patterns in `token`,
+pattern, and `drop` statements.
+A regular expression begins and ends with a `/` character.
+
+Example:
+
+```
+/#.*$/
+```
+
+Regular expressions can include many special characters:
+
+  * The `.` character matches any input character other than a newline.
+  * The `*` character matches any number of the previous regex element.
+  * The `+` character matches one or more of the previous regex element.
+  * The `?` character matches 0 or 1 of the previous regex element.
+  * The `[` character begins a character class.
+  * The `(` character begins a matching group.
+  * The `{` character begins a count qualifier.
+  * The `\` character escapes the following character and changes its meaning:
+    * The `\d` sequence matches any character `0` through `9`.
+    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
+    `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * Any other character matches itself.
+  * The `|` character creates an alternate match.
+
+Any other character just matches itself in the input stream.
+
+A character class consists of a list of character alternates or character
+ranges that can be matched by the character class.
+For example `[a-zA-Z_]` matches any lowercase character between `a` and `z` or
+any uppercase character between `A` and `Z` or the underscore `_` character.
+Character classes can also be negative character classes if the first character
+after the `[` is a `^` character.
+In this case, the set of characters matched by the character class is the
+inverse of what it otherwise would have been.
+For example, `[^0-9]` matches any character other than 0 through 9.
+
+A matching group can be used to override the pattern sequence that multiplicity
+specifiers apply to.
+For example, the pattern `/foo+/` matches "foo" or "foooo", while the pattern
+`/(foo)+/` matches "foo" or "foofoofoo", but not "foooo".
+
+A count qualifier in curly braces can be used to restrict the number of matches
+of the preceding atom to an explicit minimum and maximum range.
+For example, the pattern `\d{3}` matches exactly 3 digits 0-9.
+Both a minimum and maximum multiplicity count can be specified and separated by
+a comma.
+For example, `/a{1,5}/` matches between 1 and 5 `a` characters.
+Either the minimum or maximum count can be omitted to omit the corresponding
+restriction in the number of matches allowed.
+
+An alternate match is created with the `|` character.
+For example, the pattern `/foo|bar/` matches either the sequence "foo" or the
+sequence "bar".
+
+##> Lexer modes
+
+Lexer modes can be used to change the set of patterns that are matched by the
+lexer.
+A common use for lexer modes is to match strings.
+
+Example:
+
+```
+<<
+string mystringvalue;
+>>
+
+tokenid str;
+
+# String processing
+/"/ <<
+  mystringvalue = "";
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  mystringvalue += match;
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(str);
+>>
+```
+
+A lexer mode is defined by placing the name before a colon (`:`) character that
+precedes a token or pattern statement.
+The token or pattern statement is restricted to only applying if the named mode
+is active.
+
+By default, the active lexer mode is named `default`.
+A `$mode()` call within a lexer code block can be used to change lexer modes.
+
+In the above example, when the lexer in the default mode sees a doublequote
+(`"`) character, the lexer code block will clear the `mystringvalue` variable
+and will set the lexer mode to `string`.
+When the lexer begins looking for patterns to match against the input, it will
+now look only for patterns tagged for the `string` lexer mode.
+Any non-`"` character will be appended to the `mystringvalue` string.
+A `"` character will end the `string` lexer mode and return to the `default`
+lexer mode.
+It also returns the `str` token now that the token is complete.
+
+Note that the token name `str` above could have been `string` instead - the
+namespace for token names is distinct from the namespace for lexer modes.
+
+##> Specifying parser value types - the `ptype` statement
+
+The `ptype` statement is used to define parser value type(s).
+Example:
+
+```
+ptype void *;
+```
+
+This defines the default parser value type to be `void *` (this is, in fact,
+the default parser value type if the grammar file does not specify otherwise).
+
+Each defined lexer token type and parser rule has an associated parser value
+type.
+When the lexer runs, each lexed token has a parser value associated with it.
+When the parser runs, each instance of a reduced rule has a parser value
+associated with it.
+Propane supports using different parser value types for different rules and
+token types.
+The example `ptype` statement above defines the default parser value type.
+A parser value type name can optionally be specified following the `ptype`
+keyword.
+For example:
+
+```
+ptype Value;
+ptype array = Value[];
+ptype dict = Value[string];
+
+Object -> lbrace rbrace <<
+  $$ = new Value();
+>>
+
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>
+
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+```
+
+In this example, the default parser value type is `Value`.
+A parser value type named `array` is defined to mean `Value[]`.
+A parser value type named `dict` is defined to mean `Value[string]`.
+Any defined tokens or rules that do not specify a parser value type will have
+the default parser value type associated with them.
+To associate a different parser value type with a token or rule, write the
+parser value type name in parentheses following the name of the token or rule.
+In this example:
+
+  * a reduced `Object`'s parser value has a type of `Value`.
+  * a reduced `Values`'s parser value has a type of `Value[]`.
+  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
+
+##> Specifying a parser rule - the rule statement
+
+Rule statements create parser rules which define the grammar that will be
+parsed by the generated parser.
+
+Multiple rules with the same name can be specified.
+Rules with the same name define a rule set for that name and act as
+alternatives that the parser can accept when attempting to match a reference to
+that rule.
+
+The grammar file must define a rule with the name `Start` which will be used as
+the top-level starting rule that the parser attempts to reduce.
+
+Example:
+
+```
+ptype ulong;
+token word /[a-z]+/ <<
+  $$ = match.length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+```
+
+In the above example the `Start` rule is defined to match a single `word`
+token.
+
+Example:
+
+```
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+```
+
+A parser rule has zero or more terms on the right side of its definition.
+Each of these terms is either a token name or a rule name.
+
+In a parser rule code block, parser values for the right side terms are
+accessible as `$1` for the first term's parser value, `$2` for the second
+term's parser value, etc...
+The `$$` symbol accesses the output parser value for this rule.
+The above examples demonstrate how the parser values for the rule components
+can be used to produce the parser value for the accepted rule.
+
+##> Specifying the parser module name - the `module` statement
+
+The `module` statement can be used to specify the module name for a generated
+D module.
+
+```
+module proj.parser;
+```
+
+If a module statement is not present, then the generated D module will not
+contain a module statement and the default module name will be used.
+
+##> Specifying the generated API prefix - the `prefix` statement
+
+By default the public API (types, constants, and functions) of the generated
+lexer and parser uses a prefix of `p_`.
+
+This prefix can be changed with the `prefix` statement.
+
+Example:
+
+```
+prefix myparser_;
+```
+
+With a parser generated with this `prefix` statement, instead of calling
+`p_context_init()` you would call `myparser_context_init()`.
+
+The `prefix` statement can be optionally used if you would like to change the
+prefix used by your generated lexer and parser to something other than the
+default.
+
+It can also be used when generating multiple lexers/parsers to be used in the
+same program to avoid symbol collisions.
+
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane.rb

@@ -1,6 +1,7 @@
 require "erb"
 require "set"
 require "stringio"
+require_relative "propane/assets"
 require_relative "propane/cli"
 require_relative "propane/code_point_range"
 require_relative "propane/fa"

lib/propane/assets.rb

@@ -0,0 +1,10 @@
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
+        File.binread(path)
+      end
+    end
+  end
+end

lib/propane/generator.rb

@@ -10,7 +10,6 @@ class Propane
       else
         @log = StringIO.new
       end
-      @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
       @language =
         if output_file =~ /\.([a-z]+)$/
           $1
@@ -26,7 +25,8 @@ class Propane
         extensions += %w[h]
       end
       extensions.each do |extension|
-        erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.#{extension}.erb")), trim_mode: "<>")
+        template = Assets.get("parser.#{extension}.erb")
+        erb = ERB.new(template, trim_mode: "<>")
         output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         result = erb.result(binding.clone)
         File.open(output_file, "wb") do |fh|

lib/propane/grammar.rb

@@ -5,7 +5,6 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
-    attr_reader :classname
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
@@ -18,7 +17,7 @@ class Propane
       @patterns = []
       @tokens = []
       @rules = []
-      @code_blocks = []
+      @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
       @mode = nil
@@ -49,7 +48,6 @@ class Propane
       elsif parse_comment_line!
       elsif @mode.nil? && parse_mode_label!
       elsif parse_module_statement!
-      elsif parse_class_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
       elsif parse_token_statement!
@@ -90,16 +88,6 @@ class Propane
       end
     end
 
-    def parse_class_statement!
-      if consume!(/class\s+/)
-        md = consume!(/([\w.]+)\s*/, "expected class name")
-        @classname = md[1]
-        consume!(/;/, "expected `;'")
-        @mode = nil
-        true
-      end
-    end
-
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
         name = "default"
@@ -191,8 +179,13 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if code = parse_code_block!
+      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
-        @code_blocks << code
+        name, code = md[1..2]
+        if @code_blocks[name]
+          @code_blocks[name] += code
+        else
+          @code_blocks[name] = code
+        end
         @mode = nil
         true
       end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

propane.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec bundle exec ruby -Ilib bin/propane "$@"

rb/build_dist.rb

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require "erb"
 require "fileutils"
 require "digest/md5"
 
@@ -13,6 +14,24 @@ START_FILE = "bin/#{PROG_NAME}"
 LIB_DIR = "lib"
 DIST = "dist"
 
+ASSETS_TEMPLATE = <<EOF
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        case name
+<% Dir.glob("assets/*").each  do |asset_file| %>
+        when <%= File.basename(asset_file).inspect %>
+          <%= File.binread(asset_file).inspect %>
+<% end %>
+        end
+      end
+    end
+  end
+end
+EOF
+
+assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
 files_processed = {}
 combined_file = []
 
@@ -25,8 +44,12 @@ combine_files = lambda do |file|
         if File.exist?(path)
           unless files_processed[path]
             files_processed[path] = true
+            if require_name == "propane/assets"
+              combined_file << assets_module
+            else
               combine_files[path]
             end
+          end
         else
           raise "require path #{path.inspect} not found"
         end

spec/json_parser.c.propane

@@ -0,0 +1,183 @@
+<<header
+#include "json_types.h"
+#include "testutils.h"
+>>
+<<
+#include "math.h"
+#include <stdbool.h>
+static str_t string_value;
+>>
+
+ptype JSONValue *;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n = 0.0;
+    bool negative = false;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative = false;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp = 0.0;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = JSONValue_new(JSON_NUMBER);
+    $$->number = n;
+>>
+token true <<
+  $$ = JSONValue_new(JSON_TRUE);
+>>
+token false <<
+  $$ = JSONValue_new(JSON_FALSE);
+>>
+token null <<
+  $$ = JSONValue_new(JSON_NULL);
+>>
+/"/ <<
+  $mode(string);
+  str_init(&string_value, "");
+>>
+string: token string /"/ <<
+  $$ = JSONValue_new(JSON_STRING);
+  $$->string = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  str_append(&string_value, "\"");
+>>
+string: /\\\\/ <<
+  str_append(&string_value, "\\");
+>>
+string: /\\\// <<
+  str_append(&string_value, "/");
+>>
+string: /\\b/ <<
+  str_append(&string_value, "\b");
+>>
+string: /\\f/ <<
+  str_append(&string_value, "\f");
+>>
+string: /\\n/ <<
+  str_append(&string_value, "\n");
+>>
+string: /\\r/ <<
+  str_append(&string_value, "\r");
+>>
+string: /\\t/ <<
+  str_append(&string_value, "\t");
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  str_append(&string_value, s);
+>>
+string: /[^\\]/ <<
+  char s[] = {match[0], 0};
+  str_append(&string_value, s);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONObject_new();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
+  $$ = $1;
+>>
+KeyValue -> string colon Value <<
+  $$ = JSONObject_new();
+  JSONObject_append($$, str_cstr(&$1->string), $3);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONArray_new();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  JSONArray_append($1, $3);
+  $$ = $1;
+>>

spec/json_types.c

@@ -0,0 +1,64 @@
+#include "json_types.h"
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+JSONValue * JSONValue_new(size_t id)
+{
+    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    jv->id = id;
+    return jv;
+}
+
+JSONValue * JSONObject_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_OBJECT);
+    jv->object.size = 0u;
+    return jv;
+}
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
+{
+    size_t const size = object->object.size;
+    for (size_t i = 0u; i < size; i++)
+    {
+        if (strcmp(name, object->object.entries[i].name) == 0)
+        {
+            object->object.entries[i].value = value;
+            return;
+        }
+    }
+    size_t const new_size = size + 1;
+    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    if (size > 0)
+    {
+        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
+        free(object->object.entries);
+    }
+    object->object.entries = new_entries;
+    object->object.entries[size].name = name;
+    object->object.entries[size].value = value;
+    object->object.size = new_size;
+}
+
+JSONValue * JSONArray_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_ARRAY);
+    jv->array.size = 0u;
+    return jv;
+}
+
+void JSONArray_append(JSONValue * array, JSONValue * value)
+{
+    size_t const size = array->array.size;
+    size_t const new_size = size + 1;
+    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    if (array->array.size > 0)
+    {
+        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
+        free(array->array.entries);
+    }
+    array->array.entries = new_entries;
+    array->array.entries[size] = value;
+    array->array.size = new_size;
+}

spec/json_types.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <stddef.h>
+#include "testutils.h"
+
+#define JSON_OBJECT 0u
+#define JSON_ARRAY 1u
+#define JSON_NUMBER 2u
+#define JSON_STRING 3u
+#define JSON_TRUE 4u
+#define JSON_FALSE 5u
+#define JSON_NULL 6u
+
+typedef struct JSONValue_s
+{
+    size_t id;
+    union
+    {
+        struct
+        {
+            size_t size;
+            struct
+            {
+                char const * name;
+                struct JSONValue_s * value;
+            } * entries;
+        } object;
+        struct
+        {
+            size_t size;
+            struct JSONValue_s ** entries;
+        } array;
+        double number;
+        str_t string;
+    };
+} JSONValue;
+
+JSONValue * JSONValue_new(size_t id);
+
+JSONValue * JSONObject_new(void);
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
+
+JSONValue * JSONArray_new(void);
+
+void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane/grammar_spec.rb

@@ -5,7 +5,6 @@ class Propane
 # Comment line
 
 module a.b;
-class Foobar;
 ptype   XYZ  *  ;
 
 token while;
@@ -30,7 +29,6 @@ B -> <<
 >>
 EOF
       grammar = Grammar.new(input)
-      expect(grammar.classname).to eq "Foobar"
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -38,44 +36,44 @@ EOF
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 7
+      expect(o.line_number).to eq 6
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "while"
-      expect(o.line_number).to eq 7
+      expect(o.line_number).to eq 6
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "id"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 10
+      expect(o.line_number).to eq 9
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
-      expect(o.line_number).to eq 10
+      expect(o.line_number).to eq 9
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "token_with_code"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 12
+      expect(o.line_number).to eq 11
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
-      expect(o.line_number).to eq 12
+      expect(o.line_number).to eq 11
       expect(o.code).to eq "Code for the token\n"
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 16
+      expect(o.line_number).to eq 15
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 18
+      expect(o.line_number).to eq 17
       expect(o.token).to be_nil
       expect(o.code).to be_nil
 
@@ -84,19 +82,19 @@ EOF
       o = grammar.rules[0]
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
-      expect(o.line_number).to eq 20
+      expect(o.line_number).to eq 19
       expect(o.code).to eq "  a = 42;\n"
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
       expect(o.components).to eq %w[C while id]
-      expect(o.line_number).to eq 23
+      expect(o.line_number).to eq 22
       expect(o.code).to be_nil
 
       o = grammar.rules[2]
       expect(o.name).to eq "B"
       expect(o.components).to eq []
-      expect(o.line_number).to eq 24
+      expect(o.line_number).to eq 23
       expect(o.code).to eq "  b = 0;\n"
     end
 

spec/propane_spec.rb

@@ -4,14 +4,52 @@ require "open3"
 Results = Struct.new(:stdout, :stderr, :status)
 
 describe Propane do
+  before(:all) do
+    @statics = {}
+  end
+
   def write_grammar(grammar, options = {})
     options[:name] ||= ""
     File.write("spec/run/testparser#{options[:name]}.propane", grammar)
   end
 
   def build_parser(options = {})
-    options[:name] ||= ""
+    @statics[:build_test_id] ||= 0
-    command = %W[./propane.sh spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+    @statics[:build_test_id] += 1
+    if ENV["dist_specs"]
+      command = %W[dist/propane]
+    else
+      command = %W[ruby -I spec/run -r _simplecov_setup -I lib bin/propane]
+      command_prefix =
+        if ENV["partial_specs"]
+          "p"
+        else
+          "b"
+        end
+      command_name = "#{command_prefix}#{@statics[:build_test_id]}"
+      File.open("spec/run/_simplecov_setup.rb", "w") do |fh|
+        fh.puts <<EOF
+require "bundler"
+Bundler.setup
+require "simplecov"
+class MyFormatter
+  def format(*args)
+  end
+end
+SimpleCov.start do
+  command_name(#{command_name.inspect})
+  filters.clear
+  add_filter do |src|
+    !(src.filename[SimpleCov.root])
+  end
+  formatter(MyFormatter)
+end
+# force color off
+ENV["TERM"] = nil
+EOF
+      end
+    end
+    command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -665,7 +703,7 @@ EOF
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.propane"))
+        write_grammar(File.read("spec/json_parser.#{language}.propane"))
         build_parser(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end

spec/spec_helper.rb

@@ -1,11 +1,27 @@
-require "bundler/setup"
+unless ENV["dist_specs"]
-require "propane"
+  require "bundler/setup"
+  require "simplecov"
 
-RSpec.configure do |config|
+  SimpleCov.start do
+    add_filter "/spec/"
+    add_filter "/.bundle/"
+    if ENV["partial_specs"]
+      command_name "RSpec-partial"
+    else
+      command_name "RSpec"
+    end
+    project_name "Propane"
+    merge_timeout 3600
+  end
+
+  RSpec.configure do |config|
     # Enable flags like --only-failures and --next-failure
     config.example_status_persistence_file_path = ".rspec_status"
 
     config.expect_with :rspec do |c|
       c.syntax = :expect
     end
+  end
 end
+
+require "propane"

spec/test_parsing_json.c

@@ -0,0 +1,56 @@
+#include "testparser.h"
+#include "json_types.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "{}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_OBJECT);
+
+    input = "[]";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_ARRAY);
+
+    input = "-45.6";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == -45.6);
+
+    input = "2E-2";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == 0.02);
+
+    input = "{\"hi\":true}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    JSONValue * o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(1, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "hi") == 0);
+    assert(o->object.entries[0].value->id == JSON_TRUE);
+
+    input = "{\"ff\": false, \"nn\": null}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(2, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "ff") == 0);
+    assert(o->object.entries[0].value->id == JSON_FALSE);
+    assert(strcmp(o->object.entries[1].name, "nn") == 0);
+    assert(o->object.entries[1].value->id == JSON_NULL);
+
+    return 0;
+}

spec/testutils.c

@@ -1,6 +1,9 @@
 #include <stdio.h>
 #include <assert.h>
 #include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include "testutils.h"
 
 void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
 {
@@ -10,3 +13,26 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
         assert(false);
     }
 }
+
+void str_init(str_t * str, char const * cs)
+{
+    size_t length = strlen(cs);
+    str->cs = malloc(length + 1u);
+    strcpy(str->cs, cs);
+}
+
+void str_append(str_t * str, char const * cs)
+{
+    size_t length = strlen(str->cs);
+    size_t length2 = strlen(cs);
+    char * new_cs = malloc(length + length2 + 1u);
+    memcpy(new_cs, str->cs, length);
+    strcpy(&new_cs[length], cs);
+    free(str->cs);
+    str->cs = new_cs;
+}
+
+void str_free(str_t * str)
+{
+    free(str->cs);
+}

spec/testutils.h

@@ -5,3 +5,15 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 #define assert_eq(expected, actual) \
     assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
 
+typedef struct
+{
+    char * cs;
+} str_t;
+
+void str_init(str_t * str, char const * cs);
+void str_append(str_t * str, char const * cs);
+void str_free(str_t * str);
+static inline char * str_cstr(str_t * str)
+{
+    return str->cs;
+}