v1.5.1

Lookahead tokens are only need if either:
(1) There is more than one rule that could be reduced in a given parser
state, or
(2) There are shift actions for a state and at least one rule that could
be reduced in the same state (to warn about shift/reduce conflicts).

.gitignore

@@ -2,7 +2,8 @@
 /.yardoc
 /_yardoc/
 /coverage/
-/doc/
+/dist/
+/gen/
 /pkg/
 /spec/reports/
 /tmp/

CHANGELOG.md

@@ -0,0 +1,62 @@
+## v1.5.1
+
+### Improvements
+
+- Improve performance (#28)
+
+## v1.5.0
+
+### New Features
+
+- Track start and end text positions for tokens and rules in AST node structures (#27)
+- Add warnings for shift/reduce conflicts to log file (#25)
+- Add -w command line switch to treat warnings as errors and output to stderr (#26)
+- Add rule field aliases (#24)
+
+### Improvements
+
+- Show line numbers of rules on conflict (#23)
+
+## v1.4.0
+
+### New Features
+
+- Allow user to specify AST node name prefix or suffix
+- Allow specifying the start rule name
+- Allow rule terms to be marked as optional
+
+### Improvements
+
+- Give a better error message when a referenced ptype has not been declared
+
+## v1.3.0
+
+### New Features
+
+- Add AST generation (#22)
+
+## v1.2.0
+
+### New Features
+
+- Allow one line user code blocks (#21)
+- Add backslash escape codes (#19)
+- Add API to access unexpected token found (#18)
+- Add token_names API (#17)
+- Add D example to user guide for p_context_init() (#16)
+- Allow user termination from lexer code blocks (#15)
+
+### Fixes
+
+- Fix generator hang when state transition cycle is present (#20)
+
+## v1.1.0
+
+### New Features
+
+- Add user parser terminations (#13)
+- Document generated parser API in user guide (#14)
+
+## v1.0.0
+
+- Initial release

Gemfile

@@ -2,3 +2,7 @@ source "https://rubygems.org"
 
 gem "rake"
 gem "rspec"
+gem "rdoc"
+gem "redcarpet"
+gem "syntax"
+gem "simplecov"

Gemfile.lock

@@ -2,27 +2,45 @@ GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.5.0)
+    docile (1.4.0)
+    psych (5.1.0)
+      stringio
     rake (13.0.6)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rdoc (6.5.0)
+      psych (>= 4.0.0)
+    redcarpet (3.6.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    stringio (3.0.7)
+    syntax (1.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   rake
+  rdoc
+  redcarpet
   rspec
+  simplecov
+  syntax
 
 BUNDLED WITH
-   2.4.0.dev
+   2.3.7

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2022 Josh Holtrop
+Copyright (c) 2010-2024 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,21 +1,104 @@
 # The Propane Parser Generator
 
-Propane is an LR Parser Generator (LPG) which:
+Propane is a LALR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven parser to parse input in linear time
+  * generates a table-driven shift/reduce parser to parse input in linear time
+  * targets C or D language outputs
+  * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
 ## Installation
 
-TODO
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
 
 ## Usage
 
-TODO: Write usage instructions here
+### Command Line Interface
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      -h, --help  Show this usage and exit.
+      --log LOG   Write log file. This will show all parser states and their
+                  associated shifts and reduces. It can be helpful when
+                  debugging a grammar.
+      --version   Show program version and exit.
+      -w          Treat warnings as errors. This option will treat shift/reduce
+                  conflicts as fatal errors and will print them to stderr in
+                  addition to the log file.
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+### Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+Example grammar file:
+
+```
+<<
+import std.math;
+>>
+
+# Parser values are unsigned integers.
+ptype ulong;
+
+# A few basic arithmetic operators.
+token plus /\+/;
+token times /\*/;
+token power /\*\*/;
+token integer /\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+token lparen /\(/;
+token rparen /\)/;
+# Drop whitespace.
+drop /\s+/;
+
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
+```
+
+Grammar files can contain comment lines beginning with `#` which are ignored.
+White space in the grammar file is also ignored.
+
+It is convention to use the extension `.propane` for the Propane grammar file,
+however any file name is accepted by Propane.
+
+See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
 
 ## Development
 

Rakefile

@@ -1,9 +1,32 @@
+require "rake/clean"
 require "rspec/core/rake_task"
 
+CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
+
+task :build_dist do
+  sh "ruby rb/build_dist.rb"
+end
+
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
   end
 end
 
+# dspec task is useful to test the distributable release script, but is not
+# useful for coverage information.
+desc "Dist Specs"
+task :dspec, [:example_string] => :build_dist do |task, args|
+  ENV["dist_specs"] = "1"
+  Rake::Task["spec"].execute(args)
+  ENV.delete("dist_specs")
+end
+
 task :default => :spec
+
+desc "Build user guide"
+task :user_guide do
+  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
+end
+
+task :all => [:spec, :dspec, :user_guide]

assets/parser.h.erb

@@ -0,0 +1,198 @@
+/**
+ * @file
+ *
+ * This file is generated by Propane.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+
+/**************************************************************************
+ * Public types
+ *************************************************************************/
+
+/* Result codes. */
+#define <%= @grammar.prefix.upcase %>SUCCESS 0u
+#define <%= @grammar.prefix.upcase %>DECODE_ERROR 1u
+#define <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT 2u
+#define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
+#define <%= @grammar.prefix.upcase %>DROP 4u
+#define <%= @grammar.prefix.upcase %>EOF 5u
+#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
+
+/** Token type. */
+typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
+
+/** Token IDs. */
+<% @grammar.tokens.each_with_index do |token, index| %>
+#define TOKEN_<%= token.code_name %> <%= index %>u
+<%   unless token.id == index %>
+<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
+<%   end %>
+<% end %>
+#define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
+
+/** Code point type. */
+typedef uint32_t <%= @grammar.prefix %>code_point_t;
+
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+typedef struct
+{
+    /** Input text row (0-based). */
+    uint32_t row;
+
+    /** Input text column (0-based). */
+    uint32_t col;
+} <%= @grammar.prefix %>position_t;
+
+/** Return whether the position is valid. */
+#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0xFFFFFFFFu)
+
+/** User header code blocks. */
+<%= @grammar.code_blocks.fetch("header", "") %>
+
+<% if @grammar.ast %>
+/** Parser values type. */
+typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
+<% else %>
+/** Parser values type(s). */
+typedef union
+{
+<%   @grammar.ptypes.each do |name, typestring| %>
+    <%= typestring %> v_<%= name %>;
+<%   end %>
+} <%= @grammar.prefix %>value_t;
+<% end %>
+
+<% if @grammar.ast %>
+/** AST node types. @{ */
+typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
+{
+    /* ASTNode fields must be present in the same order here. */
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+struct <%= name %>;
+<%   end %>
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+<%     rule_set.ast_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        struct <%= type %> * <%= field_name %>;
+<%       end %>
+    };
+<%     end %>
+} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>;
+
+<%   end %>
+/** @} */
+<% end %>
+
+/** Lexed token information. */
+typedef struct
+{
+    /** Text position of first code point in token. */
+    <%= @grammar.prefix %>position_t position;
+
+    /** Text position of last code point in token. */
+    <%= @grammar.prefix %>position_t end_position;
+
+    /** Number of input bytes used by the token. */
+    size_t length;
+
+    /** Token that was lexed. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** Parser value associated with the token. */
+    <%= @grammar.prefix %>value_t pvalue;
+} <%= @grammar.prefix %>token_info_t;
+
+/**
+ * Lexer and parser context.
+ *
+ * The user must allocate an instance of this structure and pass it to any
+ * public API function.
+ */
+typedef struct
+{
+    /* Lexer context data. */
+
+    /** Input text. */
+    uint8_t const * input;
+
+    /** Input text length. */
+    size_t input_length;
+
+    /** Input text index (byte offset). */
+    size_t input_index;
+
+    /** Input text position (row/column). */
+    <%= @grammar.prefix %>position_t text_position;
+
+    /** Current lexer mode. */
+    size_t mode;
+
+    /* Parser context data. */
+
+    /** Parse result value. */
+<% if @grammar.ast %>
+    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+<% else %>
+    <%= @grammar.prefix %>value_t parse_result;
+<% end %>
+
+    /** Unexpected token received. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** User terminate code. */
+    size_t user_terminate_code;
+} <%= @grammar.prefix %>context_t;
+
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+extern const char * <%= @grammar.prefix %>token_names[];
+
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+
+size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
+        <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
+
+size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
+
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
+
+<% if @grammar.ast %>
+<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% else %>
+<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% end %>
+
+<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
+
+size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

extra/vim/ftdetect/propane.vim

@@ -0,0 +1 @@
+au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -0,0 +1,33 @@
+" Vim syntax file for Propane
+" Language: propane
+" Maintainer: Josh Holtrop
+" URL: https://github.com/holtrop/propane
+
+if exists("b:current_syntax")
+  finish
+endif
+
+if !exists("b:propane_subtype")
+  let b:propane_subtype = "d"
+endif
+
+exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
+
+syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
+
+syn match propaneComment "#.*"
+syn match propaneOperator "->"
+syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
+syn match propaneFieldOperator ":" contained
+syn match propaneOperator "?"
+syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
+
+syn region propaneRegex start="/" end="/" skip="\\/"
+
+hi def link propaneComment Comment
+hi def link propaneKeyword Keyword
+hi def link propaneRegex String
+hi def link propaneOperator Operator
+hi def link propaneFieldOperator Operator
+hi def link propaneDelimiter Delimiter
+hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -1,130 +1,41 @@
 require "erb"
 require "set"
+require "stringio"
+require_relative "propane/assets"
 require_relative "propane/cli"
 require_relative "propane/code_point_range"
 require_relative "propane/fa"
 require_relative "propane/fa/state"
 require_relative "propane/fa/state/transition"
+require_relative "propane/generator"
+require_relative "propane/grammar"
 require_relative "propane/lexer"
 require_relative "propane/lexer/dfa"
 require_relative "propane/parser"
 require_relative "propane/parser/item"
 require_relative "propane/parser/item_set"
+require_relative "propane/pattern"
 require_relative "propane/regex"
 require_relative "propane/regex/nfa"
 require_relative "propane/regex/unit"
+require_relative "propane/rule_set"
 require_relative "propane/rule"
 require_relative "propane/token"
+require_relative "propane/util"
 require_relative "propane/version"
 
 class Propane
 
-  # EOF.
-  TOKEN_EOF = 0xFFFFFFFC
-
-  # Decoding error.
-  TOKEN_DECODE_ERROR = 0xFFFFFFFD
-
-  # Token ID for a "dropped" token.
-  TOKEN_DROP = 0xFFFFFFFE
-
-  # Invalid token ID.
-  TOKEN_NONE = 0xFFFFFFFF
-
   class Error < RuntimeError
   end
 
-  def initialize(input)
-    @tokens = {}
-    @rules = {}
-    input = input.gsub("\r\n", "\n")
-    while !input.empty?
-      parse_grammar(input)
-    end
-  end
-
-  def generate(output_file, log_file)
-    expand_rules
-    lexer = Lexer.new(@tokens)
-    parser = Parser.new(@tokens, @rules)
-    classname = @classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
-    erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../assets/parser.d.erb")), trim_mode: "<>")
-    result = erb.result(binding.clone)
-    File.open(output_file, "wb") do |fh|
-      fh.write(result)
-    end
-  end
-
-  private
-
-  def parse_grammar(input)
-    if input.slice!(/\A\s+/)
-      # Skip white space.
-    elsif input.slice!(/\A#.*\n/)
-      # Skip comment lines.
-    elsif input.slice!(/\Amodule\s+(\S+)\n/)
-      @modulename = $1
-    elsif input.slice!(/\Aclass\s+(\S+)\n/)
-      @classname = $1
-    elsif input.slice!(/\Atoken\s+(\S+)(?:\s+(\S+))?\n/)
-      name, pattern = $1, $2
-      if pattern.nil?
-        pattern = name
-      end
-      unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-        raise Error.new("Invalid token name #{name}")
-      end
-      if @tokens[name]
-        raise Error.new("Duplicate token name #{name}")
-      else
-        @tokens[name] = Token.new(name, pattern, @tokens.size)
-      end
-    elsif input.slice!(/\Adrop\s+(\S+)\n/)
-      pattern = $1
-      @tokens[name] = Token.new(nil, pattern, @tokens.size)
-    elsif input.slice!(/\A(\S+)\s*:\s*\[(.*?)\] <<\n(.*?)^>>\n/m)
-      rule_name, components, code = $1, $2, $3
-      components = components.strip.split(/\s+/)
-      @rules[rule_name] ||= Rule.new(rule_name, @rules.size)
-      @rules[rule_name].add_pattern(components, code)
-    else
-      if input.size > 25
-        input = input.slice(0..20) + "..."
-      end
-      raise Error.new("Unexpected grammar input: #{input}")
-    end
-  end
-
-  def expand_rules
-    @rules.each do |rule_name, rule|
-      if @tokens.include?(rule_name)
-        raise Error.new("Rule name collides with token name #{rule_name}")
-      end
-    end
-    unless @rules["Start"]
-      raise Error.new("Start rule not found")
-    end
-    @rules.each do |rule_name, rule|
-      rule.patterns.each do |rule|
-        rule.components.map! do |component|
-          if @tokens[component]
-            @tokens[component]
-          elsif @rules[component]
-            @rules[component]
-          else
-            raise Error.new("Symbol #{component} not found")
-          end
-        end
-      end
-    end
-  end
-
   class << self
 
-    def run(input_file, output_file, log_file)
+    def run(input_file, output_file, log_file, options)
       begin
-        propane = Propane.new(File.read(input_file))
-        propane.generate(output_file, log_file)
+        grammar = Grammar.new(File.read(input_file))
+        generator = Generator.new(grammar, output_file, log_file, options)
+        generator.generate
       rescue Error => e
         $stderr.puts e.message
         return 2

lib/propane/assets.rb

@@ -0,0 +1,10 @@
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
+        File.binread(path)
+      end
+    end
+  end
+end

lib/propane/cli.rb

@@ -4,15 +4,21 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  --log LOG   Write log file
-  --version   Show program version and exit
-  -h, --help  Show this usage and exit
+  -h, --help  Show this usage and exit.
+  --log LOG   Write log file. This will show all parser states and their
+              associated shifts and reduces. It can be helpful when
+              debugging a grammar.
+  --version   Show program version and exit.
+  -w          Treat warnings as errors. This option will treat shift/reduce
+              conflicts as fatal errors and will print them to stderr in
+              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
+        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -24,11 +30,13 @@ EOF
               log_file = args[i]
             end
           when "--version"
-            puts "propane v#{VERSION}"
+            puts "propane version #{VERSION}"
             return 0
           when "-h", "--help"
             puts USAGE
             return 0
+          when "-w"
+            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -45,7 +53,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file)
+        Propane.run(*params, log_file, options)
       end
 
     end

lib/propane/code_point_range.rb

@@ -1,7 +1,7 @@
 class Propane
   class CodePointRange
 
-    MAX_CODE_POINT = 0xFFFFFFFF
+    MAX_CODE_POINT = 0x7FFFFFFF
 
     attr_reader :first
     attr_reader :last

lib/propane/generator.rb

@@ -0,0 +1,372 @@
+class Propane
+
+  class Generator
+
+    def initialize(grammar, output_file, log_file, options)
+      @grammar = grammar
+      @output_file = output_file
+      if log_file
+        @log = File.open(log_file, "wb")
+      else
+        @log = StringIO.new
+      end
+      @language =
+        if output_file =~ /\.([a-z]+)$/
+          $1
+        else
+          "d"
+        end
+      @options = options
+      process_grammar!
+    end
+
+    def generate
+      extensions = [@language]
+      if @language == "c"
+        extensions += %w[h]
+      end
+      extensions.each do |extension|
+        template = Assets.get("parser.#{extension}.erb")
+        erb = ERB.new(template, trim_mode: "<>")
+        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        result = erb.result(binding.clone)
+        File.open(output_file, "wb") do |fh|
+          fh.write(result)
+        end
+      end
+      @log.close
+    end
+
+    private
+
+    def process_grammar!
+      # Assign default pattern mode to patterns without a mode assigned.
+      found_default = false
+      @grammar.patterns.each do |pattern|
+        if pattern.mode.nil?
+          pattern.mode = "default"
+          found_default = true
+        end
+        pattern.ptypename ||= "default"
+      end
+      unless found_default
+        raise Error.new("No patterns found for default mode")
+      end
+      check_ptypes!
+      # Add EOF token.
+      @grammar.tokens << Token.new("$EOF", nil, nil)
+      tokens_by_name = {}
+      @grammar.tokens.each_with_index do |token, token_id|
+        # Assign token ID.
+        token.id = token_id
+        # Set default ptypename if none given.
+        token.ptypename ||= "default"
+        # Check for token name conflicts.
+        if tokens_by_name.include?(token.name)
+          raise Error.new("Duplicate token name #{token.name.inspect}")
+        end
+        tokens_by_name[token.name] = token
+      end
+      # Check for user start rule.
+      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
+        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
+      end
+      # Add "real" start rule.
+      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
+      # Generate and add rules for optional components.
+      generate_optional_component_rules!(tokens_by_name)
+      # Build rule sets.
+      rule_sets = {}
+      rule_set_id = @grammar.tokens.size
+      @grammar.rules.each_with_index do |rule, rule_id|
+        # Assign rule ID.
+        rule.id = rule_id
+        # Check for token/rule name conflict.
+        if tokens_by_name.include?(rule.name)
+          raise Error.new("Rule name collides with token name #{rule.name.inspect}")
+        end
+        # Build rule sets of all rules with the same name.
+        unless rule_sets[rule.name]
+          rule_sets[rule.name] = RuleSet.new(rule.name, rule_set_id)
+          rule_set_id += 1
+        end
+        rule_set = rule_sets[rule.name]
+        if rule_set.ptypename && rule.ptypename && rule_set.ptypename != rule.ptypename
+          raise Error.new("Conflicting ptypes for rule #{rule.name}")
+        end
+        rule_set.ptypename ||= rule.ptypename
+        rule.rule_set = rule_set
+        rule_set << rule
+      end
+      rule_sets.each do |name, rule_set|
+        rule_set.ptypename ||= "default"
+        # Assign rule set ptypenames back to rules.
+        rule_set.rules.each do |rule|
+          rule.ptypename = rule_set.ptypename
+        end
+      end
+      # Generate lexer user code IDs for lexer patterns with user code blocks.
+      @grammar.patterns.select do |pattern|
+        pattern.code
+      end.each_with_index do |pattern, code_id|
+        pattern.code_id = code_id
+      end
+      # Map rule components from names to Token/RuleSet objects.
+      @grammar.rules.each do |rule|
+        rule.components.map! do |component|
+          if tokens_by_name[component]
+            tokens_by_name[component]
+          elsif rule_sets[component]
+            rule_sets[component]
+          else
+            raise Error.new("Symbol #{component} not found")
+          end
+        end
+      end
+      determine_possibly_empty_rulesets!(rule_sets)
+      rule_sets.each do |name, rule_set|
+        rule_set.finalize(@grammar)
+      end
+      # Generate the lexer.
+      @lexer = Lexer.new(@grammar)
+      # Generate the parser.
+      @parser = Parser.new(@grammar, rule_sets, @log, @options)
+    end
+
+    # Check that any referenced ptypes have been defined.
+    def check_ptypes!
+      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
+        if potor.ptypename
+          unless @grammar.ptypes.include?(potor.ptypename)
+            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
+          end
+        end
+      end
+    end
+
+    # Generate and add rules for any optional components.
+    def generate_optional_component_rules!(tokens_by_name)
+      optional_rules_added = Set.new
+      @grammar.rules.each do |rule|
+        rule.components.each do |component|
+          if component =~ /^(.*)\?$/
+            c = $1
+            unless optional_rules_added.include?(component)
+              # Create two rules for the optional component: one empty and
+              # one just matching the component.
+              # We need to find the ptypename for the optional component in
+              # order to copy it to the generated rules.
+              if tokens_by_name[c]
+                # The optional component is a token.
+                ptypename = tokens_by_name[c].ptypename
+              else
+                # The optional component must be a rule, so find any instance
+                # of that rule that specifies a ptypename.
+                ptypename = @grammar.rules.reduce(nil) do |result, rule|
+                  rule.name == c && rule.ptypename ? rule.ptypename : result
+                end
+              end
+              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
+              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optional_rules_added << component
+            end
+          end
+        end
+      end
+    end
+
+    # Determine which grammar rules could expand to empty sequences.
+    #
+    # @param rule_sets [Hash]
+    #   RuleSets.
+    #
+    # @return [void]
+    def determine_possibly_empty_rulesets!(rule_sets)
+      begin
+        newly_discovered_empty_rulesets = false
+        rule_sets.each do |name, rule_set|
+          unless rule_set.could_be_empty?
+            if could_rule_set_be_empty?(rule_set)
+              newly_discovered_empty_rulesets = true
+              rule_set.could_be_empty = true
+            end
+          end
+        end
+      end while newly_discovered_empty_rulesets
+    end
+
+    # Determine whether a RuleSet could be empty.
+    #
+    # @param rule_set [RuleSet]
+    #   RuleSet to test.
+    #
+    # @return [Boolean]
+    #   Whether the RuleSet could be empty.
+    def could_rule_set_be_empty?(rule_set)
+      rule_set.rules.any? do |rule|
+        could_rule_be_empty?(rule)
+      end
+    end
+
+    # Determine whether a Rule could be empty.
+    #
+    # @param rule [Rule]
+    #   Rule to test.
+    #
+    # @return [Boolean]
+    #   Whether the Rule could be empty.
+    def could_rule_be_empty?(rule)
+      i = 0
+      loop do
+        if i == rule.components.size
+          return true
+        end
+        if rule.components[i].is_a?(Token)
+          return false
+        end
+        if !rule.components[i].could_be_empty?
+          return false
+        end
+        i += 1
+      end
+    end
+
+    # Expand expansions in user code block.
+    #
+    # @param code [String]
+    #   User code block.
+    # @param parser [Boolean]
+    #   Whether the user code is for the parser or lexer.
+    # @param rule [Rule, nil]
+    #   The Rule associated with the user code if user code is for the parser.
+    # @param pattern [Pattern, nil]
+    #   The Pattern associated with the user code if user code is for the lexer.
+    #
+    # @return [String]
+    #   Expanded user code block.
+    def expand_code(code, parser, rule, pattern)
+      code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
+        "TOKEN_#{Token.code_name($1)}"
+      end
+      code = code.gsub(/\$terminate\((.*)\);/) do |match|
+        user_terminate_code = $1
+        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
+        case @language
+        when "c"
+          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "d"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        end
+      end
+      if parser
+        code = code.gsub(/\$\$/) do |match|
+          case @language
+          when "c"
+            "_pvalue->v_#{rule.ptypename}"
+          when "d"
+            "_pvalue.v_#{rule.ptypename}"
+          end
+        end
+        code = code.gsub(/\$(\d+)/) do |match|
+          index = $1.to_i
+          case @language
+          when "c"
+            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
+          when "d"
+            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
+          end
+        end
+        code = code.gsub(/\$\{(\w+)\}/) do |match|
+          aliasname = $1
+          if index = rule.aliases[aliasname]
+            case @language
+            when "c"
+              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
+            when "d"
+              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
+            end
+          else
+            raise Error.new("Field alias '#{aliasname}' not found")
+          end
+        end
+      else
+        code = code.gsub(/\$\$/) do |match|
+          if @grammar.ast
+            case @language
+            when "c"
+              "out_token_info->pvalue"
+            when "d"
+              "out_token_info.pvalue"
+            end
+          else
+            case @language
+            when "c"
+              "out_token_info->pvalue.v_#{pattern.ptypename}"
+            when "d"
+              "out_token_info.pvalue.v_#{pattern.ptypename}"
+            end
+          end
+        end
+        code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
+          mode_name = $1
+          mode_id = @lexer.mode_id(mode_name)
+          unless mode_id
+            raise Error.new("Lexer mode '#{mode_name}' not found")
+          end
+          case @language
+          when "c"
+            "context->mode = #{mode_id}u"
+          when "d"
+            "context.mode = #{mode_id}u"
+          end
+        end
+      end
+      code
+    end
+
+    # Get the parser value type for the start rule.
+    #
+    # @return [Array<String>]
+    #   Start rule parser value type name and type string.
+    def start_rule_type
+      start_rule = @grammar.rules.find do |rule|
+        rule.name == @grammar.start_rule
+      end
+      [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
+    end
+
+    # Get an unsigned integer type that can hold the given maximum value.
+    #
+    # @param max [Integer]
+    #   Maximum value to store.
+    #
+    # @return [String]
+    #   Type.
+    def get_type_for(max)
+      if max <= 0xFF
+        case @language
+        when "c"
+          "uint8_t"
+        when "d"
+          "ubyte"
+        end
+      elsif max <= 0xFFFF
+        case @language
+        when "c"
+          "uint16_t"
+        when "d"
+          "ushort"
+        end
+      else
+        case @language
+        when "c"
+          "uint32_t"
+        else
+          "uint"
+        end
+      end
+    end
+
+  end
+
+end

lib/propane/grammar.rb

@@ -0,0 +1,320 @@
+class Propane
+
+  class Grammar
+
+    # Reserve identifiers beginning with a double-underscore for internal use.
+    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
+
+    attr_reader :ast
+    attr_reader :ast_prefix
+    attr_reader :ast_suffix
+    attr_reader :modulename
+    attr_reader :patterns
+    attr_reader :rules
+    attr_reader :start_rule
+    attr_reader :tokens
+    attr_reader :code_blocks
+    attr_reader :ptypes
+    attr_reader :prefix
+
+    def initialize(input)
+      @patterns = []
+      @start_rule = "Start"
+      @tokens = []
+      @rules = []
+      @code_blocks = {}
+      @line_number = 1
+      @next_line_number = @line_number
+      @mode = nil
+      @input = input.gsub("\r\n", "\n")
+      @ptypes = {"default" => "void *"}
+      @prefix = "p_"
+      @ast = false
+      @ast_prefix = ""
+      @ast_suffix = ""
+      parse_grammar!
+    end
+
+    def ptype
+      @ptypes["default"]
+    end
+
+    def invalid_token_id
+      @tokens.size
+    end
+
+    def terminate_token_id
+      @tokens.size + 1
+    end
+
+    private
+
+    def parse_grammar!
+      while @input.size > 0
+        parse_statement!
+      end
+    end
+
+    def parse_statement!
+      if parse_white_space!
+      elsif parse_comment_line!
+      elsif @mode.nil? && parse_mode_label!
+      elsif parse_ast_statement!
+      elsif parse_ast_prefix_statement!
+      elsif parse_ast_suffix_statement!
+      elsif parse_module_statement!
+      elsif parse_ptype_statement!
+      elsif parse_pattern_statement!
+      elsif parse_start_statement!
+      elsif parse_token_statement!
+      elsif parse_tokenid_statement!
+      elsif parse_drop_statement!
+      elsif parse_rule_statement!
+      elsif parse_code_block_statement!
+      elsif parse_prefix_statement!
+      else
+        if @input.size > 25
+          @input = @input.slice(0..20) + "..."
+        end
+        raise Error.new("Unexpected grammar input at line #{@line_number}: #{@input.chomp}")
+      end
+    end
+
+    def parse_mode_label!
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
+        @mode = md[1]
+      end
+    end
+
+    def parse_white_space!
+      consume!(/\s+/)
+    end
+
+    def parse_comment_line!
+      consume!(/#.*\n/)
+    end
+
+    def parse_ast_statement!
+      if consume!(/ast\s*;/)
+        @ast = true
+      end
+    end
+
+    def parse_ast_prefix_statement!
+      if md = consume!(/ast_prefix\s+(\w+)\s*;/)
+        @ast_prefix = md[1]
+      end
+    end
+
+    def parse_ast_suffix_statement!
+      if md = consume!(/ast_suffix\s+(\w+)\s*;/)
+        @ast_suffix = md[1]
+      end
+    end
+
+    def parse_module_statement!
+      if consume!(/module\s+/)
+        md = consume!(/([\w.]+)\s*/, "expected module name")
+        @modulename = md[1]
+        consume!(/;/, "expected `;'")
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_ptype_statement!
+      if consume!(/ptype\s+/)
+        name = "default"
+        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
+          name = md[1]
+        end
+        md = consume!(/([^;]+);/, "expected parser result type expression")
+        @ptypes[name] = md[1].strip
+      end
+    end
+
+    def parse_token_statement!
+      if consume!(/token\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
+        name = md[1]
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
+          ptypename = md[1]
+        end
+        pattern = parse_pattern! || name
+        consume!(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
+        end
+        token = Token.new(name, ptypename, @line_number)
+        @tokens << token
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @patterns << pattern
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_tokenid_statement!
+      if md = consume!(/tokenid\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
+        name = md[1]
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
+          ptypename = md[1]
+        end
+        consume!(/;/, "expected `;'");
+        token = Token.new(name, ptypename, @line_number)
+        @tokens << token
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_drop_statement!
+      if md = consume!(/drop\s+/)
+        pattern = parse_pattern!
+        unless pattern
+          raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
+        end
+        consume!(/\s+/)
+        consume!(/;/, "expected `;'")
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_rule_statement!
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
+        rule_name, ptypename = *md[1, 2]
+        if @ast && ptypename
+          raise Error.new("Multiple ptypes are unsupported in AST mode")
+        end
+        md = consume!(/((?:#{IDENTIFIER_REGEX}(?::#{IDENTIFIER_REGEX})?\??\s*)*)\s*/, "expected rule component list")
+        components = md[1].strip.split(/\s+/)
+        if @ast
+          consume!(/;/, "expected `;'")
+        else
+          unless code = parse_code_block!
+            consume!(/;/, "expected `;' or code block")
+          end
+        end
+        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_pattern_statement!
+      if pattern = parse_pattern!
+        consume!(/\s+/)
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
+          ptypename = md[1]
+        end
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block to follow pattern")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_start_statement!
+      if md = consume!(/start\s+(\w+)\s*;/)
+        @start_rule = md[1]
+      end
+    end
+
+    def parse_code_block_statement!
+      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
+        name, code = md[1..2]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
+        if @code_blocks[name]
+          @code_blocks[name] += code
+        else
+          @code_blocks[name] = code
+        end
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_prefix_statement!
+      if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
+        @prefix = md[1]
+        true
+      end
+    end
+
+    def parse_pattern!
+      if md = consume!(%r{/})
+        pattern = ""
+        while !consume!(%r{/})
+          if consume!(%r{\\})
+            pattern += "\\"
+            if md = consume!(%r{(.)})
+              pattern += md[1]
+            else
+              raise Error.new("Line #{@line_number}: unterminated escape sequence")
+            end
+          elsif md = consume!(%r{(.)})
+            pattern += md[1]
+          end
+        end
+        pattern
+      end
+    end
+
+    def parse_code_block!
+      if md = consume!(/<<(.*?)>>\n/m)
+        code = md[1]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
+        code
+      end
+    end
+
+    # Check if the input string matches the given regex.
+    #
+    # If so, remove the match from the input string, and update the line
+    # number. If the regex is not matched and an error message is provided,
+    # the error is raised.
+    #
+    # @param regex [Regexp]
+    #   Regex to attempt to match.
+    # @param error_message [String, nil]
+    #   Error message to display if the regex is not matched. If nil and the
+    #   regex is not matched, an error is not raised.
+    #
+    # @return [MatchData, nil]
+    #   MatchData for the given regex if it was matched and removed from the
+    #   input.
+    def consume!(regex, error_message = nil)
+      @line_number = @next_line_number
+      if md = @input.match(/\A#{regex}/)
+        @input.slice!(0, md[0].size)
+        @next_line_number += md[0].count("\n")
+        md
+      elsif error_message
+        raise Error.new("Line #{@line_number}: Error: #{error_message}")
+      else
+        false
+      end
+    end
+
+  end
+
+end

lib/propane/lexer.rb

@@ -1,12 +1,67 @@
 class Propane
   class Lexer
 
-    # @return [DFA]
-    #   Lexer DFA.
-    attr_accessor :dfa
+    attr_reader :state_table
+    attr_reader :transition_table
+    attr_reader :mode_table
 
-    def initialize(tokens)
-      @dfa = DFA.new(tokens)
+    def initialize(grammar)
+      @grammar = grammar
+      build_tables!
+    end
+
+    # Get ID for a mode.
+    #
+    # @param mode_name [String]
+    #   Mode name.
+    #
+    # @return [Integer, nil]
+    #   Mode ID.
+    def mode_id(mode_name)
+      if mode_info = @modes[mode_name]
+        mode_info[:id]
+      end
+    end
+
+    private
+
+    def build_tables!
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
+      @state_table = []
+      @transition_table = []
+      @mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = @state_table.size
+        @mode_table << {
+          state_table_offset: state_table_offset,
+        }
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          token = state.accepts && state.accepts.token && state.accepts.token.id
+          code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
+          @state_table << {
+            transition_table_index: @transition_table.size,
+            n_transitions: state.transitions.size,
+            accepts: !!state.accepts,
+            token: token,
+            code_id: code_id,
+          }
+          state.transitions.each do |transition|
+            @transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
+        end
+      end
     end
 
   end

lib/propane/lexer/dfa.rb

@@ -3,11 +3,11 @@ class Propane
 
     class DFA < FA
 
-      def initialize(tokens)
+      def initialize(patterns)
         super()
         start_nfa = Regex::NFA.new
-        tokens.each do |name, token|
-          start_nfa.start_state.add_transition(nil, token.nfa.start_state)
+        patterns.each do |pattern|
+          start_nfa.start_state.add_transition(nil, pattern.nfa.start_state)
         end
         @nfa_state_sets = {}
         @states = []
@@ -22,35 +22,6 @@ class Propane
         @start_state = @states[0]
       end
 
-      def build_tables
-        transition_table = []
-        state_table = []
-        states = enumerate
-        states.each do |state, id|
-          accepts =
-            if state.accepts.nil?
-              TOKEN_NONE
-            elsif state.accepts.name
-              state.accepts.id
-            else
-              TOKEN_DROP
-            end
-          state_table << {
-            transition_table_index: transition_table.size,
-            n_transitions: state.transitions.size,
-            accepts: accepts,
-          }
-          state.transitions.each do |transition|
-            transition_table << {
-              first: transition.code_point_range.first,
-              last: transition.code_point_range.last,
-              destination: states[transition.destination],
-            }
-          end
-        end
-        [transition_table, state_table]
-      end
-
       private
 
       def register_nfa_state_set(nfa_state_set)
@@ -69,7 +40,7 @@ class Propane
           nfa_state_set.each do |nfa_state|
             if nfa_state.accepts
               if state.accepts
-                if nfa_state.accepts.id < state.accepts.id
+                if nfa_state.accepts.line_number < state.accepts.line_number
                   state.accepts = nfa_state.accepts
                 end
               else

lib/propane/parser.rb

@@ -2,30 +2,33 @@ class Propane
 
   class Parser
 
-    def initialize(tokens, rules)
-      @token_eof = Token.new("$", nil, TOKEN_EOF)
+    attr_reader :state_table
+    attr_reader :shift_table
+    attr_reader :reduce_table
+    attr_reader :rule_sets
+
+    def initialize(grammar, rule_sets, log, options)
+      @grammar = grammar
+      @rule_sets = rule_sets
+      @log = log
       @item_sets = []
       @item_sets_set = {}
-      start_items = rules["Start"].patterns.map do |pattern|
-        pattern.components << @token_eof
-        Item.new(pattern, 0)
-      end
-      eval_item_sets = Set.new
-      eval_item_sets << ItemSet.new(start_items)
+      @warnings = Set.new
+      @options = options
+      start_item = Item.new(grammar.rules.first, 0)
+      eval_item_sets = Set[ItemSet.new([start_item])]
 
       while eval_item_sets.size > 0
-        this_eval_item_sets = eval_item_sets
-        eval_item_sets = Set.new
-        this_eval_item_sets.each do |item_set|
-          unless @item_sets_set.include?(item_set)
-            item_set.id = @item_sets.size
-            @item_sets << item_set
-            @item_sets_set[item_set] = item_set
-            item_set.follow_symbols.each do |follow_symbol|
-              unless follow_symbol == @token_eof
-                follow_set = item_set.build_follow_set(follow_symbol)
-                eval_item_sets << follow_set
-              end
+        item_set = eval_item_sets.first
+        eval_item_sets.delete(item_set)
+        unless @item_sets_set.include?(item_set)
+          item_set.id = @item_sets.size
+          @item_sets << item_set
+          @item_sets_set[item_set] = item_set
+          item_set.next_symbols.each do |next_symbol|
+            unless next_symbol.name == "$EOF"
+              next_item_set = item_set.build_next_item_set(next_symbol)
+              eval_item_sets << next_item_set
             end
           end
         end
@@ -33,48 +36,260 @@ class Propane
 
       @item_sets.each do |item_set|
         process_item_set(item_set)
-        puts "Item set #{item_set.id}:"
-        ids = item_set.in_sets.map(&:id)
-        if ids.size > 0
-          puts "    (in from #{ids.join(", ")})"
-        end
-        puts item_set
-        item_set.follow_item_set.each do |follow_symbol, follow_item_set|
-          puts " #{follow_symbol.name} => #{follow_item_set.id}"
-        end
-        puts
       end
-    end
 
-    def build_tables
-      shift_table = []
-      state_table = []
-      @item_sets.each do |item_set|
-        shift_entries = item_set.follow_symbols.select do |follow_symbol|
-          follow_symbol.is_a?(Token)
-        end.map do |follow_symbol|
-          {
-            token_id: follow_symbol.id,
-            state_id: item_set.follow_item_set[follow_symbol].id,
-          }
-        end
-        state_table << {
-          shift_index: shift_table.size,
-          n_shifts: shift_entries.size,
-        }
-        shift_table += shift_entries
+      build_reduce_actions!
+      build_tables!
+      write_log!
+      if @warnings.size > 0 && @options[:warnings_as_errors]
+        raise Error.new("Fatal errors (-w):\n" + @warnings.join("\n"))
       end
-      [state_table, shift_table]
     end
 
     private
 
+    def build_tables!
+      @state_table = []
+      @shift_table = []
+      @reduce_table = []
+      @item_sets.each do |item_set|
+        shift_entries = item_set.next_symbols.map do |next_symbol|
+          state_id =
+            if next_symbol.name == "$EOF"
+              0
+            else
+              item_set.next_item_set[next_symbol].id
+            end
+          {
+            symbol: next_symbol,
+            state_id: state_id,
+          }
+        end
+        unless item_set.reduce_rules.empty?
+          shift_entries.each do |shift_entry|
+            token = shift_entry[:symbol]
+            if get_lookahead_reduce_actions_for_item_set(item_set).include?(token)
+              rule = item_set.reduce_actions[token]
+              @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+            end
+          end
+        end
+        reduce_entries =
+          if rule = item_set.reduce_rule
+            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
+              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+              propagate_optional_target: rule.optional? && rule.components.size == 1}]
+          elsif reduce_actions = item_set.reduce_actions
+            reduce_actions.map do |token, rule|
+              {token_id: token.id, rule_id: rule.id, rule: rule,
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               propagate_optional_target: rule.optional? && rule.components.size == 1}
+            end
+          else
+            []
+          end
+        @state_table << {
+          shift_index: @shift_table.size,
+          n_shifts: shift_entries.size,
+          reduce_index: @reduce_table.size,
+          n_reduces: reduce_entries.size,
+        }
+        @shift_table += shift_entries
+        @reduce_table += reduce_entries
+      end
+    end
+
     def process_item_set(item_set)
-      item_set.follow_symbols.each do |follow_symbol|
-        unless follow_symbol == @token_eof
-          follow_set = @item_sets_set[item_set.build_follow_set(follow_symbol)]
-          item_set.follow_item_set[follow_symbol] = follow_set
-          follow_set.in_sets << item_set
+      item_set.next_symbols.each do |next_symbol|
+        unless next_symbol.name == "$EOF"
+          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
+          item_set.next_item_set[next_symbol] = next_item_set
+          next_item_set.in_sets << item_set
+        end
+      end
+    end
+
+    # Build the reduce actions for each ItemSet.
+    #
+    # @return [void]
+    def build_reduce_actions!
+      @item_sets.each do |item_set|
+        build_reduce_actions_for_item_set(item_set)
+      end
+    end
+
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [void]
+    def build_reduce_actions_for_item_set(item_set)
+      # To build the reduce actions, we start by looking at any
+      # "complete" items, i.e., items where the parse position is at the
+      # end of a rule. These are the only rules that are candidates for
+      # reduction in the current ItemSet.
+      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+
+      if item_set.reduce_rules.size == 1
+        item_set.reduce_rule = item_set.reduce_rules.first
+      end
+
+      if item_set.reduce_rules.size > 1
+        # Force item_set.reduce_actions to be built to store the lookahead
+        # tokens for the possible reduce rules if there is more than one.
+        get_lookahead_reduce_actions_for_item_set(item_set)
+      end
+    end
+
+    # Get the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def get_lookahead_reduce_actions_for_item_set(item_set)
+      item_set.reduce_actions ||= build_lookahead_reduce_actions_for_item_set(item_set)
+    end
+
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def build_lookahead_reduce_actions_for_item_set(item_set)
+      # We will be looking for all possible tokens that can follow instances of
+      # these rules. Rather than looking through the entire grammar for the
+      # possible following tokens, we will only look in the item sets leading
+      # up to this one. This restriction gives us a more precise lookahead set,
+      # and allows us to parse LALR grammars.
+      item_sets = Set[item_set] + item_set.leading_item_sets
+      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+        lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
+        lookahead_tokens_for_rule.each do |lookahead_token|
+          if existing_reduce_rule = reduce_actions[lookahead_token]
+            raise Error.new("Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number})")
+          end
+          reduce_actions[lookahead_token] = reduce_rule
+        end
+        reduce_actions
+      end
+    end
+
+    # Build the set of lookahead Tokens that should cause the given Rule to be
+    # reduced in the given context of ItemSets.
+    #
+    # @param rule [Rule]
+    #   Rule to reduce.
+    # @param item_sets [Set<ItemSet>]
+    #   ItemSets to consider for the context in which to reduce this Rule.
+    #
+    # @return [Set<Token>]
+    #   Possible lookahead Tokens for the given Rule within the context of the
+    #   given ItemSets.
+    def build_lookahead_tokens_to_reduce(rule, item_sets)
+      # We need to look for possible following tokens for this reduce rule. We
+      # do this by looking for tokens that follow the reduce rule, or the
+      # start token set for any other rule that follows the reduce rule.
+      # While doing this, the following situations could arise:
+      # 1. We may come across a following rule that could be empty. In this
+      #    case, in addition to the start token set for that rule, we must also
+      #    continue to the next following symbol after the potentially empty
+      #    rule and continue the search for potential following tokens.
+      # 2. We may reach the end of a rule that was not one of the original
+      #    reduce rules. In this case, we must also search for all potential
+      #    following tokens for this rule as well.
+      lookahead_tokens = Set.new
+      rule_sets_to_check_after = [rule.rule_set]
+      checked_rule_sets = Set.new
+      while !rule_sets_to_check_after.empty?
+        rule_set = rule_sets_to_check_after.slice!(0)
+        checked_rule_sets << rule_set
+        # For each RuleSet we're checking, we're going to look through all
+        # items in the item sets of interest and gather all possible following
+        # tokens to form the lookahead token set.
+        item_sets.each do |item_set|
+          item_set.items.each do |item|
+            if item.next_symbol == rule_set
+              (1..).each do |offset|
+                case symbol = item.next_symbol(offset)
+                when nil
+                  rule_set = item.rule.rule_set
+                  unless checked_rule_sets.include?(rule_set)
+                    rule_sets_to_check_after << rule_set
+                  end
+                  break
+                when Token
+                  lookahead_tokens << symbol
+                  break
+                when RuleSet
+                  lookahead_tokens += symbol.start_token_set
+                  unless symbol.could_be_empty?
+                    break
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      lookahead_tokens
+    end
+
+    def write_log!
+      @log.puts Util.banner("Parser Rules")
+      @grammar.rules.each do |rule|
+        @log.puts
+        @log.puts "Rule #{rule.id}:"
+        @log.puts "  #{rule}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser Tokens")
+      @log.puts
+      @grammar.tokens.each do |token|
+        @log.puts "Token #{token.id}: #{token.name}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser Rule Sets")
+      @rule_sets.each do |rule_set_name, rule_set|
+        @log.puts
+        @log.puts "Rule Set #{rule_set.id}: #{rule_set_name}"
+        @log.puts "  Start token set: #{rule_set.start_token_set.map(&:name).join(", ")}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser States")
+      @item_sets.each do |item_set|
+        @log.puts
+        @log.puts "State #{item_set.id}:"
+        @log.puts item_set.to_s.gsub(/^/, "  ")
+        incoming_ids = item_set.in_sets.map(&:id)
+        @log.puts
+        @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
+        @log.puts "  Outgoing states:"
+        item_set.next_item_set.each do |next_symbol, next_item_set|
+          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
+        end
+        @log.puts
+        @log.puts "  Reduce actions:"
+        if item_set.reduce_rule
+          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
+        elsif item_set.reduce_actions
+          item_set.reduce_actions.each do |token, rule|
+            @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
+          end
+        end
+      end
+      if @warnings.size > 0
+        @log.puts
+        @log.puts "Warnings:"
+        @warnings.each do |warning|
+          @log.puts "  #{warning}"
         end
       end
     end

lib/propane/parser/item.rb

@@ -1,66 +1,143 @@
 class Propane
   class Parser
 
+    # Represent a parser "item", which is a position in a Rule that the parser
+    # could potentially be at.
     class Item
 
-      attr_reader :pattern
+      # @return [Rule]
+      #   The Rule corresponding to this Item.
+      attr_reader :rule
+
+      # @return [Integer]
+      #   The parse position in this item.
       attr_reader :position
 
-      def initialize(pattern, position)
-        @pattern = pattern
+      # Construct an Item.
+      #
+      # @param rule [Rule]
+      #   The Rule corresponding to this Item.
+      # @param position [Integer]
+      #   The parse position in this Item.
+      def initialize(rule, position)
+        @rule = rule
         @position = position
       end
 
-      def next_component
-        @pattern.components[@position]
-      end
-
+      # Hash function.
+      #
+      # @return [Integer]
+      #   Hash code.
       def hash
-        [@pattern, @position].hash
+        [@rule, @position].hash
       end
 
+      # Compare Item objects.
+      #
+      # @param other [Item]
+      #   Item to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the Items are equal.
       def ==(other)
-        @pattern == other.pattern && @position == other.position
+        @rule == other.rule && @position == other.position
       end
 
+      # Compare Item objects.
+      #
+      # @param other [Item]
+      #   Item to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the Items are equal.
       def eql?(other)
         self == other
       end
 
+      # Return the set of Items obtained by "closing" the current item.
+      #
+      # If the next symbol for the current item is another Rule name, then
+      # this method will return all Items for that Rule with a position of 0.
+      # Otherwise, an empty Array is returned.
+      #
+      # @return [Array<Item>]
+      #   Items obtained by "closing" the current item.
       def closed_items
-        if @pattern.components[@position].is_a?(Rule)
-          @pattern.components[@position].patterns.map do |pattern|
-            Item.new(pattern, 0)
+        if @rule.components[@position].is_a?(RuleSet)
+          @rule.components[@position].rules.map do |rule|
+            Item.new(rule, 0)
           end
         else
           []
         end
       end
 
-      def follow_symbol
-        @pattern.components[@position]
+      # Return whether the item is "complete", meaning that the parse position
+      # marker is at the end of the rule.
+      #
+      # @return [Boolean]
+      #   Whether the item is "complete".
+      def complete?
+        @position == @rule.components.size
       end
 
-      def followed_by?(symbol)
-        follow_symbol == symbol
+      # Get the next symbol for the Item.
+      #
+      # That is, the symbol which is after the parse position marker in the
+      # current Item.
+      #
+      # @param offset [Integer]
+      #   Offset from current parse position to examine.
+      #
+      # @return [Token, RuleSet, nil]
+      #   Next symbol for the Item.
+      def next_symbol(offset = 0)
+        @rule.components[@position + offset]
       end
 
-      def next_position
-        Item.new(@pattern, @position + 1)
+      # Get the previous symbol for the Item.
+      #
+      # That is, the symbol which precedes the parse position marker in the
+      # current Item.
+      #
+      # @return [Token, RuleSet, nil]
+      #   Previous symbol for this Item.
+      def previous_symbol
+        if @position > 0
+          @rule.components[@position - 1]
+        end
       end
 
+      # Get whether this Item's next symbol is the given symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Symbol to query.
+      #
+      # @return [Boolean]
+      #   Whether this Item's next symbol is the given symbol.
+      def next_symbol?(symbol)
+        next_symbol == symbol
+      end
+
+      # Get the next item for this Item.
+      #
+      # That is, the Item formed by moving the parse position marker one place
+      # forward from its position in this Item.
+      #
+      # @return [Item]
+      #   The next item for this Item.
+      def next_item
+        Item.new(@rule, @position + 1)
+      end
+
+      # Represent the Item as a String.
+      #
+      # @return [String]
+      #   The Item represented as a String.
       def to_s
-        parts = []
-        @pattern.components.each_with_index do |symbol, index|
-          if @position == index
-            parts << "."
-          end
-          parts << symbol.name
-        end
-        if @position == @pattern.components.size
-          parts << "."
-        end
-        "#{@pattern.rule.name} -> #{parts.join(" ")}"
+        parts = @rule.components.map(&:name)
+        parts[@position, 0] = "."
+        "#{@rule.name} -> #{parts.join(" ")}"
       end
 
     end

lib/propane/parser/item_set.rb

@@ -1,72 +1,164 @@
 class Propane
   class Parser
 
+    # Represent a parser "item set", which is a set of possible items that the
+    # parser could currently be parsing. This is equivalent to a parser state.
     class ItemSet
 
+      # @return [Set<Item>]
+      #   Items in this ItemSet.
       attr_reader :items
 
+      # @return [Integer]
+      #   ID of this ItemSet.
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a follow symbol to its item set.
-      attr_reader :follow_item_set
+      #   Maps a next symbol to its ItemSet.
+      attr_reader :next_item_set
 
-      # @return [Set]
-      #   Item sets leading to this item set.
+      # @return [Set<ItemSet>]
+      #   ItemSets leading to this item set.
       attr_reader :in_sets
 
+      # @return [nil, Rule]
+      #   Rule to reduce if there is only one possibility.
+      attr_accessor :reduce_rule
+
+      # @return [Set<Rule>]
+      #   Set of rules that could be reduced in this parser state.
+      attr_accessor :reduce_rules
+
+      # @return [nil, Hash]
+      #   Reduce actions, mapping lookahead tokens to rules, if there is
+      #   more than one rule that could be reduced.
+      attr_accessor :reduce_actions
+
+      # Build an ItemSet.
+      #
+      # @param items [Array<Item>]
+      #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @follow_item_set = {}
+        @next_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      def follow_symbols
-        Set.new(@items.map(&:follow_symbol).compact)
+      # Get the set of next symbols for all Items in this ItemSet.
+      #
+      # @return [Set<Token, RuleSet>]
+      #   Set of next symbols for all Items in this ItemSet.
+      def next_symbols
+        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
       end
 
-      def build_follow_set(symbol)
-        ItemSet.new(items_followed_by(symbol).map(&:next_position))
+      # Build a next ItemSet for the given next symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Next symbol to build the next ItemSet for.
+      #
+      # @return [ItemSet]
+      #   Next ItemSet for the given next symbol.
+      def build_next_item_set(symbol)
+        ItemSet.new(items_with_next(symbol).map(&:next_item))
       end
 
+      # Hash function.
+      #
+      # @return [Integer]
+      #   Hash code.
       def hash
         @items.hash
       end
 
+      # Compare ItemSet objects.
+      #
+      # @param other [ItemSet]
+      #   ItemSet to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the ItemSets are equal.
       def ==(other)
         @items.eql?(other.items)
       end
 
+      # Compare ItemSet objects.
+      #
+      # @param other [ItemSet]
+      #   ItemSet to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the ItemSets are equal.
       def eql?(other)
         self == other
       end
 
+      # Set of ItemSets that lead to this ItemSet.
+      #
+      # @return [Set<ItemSet>]
+      #   Set of all ItemSets that lead up to this ItemSet.
+      def leading_item_sets
+        @_leading_item_sets ||=
+          begin
+            result = Set.new
+            eval_sets = Set[self]
+            evaled = Set.new
+            while eval_sets.size > 0
+              eval_set = eval_sets.first
+              eval_sets.delete(eval_set)
+              evaled << eval_set
+              eval_set.in_sets.each do |in_set|
+                result << in_set
+                unless evaled.include?(in_set)
+                  eval_sets << in_set
+                end
+              end
+            end
+            result
+          end
+      end
+
+      # Represent the ItemSet as a String.
+      #
+      # @return [String]
+      #   The ItemSet represented as a String.
       def to_s
         @items.map(&:to_s).join("\n")
       end
 
       private
 
+      # Close the ItemSet.
+      #
+      # This is done by recursively adding the closed Items for each Item in
+      # the ItemSet.
+      #
+      # @return [void]
       def close!
-        eval_items = @items
+        eval_items = @items.dup
         while eval_items.size > 0
-          this_eval_items = eval_items
-          eval_items = Set.new
-          this_eval_items.each do |item|
-            item.closed_items.each do |new_item|
-              unless @items.include?(new_item)
-                eval_items << new_item
-              end
+          item = eval_items.first
+          eval_items.delete(item)
+          item.closed_items.each do |new_item|
+            unless @items.include?(new_item)
+              eval_items << new_item
             end
           end
           @items += eval_items
         end
       end
 
-      def items_followed_by(symbol)
+      # Get the Items with the given next symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Next symbol.
+      #
+      # @return [Array<Item>]
+      #   Items with the given next symbol.
+      def items_with_next(symbol)
         @items.select do |item|
-          item.followed_by?(symbol)
+          item.next_symbol?(symbol)
         end
       end
 

lib/propane/pattern.rb

@@ -0,0 +1,65 @@
+class Propane
+
+  class Pattern
+
+    # @return [String, nil]
+    #   Code block to execute when the pattern is matched.
+    attr_reader :code
+
+    # @option options [Integer, nil] :code_id
+    #   Code block ID.
+    attr_accessor :code_id
+
+    # @return [String, nil]
+    #   Pattern.
+    attr_reader :pattern
+
+    # @return [Token, nil]
+    #   Token to be returned by this pattern.
+    attr_reader :token
+
+    # @return [Integer, nil]
+    #   Line number where the pattern was defined in the input grammar.
+    attr_reader :line_number
+
+    # @return [Regex::NFA, nil]
+    #   Regex NFA for matching the pattern.
+    attr_reader :nfa
+
+    # @return [String, nil]
+    #   Lexer mode for this pattern.
+    attr_accessor :mode
+
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
+
+    # Construct a Pattern.
+    #
+    # @param options [Hash]
+    #   Optional parameters.
+    # @option options [String, nil] :code
+    #   Code block to execute when the pattern is matched.
+    # @option options [String, nil] :pattern
+    #   Pattern.
+    # @option options [Token, nil] :token
+    #   Token to be returned by this pattern.
+    # @option options [Integer, nil] :line_number
+    #   Line number where the token was defined in the input grammar.
+    # @option options [String, nil] :mode
+    #   Lexer mode for this pattern.
+    def initialize(options)
+      @code = options[:code]
+      @pattern = options[:pattern]
+      @token = options[:token]
+      @line_number = options[:line_number]
+      @mode = options[:mode]
+      @ptypename = options[:ptypename]
+      regex = Regex.new(@pattern)
+      regex.nfa.end_state.accepts = self
+      @nfa = regex.nfa
+    end
+
+  end
+
+end

lib/propane/regex.rb

@@ -134,8 +134,18 @@ class Propane
       else
         c = @pattern.slice!(0)
         case c
+        when "a"
+          CharacterRangeUnit.new("\a", "\a")
+        when "b"
+          CharacterRangeUnit.new("\b", "\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "f"
+          CharacterRangeUnit.new("\f", "\f")
+        when "n"
+          CharacterRangeUnit.new("\n", "\n")
+        when "r"
+          CharacterRangeUnit.new("\r", "\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -145,6 +155,10 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "t"
+          CharacterRangeUnit.new("\t", "\t")
+        when "v"
+          CharacterRangeUnit.new("\v", "\v")
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/rule.rb

@@ -2,36 +2,111 @@ class Propane
 
   class Rule
 
-    class Pattern
+    # @return [Array<Token, RuleSet>]
+    #   Rule components.
+    attr_reader :components
 
-      attr_reader :rule
+    # @return [Hash]
+    #   Field aliases.
+    attr_reader :aliases
 
-      attr_reader :components
+    # @return [String]
+    #   User code associated with the rule.
+    attr_reader :code
 
-      attr_reader :code
+    # @return [Integer]
+    #   Rule ID.
+    attr_accessor :id
 
-      def initialize(rule, components, code)
-        @rule = rule
-        @components = components
-        @code = code
-      end
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
 
-    end
-
-    attr_reader :id
+    # @return [Integer]
+    #   Line number where the rule was defined in the input grammar.
+    attr_reader :line_number
 
+    # @return [String]
+    #   Rule name.
     attr_reader :name
 
-    attr_reader :patterns
+    # @return [RuleSet]
+    #   The RuleSet that this Rule is a part of.
+    attr_accessor :rule_set
 
-    def initialize(name, id)
+    # @return [Array<Integer>]
+    #   Map this rule's components to their positions in the parent RuleSet's
+    #   node field pointer array. This is used for AST construction.
+    attr_accessor :rule_set_node_field_index_map
+
+    # Construct a Rule.
+    #
+    # @param name [String]
+    #   Rule name.
+    # @param components [Array<String>]
+    #   Rule components.
+    # @param code [String]
+    #   User code associated with the rule.
+    # @param ptypename [String, nil]
+    #   Parser type name.
+    # @param line_number [Integer]
+    #   Line number where the rule was defined in the input grammar.
+    def initialize(name, components, code, ptypename, line_number)
       @name = name
-      @id = id
-      @patterns = []
+      @aliases = {}
+      @components = components.each_with_index.map do |component, i|
+        if component =~ /(\S+):(\S+)/
+          c, aliasname = $1, $2
+          if @aliases[aliasname]
+            raise Error.new("Error: duplicate field alias `#{aliasname}` for rule #{name} defined on line #{line_number}")
+          end
+          @aliases[aliasname] = i
+          c
+        else
+          component
+        end
+      end
+      @rule_set_node_field_index_map = components.map {0}
+      @code = code
+      @ptypename = ptypename
+      @line_number = line_number
     end
 
-    def add_pattern(components, code)
-      @patterns << Pattern.new(self, components, code)
+    # Return whether the Rule is empty.
+    #
+    # A Rule is empty if it has no components.
+    #
+    # @return [Boolean]
+    #   Whether the Rule is empty.
+    def empty?
+      @components.empty?
+    end
+
+    # Return whether this is an optional Rule.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional Rule.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # Represent the Rule as a String.
+    #
+    # @return [String]
+    #   Rule represented as a String.
+    def to_s
+      "#{@name} -> #{@components.map(&:name).join(" ")}"
+    end
+
+    # Check whether the rule set node field index map is just a 1:1 mapping.
+    #
+    # @return [Boolean]
+    #   Boolean indicating whether the rule set node field index map is just a
+    #   1:1 mapping.
+    def flat_rule_set_node_field_index_map?
+      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
+        v == i
+      end
     end
 
   end

lib/propane/rule_set.rb

@@ -0,0 +1,169 @@
+class Propane
+
+  # A RuleSet collects all grammar rules of the same name.
+  class RuleSet
+
+    # @return [Array<Hash>]
+    #   AST fields.
+    attr_reader :ast_fields
+
+    # @return [Integer]
+    #   ID of the RuleSet.
+    attr_reader :id
+
+    # @return [String]
+    #   Name of the RuleSet.
+    attr_reader :name
+
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
+
+    # @return [Array<Rule>]
+    #   Rules in the RuleSet.
+    attr_reader :rules
+
+    # @return [Boolean]
+    #   Whether the RuleSet could expand to an empty sequence.
+    attr_writer :could_be_empty
+
+    # Construct a RuleSet.
+    #
+    # @param name [String]
+    #   Name of the RuleSet.
+    # @param id [Integer]
+    #   ID of the RuleSet.
+    def initialize(name, id)
+      @id = id
+      @name = name
+      @rules = []
+      @could_be_empty = false
+    end
+
+    # Add a Rule to the RuleSet.
+    #
+    # @param rule [Rule]
+    #   Rule to add.
+    def <<(rule)
+      @rules << rule
+    end
+
+    # Return whether any Rule in the RuleSet is empty.
+    #
+    # @return [Boolean]
+    #   Whether any rule in the RuleSet is empty.
+    def could_be_empty?
+      @could_be_empty
+    end
+
+    # Return whether this is an optional RuleSet.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional RuleSet.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # For optional rule sets, return the underlying component that is optional.
+    def option_target
+      @rules.each do |rule|
+        if rule.components.size > 0
+          return rule.components[0]
+        end
+      end
+      raise "Optional rule target not found"
+    end
+
+    # Build the start token set for the RuleSet.
+    #
+    # @return [Set<Token>]
+    #   Start token set for the RuleSet.
+    def start_token_set
+      if @_start_token_set.nil?
+        @_start_token_set = Set.new
+        @rules.each do |rule|
+          rule.components.each do |component|
+            if component.is_a?(Token)
+              @_start_token_set << component
+              break
+            else
+              @_start_token_set += component.start_token_set
+              unless component.could_be_empty?
+                break
+              end
+            end
+          end
+        end
+      end
+      @_start_token_set
+    end
+
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize(grammar)
+      if grammar.ast
+        build_ast_fields(grammar)
+      end
+    end
+
+    private
+
+    # Build the set of AST fields for this RuleSet.
+    #
+    # This is an Array of Hashes. Each entry in the Array corresponds to a
+    # field location in the AST node. The entry is a Hash. It could have one or
+    # two keys. It will always have the field name with a positional suffix as
+    # a key. It may also have the field name without the positional suffix if
+    # that field only exists in one position across all Rules in the RuleSet.
+    #
+    # @return [void]
+    def build_ast_fields(grammar)
+      field_ast_node_indexes = {}
+      field_indexes_across_all_rules = {}
+      @ast_fields = []
+      @rules.each do |rule|
+        rule.components.each_with_index do |component, i|
+          if component.is_a?(RuleSet) && component.optional?
+            component = component.option_target
+          end
+          if component.is_a?(Token)
+            node_name = "Token"
+          else
+            node_name = component.name
+          end
+          struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+          field_name = "p#{node_name}#{i + 1}"
+          unless field_ast_node_indexes[field_name]
+            field_ast_node_indexes[field_name] = @ast_fields.size
+            @ast_fields << {field_name => struct_name}
+          end
+          field_indexes_across_all_rules[node_name] ||= Set.new
+          field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
+          rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
+        end
+      end
+      field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
+        if indexes_across_all_rules.size == 1
+          # If this field was only seen in one position across all rules,
+          # then add an alias to the positional field name that does not
+          # include the position.
+          @ast_fields[indexes_across_all_rules.first]["p#{node_name}"] =
+            "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+        end
+      end
+      # Now merge in the field aliases as given by the user in the
+      # grammar.
+      field_aliases = {}
+      @rules.each do |rule|
+        rule.aliases.each do |alias_name, index|
+          if field_aliases[alias_name] && field_aliases[alias_name] != index
+            raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}`")
+          end
+          field_aliases[alias_name] = index
+          @ast_fields[index][alias_name] = @ast_fields[index].first[1]
+        end
+      end
+    end
+
+  end
+
+end

lib/propane/token.rb

@@ -2,35 +2,54 @@ class Propane
 
   class Token
 
-    # @return [String]
+    class << self
+
+      # Name of the token to use in code (special characters replaced).
+      #
+      # @return [String]
+      #   Name of the token to use in code (special characters replaced).
+      def code_name(name)
+        name.sub(/^\$/, "__")
+      end
+
+    end
+
+    # @return [String, nil]
     #   Token name.
     attr_reader :name
 
-    # @return [String]
-    #   Token pattern.
-    attr_reader :pattern
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
 
-    # @return [Integer]
+    # @return [Integer, nil]
     #   Token ID.
-    attr_reader :id
+    attr_accessor :id
 
-    # @return [Regex::NFA]
-    #   Regex NFA for matching the token.
-    attr_reader :nfa
+    # @return [Integer, nil]
+    #   Line number where the token was defined in the input grammar.
+    attr_reader :line_number
 
-    def initialize(name, pattern, id)
+    # Construct a Token.
+    #
+    # @param name [String, nil]
+    #   Token name.
+    # @param ptypename [String, nil]
+    #   Parser value type for this token.
+    # @param line_number [Integer, nil]
+    #   Line number where the token was defined in the input grammar.
+    def initialize(name, ptypename, line_number)
       @name = name
-      @pattern = pattern
-      @id = id
-      unless pattern.nil?
-        regex = Regex.new(pattern)
-        regex.nfa.end_state.accepts = self
-        @nfa = regex.nfa
-      end
+      @ptypename = ptypename
+      @line_number = line_number
     end
 
-    def c_name
-      @name.upcase
+    # Name of the token to use in code (special characters replaced).
+    #
+    # @return [String]
+    #   Name of the token to use in code (special characters replaced).
+    def code_name
+      self.class.code_name(@name)
     end
 
     def to_s

lib/propane/util.rb

@@ -0,0 +1,17 @@
+class Propane
+
+  # Utility methods.
+  module Util
+
+    class << self
+
+      def banner(message)
+        s = "*" * (message.size + 4)
+        "#{s}\n* #{message} *\n#{s}\n"
+      end
+
+    end
+
+  end
+
+end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "0.1.0"
+  VERSION = "1.5.1"
 end

propane.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec bundle exec ruby -Ilib bin/propane "$@"

rb/assets/user_guide.html.erb

@@ -0,0 +1,68 @@
+<html>
+  <head>
+    <title>Propane User Guide<%= subpage_title %> - Version <%= Propane::VERSION %></title>
+    <style>
+      body {
+        background-color: #CCC;
+        margin: 0px;
+        padding: 0px;
+      }
+      #body-content {
+        margin-left: auto;
+        margin-right: auto;
+        margin-top: 1em;
+        margin-bottom: 1em;
+        border: 1px solid black;
+        background-color: #FFF;
+        width: 120ex;
+        padding: 2ex;
+      }
+      a {
+        text-decoration: none;
+      }
+      .separator {
+        height: 2em;
+      }
+      .img_block_center {
+        display: block;
+        margin-left: auto;
+        margin-right: auto;
+      }
+      .page_nav {
+        width: 100%;
+        table-layout: fixed;
+      }
+      .page_nav_toc {
+        text-align: center;
+      }
+      .page_nav_next {
+        text-align: right;
+      }
+      .code {
+        padding-left: 2ex;
+        width: 116ex;
+        overflow-x: auto;
+      }
+      .ruby_code .normal {}
+      .ruby_code .comment { color: #005; font-style: italic; }
+      .ruby_code .keyword { color: #A00; font-weight: bold; }
+      .ruby_code .method { color: #077; }
+      .ruby_code .class { color: #074; }
+      .ruby_code .module { color: #050; }
+      .ruby_code .punct { color: #447; font-weight: bold; }
+      .ruby_code .symbol { color: #099; }
+      .ruby_code .string { color: #090; }
+      .ruby_code .char { color: #F07; }
+      .ruby_code .ident { color: #004; }
+      .ruby_code .constant { color: #07F; }
+      .ruby_code .regex { color: #B66; }
+      .ruby_code .number { color: #D55; }
+      .ruby_code .attribute { color: #377; }
+      .ruby_code .global { color: #3B7; }
+      .ruby_code .expr { color: #227; }
+    </style>
+  </head>
+  <body>
+    <div id="body-content"><%= content %></div>
+  </body>
+</html>

rb/build_dist.rb

@@ -0,0 +1,129 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "fileutils"
+require "digest/md5"
+
+if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
+  VERSION = $1
+else
+  raise "Could not determine version."
+end
+PROG_NAME = "propane"
+START_FILE = "bin/#{PROG_NAME}"
+LIB_DIR = "lib"
+DIST = "dist"
+
+ASSETS_TEMPLATE = <<EOF
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        case name
+<% Dir.glob("assets/*").each  do |asset_file| %>
+        when <%= File.basename(asset_file).inspect %>
+          <%= File.binread(asset_file).inspect %>
+<% end  %>
+        end
+      end
+    end
+  end
+end
+EOF
+
+assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
+files_processed = {}
+combined_file = []
+
+combine_files = lambda do |file|
+  File.read(file, mode: "rb").each_line do |line|
+    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
+      require_name = $1
+      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
+        path = "#{LIB_DIR}/#{require_name}.rb"
+        if File.exist?(path)
+          unless files_processed[path]
+            files_processed[path] = true
+            if require_name == "propane/assets"
+              combined_file << assets_module
+            else
+              combine_files[path]
+            end
+          end
+        else
+          raise "require path #{path.inspect} not found"
+        end
+      else
+        combined_file << line
+      end
+    else
+      combined_file << line
+    end
+  end
+end
+
+combine_files[START_FILE]
+
+# Strip Ruby comment lines and empty lines to save some space, but do not
+# remove lines that are in heredoc sections. This isn't terribly robust to be
+# used in the wild, but works for the heredoc instances for this project.
+stripped = []
+heredoc_end = nil
+combined_file.each do |line|
+  if line =~ /<<-?([A-Z]+)/
+    heredoc_end = $1
+  end
+  if heredoc_end and line =~ /^\s*#{heredoc_end}/
+    heredoc_end = nil
+  end
+  if line !~ /#\sspecs/
+    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
+      stripped << line
+    end
+  end
+end
+
+license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
+  if line.size > 0
+    "# #{line}"
+  else
+    "#"
+  end
+end
+
+require "zlib"
+require "base64"
+compressed_script = Zlib::Deflate.deflate(stripped.join)
+encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
+hash = Digest::MD5.hexdigest(encoded_compressed_script)
+
+FileUtils.rm_rf(DIST)
+FileUtils.mkdir_p(DIST)
+File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
+  fh.write(<<EOF)
+#!/usr/bin/env ruby
+
+#{license}
+
+script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
+unless File.exist?(script)
+  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
+    require "zlib"
+    require "base64"
+    encoded_compressed = $1
+    unescaped_compressed = Base64.decode64(encoded_compressed)
+    inflated = Zlib::Inflate.inflate(unescaped_compressed)
+    File.open(script, "wb") do |fh|
+      fh.write(inflated)
+    end
+  else
+    raise "Could not decompress."
+  end
+end
+load script
+if __FILE__ == $0
+  Propane::CLI.new.run(ARGV)
+end
+#==>#{encoded_compressed_script}
+EOF
+end

rb/gen_user_guide.rb

@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "fileutils"
+require "redcarpet"
+require "syntax"
+require "syntax/convertors/html"
+require "propane/version"
+
+def load_file(file_name)
+  contents = File.read(file_name)
+  contents.gsub(/\$\{include (.*?)\}/) do |match|
+    include_file_name = $1
+    File.read(include_file_name)
+  end
+end
+
+class Generator
+  class Section
+    attr_reader :number
+    attr_reader :title
+    attr_reader :page
+    attr_reader :contents
+    attr_reader :anchor
+    def initialize(number, title, page, anchor)
+      @number = number
+      @title = title.gsub(/[`]/, "")
+      @page = page
+      @anchor = anchor
+      @contents = ""
+    end
+    def append(contents)
+      @contents += contents
+    end
+  end
+
+  class Page
+    attr_reader :name
+    attr_reader :title
+    attr_accessor :contents
+    def initialize(name, title, contents = "")
+      @name = name
+      @title = title
+      @contents = contents
+    end
+  end
+
+  def initialize(input, output_file, multi_page)
+    current_page =
+      if multi_page
+        nil
+      else
+        File.basename(output_file).sub(/\.html$/, "")
+      end
+    @sections = []
+    @current_section_number = [0]
+    @lines = input.lines
+    while @lines.size > 0
+      line = @lines.slice!(0)
+      if line =~ /^```(.*)$/
+        @sections.last.append(gather_code_section($1))
+      elsif line.chomp == "${remove}"
+        remove_section
+      elsif line =~ /^(#+)(>)?\s*(.*)$/
+        level_text, new_page_text, title_text = $1, $2, $3
+        level = $1.size
+        new_page = !new_page_text.nil?
+        section_number = get_next_section_number(level)
+        anchor = make_anchor(section_number, title_text)
+        if multi_page and (new_page or current_page.nil?)
+          current_page = anchor
+        end
+        @sections << Section.new(section_number, title_text, current_page, anchor)
+        @sections.last.append("#{level_text} #{section_number} #{title_text}")
+      elsif @sections.size > 0
+        @sections.last.append(line)
+      end
+    end
+
+    renderer = Redcarpet::Render::HTML.new
+    @markdown_renderer = Redcarpet::Markdown.new(renderer)
+    changelog = @markdown_renderer.render(File.read("CHANGELOG.md"))
+
+    @pages = [Page.new("toc", "Table of Contents", render_toc)]
+    @sections.each do |section|
+      unless @pages.last.name == section.page
+        @pages << Page.new(section.page, "#{section.number} #{section.title}")
+      end
+      @pages.last.contents += render_section(section)
+    end
+
+    @pages.each do |page|
+      page.contents.gsub!("${changelog}", changelog)
+      page.contents.gsub!(%r{\$\{#(.+?)\}}) do |match|
+        section_name = $1
+        href = get_link_to_section(section_name)
+        %[<a href="#{href}">#{section_name}</a>]
+      end
+    end
+
+    template = File.read("rb/assets/user_guide.html.erb")
+    erb = ERB.new(template, trim_mode: "<>")
+
+    if multi_page
+      @pages.each_with_index do |page, page_index|
+        subpage_title = " - #{page.title}"
+        page_nav_bar = render_page_nav_bar(page_index)
+        content = page_nav_bar + separator + page.contents + separator + page_nav_bar
+        html_result = erb.result(binding.clone)
+        File.open(File.join(output_file, "#{page.name}.html"), "w") do |fh|
+          fh.write(html_result)
+        end
+      end
+    else
+      subpage_title = ""
+      content = @pages.reduce("") do |result, page|
+        result + page.contents
+      end
+      html_result = erb.result(binding.clone)
+      File.open(output_file, "w") do |fh|
+        fh.write(html_result)
+      end
+    end
+  end
+
+  def separator
+    %[<div class="separator"></div>]
+  end
+
+  def render_page_nav_bar(page_index)
+    page_nav_prev =
+      if page_index > 1
+        %[<a href="#{@pages[page_index - 1].name}.html">&laquo; Prev<br/>#{@pages[page_index - 1].title}</a>]
+      else
+        ""
+      end
+    page_nav_toc =
+      if page_index > 0
+        %[<a href="toc.html">Table of Contents</a>]
+      else
+        ""
+      end
+    page_nav_next =
+      if page_index < @pages.size - 1
+        %[<a href="#{@pages[page_index + 1].name}.html">Next &raquo<br/>#{@pages[page_index + 1].title}</a>]
+      else
+        ""
+      end
+    %[<table class="page_nav"><tr>] + \
+    %[<td class="page_nav_prev">#{page_nav_prev}</td>] + \
+    %[<td class="page_nav_toc">#{page_nav_toc}</td>] + \
+    %[<td class="page_nav_next">#{page_nav_next}</td>] + \
+    %[</tr></table>]
+  end
+
+  def render_toc
+    toc_content = %[<h1>Table of Contents</h1>\n]
+    @sections.each do |section|
+      indent = section.number.split(".").size - 1
+      toc_content += %[<span style="padding-left: #{4 * indent}ex;">]
+      toc_content += %[<a href="#{section.page}.html##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
+      toc_content += %[</span>]
+    end
+    toc_content
+  end
+
+  def render_section(section)
+    %[<a name="#{section.anchor}" />] + \
+      @markdown_renderer.render(section.contents)
+  end
+
+  def make_anchor(section_number, section_title)
+    "s" + ("#{section_number} #{section_title}").gsub(/[^a-zA-Z0-9]/, "_")
+  end
+
+  def get_link_to_section(section_name)
+    section = @sections.find do |section|
+      section.title == section_name
+    end
+    raise "Could not find section #{section_name}" unless section
+    "#{section.page}.html##{section.anchor}"
+  end
+
+  def gather_code_section(syntax)
+    code = ""
+    loop do
+      line = @lines.slice!(0)
+      if line =~ /^```/
+        break
+      end
+      code += line
+    end
+    if syntax != ""
+      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
+      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
+    else
+      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
+    end
+  end
+
+  def remove_section
+    loop do
+      line = @lines.slice!(0)
+      if line.chomp == "${/remove}"
+        break
+      end
+    end
+  end
+
+  def get_next_section_number(level)
+    if @current_section_number.size == level - 1
+      @current_section_number << 1
+    elsif @current_section_number.size >= level
+      @current_section_number[level - 1] += 1
+      @current_section_number.slice!(level, @current_section_number.size)
+    else
+      raise "Section level change from #{@current_section_number.size} to #{level}"
+    end
+    @current_section_number.join(".")
+  end
+end
+
+input = load_file("doc/user_guide.md")
+FileUtils.rm_rf("gen/user_guide")
+FileUtils.mkdir_p("gen/user_guide")
+Generator.new(input, "gen/user_guide/user_guide.html", false)
+Generator.new(input, "gen/user_guide", true)

spec/json_parser.c.propane

@@ -0,0 +1,183 @@
+<<header
+#include "json_types.h"
+#include "testutils.h"
+>>
+<<
+#include "math.h"
+#include <stdbool.h>
+static str_t string_value;
+>>
+
+ptype JSONValue *;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n = 0.0;
+    bool negative = false;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative = false;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp = 0.0;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = JSONValue_new(JSON_NUMBER);
+    $$->number = n;
+>>
+token true <<
+  $$ = JSONValue_new(JSON_TRUE);
+>>
+token false <<
+  $$ = JSONValue_new(JSON_FALSE);
+>>
+token null <<
+  $$ = JSONValue_new(JSON_NULL);
+>>
+/"/ <<
+  $mode(string);
+  str_init(&string_value, "");
+>>
+string: token string /"/ <<
+  $$ = JSONValue_new(JSON_STRING);
+  $$->string = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  str_append(&string_value, "\"");
+>>
+string: /\\\\/ <<
+  str_append(&string_value, "\\");
+>>
+string: /\\\// <<
+  str_append(&string_value, "/");
+>>
+string: /\\b/ <<
+  str_append(&string_value, "\b");
+>>
+string: /\\f/ <<
+  str_append(&string_value, "\f");
+>>
+string: /\\n/ <<
+  str_append(&string_value, "\n");
+>>
+string: /\\r/ <<
+  str_append(&string_value, "\r");
+>>
+string: /\\t/ <<
+  str_append(&string_value, "\t");
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  str_append(&string_value, s);
+>>
+string: /[^\\]/ <<
+  char s[] = {match[0], 0};
+  str_append(&string_value, s);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONObject_new();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
+  $$ = $1;
+>>
+KeyValue -> string colon Value <<
+  $$ = JSONObject_new();
+  JSONObject_append($$, str_cstr(&$1->string), $3);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONArray_new();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  JSONArray_append($1, $3);
+  $$ = $1;
+>>

spec/json_parser.d.propane

@@ -0,0 +1,179 @@
+<<
+import std.math;
+import json_types;
+private string string_value;
+>>
+
+ptype JSONValue;
+ptype array = JSONValue[];
+ptype dict = JSONValue[string];
+ptype string = string;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n;
+    bool negative;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = new JSONNumber(n);
+>>
+token true <<
+  $$ = new JSONTrue();
+>>
+token false <<
+  $$ = new JSONFalse();
+>>
+token null <<
+  $$ = new JSONNull();
+>>
+/"/ <<
+  $mode(string);
+  string_value = "";
+>>
+string: token string (string) /"/ <<
+  $$ = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  string_value ~= "\"";
+>>
+string: /\\\\/ <<
+  string_value ~= "\\";
+>>
+string: /\\\// <<
+  string_value ~= "/";
+>>
+string: /\\b/ <<
+  string_value ~= "\b";
+>>
+string: /\\f/ <<
+  string_value ~= "\f";
+>>
+string: /\\n/ <<
+  string_value ~= "\n";
+>>
+string: /\\r/ <<
+  string_value ~= "\r";
+>>
+string: /\\t/ <<
+  string_value ~= "\t";
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  string_value ~= "{" ~ match[2..6] ~ "}";
+>>
+string: /[^\\]/ <<
+  string_value ~= match;
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = new JSONString($1);
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = new JSONObject();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = new JSONObject($2);
+>>
+KeyValues (dict) -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  foreach (key, value; $3)
+  {
+    $1[key] = value;
+  }
+  $$ = $1;
+>>
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+Array -> lbracket rbracket <<
+  $$ = new JSONArray();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = new JSONArray($2);
+>>
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>

spec/json_types.c

@@ -0,0 +1,64 @@
+#include "json_types.h"
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+JSONValue * JSONValue_new(size_t id)
+{
+    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    jv->id = id;
+    return jv;
+}
+
+JSONValue * JSONObject_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_OBJECT);
+    jv->object.size = 0u;
+    return jv;
+}
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
+{
+    size_t const size = object->object.size;
+    for (size_t i = 0u; i < size; i++)
+    {
+        if (strcmp(name, object->object.entries[i].name) == 0)
+        {
+            object->object.entries[i].value = value;
+            return;
+        }
+    }
+    size_t const new_size = size + 1;
+    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    if (size > 0)
+    {
+        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
+        free(object->object.entries);
+    }
+    object->object.entries = new_entries;
+    object->object.entries[size].name = name;
+    object->object.entries[size].value = value;
+    object->object.size = new_size;
+}
+
+JSONValue * JSONArray_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_ARRAY);
+    jv->array.size = 0u;
+    return jv;
+}
+
+void JSONArray_append(JSONValue * array, JSONValue * value)
+{
+    size_t const size = array->array.size;
+    size_t const new_size = size + 1;
+    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    if (array->array.size > 0)
+    {
+        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
+        free(array->array.entries);
+    }
+    array->array.entries = new_entries;
+    array->array.entries[size] = value;
+    array->array.size = new_size;
+}

spec/json_types.d

@@ -0,0 +1,63 @@
+class JSONValue
+{
+}
+
+class JSONObject : JSONValue
+{
+    JSONValue[string] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[string] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONArray : JSONValue
+{
+    JSONValue[] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONNumber : JSONValue
+{
+    double value;
+
+    this(double value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONString : JSONValue
+{
+    string value;
+
+    this(string value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONTrue : JSONValue
+{
+}
+
+class JSONFalse : JSONValue
+{
+}
+
+class JSONNull : JSONValue
+{
+}

spec/json_types.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <stddef.h>
+#include "testutils.h"
+
+#define JSON_OBJECT 0u
+#define JSON_ARRAY 1u
+#define JSON_NUMBER 2u
+#define JSON_STRING 3u
+#define JSON_TRUE 4u
+#define JSON_FALSE 5u
+#define JSON_NULL 6u
+
+typedef struct JSONValue_s
+{
+    size_t id;
+    union
+    {
+        struct
+        {
+            size_t size;
+            struct
+            {
+                char const * name;
+                struct JSONValue_s * value;
+            } * entries;
+        } object;
+        struct
+        {
+            size_t size;
+            struct JSONValue_s ** entries;
+        } array;
+        double number;
+        str_t string;
+    };
+} JSONValue;
+
+JSONValue * JSONValue_new(size_t id);
+
+JSONValue * JSONObject_new(void);
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
+
+JSONValue * JSONArray_new(void);
+
+void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane/code_point_range_spec.rb

@@ -54,10 +54,10 @@ class Propane
            CodePointRange.new(200, 300)])).to eq [
              CodePointRange.new(0, 59),
              CodePointRange.new(102, 199),
-             CodePointRange.new(301, 0xFFFFFFFF)]
+             CodePointRange.new(301, 0x7FFFFFFF)]
         expect(CodePointRange.invert_ranges(
           [CodePointRange.new(0, 500),
-           CodePointRange.new(7000, 0xFFFFFFFF)])).to eq [
+           CodePointRange.new(7000, 0x7FFFFFFF)])).to eq [
              CodePointRange.new(501, 6999)]
       end
     end

spec/propane/grammar_spec.rb

@@ -0,0 +1,231 @@
+class Propane
+  describe Grammar do
+    it "parses a user grammar" do
+      input = <<EOF
+# Comment line
+
+module a.b;
+ptype   XYZ  *  ;
+
+token while;
+
+token id
+  /[a-zA-Z_][a-zA-Z_0-9]*/;
+
+token token_with_code <<
+Code for the token
+>>
+
+tokenid token_with_no_pattern;
+
+drop /\\s+/;
+
+A -> B <<
+  a = 42;
+>>
+B -> C while id;
+B -> <<
+  b = 0;
+>>
+EOF
+      grammar = Grammar.new(input)
+      expect(grammar.modulename).to eq "a.b"
+      expect(grammar.ptype).to eq "XYZ  *"
+      expect(grammar.ptypes).to eq("default" => "XYZ  *")
+      expect(grammar.prefix).to eq "p_"
+
+      o = grammar.tokens.find {|token| token.name == "while"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 6
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "while"
+      expect(o.line_number).to eq 6
+      expect(o.code).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "id"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 9
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
+      expect(o.line_number).to eq 9
+      expect(o.code).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "token_with_code"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 11
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "token_with_code"
+      expect(o.line_number).to eq 11
+      expect(o.code).to eq "Code for the token\n"
+
+      o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 15
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 17
+      expect(o.token).to be_nil
+      expect(o.code).to be_nil
+
+      expect(grammar.rules.size).to eq 3
+
+      o = grammar.rules[0]
+      expect(o.name).to eq "A"
+      expect(o.components).to eq %w[B]
+      expect(o.line_number).to eq 19
+      expect(o.code).to eq "  a = 42;\n"
+
+      o = grammar.rules[1]
+      expect(o.name).to eq "B"
+      expect(o.components).to eq %w[C while id]
+      expect(o.line_number).to eq 22
+      expect(o.code).to be_nil
+
+      o = grammar.rules[2]
+      expect(o.name).to eq "B"
+      expect(o.components).to eq []
+      expect(o.line_number).to eq 23
+      expect(o.code).to eq "  b = 0;\n"
+    end
+
+    it "parses code segments with semicolons" do
+      input = <<EOF
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input)
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 1
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq "  a = b;\n  return c;\n"
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 6
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[  writeln("Hello there");\n]
+    end
+
+    it "supports mode labels" do
+      input = <<EOF
+token a;
+m1: token b;
+/foo/ <<
+>>
+m2: /bar/ <<
+>>
+drop /q/;
+m3: drop /r/;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "a"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "b"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m1"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m2"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m3"
+    end
+
+    it "allows assigning ptypes to tokens and rules" do
+      input = <<EOF
+ptype Subnode *;
+ptype string = char *;
+ptype integer = int;
+ptype node = Node *;
+
+token abc(string);
+token bar;
+tokenid int(integer);
+
+/xyz/ (string) <<
+>>
+
+/z28/ <<
+>>
+
+Start (node) -> R;
+R -> abc int;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "abc"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.tokens.find {|token| token.name == "bar"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "int"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "integer"
+
+      o = grammar.rules.find {|rule| rule.name == "Start"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "node"
+
+      o = grammar.rules.find {|rule| rule.name == "R"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "xyz"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "z28"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+    end
+  end
+end

spec/propane/lexer/dfa_spec.rb

@@ -35,7 +35,8 @@ class TestLexer
       end
     end
     if last_accepts
-      [last_accepts.name, last_s]
+      name = last_accepts.token ? last_accepts.token.name : nil
+      [name, last_s]
     end
   end
 
@@ -50,8 +51,8 @@ class TestLexer
 end
 
 def run(grammar, input)
-  propane = Propane.new(grammar)
-  token_dfa = Propane::Lexer::DFA.new(propane.instance_variable_get(:@tokens))
+  grammar = Propane::Grammar.new(grammar)
+  token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)
 end
@@ -59,7 +60,7 @@ end
 describe Propane::Lexer::DFA do
   it "lexes a simple token" do
     expect(run(<<EOF, "foo")).to eq [["foo", "foo"]]
-token foo
+token foo;
 EOF
   end
 
@@ -69,8 +70,8 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo
-token bar
+token foo;
+token bar;
 EOF
   end
 
@@ -79,17 +80,17 @@ EOF
       ["identifier", "foobar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo
-token bar
-token identifier [a-z]+
+token foo;
+token bar;
+token identifier /[a-z]+/;
 EOF
     expected = [
       ["plusplus", "++"],
       ["plus", "+"],
     ]
     expect(run(<<EOF, "+++")).to eq expected
-token plus \\+
-token plusplus \\+\\+
+token plus /\\+/;
+token plusplus /\\+\\+/;
 EOF
   end
 
@@ -100,9 +101,9 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo
-token bar
-token WS \\s+
+token foo;
+token bar;
+token WS /\\s+/;
 EOF
   end
 
@@ -113,9 +114,18 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo
-token bar
-drop \\s+
+token foo;
+token bar;
+drop /\\s+/;
+EOF
+  end
+
+  it "matches a semicolon" do
+    expected = [
+      ["semicolon", ";"],
+    ]
+    expect(run(<<EOF, ";")).to eq expected
+token semicolon /;/;
 EOF
   end
 end

spec/spec_helper.rb

@@ -1,11 +1,27 @@
-require "bundler/setup"
-require "propane"
+unless ENV["dist_specs"]
+  require "bundler/setup"
+  require "simplecov"
 
-RSpec.configure do |config|
-  # Enable flags like --only-failures and --next-failure
-  config.example_status_persistence_file_path = ".rspec_status"
+  SimpleCov.start do
+    add_filter "/spec/"
+    add_filter "/.bundle/"
+    if ENV["partial_specs"]
+      command_name "RSpec-partial"
+    else
+      command_name "RSpec"
+    end
+    project_name "Propane"
+    merge_timeout 3600
+  end
 
-  config.expect_with :rspec do |c|
-    c.syntax = :expect
+  RSpec.configure do |config|
+    # Enable flags like --only-failures and --next-failure
+    config.example_status_persistence_file_path = ".rspec_status"
+
+    config.expect_with :rspec do |c|
+      c.syntax = :expect
+    end
   end
 end
+
+require "propane"

spec/test_ast.c

@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    Items * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    ItemsMore * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}

spec/test_ast.d

@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    Items * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    ItemsMore * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}

spec/test_ast_field_aliases.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "\na\nb\nc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(TOKEN_a, start->first->pToken->token);
+    assert_eq(TOKEN_b, start->second->pToken->token);
+    assert_eq(TOKEN_c, start->third->pToken->token);
+
+    return 0;
+}

spec/test_ast_field_aliases.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\na\nb\nc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(TOKEN_a, start.first.pToken.token);
+    assert_eq(TOKEN_b, start.second.pToken.token);
+    assert_eq(TOKEN_c, start.third.pToken.token);
+}

spec/test_ast_invalid_positions.c

@@ -0,0 +1,102 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "\na\n  bb ccc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(2, start->pT1->pA->position.col);
+    assert_eq(2, start->pT1->pA->end_position.row);
+    assert_eq(7, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(0, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(7, start->pT1->end_position.col);
+
+    assert_eq(1, start->position.row);
+    assert_eq(0, start->position.col);
+    assert_eq(2, start->end_position.row);
+    assert_eq(7, start->end_position.col);
+
+    input = "a\nbb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT1->pToken->end_position.row);
+    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(1, start->pT1->pA->position.row);
+    assert_eq(0, start->pT1->pA->position.col);
+    assert_eq(1, start->pT1->pA->end_position.row);
+    assert_eq(1, start->pT1->pA->end_position.col);
+    assert_eq(0, start->pT1->position.row);
+    assert_eq(0, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
+
+    assert_eq(0, start->position.row);
+    assert_eq(0, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(1, start->end_position.col);
+
+    input = "a\nc\nc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT1->pToken->end_position.row);
+    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(1, start->pT1->pA->position.row);
+    assert_eq(0, start->pT1->pA->position.col);
+    assert_eq(2, start->pT1->pA->end_position.row);
+    assert_eq(0, start->pT1->pA->end_position.col);
+    assert_eq(0, start->pT1->position.row);
+    assert_eq(0, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(0, start->pT1->end_position.col);
+
+    assert_eq(0, start->position.row);
+    assert_eq(0, start->position.col);
+    assert_eq(2, start->end_position.row);
+    assert_eq(0, start->end_position.col);
+
+    input = "a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT1->pToken->end_position.row);
+    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert(!p_position_valid(start->pT1->pA->position));
+    assert_eq(0, start->pT1->position.row);
+    assert_eq(0, start->pT1->position.col);
+    assert_eq(0, start->pT1->end_position.row);
+    assert_eq(0, start->pT1->end_position.col);
+
+    assert_eq(0, start->position.row);
+    assert_eq(0, start->position.col);
+    assert_eq(0, start->end_position.row);
+    assert_eq(0, start->end_position.col);
+
+    return 0;
+}

spec/test_ast_invalid_positions.d

@@ -0,0 +1,104 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\na\n  bb ccc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(2, start.pT1.pA.position.col);
+    assert_eq(2, start.pT1.pA.end_position.row);
+    assert_eq(7, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(0, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(7, start.pT1.end_position.col);
+
+    assert_eq(1, start.position.row);
+    assert_eq(0, start.position.col);
+    assert_eq(2, start.end_position.row);
+    assert_eq(7, start.end_position.col);
+
+    input = "a\nbb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT1.pToken.end_position.row);
+    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(1, start.pT1.pA.position.row);
+    assert_eq(0, start.pT1.pA.position.col);
+    assert_eq(1, start.pT1.pA.end_position.row);
+    assert_eq(1, start.pT1.pA.end_position.col);
+    assert_eq(0, start.pT1.position.row);
+    assert_eq(0, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
+
+    assert_eq(0, start.position.row);
+    assert_eq(0, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(1, start.end_position.col);
+
+    input = "a\nc\nc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT1.pToken.end_position.row);
+    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(1, start.pT1.pA.position.row);
+    assert_eq(0, start.pT1.pA.position.col);
+    assert_eq(2, start.pT1.pA.end_position.row);
+    assert_eq(0, start.pT1.pA.end_position.col);
+    assert_eq(0, start.pT1.position.row);
+    assert_eq(0, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(0, start.pT1.end_position.col);
+
+    assert_eq(0, start.position.row);
+    assert_eq(0, start.position.col);
+    assert_eq(2, start.end_position.row);
+    assert_eq(0, start.end_position.col);
+
+    input = "a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT1.pToken.end_position.row);
+    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert(!start.pT1.pA.position.valid);
+    assert_eq(0, start.pT1.position.row);
+    assert_eq(0, start.pT1.position.col);
+    assert_eq(0, start.pT1.end_position.row);
+    assert_eq(0, start.pT1.end_position.col);
+
+    assert_eq(0, start.position.row);
+    assert_eq(0, start.position.col);
+    assert_eq(0, start.end_position.row);
+    assert_eq(0, start.end_position.col);
+}

spec/test_ast_ps.c

@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    PItemsS * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    PItemsMoreS * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}

spec/test_ast_ps.d

@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    PItemsS * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    PItemsMoreS * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}

spec/test_ast_token_positions.c

@@ -0,0 +1,84 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "abbccc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT1->pToken->end_position.row);
+    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(0, start->pT1->position.row);
+    assert_eq(0, start->pT1->position.col);
+    assert_eq(0, start->pT1->end_position.row);
+    assert_eq(0, start->pT1->end_position.col);
+
+    assert_eq(0, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(0, start->pT2->pToken->end_position.row);
+    assert_eq(2, start->pT2->pToken->end_position.col);
+    assert_eq(0, start->pT2->position.row);
+    assert_eq(1, start->pT2->position.col);
+    assert_eq(0, start->pT2->end_position.row);
+    assert_eq(2, start->pT2->end_position.col);
+
+    assert_eq(0, start->pT3->pToken->position.row);
+    assert_eq(3, start->pT3->pToken->position.col);
+    assert_eq(0, start->pT3->pToken->end_position.row);
+    assert_eq(5, start->pT3->pToken->end_position.col);
+    assert_eq(0, start->pT3->position.row);
+    assert_eq(3, start->pT3->position.col);
+    assert_eq(0, start->pT3->end_position.row);
+    assert_eq(5, start->pT3->end_position.col);
+
+    assert_eq(0, start->position.row);
+    assert_eq(0, start->position.col);
+    assert_eq(0, start->end_position.row);
+    assert_eq(5, start->end_position.col);
+
+    input = "\n\n  bb\nc\ncc\n\n     a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(2, start->pT1->pToken->position.col);
+    assert_eq(2, start->pT1->pToken->end_position.row);
+    assert_eq(3, start->pT1->pToken->end_position.col);
+    assert_eq(2, start->pT1->position.row);
+    assert_eq(2, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(3, start->pT1->end_position.col);
+
+    assert_eq(3, start->pT2->pToken->position.row);
+    assert_eq(0, start->pT2->pToken->position.col);
+    assert_eq(4, start->pT2->pToken->end_position.row);
+    assert_eq(1, start->pT2->pToken->end_position.col);
+    assert_eq(3, start->pT2->position.row);
+    assert_eq(0, start->pT2->position.col);
+    assert_eq(4, start->pT2->end_position.row);
+    assert_eq(1, start->pT2->end_position.col);
+
+    assert_eq(6, start->pT3->pToken->position.row);
+    assert_eq(5, start->pT3->pToken->position.col);
+    assert_eq(6, start->pT3->pToken->end_position.row);
+    assert_eq(5, start->pT3->pToken->end_position.col);
+    assert_eq(6, start->pT3->position.row);
+    assert_eq(5, start->pT3->position.col);
+    assert_eq(6, start->pT3->end_position.row);
+    assert_eq(5, start->pT3->end_position.col);
+
+    assert_eq(2, start->position.row);
+    assert_eq(2, start->position.col);
+    assert_eq(6, start->end_position.row);
+    assert_eq(5, start->end_position.col);
+
+    return 0;
+}

spec/test_ast_token_positions.d

@@ -0,0 +1,86 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abbccc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT1.pToken.end_position.row);
+    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(0, start.pT1.position.row);
+    assert_eq(0, start.pT1.position.col);
+    assert_eq(0, start.pT1.end_position.row);
+    assert_eq(0, start.pT1.end_position.col);
+
+    assert_eq(0, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(0, start.pT2.pToken.end_position.row);
+    assert_eq(2, start.pT2.pToken.end_position.col);
+    assert_eq(0, start.pT2.position.row);
+    assert_eq(1, start.pT2.position.col);
+    assert_eq(0, start.pT2.end_position.row);
+    assert_eq(2, start.pT2.end_position.col);
+
+    assert_eq(0, start.pT3.pToken.position.row);
+    assert_eq(3, start.pT3.pToken.position.col);
+    assert_eq(0, start.pT3.pToken.end_position.row);
+    assert_eq(5, start.pT3.pToken.end_position.col);
+    assert_eq(0, start.pT3.position.row);
+    assert_eq(3, start.pT3.position.col);
+    assert_eq(0, start.pT3.end_position.row);
+    assert_eq(5, start.pT3.end_position.col);
+
+    assert_eq(0, start.position.row);
+    assert_eq(0, start.position.col);
+    assert_eq(0, start.end_position.row);
+    assert_eq(5, start.end_position.col);
+
+    input = "\n\n  bb\nc\ncc\n\n     a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(2, start.pT1.pToken.position.col);
+    assert_eq(2, start.pT1.pToken.end_position.row);
+    assert_eq(3, start.pT1.pToken.end_position.col);
+    assert_eq(2, start.pT1.position.row);
+    assert_eq(2, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(3, start.pT1.end_position.col);
+
+    assert_eq(3, start.pT2.pToken.position.row);
+    assert_eq(0, start.pT2.pToken.position.col);
+    assert_eq(4, start.pT2.pToken.end_position.row);
+    assert_eq(1, start.pT2.pToken.end_position.col);
+    assert_eq(3, start.pT2.position.row);
+    assert_eq(0, start.pT2.position.col);
+    assert_eq(4, start.pT2.end_position.row);
+    assert_eq(1, start.pT2.end_position.col);
+
+    assert_eq(6, start.pT3.pToken.position.row);
+    assert_eq(5, start.pT3.pToken.position.col);
+    assert_eq(6, start.pT3.pToken.end_position.row);
+    assert_eq(5, start.pT3.pToken.end_position.col);
+    assert_eq(6, start.pT3.position.row);
+    assert_eq(5, start.pT3.position.col);
+    assert_eq(6, start.pT3.end_position.row);
+    assert_eq(5, start.pT3.end_position.col);
+
+    assert_eq(2, start.position.row);
+    assert_eq(2, start.position.col);
+    assert_eq(6, start.end_position.row);
+    assert_eq(5, start.end_position.col);
+}

spec/test_basic_math_grammar.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "1 + 2 * 3 + 4";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(11, p_result(&context));
+
+    input = "1 * 2 ** 4 * 3";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(48, p_result(&context));
+
+    input = "(1 + 2) * 3 + 4";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(13, p_result(&context));
+
+    input = "(2 * 2) ** 3 + 4 + 5";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(73, p_result(&context));
+
+    return 0;
+}

spec/test_basic_math_grammar.d

@@ -0,0 +1,32 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "1 + 2 * 3 + 4";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(11, p_result(&context));
+
+    input = "1 * 2 ** 4 * 3";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(48, p_result(&context));
+
+    input = "(1 + 2) * 3 + 4";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(13, p_result(&context));
+
+    input = "(2 * 2) ** 3 + 4 + 5";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(73, p_result(&context));
+}

spec/test_d_lexer.d

@@ -1,66 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    alias DCP = Testparser.Decoder.DecodedCodePoint;
-    string inputstring = "5+\n 66";
-    const(ubyte) * input = cast(const(ubyte) *)inputstring.ptr;
-    size_t input_length = inputstring.length;
-    DCP dcp;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('5', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('+', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('\n', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(' ', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('6', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP('6', 1u));
-    input += dcp.code_point_length;
-    input_length -= dcp.code_point_length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
-
-    inputstring = "\xf0\x9f\xa7\xa1";
-    input = cast(const(ubyte) *)inputstring.ptr;
-    input_length = inputstring.length;
-    dcp = Testparser.Decoder.decode_code_point(input, input_length);
-    assert(dcp == DCP(0x1F9E1, 4u));
-}
-
-unittest
-{
-    alias LT = Testparser.Lexer.LexedToken;
-    string input = "5 + 4 * \n677 + 567";
-    Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
-    assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_INT));
-    assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_PLUS));
-    assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_INT));
-    assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_TIMES));
-    assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_INT));
-    assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_PLUS));
-    assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_INT));
-    assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_EOF));
-
-    lexer = new Testparser.Lexer(null, 0u);
-    assert(lexer.lex_token() == LT(0, 0, 0, Testparser.TOKEN_EOF));
-}

spec/test_error_positions.c

@@ -0,0 +1,42 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a 42";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "a\n123\na  a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 3);
+    assert(p_token(&context) == TOKEN_a);
+
+    input = "12";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 0);
+    assert(p_token(&context) == TOKEN_num);
+
+    input = "a 12\n\nab";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 1);
+
+    input = "a 12\n\na\n\n77\na   \xAA";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_DECODE_ERROR);
+    assert(p_position(&context).row == 5);
+    assert(p_position(&context).col == 4);
+
+    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
+    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+
+    return 0;
+}

spec/test_error_positions.d

@@ -0,0 +1,40 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a 42";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "a\n123\na  a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(2, 3));
+    assert(p_token(&context) == TOKEN_a);
+
+    input = "12";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(0, 0));
+    assert(p_token(&context) == TOKEN_num);
+
+    input = "a 12\n\nab";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    assert(p_position(&context) == p_position_t(2, 1));
+
+    input = "a 12\n\na\n\n77\na   \xAA";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_DECODE_ERROR);
+    assert(p_position(&context) == p_position_t(5, 4));
+
+    assert(p_token_names[TOKEN_a] == "a");
+    assert(p_token_names[TOKEN_num] == "num");
+}

spec/test_field_aliases.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "foo1\nbar2";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    return 0;
+}

spec/test_field_aliases.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "foo1\nbar2";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_lexer.c

@@ -0,0 +1,110 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    size_t result;
+    p_code_point_t code_point;
+    uint8_t code_point_length;
+
+    result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == '5');
+    assert(code_point_length == 1u);
+
+    result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
+    assert(result == P_EOF);
+
+    result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0xA9u);
+    assert(code_point_length == 2u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0x1F9E1u);
+    assert(code_point_length == 4u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+
+    p_token_info_t token_info;
+    char const * input = "5 + 4 * \n677 + 567";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.end_position.row == 0u);
+    assert(token_info.end_position.col == 0u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 2u);
+    assert(token_info.end_position.row == 0u);
+    assert(token_info.end_position.col == 2u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.end_position.row == 0u);
+    assert(token_info.end_position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.end_position.row == 0u);
+    assert(token_info.end_position.col == 6u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_times);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 2u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 8u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 9u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 9u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    p_context_init(&context, (uint8_t const *)"", 0u);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.end_position.row == 0u);
+    assert(token_info.end_position.col == 0u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    return 0;
+}

spec/test_lexer.d

@@ -0,0 +1,69 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    size_t result;
+    p_code_point_t code_point;
+    ubyte code_point_length;
+
+    result = p_decode_code_point("5", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == '5');
+    assert(code_point_length == 1u);
+
+    result = p_decode_code_point("", &code_point, &code_point_length);
+    assert(result == P_EOF);
+
+    result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0xA9u);
+    assert(code_point_length == 2u);
+
+    result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0x1F9E1u);
+    assert(code_point_length == 4u);
+
+    result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point("\xfe", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "5 + 4 * \n677 + 567";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 1, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 2), p_position_t(0, 2), 1, TOKEN_plus));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 4), p_position_t(0, 4), 1, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 6), p_position_t(0, 6), 1, TOKEN_times));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 0), p_position_t(1, 2), 3, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 4), p_position_t(1, 4), 1, TOKEN_plus));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 6), p_position_t(1, 8), 3, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 9), p_position_t(1, 9), 0, TOKEN___EOF));
+
+    p_context_init(&context, "");
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 0, TOKEN___EOF));
+}

spec/test_lexer_match_text.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "identifier_123";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    return 0;
+}

spec/test_lexer_match_text.d

@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `identifier_123`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+}

spec/test_lexer_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc \"a string\" def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc \"abc def\" def";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc "a string" def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc "abc def" def`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_lexer_result_value.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "fabulous";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 8u);
+
+    return 0;
+}

spec/test_lexer_result_value.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `x`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = `fabulous`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 8u);
+}

spec/test_lexer_unknown_character.c

@@ -0,0 +1,18 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+
+    input = "123";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 123u);
+
+    return 0;
+}

spec/test_lexer_unknown_character.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `x`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+
+    input = `123`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 123u);
+}

spec/test_match_backslashes.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_match_backslashes.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_multiple_parsers.c

@@ -0,0 +1,19 @@
+#include "testparsermyp1.h"
+#include "testparsermyp2.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input1 = "a\n1";
+    myp1_context_t context1;
+    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+
+    char const * input2 = "bcb";
+    myp2_context_t context2;
+    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+
+    return 0;
+}

spec/test_multiple_parsers.d

@@ -0,0 +1,21 @@
+import testparsermyp1;
+import testparsermyp2;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input1 = "a\n1";
+    myp1_context_t context1;
+    myp1_context_init(&context1, input1);
+    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+
+    string input2 = "bcb";
+    myp2_context_t context2;
+    myp2_context_init(&context2, input2);
+    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+}

spec/test_optional_rule_component.c

@@ -0,0 +1,22 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
+

spec/test_optional_rule_component.d

@@ -0,0 +1,23 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_optional_rule_component_ast.c

@@ -0,0 +1,42 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->pR == start->pR3);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->pR != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    return 0;
+}
+

spec/test_optional_rule_component_ast.d

@@ -0,0 +1,43 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.pR == start.pR3);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_parser_identical_rules_lookahead.c

@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "aba";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parser_identical_rules_lookahead.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aba";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_parser_rule_from_multiple_states.c

@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 1);
+    assert(context.token == TOKEN___EOF);
+
+    input = "a b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "bb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parser_rule_from_multiple_states.d

@@ -0,0 +1,25 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(0, 1));
+    assert(context.token == TOKEN___EOF);
+
+    input = "a b";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "bb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_parser_rule_user_code.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_parser_rule_user_code.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_parsing_json.c

@@ -0,0 +1,56 @@
+#include "testparser.h"
+#include "json_types.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "{}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_OBJECT);
+
+    input = "[]";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_ARRAY);
+
+    input = "-45.6";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == -45.6);
+
+    input = "2E-2";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == 0.02);
+
+    input = "{\"hi\":true}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    JSONValue * o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(1, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "hi") == 0);
+    assert(o->object.entries[0].value->id == JSON_TRUE);
+
+    input = "{\"ff\": false, \"nn\": null}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(2, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "ff") == 0);
+    assert(o->object.entries[0].value->id == JSON_FALSE);
+    assert(strcmp(o->object.entries[1].name, "nn") == 0);
+    assert(o->object.entries[1].value->id == JSON_NULL);
+
+    return 0;
+}

spec/test_parsing_json.d

@@ -0,0 +1,56 @@
+import testparser;
+import std.stdio;
+import json_types;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = ``;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = `{}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+
+    input = `[]`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONArray)p_result(&context));
+
+    input = `-45.6`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(&context));
+    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+
+    input = `2E-2`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(&context));
+    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+
+    input = `{"hi":true}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+    JSONObject o = cast(JSONObject)p_result(&context);
+    assert(o.value["hi"]);
+    assert(cast(JSONTrue)o.value["hi"]);
+
+    input = `{"ff": false, "nn": null}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+    o = cast(JSONObject)p_result(&context);
+    assert(o.value["ff"]);
+    assert(cast(JSONFalse)o.value["ff"]);
+    assert(o.value["nn"]);
+    assert(cast(JSONNull)o.value["nn"]);
+}

spec/test_parsing_lists.c

@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 16u);
+
+    return 0;
+}

spec/test_parsing_lists.d

@@ -0,0 +1,26 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 16u);
+}

spec/test_pattern.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "defabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_pattern.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = "defabcdef";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_return_token_from_pattern.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "defghidef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_return_token_from_pattern.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "defghidef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_start_rule.c

@@ -0,0 +1,9 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule.d

@@ -0,0 +1,8 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule_ast.c

@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "hi";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
+
+    return 0;
+}

spec/test_start_rule_ast.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "hi";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top.pToken !is null);
+    assert_eq(TOKEN_hi, top.pToken.token);
+}

spec/test_user_code.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abcabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_user_code.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = "abcabcdef";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}