Bump version to 1.0.0

Update README with some user guide contents
Add all task
2023-09-25 20:02:02 -04:00 · 2023-09-25 19:57:49 -04:00 · 2023-09-25 19:51:47 -04:00 · 2023-09-25 19:51:41 -04:00 · 2023-09-25 19:50:13 -04:00 · 2023-09-25 19:38:55 -04:00
103 changed files with 8567 additions and 1317 deletions
@@ -1,9 +1,11 @@
-imbecile
-tags
-*.o
-.*.swp
-*.dep
-tmpl.*
-tests/*/itest.cc
-tests/*/itest.h
-tests/*/test
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/dist/
+/gen/
+/pkg/
+/spec/reports/
+/tmp/
+/.rspec_status
+/spec/run/
@@ -1,3 +0,0 @@
-[submodule "refptr"]
-	path = refptr
-	url = http://github.com/holtrop/refptr.git
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
@@ -0,0 +1,3 @@
+## v1.0.0
+
+- Initial release
@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+gem "rake"
+gem "rspec"
+gem "rdoc"
+gem "redcarpet"
+gem "syntax"
+gem "simplecov"
@@ -0,0 +1,46 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.5.0)
+    docile (1.4.0)
+    psych (5.1.0)
+      stringio
+    rake (13.0.6)
+    rdoc (6.5.0)
+      psych (>= 4.0.0)
+    redcarpet (3.6.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    stringio (3.0.7)
+    syntax (1.2.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  rdoc
+  redcarpet
+  rspec
+  simplecov
+  syntax
+
+BUNDLED WITH
+   2.3.7
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2010-2023 Josh Holtrop
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
@@ -1,61 +0,0 @@
-
-TARGET   := imbecile
-CXXOBJS  := $(patsubst %.cc,%.o,$(wildcard *.cc)) tmpl.o
-CXXDEPS  := $(patsubst %.o,.%.dep,$(CXXOBJS))
-CXXFLAGS := -O2
-DEPS     := $(CXXDEPS)
-OBJS     := $(CXXOBJS)
-LDFLAGS  := -lpcre
-CPPFLAGS := -I$(shell pwd)/refptr
-
-all: submodule_check tmpl.h $(TARGET)
-
-.PHONY: submodule_check
-submodule_check:
-	@if [ ! -e refptr/refptr.h ]; then \
-		echo Error: \"refptr\" folder is not populated.; \
-		echo Perhaps you forgot to do \"git checkout --recursive\"?; \
-		echo You can remedy the situation with \"git submodule update --init\".; \
-		exit 1; \
-	fi
-
-$(TARGET): $(OBJS)
-	$(CXX) -o $@ $^ $(LDFLAGS)
-
-# Object file rules
-%.o: %.cc
-	$(CXX) -c -o $@ $(CPPFLAGS) $(CXXFLAGS) $<
-
-# Make dependency files
-.%.dep: %.c
-	@set -e; rm -f $@; \
-	  $(CC) -MM $(CPPFLAGS) $< | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' > $@
-
-.%.dep: %.cc tmpl.h
-	@set -e; rm -f $@; \
-	  $(CXX) -MM $(CPPFLAGS) $< | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' > $@
-
-tmpl.cc: $(wildcard tmpl/*)
-	echo -n > $@
-	for f in $*/*; \
-		do xxd -i $$f >> $@; \
-	done
-
-tmpl.h: tmpl.cc
-	echo '#ifndef $*_h' > $@
-	echo '#define $*_h' >> $@
-	grep '$*_' $^ | sed -e 's/^/extern /' -e 's/ =.*/;/' >> $@
-	echo '#endif' >> $@
-
-.PHONY: tests
-tests: PATH := $(shell pwd):$(PATH)
-tests: all
-	$(MAKE) -C $@
-
-tests-clean:
-	$(MAKE) -C tests clean
-
-clean: tests-clean
-	-rm -f $(TARGET) *.o .*.dep tmpl.cc tmpl.h
-
-include $(CXXDEPS)
@@ -1,423 +0,0 @@
-
-#include <stdio.h>
-#include <string.h>
-#include <pcre.h>
-#include <ctype.h>                  /* toupper() */
-
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <map>
-
-#include "Parser.h"
-#include "TokenDefinition.h"
-#include "RuleDefinition.h"
-#include "tmpl.h"
-
-using namespace std;
-
-#define DEBUG
-
-Parser::Parser()
-    : m_classname("Parser"), m_namespace(""), m_extension("cc"),
-    m_token_data(new string()), m_token_code(new string()),
-    m_defines(new string())
-{
-}
-
-void Parser::makeDefine(const string & defname, const string & definition)
-{
-    *m_defines += string("#define ") + defname + " " + definition + "\n";
-}
-
-bool Parser::write(const string & fname)
-{
-    if (m_tokens.size() < 1 || m_rules.size() < 1)
-        return false;
-
-    string header_fname = fname + ".h";
-    string body_fname = fname + "." + m_extension;
-
-    ofstream header(header_fname.c_str());
-    ofstream body(body_fname.c_str());
-
-    /* process data */
-    refptr<string> token_classes = new string();
-    refptr<string> token_classes_code = new string();
-    int i = 0;
-    for (list<TokenDefinitionRef>::const_iterator it = m_tokens.begin();
-            it != m_tokens.end();
-            it++)
-    {
-        char buff[20];
-        sprintf(buff, "%d", i++);
-        makeDefine((*it)->getIdentifier(), buff);
-        *token_classes += (*it)->getClassDefinition();
-        *token_classes_code += (*it)->getProcessMethod();
-    }
-    if (m_namespace != "")
-    {
-        makeDefine("I_NAMESPACE", m_namespace);
-    }
-    makeDefine("I_CLASSNAME", m_classname);
-
-    /* set up replacements */
-    setReplacement("token_list", buildTokenList());
-    setReplacement("buildToken", buildBuildToken());
-    setReplacement("header_name",
-            new string(string("\"") + header_fname + "\""));
-    setReplacement("token_code", m_token_code);
-    setReplacement("token_data", m_token_data);
-    setReplacement("defines", m_defines);
-    setReplacement("token_classes", token_classes);
-    setReplacement("token_classes_code", token_classes_code);
-
-    /* write the header */
-    writeTmpl(header, (char *) tmpl_parser_h, tmpl_parser_h_len);
-
-    /* write the body */
-    writeTmpl(body, (char *) tmpl_parser_cc, tmpl_parser_cc_len);
-
-    header.close();
-    body.close();
-
-    return true;
-}
-
-bool Parser::writeTmpl(std::ostream & out, char * dat, int len)
-{
-    char * newline;
-    char * data = dat;
-    const char * errptr;
-    int erroffset;
-    data[len-1] = '\n';
-    const int ovec_size = 6;
-    int ovector[ovec_size];
-    pcre * replace = pcre_compile("{%(\\w+)%}", 0, &errptr, &erroffset, NULL);
-    while (data < (dat + len) && (newline = strstr(data, "\n")) != NULL)
-    {
-        if (pcre_exec(replace, NULL, data, newline - data,
-                    0, 0, ovector, ovec_size) >= 0)
-        {
-            if (ovector[0] > 0)
-            {
-                out.write(data, ovector[0]);
-            }
-            out << *getReplacement(string(data, ovector[2],
-                        ovector[3] - ovector[2]));
-            if (ovector[1] < newline - data)
-            {
-                out.write(data + ovector[1], newline - data - ovector[1]);
-            }
-        }
-        else
-        {
-            out.write(data, newline - data);
-        }
-        out << '\n';
-        data = newline + 1;
-    }
-}
-
-refptr<std::string> Parser::getReplacement(const std::string & name)
-{
-    if (m_replacements.find(name) != m_replacements.end())
-    {
-        return m_replacements[name];
-    }
-#ifdef DEBUG
-    cerr << "No replacement found for \"" << name << "\"" << endl;
-#endif
-    return new string("");
-}
-
-refptr<string> Parser::buildTokenList()
-{
-    refptr<string> tokenlist = new string();
-    for (list<TokenDefinitionRef>::const_iterator t = m_tokens.begin();
-            t != m_tokens.end();
-            t++)
-    {
-        if (t != m_tokens.begin())
-            *tokenlist += "    ";
-        *tokenlist += "{ \"" + (*t)->getName() + "\", \""
-            + (*t)->getCString() + "\", "
-            + ((*t)->getProcessFlag() ? "true" : "false") + " }";
-        if (({typeof(t) tmp = t; ++tmp;}) != m_tokens.end())
-            *tokenlist += ",\n";
-    }
-    return tokenlist;
-}
-
-refptr<string> Parser::buildBuildToken()
-{
-    refptr<string> buildToken = new string();
-    for (list<TokenDefinitionRef>::const_iterator t = m_tokens.begin();
-            t != m_tokens.end();
-            t++)
-    {
-        *buildToken += "case " + (*t)->getIdentifier() + ":\n";
-        *buildToken += "    token = new " + (*t)->getClassName() + "();\n";
-        *buildToken += "    break;\n";
-    }
-    return buildToken;
-}
-
-bool Parser::parseInputFile(char * buff, int size)
-{
-    typedef pcre * pcre_ptr;
-    enum { none, tokens, rules };
-    pcre_ptr empty, comment, section_name, token, rule,
-             data_begin, data_end, code_begin, code_end;
-    struct { pcre_ptr * re; const char * pattern; } exprs[] = {
-        {&empty,        "^\\s*$"},
-        {&comment,      "^\\s*#"},
-        {&section_name, "^\\s*\\[([^\\]]+?)\\]\\s*$"},
-        {&token,        "^\\s*"                     /* possible leading ws */
-                        "([a-zA-Z_][a-zA-Z_0-9]*)"  /* 1: token name */
-                        "\\s+"                      /* required whitespace */
-                        "((?:[^\\\\\\s]|\\\\.)+)"}, /* 2: token RE */
-        {&rule,         "^\\s*(\\S+)\\s*:=(.*)$"},
-        {&data_begin,   "^\\s*\\${"},
-        {&data_end,     "\\$}"},
-        {&code_begin,   "^\\s*%{"},
-        {&code_end,     "%}"}
-    };
-    const int ovec_size = 3 * 10;
-    int ovector[ovec_size];
-    int lineno = 0;
-    char * newline;
-    char * input = buff;
-    string current_section_name;
-    map<string, int> sections;
-    sections["none"] = none;
-    sections["tokens"] = tokens;
-    sections["rules"] = rules;
-    int section = none;
-    string line;
-    bool append_line = false;
-    bool gathering_data = false;
-    bool gathering_code = false;
-    string gather;
-    bool continue_line = false;
-    TokenDefinitionRef current_token;
-
-    for (int i = 0; i < sizeof(exprs)/sizeof(exprs[0]); i++)
-    {
-        const char * errptr;
-        int erroffset;
-        *exprs[i].re = pcre_compile(exprs[i].pattern, 0,
-                &errptr, &erroffset, NULL);
-        if (*exprs[i].re == NULL)
-        {
-            cerr << "Error compiling regex '" << exprs[i].pattern <<
-                "': " << errptr << " at position " << erroffset << endl;
-            return false;
-        }
-    }
-
-    for (;;)
-    {
-        if (continue_line)
-        {
-            continue_line = false;
-        }
-        else
-        {
-            if ((newline = strstr(input, "\n")) == NULL)
-                break;
-            int line_length = newline - input;
-            if (line_length >= 1 && newline[-1] == '\r')
-            {
-                newline[-1] = '\n';
-                line_length--;
-            }
-            lineno++;
-
-            if (append_line)
-            {
-                line += string(input, line_length);
-            }
-            else
-            {
-                line = string(input, line_length);
-            }
-            input = newline + 1;        /* set up for next loop iteration */
-        }
-
-        if ( (pcre_exec(empty, NULL, line.c_str(), line.size(),
-                  0, 0, ovector, ovec_size) >= 0)
-          || (pcre_exec(comment, NULL, line.c_str(), line.size(),
-                  0, 0, ovector, ovec_size) >= 0)
-           )
-        {
-            /* skip empty or comment lines */;
-            continue;
-        }
-
-        if (! (gathering_code || gathering_data) )
-        {
-            if (line.size() > 0 && line[line.size()-1] == '\\')
-            {
-                line[line.size()-1] = ' ';
-                append_line = true;
-                continue;
-            }
-            else
-            {
-                append_line = false;
-            }
-
-            if (pcre_exec(section_name, NULL, line.c_str(), line.size(),
-                        0, 0, ovector, ovec_size) >= 0)
-            {
-                current_section_name
-                    = string(line, ovector[2], ovector[3] - ovector[2]);
-                if (sections.find(current_section_name) != sections.end())
-                {
-                    section = sections[current_section_name];
-                }
-                else
-                {
-                    cerr << "Unknown section name '" << current_section_name
-                        << "'!" << endl;
-                    return false;
-                }
-                continue;
-            }
-        }
-
-        switch (section)
-        {
-            case none:
-                cerr << "Unrecognized input on line " << lineno << endl;
-                return false;
-            case tokens:
-                if      (gathering_data)
-                {
-                    if (pcre_exec(data_end, NULL, line.c_str(), line.size(),
-                                0, 0, ovector, ovec_size) >= 0)
-                    {
-                        gather += string(line, 0, ovector[0]) + "\n";
-                        gathering_data = false;
-                        line = string(line, ovector[1]);
-                        continue_line = true;
-                        if (current_token.isNull())
-                        {
-                            *m_token_data += gather;
-                        }
-                        else
-                        {
-                            current_token->addData(gather);
-                        }
-                    }
-                    else
-                    {
-                        gather += line + "\n";
-                    }
-                    continue;
-                }
-                else if (gathering_code)
-                {
-                    if (pcre_exec(code_end, NULL, line.c_str(), line.size(),
-                                0, 0, ovector, ovec_size) >= 0)
-                    {
-                        gather += string(line, 0, ovector[0]) + "\n";
-                        gathering_code = false;
-                        line = string(line, ovector[1]);
-                        continue_line = true;
-                        if (current_token.isNull())
-                        {
-                            *m_token_code += gather;
-                        }
-                        else
-                        {
-                            current_token->addCode(gather);
-                        }
-                    }
-                    else
-                    {
-                        gather += line + "\n";
-                    }
-                    continue;
-                }
-                else if (pcre_exec(data_begin, NULL, line.c_str(), line.size(),
-                            0, 0, ovector, ovec_size) >= 0)
-                {
-                    gathering_data = true;
-                    gather = "";
-                    line = string(line, ovector[1]);
-                    continue_line = true;
-                    continue;
-                }
-                else if (pcre_exec(code_begin, NULL, line.c_str(), line.size(),
-                            0, 0, ovector, ovec_size) >= 0)
-                {
-                    gathering_code = true;
-                    gather = "";
-                    line = string(line, ovector[1]);
-                    continue_line = true;
-                    continue;
-                }
-                else if (pcre_exec(token, NULL, line.c_str(), line.size(),
-                            0, 0, ovector, ovec_size) >= 0)
-                {
-                    string name(line, ovector[2], ovector[3] - ovector[2]);
-                    string definition(line,
-                            ovector[4], ovector[5] - ovector[4]);
-                    current_token = new TokenDefinition();
-                    if (current_token->create(name, definition))
-                    {
-                        addTokenDefinition(current_token);
-                    }
-                    else
-                    {
-                        cerr << "Error in token definition ending on line "
-                            << lineno << endl;
-                        return false;
-                    }
-                    line = string(line, ovector[1]);
-                    continue_line = true;
-                    continue;
-                }
-                else
-                {
-                    cerr << "Unrecognized input on line " << lineno << endl;
-                    return false;
-                }
-                break;
-            case rules:
-                if (pcre_exec(rule, NULL, line.c_str(), line.size(),
-                            0, 0, ovector, ovec_size) >= 0)
-                {
-                    string name(line, ovector[2], ovector[3] - ovector[2]);
-                    string definition(line,
-                            ovector[4], ovector[5] - ovector[4]);
-                    refptr<RuleDefinition> rd = new RuleDefinition();
-                    if (rd->create(name, definition))
-                    {
-                        addRuleDefinition(rd);
-                    }
-                    else
-                    {
-                        cerr << "Error in rule definition ending on line "
-                            << lineno << endl;
-                        return false;
-                    }
-                }
-                else
-                {
-                    cerr << "Unrecognized input on line " << lineno << endl;
-                    return false;
-                }
-                break;
-        }
-    }
-
-    for (int i = 0; i < sizeof(exprs)/sizeof(exprs[0]); i++)
-    {
-        pcre_free(*exprs[i].re);
-    }
-
-    return true;
-}
@@ -1,61 +0,0 @@
-
-#ifndef PARSER_H
-#define PARSER_H
-
-#include <vector>
-#include <string>
-#include <list>
-#include <map>
-
-#include "refptr.h"
-#include "TokenDefinition.h"
-#include "RuleDefinition.h"
-
-class Parser
-{
-    public:
-        Parser();
-        void addTokenDefinition(refptr<TokenDefinition> td)
-        {
-            m_tokens.push_back(td);
-        }
-        void addRuleDefinition(refptr<RuleDefinition> rd)
-        {
-            m_rules.push_back(rd);
-        }
-        bool write(const std::string & fname);
-        bool parseInputFile(char * buff, int size);
-
-        void setClassName(const std::string & cn) { m_classname = cn; }
-        std::string getClassName() { return m_classname; }
-
-        void setNamespace(const std::string & ns) { m_namespace = ns; }
-        std::string getNamespace() { return m_namespace; }
-
-        void setExtension(const std::string & e) { m_extension = e; }
-        std::string getExtension() { return m_extension; }
-
-    protected:
-        refptr<std::string> buildTokenList();
-        refptr<std::string> buildBuildToken();
-        bool writeTmpl(std::ostream & out, char * dat, int len);
-        refptr<std::string> getReplacement(const std::string & name);
-        void setReplacement(const std::string & name, refptr<std::string> val)
-        {
-            m_replacements[name] = val;
-        }
-        void makeDefine(const std::string & defname,
-                const std::string & definition);
-
-        std::list<TokenDefinitionRef> m_tokens;
-        std::vector< refptr< RuleDefinition > > m_rules;
-        std::string m_classname;
-        std::string m_namespace;
-        std::string m_extension;
-        std::map< std::string, refptr<std::string> > m_replacements;
-        refptr<std::string> m_token_data;
-        refptr<std::string> m_token_code;
-        refptr<std::string> m_defines;
-};
-
-#endif
@@ -1,5 +0,0 @@
-Imbecile is a bottom-up parser generator. It targets C++ and automatically
-generates a class heirarchy for interacting with the parser.
-
-Imbecile generates both a lexer and a parser based on the rules given to
-it in the input file.
@@ -0,0 +1,123 @@
+# The Propane Parser Generator
+
+Propane is a LALR Parser Generator (LPG) which:
+
+  * accepts LR(0), SLR, and LALR grammars
+  * generates a built-in lexer to tokenize input
+  * supports UTF-8 lexer inputs
+  * generates a table-driven shift/reduce parser to parse input in linear time
+  * target C or D language outputs
+  * is MIT-licensed
+  * is distributable as a standalone Ruby script
+
+## Installation
+
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
+
+## Usage
+
+### Command Line Interface
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+### Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+Example grammar file:
+
+```
+<<
+import std.math;
+>>
+
+# Parser values are unsigned integers.
+ptype ulong;
+
+# A few basic arithmetic operators.
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+# Drop whitespace.
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+```
+
+Grammar files can contain comment lines beginning with `#` which are ignored.
+White space in the grammar file is also ignored.
+
+It is convention to use the extension `.propane` for the Propane grammar file,
+however any file name is accepted by Propane.
+
+See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
+
+## Development
+
+After checking out the repository, run `bundle install` to install dependencies.
+Run `rake spec` to execute tests.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/holtrop/propane.
+
+## License
+
+Propane is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
@@ -0,0 +1,32 @@
+require "rake/clean"
+require "rspec/core/rake_task"
+
+CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
+
+task :build_dist do
+  sh "ruby rb/build_dist.rb"
+end
+
+RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
+  if args.example_pattern
+    task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  end
+end
+
+# dspec task is useful to test the distributable release script, but is not
+# useful for coverage information.
+desc "Dist Specs"
+task :dspec, [:example_string] => :build_dist do |task, args|
+  ENV["dist_specs"] = "1"
+  Rake::Task["spec"].execute(args)
+  ENV.delete("dist_specs")
+end
+
+task :default => :spec
+
+desc "Build user guide"
+task :user_guide do
+  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
+end
+
+task :all => [:spec, :dspec, :user_guide]
@@ -1,9 +0,0 @@
-
-#include "RuleDefinition.h"
-
-using namespace std;
-
-bool RuleDefinition::create(const string & name, const string & definition)
-{
-    m_name = name;
-}
@@ -1,16 +0,0 @@
-
-#ifndef RULEDEFINITION_H
-#define RULEDEFINITION_H
-
-#include <string>
-
-class RuleDefinition
-{
-    public:
-        bool create(const std::string & name, const std::string & definition);
-
-    protected:
-        std::string m_name;
-};
-
-#endif
@@ -1,125 +0,0 @@
-
-#include <pcre.h>
-
-#include <iostream>
-#include <string>
-#include <vector>
-
-#include "TokenDefinition.h"
-#include "refptr.h"
-
-using namespace std;
-
-#define WHITESPACE " \n\r\t\v"
-
-static string trim(string s)
-{
-    size_t lastpos = s.find_last_not_of(WHITESPACE);
-    if (lastpos == string::npos)
-        return "";
-    s.erase(lastpos + 1);
-    s.erase(0, s.find_first_not_of(WHITESPACE));
-    return s;
-}
-
-static refptr< vector<string> > split(const string & delim, string str)
-{
-    refptr< vector<string> > ret = new vector<string>();
-    size_t pos;
-    while ( (pos = str.find(delim)) != string::npos )
-    {
-        string t = str.substr(0, pos);
-        ret->push_back(t);
-        str.erase(0, pos + 1);
-    }
-    if (str != "")
-        ret->push_back(str);
-    return ret;
-}
-
-static string c_escape(const string & orig)
-{
-    string result;
-    for (string::const_iterator it = orig.begin(); it != orig.end(); it++)
-    {
-        if (*it == '\\' || *it == '"')
-            result += '\\';
-        result += *it;
-    }
-    return result;
-}
-
-
-TokenDefinition::TokenDefinition()
-    : m_process(false)
-{
-}
-
-bool TokenDefinition::create(const string & name,
-        const string & definition)
-{
-    const char * errptr;
-    int erroffset;
-    pcre * re = pcre_compile(definition.c_str(), 0, &errptr, &erroffset, NULL);
-    if (re == NULL)
-    {
-        cerr << "Error compiling regular expression '" << definition
-            << "' at position " << erroffset << ": " << errptr << endl;
-        return false;
-    }
-    m_name = name;
-    m_definition = definition;
-    pcre_free(re);
-
-#if 0
-    refptr< vector< string > > parts = split(",", flags);
-    for (int i = 0, sz = parts->size(); i < sz; i++)
-    {
-        (*parts)[i] = trim((*parts)[i]);
-        string & s = (*parts)[i];
-        if (s == "p")
-        {
-            m_process = true;
-        }
-        else
-        {
-            cerr << "Unknown token flag \"" << s << "\"" << endl;
-            return false;
-        }
-    }
-#endif
-
-    return true;
-}
-
-string TokenDefinition::getCString() const
-{
-    return c_escape(m_definition);
-}
-
-string TokenDefinition::getClassDefinition() const
-{
-    string ret = "class "+ getClassName() + " : public Token {\n";
-    ret += "public:\n";
-    if (m_process)
-    {
-        ret += "    virtual void process(const Matches & matches);\n";
-    }
-    ret += "\n";
-    ret += "protected:\n";
-    ret += m_data + "\n";
-    ret += "};\n";
-    return ret;
-}
-
-string TokenDefinition::getProcessMethod() const
-{
-    string ret;
-    if (m_code != "")
-    {
-        ret += "void " + getClassName() + "::process(const Matches & matches) {\n";
-        ret += m_code + "\n";
-        ret += "}\n";
-    }
-    return ret;
-}
@@ -1,37 +0,0 @@
-
-#ifndef TOKENDEFINITION_H
-#define TOKENDEFINITION_H
-
-#include <string>
-#include "refptr.h"
-
-class TokenDefinition
-{
-    public:
-        TokenDefinition();
-        bool create(const std::string & name,
-                const std::string & definition);
-        std::string getCString() const;
-        std::string getName() const { return m_name; }
-        bool getProcessFlag() const { return m_process; }
-        void setProcessFlag(bool p) { m_process = p; }
-        void addData(const std::string & d) { m_data += d; }
-        std::string getData() const { return m_data; }
-        void addCode(const std::string & c) { m_code += c; m_process = true; }
-        std::string getCode() const { return m_code; }
-        std::string getClassDefinition() const;
-        std::string getProcessMethod() const;
-        std::string getIdentifier() const { return "TK_" + m_name; }
-        std::string getClassName() const { return "Tk" + m_name; }
-
-    protected:
-        std::string m_name;
-        std::string m_definition;
-        bool m_process;
-        std::string m_data;
-        std::string m_code;
-};
-
-typedef refptr<TokenDefinition> TokenDefinitionRef;
-
-#endif
@@ -0,0 +1,936 @@
+#include "<%= File.basename(output_file).sub(%r{\.[a-z]+$}, "") %>.h"
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**************************************************************************
+ * User code blocks
+ *************************************************************************/
+
+<%= @grammar.code_blocks.fetch("", "") %>
+
+/**************************************************************************
+ * Private types
+ *************************************************************************/
+
+<% if @grammar.prefix.upcase != "P_" %>
+/* Result codes. */
+#define P_SUCCESS 0u
+#define P_DECODE_ERROR 1u
+#define P_UNEXPECTED_INPUT 2u
+#define P_UNEXPECTED_TOKEN 3u
+#define P_DROP 4u
+#define P_EOF 5u
+<% end %>
+
+/* An invalid ID value. */
+#define INVALID_ID ((size_t)-1)
+
+/**************************************************************************
+ * State initialization
+ *************************************************************************/
+
+/**
+ * Initialize lexer/parser context structure.
+ *
+ * @param[out] context
+ *   Lexer/parser context structure.
+ * @param input
+ *   Text input.
+ * @param input_length
+ *   Text input length.
+ */
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
+{
+    /* New default-initialized context structure. */
+    <%= @grammar.prefix %>context_t newcontext = {0};
+
+    /* Lexer initialization. */
+    newcontext.input = input;
+    newcontext.input_length = input_length;
+    newcontext.mode = <%= @lexer.mode_id("default") %>;
+
+    /* Copy to the user's context structure. */
+    *context = newcontext;
+}
+
+/**************************************************************************
+ * Decoder
+ *************************************************************************/
+
+/**
+ * Decode a UTF-8 code point.
+ *
+ * @param input
+ *   Text input to decode.
+ * @param input_length
+ *   Input text length.
+ * @param[out] out_code_point
+ *   The decoded code point is stored here if the return value is P_SUCCESS.
+ * @param[out] out_code_point_length
+ *   The number of bytes the code point used is stored here if the return value
+ *   is P_SUCCESS.
+ *
+ * @retval P_SUCCESS on a successful code point decode
+ * @retval P_DECODE_ERROR when an encoding error is observed
+ * @retval P_EOF when the end of the text input is reached
+ */
+size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
+    <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length)
+{
+    if (input_length == 0u)
+    {
+        return P_EOF;
+    }
+    char c = input[0];
+    <%= @grammar.prefix %>code_point_t code_point;
+    uint8_t code_point_length;
+    if ((c & 0x80u) == 0u)
+    {
+        code_point = c;
+        code_point_length = 1u;
+    }
+    else
+    {
+        uint8_t following_bytes;
+        if ((c & 0xE0u) == 0xC0u)
+        {
+            code_point = c & 0x1Fu;
+            following_bytes = 1u;
+        }
+        else if ((c & 0xF0u) == 0xE0u)
+        {
+            code_point = c & 0x0Fu;
+            following_bytes = 2u;
+        }
+        else if ((c & 0xF8u) == 0xF0u)
+        {
+            code_point = c & 0x07u;
+            following_bytes = 3u;
+        }
+        else if ((c & 0xFCu) == 0xF8u)
+        {
+            code_point = c & 0x03u;
+            following_bytes = 4u;
+        }
+        else if ((c & 0xFEu) == 0xFCu)
+        {
+            code_point = c & 0x01u;
+            following_bytes = 5u;
+        }
+        else
+        {
+            return P_DECODE_ERROR;
+        }
+        if (input_length <= following_bytes)
+        {
+            return P_DECODE_ERROR;
+        }
+        code_point_length = (uint8_t)(following_bytes + 1u);
+        for (size_t i = 0u; i < following_bytes; i++)
+        {
+            char b = input[i + 1u];
+            if ((b & 0xC0u) != 0x80u)
+            {
+                return P_DECODE_ERROR;
+            }
+            code_point = (code_point << 6u) | (b & 0x3Fu);
+        }
+    }
+    *out_code_point = code_point;
+    *out_code_point_length = code_point_length;
+    return P_SUCCESS;
+}
+
+/**************************************************************************
+ * Lexer
+ *************************************************************************/
+
+/** Lexer state ID type. */
+typedef <%= get_type_for(@lexer.state_table.size) %> lexer_state_id_t;
+
+/** Invalid lexer state ID. */
+#define INVALID_LEXER_STATE_ID <%= @lexer.state_table.size %>u
+
+/** Lexer user code ID type. */
+<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
+typedef <%= get_type_for(user_code_id_count) %> lexer_user_code_id_t;
+
+/** Invalid lexer user code ID. */
+#define INVALID_USER_CODE_ID <%= user_code_id_count %>u
+
+/**
+ * Lexer transition table entry.
+ *
+ * An incoming code point matching the range for a transition entry will cause
+ * the lexer to progress to the destination state.
+ */
+typedef struct
+{
+    /** First code point in the range for this transition. */
+    <%= @grammar.prefix %>code_point_t first;
+
+    /** Last code point in the range for this transition. */
+    <%= @grammar.prefix %>code_point_t last;
+
+    /** Destination lexer state ID for this transition. */
+    lexer_state_id_t destination_state;
+} lexer_transition_t;
+
+/** Lexer state table entry. */
+typedef struct
+{
+    /** Index to the transition table for this state. */
+    <%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
+
+    /** Number of transition table entries for this state. */
+    <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
+
+    /** Lexer token formed at this state. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** Lexer user code ID to execute at this state. */
+    lexer_user_code_id_t code_id;
+
+    /** Whether this state matches a lexer pattern. */
+    bool accepts;
+} lexer_state_t;
+
+/** Lexer mode table entry. */
+typedef struct
+{
+    /** Offset in the state table to be used for this mode. */
+    uint32_t state_table_offset;
+} lexer_mode_t;
+
+/**
+ * Lexer match info structure.
+ *
+ * This structure holds output values from the lexer upon a successful pattern
+ * match.
+ */
+typedef struct
+{
+    /** Number of bytes of input text used to match. */
+    size_t length;
+
+    /** Input text position delta. */
+    <%= @grammar.prefix %>position_t delta_position;
+
+    /** Accepting lexer state from the match. */
+    lexer_state_t const * accepting_state;
+} lexer_match_info_t;
+
+/** Lexer transition table. */
+static lexer_transition_t lexer_transition_table[] = {
+<% @lexer.transition_table.each do |transition_table_entry| %>
+    {<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u},
+<% end %>
+};
+
+/** Lexer state table. */
+static lexer_state_t lexer_state_table[] = {
+<% @lexer.state_table.each do |state_table_entry| %>
+    {<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>},
+<% end %>
+};
+
+/** Lexer mode table. */
+static lexer_mode_t lexer_mode_table[] = {
+<% @lexer.mode_table.each do |mode_table_entry| %>
+    {<%= mode_table_entry[:state_table_offset] %>},
+<% end %>
+};
+
+/**
+ * Execute user code associated with a lexer pattern.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param code_id
+ *   The ID of the user code block to execute.
+ * @param match
+ *   Matched text for this pattern.
+ * @param match_length
+ *   Matched text length.
+ * @param out_token_info
+ *   Lexer token info in progress.
+ *
+ * @return Token to accept, or invalid token if the user code does
+ *   not explicitly return a token.
+ */
+static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
+    lexer_user_code_id_t code_id, uint8_t const * match,
+    size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    switch (code_id)
+    {
+<% @grammar.patterns.each do |pattern| %>
+<%   if pattern.code_id %>
+    case <%= pattern.code_id %>u: {
+<%= expand_code(pattern.code, false, nil, pattern) %>
+    } break;
+<%   end %>
+<% end %>
+    default: break;
+    }
+
+    return INVALID_TOKEN_ID;
+}
+
+/**
+ * Check if there is a transition from the current lexer state to another
+ * based on the given input code point.
+ *
+ * @param current_state
+ *   Current lexer state.
+ * @param code_point
+ *   Input code point.
+ *
+ * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
+ */
+static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t code_point)
+{
+    uint32_t transition_table_index = lexer_state_table[current_state].transition_table_index;
+    for (uint32_t i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
+    {
+        if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
+            (code_point <= lexer_transition_table[transition_table_index + i].last))
+        {
+            return lexer_transition_table[transition_table_index + i].destination_state;
+        }
+    }
+    return INVALID_LEXER_STATE_ID;
+}
+
+/**
+ * Find the longest lexer pattern match at the current position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_EOF
+ *   The end of the text input was reached.
+ */
+static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
+    lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
+{
+    lexer_match_info_t longest_match = {0};
+    lexer_match_info_t attempt_match = {0};
+    *out_match_info = longest_match;
+    uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
+    for (;;)
+    {
+        size_t const input_index = context->input_index + attempt_match.length;
+        uint8_t const * input = &context->input[input_index];
+        size_t input_length = context->input_length - input_index;
+        <%= @grammar.prefix %>code_point_t code_point;
+        uint8_t code_point_length;
+        size_t result = <%= @grammar.prefix %>decode_code_point(input, input_length, &code_point, &code_point_length);
+        switch (result)
+        {
+        case P_SUCCESS:
+            lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
+            if (transition_state != INVALID_LEXER_STATE_ID)
+            {
+                attempt_match.length += code_point_length;
+                if (code_point == '\n')
+                {
+                    attempt_match.delta_position.row++;
+                    attempt_match.delta_position.col = 0u;
+                }
+                else
+                {
+                    attempt_match.delta_position.col++;
+                }
+                current_state = transition_state;
+                if (lexer_state_table[current_state].accepts)
+                {
+                    attempt_match.accepting_state = &lexer_state_table[current_state];
+                    longest_match = attempt_match;
+                }
+            }
+            else if (longest_match.length > 0)
+            {
+                *out_match_info = longest_match;
+                return P_SUCCESS;
+            }
+            else
+            {
+                *out_unexpected_input_length = attempt_match.length + code_point_length;
+                return P_UNEXPECTED_INPUT;
+            }
+            break;
+
+        case P_EOF:
+            /* We hit EOF. */
+            if (longest_match.length > 0)
+            {
+                /* We have a match, so use it. */
+                *out_match_info = longest_match;
+                return P_SUCCESS;
+            }
+            else if (attempt_match.length != 0)
+            {
+                /* There is a partial match - error! */
+                *out_unexpected_input_length = attempt_match.length;
+                return P_UNEXPECTED_INPUT;
+            }
+            else
+            {
+                /* Valid EOF return. */
+                return P_EOF;
+            }
+            break;
+
+        case P_DECODE_ERROR:
+            /* If we see a decode error, we may be partially in the middle of
+             * matching a pattern, so return the attempted match info so that
+             * the input text position can be updated. */
+            *out_match_info = attempt_match;
+            return result;
+
+        default:
+            return result;
+        }
+    }
+}
+
+/**
+ * Attempt to lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_DROP
+ *   A drop pattern was matched so the lexer should continue.
+ */
+static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    <%= @grammar.prefix %>token_info_t token_info = {0};
+    token_info.position = context->text_position;
+    token_info.token = INVALID_TOKEN_ID;
+    *out_token_info = token_info; // TODO: remove
+    lexer_match_info_t match_info;
+    size_t unexpected_input_length;
+    size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
+    switch (result)
+    {
+    case P_SUCCESS:
+        <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
+        {
+            uint8_t const * match = &context->input[context->input_index];
+            <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
+                match_info.accepting_state->code_id, match, match_info.length, &token_info);
+            /* An invalid token returned from lexer_user_code() means that the
+             * user code did not explicitly return a token. So only override
+             * the token to return if the user code does explicitly return a
+             * token. */
+            if (user_code_token != INVALID_TOKEN_ID)
+            {
+                token_to_accept = user_code_token;
+            }
+        }
+
+        /* Update the input position tracking. */
+        context->input_index += match_info.length;
+        context->text_position.row += match_info.delta_position.row;
+        if (match_info.delta_position.row != 0u)
+        {
+            context->text_position.col = match_info.delta_position.col;
+        }
+        else
+        {
+            context->text_position.col += match_info.delta_position.col;
+        }
+
+        if (token_to_accept == INVALID_TOKEN_ID)
+        {
+            return P_DROP;
+        }
+        token_info.token = token_to_accept;
+        token_info.length = match_info.length;
+        *out_token_info = token_info;
+        return P_SUCCESS;
+
+    case P_EOF:
+        token_info.token = TOKEN___EOF;
+        *out_token_info = token_info;
+        return P_SUCCESS;
+
+    case P_DECODE_ERROR:
+        /* Update the input position tracking. */
+        context->input_index += match_info.length;
+        context->text_position.row += match_info.delta_position.row;
+        if (match_info.delta_position.row != 0u)
+        {
+            context->text_position.col = match_info.delta_position.col;
+        }
+        else
+        {
+            context->text_position.col += match_info.delta_position.col;
+        }
+        return result;
+
+    default:
+        return result;
+    }
+}
+
+/**
+ * Lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ */
+size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    for (;;)
+    {
+        size_t result = attempt_lex_token(context, out_token_info);
+        if (result != P_DROP)
+        {
+            return result;
+        }
+    }
+}
+
+/**************************************************************************
+ * Parser
+ *************************************************************************/
+
+/** Reduce ID type. */
+typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
+
+/**
+ * A symbol ID can hold either a token ID or a rule set ID.
+ *
+ * Token IDs and rule set IDs share the same namespace, with rule set IDs
+ * beginning after token IDs end.
+ */
+typedef <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %> symbol_id_t;
+
+/** Parser state ID type. */
+typedef <%= get_type_for(@parser.state_table.size) %> parser_state_id_t;
+
+/** Parser rule ID type. */
+typedef <%= get_type_for(@grammar.rules.size) %> rule_id_t;
+
+/** Parser shift ID type. */
+typedef <%= get_type_for(@parser.shift_table.size) %> shift_id_t;
+
+/** Shift table entry. */
+typedef struct
+{
+    /** Token or rule set ID. */
+    symbol_id_t symbol_id;
+
+    /** Parser state to shift to. */
+    parser_state_id_t state_id;
+} shift_t;
+
+/** Reduce table entry. */
+typedef struct
+{
+    /** Lookahead token. */
+    <%= @grammar.prefix %>token_t token;
+
+    /**
+     * Rule ID.
+     *
+     * This is used to execute the parser user code block associated with a
+     * grammar rule.
+     */
+    rule_id_t rule;
+
+    /**
+     * Rule set ID.
+     *
+     * This is used as the new top symbol ID of the parse stack after this
+     * reduce action.
+     */
+    symbol_id_t rule_set;
+
+    /**
+     * Number of states leading to this reduce action.
+     *
+     * This is the number of entries popped from the parse stack after this
+     * reduce action.
+     */
+    parser_state_id_t n_states;
+} reduce_t;
+
+/** Parser state entry. */
+typedef struct
+{
+    /** First shift table entry for this parser state. */
+    shift_id_t shift_table_index;
+
+    /** Number of shift table entries for this parser state. */
+    shift_id_t n_shift_entries;
+
+    /** First reduce table entry for this parser state. */
+    reduce_id_t reduce_table_index;
+
+    /** Number of reduce table entries for this parser state. */
+    reduce_id_t n_reduce_entries;
+} parser_state_t;
+
+/**
+ * Structure to hold a state ID and value pair.
+ *
+ * A stack of these structures makes up the parse stack.
+ */
+typedef struct
+{
+    /** Parser state ID. */
+    size_t state_id;
+
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
+} state_value_t;
+
+/** Parser shift table. */
+static const shift_t parser_shift_table[] = {
+<%   @parser.shift_table.each do |shift| %>
+    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
+<%   end %>
+};
+
+/** Parser reduce table. */
+static const reduce_t parser_reduce_table[] = {
+<%   @parser.reduce_table.each do |reduce| %>
+    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
+<%   end %>
+};
+
+/** Parser state table. */
+static const parser_state_t parser_state_table[] = {
+<%   @parser.state_table.each do |state| %>
+    {<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u},
+<%   end %>
+};
+
+/* state_values stack functionality */
+
+/** state_values stack type. */
+typedef struct
+{
+    size_t length;
+    size_t capacity;
+    state_value_t * entries;
+} state_values_stack_t;
+
+/**
+ * Initialize state_values stack structure.
+ *
+ * @param stack
+ *   state_values stack structure.
+ */
+static void state_values_stack_init(state_values_stack_t * stack)
+{
+    const size_t initial_capacity = 10u;
+    stack->length = 0u;
+    stack->capacity = initial_capacity;
+    stack->entries = (state_value_t *)malloc(initial_capacity * sizeof(state_value_t));
+}
+
+/**
+ * Index a state_values stack.
+ *
+ * @param stack
+ *   state_values stack structure.
+ * @param index
+ *   Index to the stack.
+ *
+ * @return Pointer to the state value structure at the given index.
+ */
+static state_value_t * state_values_stack_index(state_values_stack_t * stack, int index)
+{
+    if (index >= 0)
+    {
+        return &stack->entries[index];
+    }
+    else
+    {
+        return &stack->entries[stack->length - (size_t)(unsigned int)(-index)];
+    }
+}
+
+/**
+ * Push a new state_value to the state_values stack.
+ *
+ * @param stack
+ *   state_values stack structure.
+ */
+static void state_values_stack_push(state_values_stack_t * stack)
+{
+    size_t const current_capacity = stack->capacity;
+    size_t const current_length = stack->length;
+    if (current_length >= current_capacity)
+    {
+        size_t const new_capacity = current_capacity * 2u;
+        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
+        memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
+        free(stack->entries);
+        stack->capacity = new_capacity;
+        stack->entries = new_entries;
+    }
+    memset(&stack->entries[current_length], 0, sizeof(state_value_t));
+    stack->length = current_length + 1u;
+}
+
+/**
+ * Pop entries from a state_values stack.
+ *
+ * @param stack
+ *   state_values stack structure.
+ * @param n
+ *   Number of states to pop.
+ */
+static void state_values_stack_pop(state_values_stack_t * stack, size_t n)
+{
+    stack->length -= n;
+}
+
+/**
+ * Free memory for a state_values stack structure.
+ *
+ * @param stack
+ *   state_values stack structure.
+ */
+static void state_values_stack_free(state_values_stack_t * stack)
+{
+     free(stack->entries);
+}
+
+/**
+ * Execute user code associated with a parser rule.
+ *
+ * @param rule The ID of the rule.
+ *
+ * @return Parse value.
+ */
+static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
+{
+    <%= @grammar.prefix %>value_t _pvalue = {0};
+
+    switch (rule)
+    {
+<% @grammar.rules.each do |rule| %>
+<%   if rule.code %>
+    case <%= rule.id %>u: {
+<%= expand_code(rule.code, true, rule, nil) %>
+    } break;
+<%   end %>
+<% end %>
+    default: break;
+    }
+
+    return _pvalue;
+}
+
+/**
+ * Check if the parser should shift to a new state.
+ *
+ * @param state_id
+ *   Parser state ID.
+ * @param symbol_id
+ *   Incoming token/rule set ID.
+ *
+ * @return State to shift to, or INVALID_ID if none.
+ */
+static size_t check_shift(size_t state_id, size_t symbol_id)
+{
+    uint32_t start = parser_state_table[state_id].shift_table_index;
+    uint32_t end = start + parser_state_table[state_id].n_shift_entries;
+    for (uint32_t i = start; i < end; i++)
+    {
+        if (parser_shift_table[i].symbol_id == symbol_id)
+        {
+            return parser_shift_table[i].state_id;
+        }
+    }
+    return INVALID_ID;
+}
+
+/**
+ * Check if the parser should reduce to a new state.
+ *
+ * @param state_id
+ *   Parser state ID.
+ * @param token
+ *   Incoming token.
+ *
+ * @return State to reduce to, or INVALID_ID if none.
+ */
+static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
+{
+    size_t start = parser_state_table[state_id].reduce_table_index;
+    size_t end = start + parser_state_table[state_id].n_reduce_entries;
+    for (size_t i = start; i < end; i++)
+    {
+        if ((parser_reduce_table[i].token == token) ||
+            (parser_reduce_table[i].token == INVALID_TOKEN_ID))
+        {
+            return i;
+        }
+    }
+    return INVALID_ID;
+}
+
+/**
+ * Run the parser.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @retval P_SUCCESS
+ *   The parser successfully matched the input text. The parse result value
+ *   can be accessed with <%= @grammar.prefix %>result().
+ * @retval P_UNEXPECTED_TOKEN
+ *   An unexpected token was encountered that does not match any grammar rule.
+ *   The value context->token holds the unexpected token.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ */
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    <%= @grammar.prefix %>token_info_t token_info;
+    <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
+    state_values_stack_t statevalues;
+    size_t reduced_rule_set = INVALID_ID;
+    <%= @grammar.prefix %>value_t reduced_parser_value;
+    state_values_stack_init(&statevalues);
+    state_values_stack_push(&statevalues);
+    size_t result;
+    for (;;)
+    {
+        if (token == INVALID_TOKEN_ID)
+        {
+            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            if (lexer_result != P_SUCCESS)
+            {
+                result = lexer_result;
+                break;
+            }
+            token = token_info.token;
+        }
+        size_t shift_state = INVALID_ID;
+        if (reduced_rule_set != INVALID_ID)
+        {
+            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, reduced_rule_set);
+        }
+        if (shift_state == INVALID_ID)
+        {
+            shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, token);
+            if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
+            {
+                /* Successful parse. */
+                context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+                result = P_SUCCESS;
+                break;
+            }
+        }
+        if (shift_state != INVALID_ID)
+        {
+            /* We have something to shift. */
+            state_values_stack_push(&statevalues);
+            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            if (reduced_rule_set == INVALID_ID)
+            {
+                /* We shifted a token, mark it consumed. */
+                token = INVALID_TOKEN_ID;
+                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+            }
+            else
+            {
+                /* We shifted a RuleSet. */
+                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                <%= @grammar.prefix %>value_t new_parse_result = {0};
+                reduced_parser_value = new_parse_result;
+                reduced_rule_set = INVALID_ID;
+            }
+            continue;
+        }
+
+        size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if (reduce_index != INVALID_ID)
+        {
+            /* We have something to reduce. */
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
+            reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
+            state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
+            continue;
+        }
+
+        /* A token was successfully lexed, so the input text position was
+         * advanced. However, this is an unexpected token, so we want to reset
+         * the context text position to point to the token rather than the text
+         * after it, so that if the caller wants to report the error position,
+         * it will point to the correct position of the unexpected token. */
+        context->text_position = token_info.position;
+        context->token = token;
+        result = P_UNEXPECTED_TOKEN;
+        break;
+    }
+    state_values_stack_free(&statevalues);
+    return result;
+}
+
+/**
+ * Get the parse result value.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Parse result value.
+ */
+<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return context->parse_result.v_<%= start_rule_type[0] %>;
+}
+
+/**
+ * Get the current text input position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current text position.
+ */
+<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
+{
+    return context->text_position;
+}
@@ -0,0 +1,951 @@
+/**
+ * @file
+ *
+ * This file is generated by Propane.
+ */
+<% if @grammar.modulename %>
+
+module <%= @grammar.modulename %>;
+<% end %>
+
+/**************************************************************************
+ * User code blocks
+ *************************************************************************/
+
+<%= @grammar.code_blocks.fetch("", "") %>
+
+/**************************************************************************
+ * Public types
+ *************************************************************************/
+
+/* Result codes. */
+public enum : size_t
+{
+    <%= @grammar.prefix.upcase %>SUCCESS,
+    <%= @grammar.prefix.upcase %>DECODE_ERROR,
+    <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT,
+    <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN,
+    <%= @grammar.prefix.upcase %>DROP,
+    <%= @grammar.prefix.upcase %>EOF,
+}
+
+/** Token type. */
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
+
+/** Token IDs. */
+public enum : <%= @grammar.prefix %>token_t
+{
+<% @grammar.tokens.each_with_index do |token, index| %>
+    TOKEN_<%= token.code_name %> = <%= index %>,
+<%   unless token.id == index %>
+<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
+<%   end %>
+<% end %>
+    INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
+}
+
+/** Code point type. */
+public alias <%= @grammar.prefix %>code_point_t = uint;
+
+/** Parser values type(s). */
+public union <%= @grammar.prefix %>value_t
+{
+<% @grammar.ptypes.each do |name, typestring| %>
+    <%= typestring %> v_<%= name %>;
+<% end %>
+}
+
+/**
+ * A structure to keep track of parser position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+public struct <%= @grammar.prefix %>position_t
+{
+    /** Input text row (0-based). */
+    uint row;
+
+    /** Input text column (0-based). */
+    uint col;
+}
+
+/** Lexed token information. */
+public struct <%= @grammar.prefix %>token_info_t
+{
+    /** Text position where the token was found. */
+    <%= @grammar.prefix %>position_t position;
+
+    /** Number of input bytes used by the token. */
+    size_t length;
+
+    /** Token that was lexed. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** Parser value associated with the token. */
+    <%= @grammar.prefix %>value_t pvalue;
+}
+
+/**
+ * Lexer and parser context.
+ *
+ * The user must allocate an instance of this structure and pass it to any
+ * public API function.
+ */
+public struct <%= @grammar.prefix %>context_t
+{
+    /* Lexer context data. */
+
+    /** Input text. */
+    string input;
+
+    /** Input text index (byte offset). */
+    size_t input_index;
+
+    /** Input text position (row/column). */
+    <%= @grammar.prefix %>position_t text_position;
+
+    /** Current lexer mode. */
+    size_t mode;
+
+    /* Parser context data. */
+
+    /** Parse result value. */
+    <%= @grammar.prefix %>value_t parse_result;
+
+    /** Unexpected token received. */
+    <%= @grammar.prefix %>token_t token;
+}
+
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+public immutable string[] <%= @grammar.prefix %>token_names = [
+<% @grammar.tokens.each_with_index do |token, index| %>
+    "<%= token.name %>",
+<% end %>
+];
+
+/**************************************************************************
+ * Private types
+ *************************************************************************/
+
+<% if @grammar.prefix.upcase != "P_" %>
+/* Result codes. */
+private enum : size_t
+{
+    P_SUCCESS,
+    P_DECODE_ERROR,
+    P_UNEXPECTED_INPUT,
+    P_UNEXPECTED_TOKEN,
+    P_DROP,
+    P_EOF,
+}
+<% end %>
+
+/* An invalid ID value. */
+private enum size_t INVALID_ID = cast(size_t)-1;
+
+/**************************************************************************
+ * State initialization
+ *************************************************************************/
+
+/**
+ * Initialize lexer/parser context structure.
+ *
+ * @param[out] context
+ *   Lexer/parser context structure.
+ * @param input
+ *   Text input.
+ */
+public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, string input)
+{
+    /* New default-initialized context structure. */
+    <%= @grammar.prefix %>context_t newcontext;
+
+    /* Lexer initialization. */
+    newcontext.input = input;
+    newcontext.mode = <%= @lexer.mode_id("default") %>;
+
+    /* Copy to the user's context structure. */
+    *context = newcontext;
+}
+
+/**************************************************************************
+ * Decoder
+ *************************************************************************/
+
+/**
+ * Decode a UTF-8 code point.
+ *
+ * @param input
+ *   Text input to decode.
+ * @param[out] out_code_point
+ *   The decoded code point is stored here if the return value is P_SUCCESS.
+ * @param[out] out_code_point_length
+ *   The number of bytes the code point used is stored here if the return value
+ *   is P_SUCCESS.
+ *
+ * @retval P_SUCCESS on a successful code point decode
+ * @retval P_DECODE_ERROR when an encoding error is observed
+ * @retval P_EOF when the end of the text input is reached
+ */
+public size_t <%= @grammar.prefix %>decode_code_point(string input,
+    <%= @grammar.prefix %>code_point_t * out_code_point, ubyte * out_code_point_length)
+{
+    if (input.length == 0u)
+    {
+        return P_EOF;
+    }
+    char c = input[0];
+    <%= @grammar.prefix %>code_point_t code_point;
+    ubyte code_point_length;
+    if ((c & 0x80u) == 0u)
+    {
+        code_point = c;
+        code_point_length = 1u;
+    }
+    else
+    {
+        ubyte following_bytes;
+        if ((c & 0xE0u) == 0xC0u)
+        {
+            code_point = c & 0x1Fu;
+            following_bytes = 1u;
+        }
+        else if ((c & 0xF0u) == 0xE0u)
+        {
+            code_point = c & 0x0Fu;
+            following_bytes = 2u;
+        }
+        else if ((c & 0xF8u) == 0xF0u)
+        {
+            code_point = c & 0x07u;
+            following_bytes = 3u;
+        }
+        else if ((c & 0xFCu) == 0xF8u)
+        {
+            code_point = c & 0x03u;
+            following_bytes = 4u;
+        }
+        else if ((c & 0xFEu) == 0xFCu)
+        {
+            code_point = c & 0x01u;
+            following_bytes = 5u;
+        }
+        else
+        {
+            return P_DECODE_ERROR;
+        }
+        if (input.length <= following_bytes)
+        {
+            return P_DECODE_ERROR;
+        }
+        code_point_length = cast(ubyte)(following_bytes + 1u);
+        for (size_t i = 0u; i < following_bytes; i++)
+        {
+            char b = input[i + 1u];
+            if ((b & 0xC0u) != 0x80u)
+            {
+                return P_DECODE_ERROR;
+            }
+            code_point = (code_point << 6u) | (b & 0x3Fu);
+        }
+    }
+    *out_code_point = code_point;
+    *out_code_point_length = code_point_length;
+    return P_SUCCESS;
+}
+
+/**************************************************************************
+ * Lexer
+ *************************************************************************/
+
+/** Lexer state ID type. */
+private alias lexer_state_id_t = <%= get_type_for(@lexer.state_table.size) %>;
+
+/** Invalid lexer state ID. */
+private enum lexer_state_id_t INVALID_LEXER_STATE_ID = <%= @lexer.state_table.size %>u;
+
+/** Lexer user code ID type. */
+<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
+private alias lexer_user_code_id_t = <%= get_type_for(user_code_id_count) %>;
+
+/** Invalid lexer user code ID. */
+private enum lexer_user_code_id_t INVALID_USER_CODE_ID = <%= user_code_id_count %>u;
+
+/**
+ * Lexer transition table entry.
+ *
+ * An incoming code point matching the range for a transition entry will cause
+ * the lexer to progress to the destination state.
+ */
+private struct lexer_transition_t
+{
+    /** First code point in the range for this transition. */
+    <%= @grammar.prefix %>code_point_t first;
+
+    /** Last code point in the range for this transition. */
+    <%= @grammar.prefix %>code_point_t last;
+
+    /** Destination lexer state ID for this transition. */
+    lexer_state_id_t destination_state;
+}
+
+/** Lexer state table entry. */
+private struct lexer_state_t
+{
+    /** Index to the transition table for this state. */
+    <%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
+
+    /** Number of transition table entries for this state. */
+    <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
+
+    /** Lexer token formed at this state. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** Lexer user code ID to execute at this state. */
+    lexer_user_code_id_t code_id;
+
+    /** Whether this state matches a lexer pattern. */
+    bool accepts;
+}
+
+/** Lexer mode table entry. */
+private struct lexer_mode_t
+{
+    /** Offset in the state table to be used for this mode. */
+    uint state_table_offset;
+}
+
+/**
+ * Lexer match info structure.
+ *
+ * This structure holds output values from the lexer upon a successful pattern
+ * match.
+ */
+private struct lexer_match_info_t
+{
+    /** Number of bytes of input text used to match. */
+    size_t length;
+
+    /** Input text position delta. */
+    <%= @grammar.prefix %>position_t delta_position;
+
+    /** Accepting lexer state from the match. */
+    const(lexer_state_t) * accepting_state;
+}
+
+/** Lexer transition table. */
+private immutable lexer_transition_t[] lexer_transition_table = [
+<% @lexer.transition_table.each do |transition_table_entry| %>
+  lexer_transition_t(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
+<% end %>
+];
+
+/** Lexer state table. */
+private immutable lexer_state_t[] lexer_state_table = [
+<% @lexer.state_table.each do |state_table_entry| %>
+    lexer_state_t(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>),
+<% end %>
+];
+
+/** Lexer mode table. */
+private immutable lexer_mode_t[] lexer_mode_table = [
+<% @lexer.mode_table.each do |mode_table_entry| %>
+    lexer_mode_t(<%= mode_table_entry[:state_table_offset] %>),
+<% end %>
+];
+
+/**
+ * Execute user code associated with a lexer pattern.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param code_id
+ *   The ID of the user code block to execute.
+ * @param match
+ *   Matched text for this pattern.
+ * @param out_token_info
+ *   Lexer token info in progress.
+ *
+ * @return Token to accept, or invalid token if the user code does
+ *   not explicitly return a token.
+ */
+private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
+    lexer_user_code_id_t code_id, string match,
+    <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    switch (code_id)
+    {
+<% @grammar.patterns.each do |pattern| %>
+<%   if pattern.code_id %>
+    case <%= pattern.code_id %>u: {
+<%= expand_code(pattern.code, false, nil, pattern) %>
+    } break;
+<%   end %>
+<% end %>
+    default: break;
+    }
+
+    return INVALID_TOKEN_ID;
+}
+
+/**
+ * Check if there is a transition from the current lexer state to another
+ * based on the given input code point.
+ *
+ * @param current_state
+ *   Current lexer state.
+ * @param code_point
+ *   Input code point.
+ *
+ * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
+ */
+private lexer_state_id_t check_lexer_transition(uint current_state, uint code_point)
+{
+    uint transition_table_index = lexer_state_table[current_state].transition_table_index;
+    for (uint i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
+    {
+        if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
+            (code_point <= lexer_transition_table[transition_table_index + i].last))
+        {
+            return lexer_transition_table[transition_table_index + i].destination_state;
+        }
+    }
+    return INVALID_LEXER_STATE_ID;
+}
+
+/**
+ * Find the longest lexer pattern match at the current position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_EOF
+ *   The end of the text input was reached.
+ */
+private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
+    lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
+{
+    lexer_match_info_t longest_match;
+    lexer_match_info_t attempt_match;
+    *out_match_info = longest_match;
+    uint current_state = lexer_mode_table[context.mode].state_table_offset;
+    for (;;)
+    {
+        string input = context.input[(context.input_index + attempt_match.length)..(context.input.length)];
+        <%= @grammar.prefix %>code_point_t code_point;
+        ubyte code_point_length;
+        size_t result = <%= @grammar.prefix %>decode_code_point(input, &code_point, &code_point_length);
+        switch (result)
+        {
+        case P_SUCCESS:
+            lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
+            if (transition_state != INVALID_LEXER_STATE_ID)
+            {
+                attempt_match.length += code_point_length;
+                if (code_point == '\n')
+                {
+                    attempt_match.delta_position.row++;
+                    attempt_match.delta_position.col = 0u;
+                }
+                else
+                {
+                    attempt_match.delta_position.col++;
+                }
+                current_state = transition_state;
+                if (lexer_state_table[current_state].accepts)
+                {
+                    attempt_match.accepting_state = &lexer_state_table[current_state];
+                    longest_match = attempt_match;
+                }
+            }
+            else if (longest_match.length > 0)
+            {
+                *out_match_info = longest_match;
+                return P_SUCCESS;
+            }
+            else
+            {
+                *out_unexpected_input_length = attempt_match.length + code_point_length;
+                return P_UNEXPECTED_INPUT;
+            }
+            break;
+
+        case P_EOF:
+            /* We hit EOF. */
+            if (longest_match.length > 0)
+            {
+                /* We have a match, so use it. */
+                *out_match_info = longest_match;
+                return P_SUCCESS;
+            }
+            else if (attempt_match.length != 0)
+            {
+                /* There is a partial match - error! */
+                *out_unexpected_input_length = attempt_match.length;
+                return P_UNEXPECTED_INPUT;
+            }
+            else
+            {
+                /* Valid EOF return. */
+                return P_EOF;
+            }
+            break;
+
+        case P_DECODE_ERROR:
+            /* If we see a decode error, we may be partially in the middle of
+             * matching a pattern, so return the attempted match info so that
+             * the input text position can be updated. */
+            *out_match_info = attempt_match;
+            return result;
+
+        default:
+            return result;
+        }
+    }
+}
+
+/**
+ * Attempt to lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_DROP
+ *   A drop pattern was matched so the lexer should continue.
+ */
+private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    <%= @grammar.prefix %>token_info_t token_info;
+    token_info.position = context.text_position;
+    token_info.token = INVALID_TOKEN_ID;
+    *out_token_info = token_info; // TODO: remove
+    lexer_match_info_t match_info;
+    size_t unexpected_input_length;
+    size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
+    switch (result)
+    {
+    case P_SUCCESS:
+        <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
+        {
+            string match = context.input[context.input_index..(context.input_index + match_info.length)];
+            <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
+                match_info.accepting_state.code_id, match, &token_info);
+            /* An invalid token returned from lexer_user_code() means that the
+             * user code did not explicitly return a token. So only override
+             * the token to return if the user code does explicitly return a
+             * token. */
+            if (user_code_token != INVALID_TOKEN_ID)
+            {
+                token_to_accept = user_code_token;
+            }
+        }
+
+        /* Update the input position tracking. */
+        context.input_index += match_info.length;
+        context.text_position.row += match_info.delta_position.row;
+        if (match_info.delta_position.row != 0u)
+        {
+            context.text_position.col = match_info.delta_position.col;
+        }
+        else
+        {
+            context.text_position.col += match_info.delta_position.col;
+        }
+
+        if (token_to_accept == INVALID_TOKEN_ID)
+        {
+            return P_DROP;
+        }
+        token_info.token = token_to_accept;
+        token_info.length = match_info.length;
+        *out_token_info = token_info;
+        return P_SUCCESS;
+
+    case P_EOF:
+        token_info.token = TOKEN___EOF;
+        *out_token_info = token_info;
+        return P_SUCCESS;
+
+    case P_DECODE_ERROR:
+        /* Update the input position tracking. */
+        context.input_index += match_info.length;
+        context.text_position.row += match_info.delta_position.row;
+        if (match_info.delta_position.row != 0u)
+        {
+            context.text_position.col = match_info.delta_position.col;
+        }
+        else
+        {
+            context.text_position.col += match_info.delta_position.col;
+        }
+        return result;
+
+    default:
+        return result;
+    }
+}
+
+/**
+ * Lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ */
+public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
+{
+    for (;;)
+    {
+        size_t result = attempt_lex_token(context, out_token_info);
+        if (result != P_DROP)
+        {
+            return result;
+        }
+    }
+}
+
+/**************************************************************************
+ * Parser
+ *************************************************************************/
+
+/** Reduce ID type. */
+private alias reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
+
+/**
+ * A symbol ID can hold either a token ID or a rule set ID.
+ *
+ * Token IDs and rule set IDs share the same namespace, with rule set IDs
+ * beginning after token IDs end.
+ */
+private alias symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
+
+/** Parser state ID type. */
+private alias parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
+
+/** Parser rule ID type. */
+private alias rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
+
+/** Parser shift ID type. */
+private alias shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
+
+/** Shift table entry. */
+private struct shift_t
+{
+    /** Token or rule set ID. */
+    symbol_id_t symbol_id;
+
+    /** Parser state to shift to. */
+    parser_state_id_t state_id;
+}
+
+/** Reduce table entry. */
+private struct reduce_t
+{
+    /** Lookahead token. */
+    <%= @grammar.prefix %>token_t token;
+
+    /**
+     * Rule ID.
+     *
+     * This is used to execute the parser user code block associated with a
+     * grammar rule.
+     */
+    rule_id_t rule;
+
+    /**
+     * Rule set ID.
+     *
+     * This is used as the new top symbol ID of the parse stack after this
+     * reduce action.
+     */
+    symbol_id_t rule_set;
+
+    /**
+     * Number of states leading to this reduce action.
+     *
+     * This is the number of entries popped from the parse stack after this
+     * reduce action.
+     */
+    parser_state_id_t n_states;
+}
+
+/** Parser state entry. */
+private struct parser_state_t
+{
+    /** First shift table entry for this parser state. */
+    shift_id_t shift_table_index;
+
+    /** Number of shift table entries for this parser state. */
+    shift_id_t n_shift_entries;
+
+    /** First reduce table entry for this parser state. */
+    reduce_id_t reduce_table_index;
+
+    /** Number of reduce table entries for this parser state. */
+    reduce_id_t n_reduce_entries;
+}
+
+/**
+ * Structure to hold a state ID and value pair.
+ *
+ * A stack of these structures makes up the parse stack.
+ */
+private struct state_value_t
+{
+    /** Parser state ID. */
+    size_t state_id;
+
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
+
+    this(size_t state_id)
+    {
+        this.state_id = state_id;
+    }
+}
+
+/** Parser shift table. */
+private immutable shift_t[] parser_shift_table = [
+<%   @parser.shift_table.each do |shift| %>
+    shift_t(<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u),
+<%   end %>
+];
+
+/** Parser reduce table. */
+private immutable reduce_t[] parser_reduce_table = [
+<%   @parser.reduce_table.each do |reduce| %>
+    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
+<%   end %>
+];
+
+/** Parser state table. */
+private immutable parser_state_t[] parser_state_table = [
+<%   @parser.state_table.each do |state| %>
+    parser_state_t(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
+<%   end %>
+];
+
+/**
+ * Execute user code associated with a parser rule.
+ *
+ * @param rule The ID of the rule.
+ *
+ * @return Parse value.
+ */
+private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[] statevalues, uint n_states)
+{
+    <%= @grammar.prefix %>value_t _pvalue;
+
+    switch (rule)
+    {
+<% @grammar.rules.each do |rule| %>
+<%   if rule.code %>
+    case <%= rule.id %>u: {
+<%= expand_code(rule.code, true, rule, nil) %>
+    } break;
+<%   end %>
+<% end %>
+    default: break;
+    }
+
+    return _pvalue;
+}
+
+/**
+ * Check if the parser should shift to a new state.
+ *
+ * @param state_id
+ *   Parser state ID.
+ * @param symbol_id
+ *   Incoming token/rule set ID.
+ *
+ * @return State to shift to, or INVALID_ID if none.
+ */
+private size_t check_shift(size_t state_id, size_t symbol_id)
+{
+    uint start = parser_state_table[state_id].shift_table_index;
+    uint end = start + parser_state_table[state_id].n_shift_entries;
+    for (uint i = start; i < end; i++)
+    {
+        if (parser_shift_table[i].symbol_id == symbol_id)
+        {
+            return parser_shift_table[i].state_id;
+        }
+    }
+    return INVALID_ID;
+}
+
+/**
+ * Check if the parser should reduce to a new state.
+ *
+ * @param state_id
+ *   Parser state ID.
+ * @param token
+ *   Incoming token.
+ *
+ * @return State to reduce to, or INVALID_ID if none.
+ */
+private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
+{
+    size_t start = parser_state_table[state_id].reduce_table_index;
+    size_t end = start + parser_state_table[state_id].n_reduce_entries;
+    for (size_t i = start; i < end; i++)
+    {
+        if ((parser_reduce_table[i].token == token) ||
+            (parser_reduce_table[i].token == INVALID_TOKEN_ID))
+        {
+            return i;
+        }
+    }
+    return INVALID_ID;
+}
+
+/**
+ * Run the parser.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @retval P_SUCCESS
+ *   The parser successfully matched the input text. The parse result value
+ *   can be accessed with <%= @grammar.prefix %>result().
+ * @retval P_UNEXPECTED_TOKEN
+ *   An unexpected token was encountered that does not match any grammar rule.
+ *   The value context.token holds the unexpected token.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ */
+public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    <%= @grammar.prefix %>token_info_t token_info;
+    <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
+    state_value_t[] statevalues = new state_value_t[](1);
+    size_t reduced_rule_set = INVALID_ID;
+    <%= @grammar.prefix %>value_t reduced_parser_value;
+    for (;;)
+    {
+        if (token == INVALID_TOKEN_ID)
+        {
+            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            if (lexer_result != P_SUCCESS)
+            {
+                return lexer_result;
+            }
+            token = token_info.token;
+        }
+        size_t shift_state = INVALID_ID;
+        if (reduced_rule_set != INVALID_ID)
+        {
+            shift_state = check_shift(statevalues[$-1].state_id, reduced_rule_set);
+        }
+        if (shift_state == INVALID_ID)
+        {
+            shift_state = check_shift(statevalues[$-1].state_id, token);
+            if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
+            {
+                /* Successful parse. */
+                context.parse_result = statevalues[$-1].pvalue;
+                return P_SUCCESS;
+            }
+        }
+        if (shift_state != INVALID_ID)
+        {
+            /* We have something to shift. */
+            statevalues ~= state_value_t(shift_state);
+            if (reduced_rule_set == INVALID_ID)
+            {
+                /* We shifted a token, mark it consumed. */
+                token = INVALID_TOKEN_ID;
+                statevalues[$-1].pvalue = token_info.pvalue;
+            }
+            else
+            {
+                /* We shifted a RuleSet. */
+                statevalues[$-1].pvalue = reduced_parser_value;
+                <%= @grammar.prefix %>value_t new_parse_result;
+                reduced_parser_value = new_parse_result;
+                reduced_rule_set = INVALID_ID;
+            }
+            continue;
+        }
+
+        size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if (reduce_index != INVALID_ID)
+        {
+            /* We have something to reduce. */
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states);
+            reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
+            statevalues.length -= parser_reduce_table[reduce_index].n_states;
+            continue;
+        }
+
+        /* A token was successfully lexed, so the input text position was
+         * advanced. However, this is an unexpected token, so we want to reset
+         * the context text position to point to the token rather than the text
+         * after it, so that if the caller wants to report the error position,
+         * it will point to the correct position of the unexpected token. */
+        context.text_position = token_info.position;
+        context.token = token;
+        return P_UNEXPECTED_TOKEN;
+    }
+}
+
+/**
+ * Get the parse result value.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Parse result value.
+ */
+public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return context.parse_result.v_<%= start_rule_type[0] %>;
+}
+
+/**
+ * Get the current text input position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current text position.
+ */
+public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
+{
+    return context.text_position;
+}
@@ -0,0 +1,125 @@
+/**
+ * @file
+ *
+ * This file is generated by Propane.
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+
+/**************************************************************************
+ * Public types
+ *************************************************************************/
+
+/* Result codes. */
+#define <%= @grammar.prefix.upcase %>SUCCESS 0u
+#define <%= @grammar.prefix.upcase %>DECODE_ERROR 1u
+#define <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT 2u
+#define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
+#define <%= @grammar.prefix.upcase %>DROP 4u
+#define <%= @grammar.prefix.upcase %>EOF 5u
+
+/** Token type. */
+typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
+
+/** Token IDs. */
+<% @grammar.tokens.each_with_index do |token, index| %>
+#define TOKEN_<%= token.code_name %> <%= index %>u
+<%   unless token.id == index %>
+<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
+<%   end %>
+<% end %>
+#define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+
+/** Code point type. */
+typedef uint32_t <%= @grammar.prefix %>code_point_t;
+
+/** User header code blocks. */
+<%= @grammar.code_blocks.fetch("header", "") %>
+
+/** Parser values type(s). */
+typedef union
+{
+<% @grammar.ptypes.each do |name, typestring| %>
+    <%= typestring %> v_<%= name %>;
+<% end %>
+} <%= @grammar.prefix %>value_t;
+
+/**
+ * A structure to keep track of parser position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+typedef struct
+{
+    /** Input text row (0-based). */
+    uint32_t row;
+
+    /** Input text column (0-based). */
+    uint32_t col;
+} <%= @grammar.prefix %>position_t;
+
+/** Lexed token information. */
+typedef struct
+{
+    /** Text position where the token was found. */
+    <%= @grammar.prefix %>position_t position;
+
+    /** Number of input bytes used by the token. */
+    size_t length;
+
+    /** Token that was lexed. */
+    <%= @grammar.prefix %>token_t token;
+
+    /** Parser value associated with the token. */
+    <%= @grammar.prefix %>value_t pvalue;
+} <%= @grammar.prefix %>token_info_t;
+
+/**
+ * Lexer and parser context.
+ *
+ * The user must allocate an instance of this structure and pass it to any
+ * public API function.
+ */
+typedef struct
+{
+    /* Lexer context data. */
+
+    /** Input text. */
+    uint8_t const * input;
+
+    /** Input text length. */
+    size_t input_length;
+
+    /** Input text index (byte offset). */
+    size_t input_index;
+
+    /** Input text position (row/column). */
+    <%= @grammar.prefix %>position_t text_position;
+
+    /** Current lexer mode. */
+    size_t mode;
+
+    /* Parser context data. */
+
+    /** Parse result value. */
+    <%= @grammar.prefix %>value_t parse_result;
+
+    /** Unexpected token received. */
+    <%= @grammar.prefix %>token_t token;
+} <%= @grammar.prefix %>context_t;
+
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+
+size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
+        <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
+
+size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
+
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
+
+<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "propane"
+
+exit Propane::CLI.run(ARGV.dup)
@@ -0,0 +1,601 @@
+${remove}
+WARNING: This user guide is meant to be preprocessed and rendered by a custom
+script.
+The markdown source file is not intended to be viewed directly and will not
+include all intended content.
+${/remove}
+
+#> Overview
+
+Propane is a LALR Parser Generator (LPG) which:
+
+  * accepts LR(0), SLR, and LALR grammars
+  * generates a built-in lexer to tokenize input
+  * supports UTF-8 lexer inputs
+  * generates a table-driven shift/reduce parser to parse input in linear time
+  * target C or D language outputs
+  * is MIT-licensed
+  * is distributable as a standalone Ruby script
+
+#> Installation
+
+Propane is designed to be distributed as a stand-alone single file script that
+can be copied into and versioned in a project's source tree.
+The only requirement to run Propane is that the system has a Ruby interpreter
+installed.
+The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+Simply copy the `propane` executable script into the desired location within
+the project to be built (typically the root of the repository) and mark it
+executable.
+
+#> Command Line Usage
+
+Propane is typically invoked from the command-line as `./propane`.
+
+    Usage: ./propane [options] <input-file> <output-file>
+    Options:
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
+
+The user must specify the path to a Propane input grammar file and a path to an
+output file.
+The generated source code will be written to the output file.
+If a log file path is specified, Propane will write a log file containing
+detailed information about the parser states and transitions.
+
+#> Propane Grammar File
+
+A Propane grammar file provides Propane with the patterns, tokens, grammar
+rules, and user code blocks from which to build the generated lexer and parser.
+
+Example grammar file:
+
+```
+<<
+import std.math;
+>>
+
+# Parser values are unsigned integers.
+ptype ulong;
+
+# A few basic arithmetic operators.
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+# Drop whitespace.
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+```
+
+Grammar files can contain comment lines beginning with `#` which are ignored.
+White space in the grammar file is also ignored.
+
+It is convention to use the extension `.propane` for the Propane grammar file,
+however any file name is accepted by Propane.
+
+This user guide follows the convention of beginning a token name with a
+lowercase character and beginning a rule name with an uppercase character.
+
+##> User Code Blocks
+
+User code blocks begin with the line following a "<<" token and end with the
+line preceding a grammar line consisting of solely the ">>" token.
+All text lines in the code block are copied verbatim into the output file.
+
+### Standalone Code Blocks
+
+C example:
+
+```
+<<
+#include &lt;stdio.h>
+>>
+```
+
+D example:
+
+```
+<<
+import std.stdio;
+>>
+```
+
+Standalone code blocks are emitted early in the output file as top-level code
+outside the context of any function.
+Standalone code blocks are a good place to include/import any other necessary
+supporting code modules.
+They can also define helper functions that can be reused by lexer or parser
+user code blocks.
+They are emitted in the order they are defined in the grammar file.
+
+For a C target, the word "header" may immediately follow the "<<" token to
+cause Propane to emit the code block in the generated header file rather than
+the generated implementation file.
+This allows including another header that may be necessary to define any types
+needed by a `ptype` directive, for example:
+
+```
+<&lt;header
+#include "mytypes.h"
+>>
+```
+
+### Lexer pattern code blocks
+
+Example:
+
+```
+ptype ulong;
+
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+```
+
+Lexer code blocks appear following a `token` or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+### Parser rule code blocks
+
+Example:
+
+```
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+```
+
+Parser rule code blocks appear following a rule expression.
+User code in a parser rule code block will be executed when the parser reduces
+the given rule.
+Assignment to the `$$` symbol will associate a parser value with the reduced
+rule.
+Parser values for the rules or tokens in the rule pattern can be accessed
+positionally with tokens `$1`, `$2`, `$3`, etc...
+
+##> Specifying tokens - the `token` statement
+
+The `token` statement allows defining a lexer token and a pattern to match that
+token.
+The name of the token must be specified immediately following the `token`
+keyword.
+A regular expression pattern may optionally follow the token name.
+If a regular expression pattern is not specified, the name of the token is
+taken to be the pattern.
+See also: ${#Regular expression syntax}.
+
+Example:
+
+```
+token for;
+```
+
+In this example, the token name is `for` and the pattern to match it is
+`/for/`.
+
+Example:
+
+```
+token lbrace /\{/;
+```
+
+In this example, the token name is `lbrace` and a single left curly brace will
+match it.
+
+The `token` statement can also include a user code block.
+The user code block will be executed whenever the token is matched by the
+lexer.
+
+Example:
+
+```
+token if <<
+  writeln("'if' keyword lexed");
+>>
+```
+
+The `token` statement is actually a shortcut statement for a combination of a
+`tokenid` statement and a pattern statement.
+To define a lexer token without an associated pattern to match it, use a
+`tokenid` statement.
+To define a lexer pattern that may or may not result in a matched token, use
+a pattern statement.
+
+##> Defining tokens without a matching pattern - the `tokenid` statement
+
+The `tokenid` statement can be used to define a token without associating it
+with a lexer pattern that matches it.
+
+Example:
+
+```
+tokenid string;
+```
+
+The `tokenid` statement can be useful when defining a token that may optionally
+be returned by user code associated with a pattern.
+
+It is also useful when lexer modes and multiple lexer patterns are required to
+build up a full token.
+A common example is parsing a string.
+See the ${#Lexer modes} chapter for more information.
+
+##> Specifying a lexer pattern - the pattern statement
+
+A pattern statement is used to define a lexer pattern that can execute user
+code but may not result in a matched token.
+
+Example:
+
+```
+/foo+/ <<
+  writeln("saw a foo pattern");
+>>
+```
+
+This can be especially useful with ${#Lexer modes}.
+
+See also ${#Regular expression syntax}.
+
+##> Ignoring input sections - the `drop` statement
+
+A `drop` statement can be used to specify a lexer pattern that when matched
+should result in the matched input being dropped and lexing continuing after
+the matched input.
+
+A common use for a `drop` statement would be to ignore whitespace sequences in
+the user input.
+
+Example:
+
+```
+drop /\s+/;
+```
+
+See also ${#Regular expression syntax}.
+
+##> Regular expression syntax
+
+A regular expression ("regex") is used to define lexer patterns in `token`,
+pattern, and `drop` statements.
+A regular expression begins and ends with a `/` character.
+
+Example:
+
+```
+/#.*$/
+```
+
+Regular expressions can include many special characters:
+
+  * The `.` character matches any input character other than a newline.
+  * The `*` character matches any number of the previous regex element.
+  * The `+` character matches one or more of the previous regex element.
+  * The `?` character matches 0 or 1 of the previous regex element.
+  * The `[` character begins a character class.
+  * The `(` character begins a matching group.
+  * The `{` character begins a count qualifier.
+  * The `\` character escapes the following character and changes its meaning:
+    * The `\d` sequence matches any character `0` through `9`.
+    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
+    `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * Any other character matches itself.
+  * The `|` character creates an alternate match.
+
+Any other character just matches itself in the input stream.
+
+A character class consists of a list of character alternates or character
+ranges that can be matched by the character class.
+For example `[a-zA-Z_]` matches any lowercase character between `a` and `z` or
+any uppercase character between `A` and `Z` or the underscore `_` character.
+Character classes can also be negative character classes if the first character
+after the `[` is a `^` character.
+In this case, the set of characters matched by the character class is the
+inverse of what it otherwise would have been.
+For example, `[^0-9]` matches any character other than 0 through 9.
+
+A matching group can be used to override the pattern sequence that multiplicity
+specifiers apply to.
+For example, the pattern `/foo+/` matches "foo" or "foooo", while the pattern
+`/(foo)+/` matches "foo" or "foofoofoo", but not "foooo".
+
+A count qualifier in curly braces can be used to restrict the number of matches
+of the preceding atom to an explicit minimum and maximum range.
+For example, the pattern `\d{3}` matches exactly 3 digits 0-9.
+Both a minimum and maximum multiplicity count can be specified and separated by
+a comma.
+For example, `/a{1,5}/` matches between 1 and 5 `a` characters.
+Either the minimum or maximum count can be omitted to omit the corresponding
+restriction in the number of matches allowed.
+
+An alternate match is created with the `|` character.
+For example, the pattern `/foo|bar/` matches either the sequence "foo" or the
+sequence "bar".
+
+##> Lexer modes
+
+Lexer modes can be used to change the set of patterns that are matched by the
+lexer.
+A common use for lexer modes is to match strings.
+
+Example:
+
+```
+<<
+string mystringvalue;
+>>
+
+tokenid str;
+
+# String processing
+/"/ <<
+  mystringvalue = "";
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  mystringvalue += match;
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(str);
+>>
+```
+
+A lexer mode is defined by placing the name before a colon (`:`) character that
+precedes a token or pattern statement.
+The token or pattern statement is restricted to only applying if the named mode
+is active.
+
+By default, the active lexer mode is named `default`.
+A `$mode()` call within a lexer code block can be used to change lexer modes.
+
+In the above example, when the lexer in the default mode sees a doublequote
+(`"`) character, the lexer code block will clear the `mystringvalue` variable
+and will set the lexer mode to `string`.
+When the lexer begins looking for patterns to match against the input, it will
+now look only for patterns tagged for the `string` lexer mode.
+Any non-`"` character will be appended to the `mystringvalue` string.
+A `"` character will end the `string` lexer mode and return to the `default`
+lexer mode.
+It also returns the `str` token now that the token is complete.
+
+Note that the token name `str` above could have been `string` instead - the
+namespace for token names is distinct from the namespace for lexer modes.
+
+##> Specifying parser value types - the `ptype` statement
+
+The `ptype` statement is used to define parser value type(s).
+Example:
+
+```
+ptype void *;
+```
+
+This defines the default parser value type to be `void *` (this is, in fact,
+the default parser value type if the grammar file does not specify otherwise).
+
+Each defined lexer token type and parser rule has an associated parser value
+type.
+When the lexer runs, each lexed token has a parser value associated with it.
+When the parser runs, each instance of a reduced rule has a parser value
+associated with it.
+Propane supports using different parser value types for different rules and
+token types.
+The example `ptype` statement above defines the default parser value type.
+A parser value type name can optionally be specified following the `ptype`
+keyword.
+For example:
+
+```
+ptype Value;
+ptype array = Value[];
+ptype dict = Value[string];
+
+Object -> lbrace rbrace <<
+  $$ = new Value();
+>>
+
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>
+
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+```
+
+In this example, the default parser value type is `Value`.
+A parser value type named `array` is defined to mean `Value[]`.
+A parser value type named `dict` is defined to mean `Value[string]`.
+Any defined tokens or rules that do not specify a parser value type will have
+the default parser value type associated with them.
+To associate a different parser value type with a token or rule, write the
+parser value type name in parentheses following the name of the token or rule.
+In this example:
+
+  * a reduced `Object`'s parser value has a type of `Value`.
+  * a reduced `Values`'s parser value has a type of `Value[]`.
+  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
+
+##> Specifying a parser rule - the rule statement
+
+Rule statements create parser rules which define the grammar that will be
+parsed by the generated parser.
+
+Multiple rules with the same name can be specified.
+Rules with the same name define a rule set for that name and act as
+alternatives that the parser can accept when attempting to match a reference to
+that rule.
+
+The grammar file must define a rule with the name `Start` which will be used as
+the top-level starting rule that the parser attempts to reduce.
+
+Example:
+
+```
+ptype ulong;
+token word /[a-z]+/ <<
+  $$ = match.length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+```
+
+In the above example the `Start` rule is defined to match a single `word`
+token.
+
+Example:
+
+```
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+```
+
+A parser rule has zero or more terms on the right side of its definition.
+Each of these terms is either a token name or a rule name.
+
+In a parser rule code block, parser values for the right side terms are
+accessible as `$1` for the first term's parser value, `$2` for the second
+term's parser value, etc...
+The `$$` symbol accesses the output parser value for this rule.
+The above examples demonstrate how the parser values for the rule components
+can be used to produce the parser value for the accepted rule.
+
+##> Specifying the parser module name - the `module` statement
+
+The `module` statement can be used to specify the module name for a generated
+D module.
+
+```
+module proj.parser;
+```
+
+If a module statement is not present, then the generated D module will not
+contain a module statement and the default module name will be used.
+
+##> Specifying the generated API prefix - the `prefix` statement
+
+By default the public API (types, constants, and functions) of the generated
+lexer and parser uses a prefix of `p_`.
+
+This prefix can be changed with the `prefix` statement.
+
+Example:
+
+```
+prefix myparser_;
+```
+
+With a parser generated with this `prefix` statement, instead of calling
+`p_context_init()` you would call `myparser_context_init()`.
+
+The `prefix` statement can be optionally used if you would like to change the
+prefix used by your generated lexer and parser to something other than the
+default.
+
+It can also be used when generating multiple lexers/parsers to be used in the
+same program to avoid symbol collisions.
+
+#> License
+
+Propane is licensed under the terms of the MIT License:
+
+```
+${include LICENSE.txt}
+```
+
+#> Contributing
+
+Propane is developed on [github](https://github.com/holtrop/propane).
+
+Issues may be submitted to [https://github.com/holtrop/propane/issues](https://github.com/holtrop/propane/issues).
+
+Pull requests may be submitted as well:
+
+  1. Fork it
+  2. Create your feature branch (`git checkout -b my-new-feature`)
+  3. Commit your changes (`git commit -am 'Add some feature'`)
+  4. Push to the branch (`git push origin my-new-feature`)
+  5. Create new Pull Request
+
+#> Change Log
+
+${changelog}
@@ -1,101 +0,0 @@
-
-#include <getopt.h>
-
-#include <iostream>
-#include <fstream>
-
-#include "refptr.h"
-#include "Parser.h"
-
-using namespace std;
-
-string buildOutputFilename(string & input_fname);
-
-int main(int argc, char * argv[])
-{
-    int longind = 1;
-    int opt;
-    Parser p;
-    string outfile;
-
-    static struct option longopts[] = {
-        /* name, has_arg, flag, val */
-        { "classname", required_argument, NULL, 'c' },
-        { "extension", required_argument, NULL, 'e' },
-        { "namespace", required_argument, NULL, 'n' },
-        { "outfile", required_argument, NULL, 'o' },
-        { NULL, 0, NULL, 0 }
-    };
-
-    while ((opt = getopt_long(argc, argv, "", longopts, &longind)) != -1)
-    {
-        switch (opt)
-        {
-            case 'c':   /* classname */
-                p.setClassName(optarg);
-                break;
-            case 'e':   /* extension */
-                p.setExtension(optarg);
-                break;
-            case 'n':   /* namespace */
-                p.setNamespace(optarg);
-                break;
-            case 'o':   /* outfile */
-                outfile = optarg;
-                break;
-        }
-    }
-
-    if (optind >= argc)
-    {
-        cerr << "Usage: imbecile [options] <input-file>" << endl;
-        return 1;
-    }
-
-    string input_fname = argv[optind];
-    ifstream ifs;
-    ifs.open(input_fname.c_str(), ios::binary);
-    if (!ifs.is_open())
-    {
-        cerr << "Error opening input file: '" << input_fname << "'";
-        return 2;
-    }
-    ifs.seekg(0, ios_base::end);
-    int size = ifs.tellg();
-    ifs.seekg(0, ios_base::beg);
-    char * buff = new char[size];
-    ifs.read(buff, size);
-    ifs.close();
-
-    if (outfile == "")
-        outfile = buildOutputFilename(input_fname);
-
-    if (!p.parseInputFile(buff, size))
-    {
-        cerr << "Error parsing " << input_fname << endl;
-        return 3;
-    }
-    if (!p.write(outfile))
-    {
-        cerr << "Error processing " << input_fname << endl;
-        return 4;
-    }
-
-    delete[] buff;
-    return 0;
-}
-
-string buildOutputFilename(string & input_fname)
-{
-    string outfile;
-    size_t len = input_fname.length();
-    if (len > 2 && input_fname.substr(len - 2) == ".I")
-    {
-        outfile = input_fname.substr(0, len - 2);
-    }
-    else
-    {
-        outfile = input_fname;
-    }
-    return outfile;
-}
@@ -0,0 +1,48 @@
+require "erb"
+require "set"
+require "stringio"
+require_relative "propane/assets"
+require_relative "propane/cli"
+require_relative "propane/code_point_range"
+require_relative "propane/fa"
+require_relative "propane/fa/state"
+require_relative "propane/fa/state/transition"
+require_relative "propane/generator"
+require_relative "propane/grammar"
+require_relative "propane/lexer"
+require_relative "propane/lexer/dfa"
+require_relative "propane/parser"
+require_relative "propane/parser/item"
+require_relative "propane/parser/item_set"
+require_relative "propane/pattern"
+require_relative "propane/regex"
+require_relative "propane/regex/nfa"
+require_relative "propane/regex/unit"
+require_relative "propane/rule_set"
+require_relative "propane/rule"
+require_relative "propane/token"
+require_relative "propane/util"
+require_relative "propane/version"
+
+class Propane
+
+  class Error < RuntimeError
+  end
+
+  class << self
+
+    def run(input_file, output_file, log_file)
+      begin
+        grammar = Grammar.new(File.read(input_file))
+        generator = Generator.new(grammar, output_file, log_file)
+        generator.generate
+      rescue Error => e
+        $stderr.puts e.message
+        return 2
+      end
+      return 0
+    end
+
+  end
+
+end
@@ -0,0 +1,10 @@
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
+        File.binread(path)
+      end
+    end
+  end
+end
@@ -0,0 +1,54 @@
+class Propane
+  module CLI
+
+    USAGE = <<EOF
+Usage: #{$0} [options] <input-file> <output-file>
+Options:
+  --log LOG   Write log file
+  --version   Show program version and exit
+  -h, --help  Show this usage and exit
+EOF
+
+    class << self
+
+      def run(args)
+        params = []
+        log_file = nil
+        i = 0
+        while i < args.size
+          arg = args[i]
+          case arg
+          when "--log"
+            if i + 1 < args.size
+              i += 1
+              log_file = args[i]
+            end
+          when "--version"
+            puts "propane v#{VERSION}"
+            return 0
+          when "-h", "--help"
+            puts USAGE
+            return 0
+          when /^-/
+            $stderr.puts "Error: unknown option #{arg}"
+            return 1
+          else
+            params << arg
+          end
+          i += 1
+        end
+        if params.size != 2
+          $stderr.puts "Error: specify input and output files"
+          return 1
+        end
+        unless File.readable?(params[0])
+          $stderr.puts "Error: cannot read #{params[0]}"
+          return 2
+        end
+        Propane.run(*params, log_file)
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,84 @@
+class Propane
+  class CodePointRange
+
+    MAX_CODE_POINT = 0x7FFFFFFF
+
+    attr_reader :first
+    attr_reader :last
+
+    include Comparable
+
+    # Build a CodePointRange
+    def initialize(first, last = nil)
+      @first = first.ord
+      if last
+        @last = last.ord
+        if @last < @first
+          raise "Invalid CodePointRange: last code point must be > first code point"
+        end
+      else
+        @last = @first
+      end
+    end
+
+    def <=>(other)
+      if self.first != other.first
+        @first <=> other.first
+      else
+        @last <=> other.last
+      end
+    end
+
+    def include?(v)
+      if v.is_a?(CodePointRange)
+        @first <= v.first && v.last <= @last
+      else
+        @first <= v && v <= @last
+      end
+    end
+
+    def size
+      @last - @first + 1
+    end
+
+    class << self
+
+      def invert_ranges(code_point_ranges)
+        new_ranges = []
+        last_cp = -1
+        code_point_ranges.sort.each do |code_point_range|
+          if code_point_range.first > (last_cp + 1)
+            new_ranges << CodePointRange.new(last_cp + 1, code_point_range.first - 1)
+            last_cp = code_point_range.last
+          else
+            last_cp = [last_cp, code_point_range.last].max
+          end
+        end
+        if last_cp < MAX_CODE_POINT
+          new_ranges << CodePointRange.new(last_cp + 1, MAX_CODE_POINT)
+        end
+        new_ranges
+      end
+
+      def first_subrange(code_point_ranges)
+        code_point_ranges.sort.reduce do |result, code_point_range|
+          if code_point_range.include?(result.first)
+            if code_point_range.last < result.last
+              code_point_range
+            else
+              result
+            end
+          else
+            if code_point_range.first <= result.last
+              CodePointRange.new(result.first, code_point_range.first - 1)
+            else
+              result
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,61 @@
+class Propane
+
+  class FA
+
+    attr_reader :start_state
+
+    def initialize
+      @start_state = State.new
+    end
+
+    def to_s
+      chr = lambda do |value|
+        if value < 32 || value > 127
+          "{#{value}}"
+        else
+          value.chr
+        end
+      end
+      rv = ""
+      states = enumerate
+      states.each do |state, id|
+        accepts_s = state.accepts ? " #{state.accepts}" : ""
+        rv += "#{id}#{accepts_s}:\n"
+        state.transitions.each do |transition|
+          if transition.nil?
+            range_s = "nil"
+          else
+            range_s = chr[transition.code_point_range.first]
+            if transition.code_point_range.size > 1
+              range_s += "-" + chr[transition.code_point_range.last]
+            end
+          end
+          accepts_s = transition.destination.accepts ? " #{transition.destination.accepts}" : ""
+          rv += "  #{range_s} => #{states[transition.destination]}#{accepts_s}\n"
+        end
+      end
+      rv
+    end
+
+    def enumerate
+      @_enumerated ||=
+        begin
+          id = 0
+          states = {}
+          visit = lambda do |state|
+            unless states.include?(state)
+              states[state] = id
+              id += 1
+              state.transitions.each do |transition|
+                visit[transition.destination]
+              end
+            end
+          end
+          visit[@start_state]
+          states
+        end
+    end
+
+  end
+
+end
@@ -0,0 +1,51 @@
+class Propane
+  class FA
+
+    class State
+
+      attr_accessor :accepts
+      attr_reader :transitions
+
+      def initialize
+        @transitions = []
+      end
+
+      def add_transition(code_point_range, destination)
+        @transitions << Transition.new(code_point_range, destination)
+      end
+
+      # Determine the set of states that can be reached by nil transitions.
+      # from this state.
+      #
+      # @return [Set<NFA::State>]
+      #   Set of states.
+      def nil_transition_states
+        states = Set[self]
+        analyze_state = lambda do |state|
+          state.nil_transitions.each do |transition|
+            unless states.include?(transition.destination)
+              states << transition.destination
+              analyze_state[transition.destination]
+            end
+          end
+        end
+        analyze_state[self]
+        states
+      end
+
+      def nil_transitions
+        @transitions.select do |transition|
+          transition.nil?
+        end
+      end
+
+      def cp_transitions
+        @transitions.reject do |transition|
+          transition.nil?
+        end
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,23 @@
+class Propane
+  class FA
+    class State
+
+      class Transition
+
+        attr_reader :code_point_range
+        attr_reader :destination
+
+        def initialize(code_point_range, destination)
+          @code_point_range = code_point_range
+          @destination = destination
+        end
+
+        def nil?
+          @code_point_range.nil?
+        end
+
+      end
+
+    end
+  end
+end
@@ -0,0 +1,285 @@
+class Propane
+
+  class Generator
+
+    def initialize(grammar, output_file, log_file)
+      @grammar = grammar
+      @output_file = output_file
+      if log_file
+        @log = File.open(log_file, "wb")
+      else
+        @log = StringIO.new
+      end
+      @language =
+        if output_file =~ /\.([a-z]+)$/
+          $1
+        else
+          "d"
+        end
+      process_grammar!
+    end
+
+    def generate
+      extensions = [@language]
+      if @language == "c"
+        extensions += %w[h]
+      end
+      extensions.each do |extension|
+        template = Assets.get("parser.#{extension}.erb")
+        erb = ERB.new(template, trim_mode: "<>")
+        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        result = erb.result(binding.clone)
+        File.open(output_file, "wb") do |fh|
+          fh.write(result)
+        end
+      end
+      @log.close
+    end
+
+    private
+
+    def process_grammar!
+      # Assign default pattern mode to patterns without a mode assigned.
+      found_default = false
+      @grammar.patterns.each do |pattern|
+        if pattern.mode.nil?
+          pattern.mode = "default"
+          found_default = true
+        end
+        pattern.ptypename ||= "default"
+      end
+      unless found_default
+        raise Error.new("No patterns found for default mode")
+      end
+      # Add EOF token.
+      @grammar.tokens << Token.new("$EOF", nil, nil)
+      tokens_by_name = {}
+      @grammar.tokens.each_with_index do |token, token_id|
+        # Assign token ID.
+        token.id = token_id
+        # Set default ptypename if none given.
+        token.ptypename ||= "default"
+        # Check for token name conflicts.
+        if tokens_by_name.include?(token.name)
+          raise Error.new("Duplicate token name #{token.name.inspect}")
+        end
+        tokens_by_name[token.name] = token
+      end
+      # Check for user start rule.
+      unless @grammar.rules.find {|rule| rule.name == "Start"}
+        raise Error.new("Start rule not found")
+      end
+      # Add "real" start rule.
+      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
+      rule_sets = {}
+      rule_set_id = @grammar.tokens.size
+      @grammar.rules.each_with_index do |rule, rule_id|
+        # Assign rule ID.
+        rule.id = rule_id
+        # Check for token/rule name conflict.
+        if tokens_by_name.include?(rule.name)
+          raise Error.new("Rule name collides with token name #{rule.name.inspect}")
+        end
+        # Build rule sets of all rules with the same name.
+        unless rule_sets[rule.name]
+          rule_sets[rule.name] = RuleSet.new(rule.name, rule_set_id)
+          rule_set_id += 1
+        end
+        rule_set = rule_sets[rule.name]
+        if rule_set.ptypename && rule.ptypename && rule_set.ptypename != rule.ptypename
+          raise Error.new("Conflicting ptypes for rule #{rule.name}")
+        end
+        rule_set.ptypename ||= rule.ptypename
+        rule.rule_set = rule_set
+        rule_set << rule
+      end
+      rule_sets.each do |name, rule_set|
+        rule_set.ptypename ||= "default"
+        # Assign rule set ptypenames back to rules.
+        rule_set.rules.each do |rule|
+          rule.ptypename = rule_set.ptypename
+        end
+      end
+      # Generate lexer user code IDs for lexer patterns with user code blocks.
+      @grammar.patterns.select do |pattern|
+        pattern.code
+      end.each_with_index do |pattern, code_id|
+        pattern.code_id = code_id
+      end
+      # Map rule components from names to Token/RuleSet objects.
+      @grammar.rules.each do |rule|
+        rule.components.map! do |component|
+          if tokens_by_name[component]
+            tokens_by_name[component]
+          elsif rule_sets[component]
+            rule_sets[component]
+          else
+            raise Error.new("Symbol #{component} not found")
+          end
+        end
+      end
+      determine_possibly_empty_rulesets!(rule_sets)
+      # Generate the lexer.
+      @lexer = Lexer.new(@grammar)
+      # Generate the parser.
+      @parser = Parser.new(@grammar, rule_sets, @log)
+    end
+
+    # Determine which grammar rules could expand to empty sequences.
+    #
+    # @param rule_sets [Hash]
+    #   RuleSets.
+    #
+    # @return [void]
+    def determine_possibly_empty_rulesets!(rule_sets)
+      begin
+        newly_discovered_empty_rulesets = false
+        rule_sets.each do |name, rule_set|
+          unless rule_set.could_be_empty?
+            if could_rule_set_be_empty?(rule_set)
+              newly_discovered_empty_rulesets = true
+              rule_set.could_be_empty = true
+            end
+          end
+        end
+      end while newly_discovered_empty_rulesets
+    end
+
+    # Determine whether a RuleSet could be empty.
+    #
+    # @param rule_set [RuleSet]
+    #   RuleSet to test.
+    #
+    # @return [Boolean]
+    #   Whether the RuleSet could be empty.
+    def could_rule_set_be_empty?(rule_set)
+      rule_set.rules.any? do |rule|
+        could_rule_be_empty?(rule)
+      end
+    end
+
+    # Determine whether a Rule could be empty.
+    #
+    # @param rule [Rule]
+    #   Rule to test.
+    #
+    # @return [Boolean]
+    #   Whether the Rule could be empty.
+    def could_rule_be_empty?(rule)
+      i = 0
+      loop do
+        if i == rule.components.size
+          return true
+        end
+        if rule.components[i].is_a?(Token)
+          return false
+        end
+        if !rule.components[i].could_be_empty?
+          return false
+        end
+        i += 1
+      end
+    end
+
+    # Expand expansions in user code block.
+    #
+    # @param code [String]
+    #   User code block.
+    # @param parser [Boolean]
+    #   Whether the user code is for the parser or lexer.
+    # @param rule [Rule, nil]
+    #   The Rule associated with the user code if user code is for the parser.
+    # @param pattern [Pattern, nil]
+    #   The Pattern associated with the user code if user code is for the lexer.
+    #
+    # @return [String]
+    #   Expanded user code block.
+    def expand_code(code, parser, rule, pattern)
+      code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
+        "TOKEN_#{Token.code_name($1)}"
+      end
+      if parser
+        code = code.gsub(/\$\$/) do |match|
+          "_pvalue.v_#{rule.ptypename}"
+        end
+        code = code.gsub(/\$(\d+)/) do |match|
+          index = $1.to_i
+          case @language
+          when "c"
+            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
+          when "d"
+            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
+          end
+        end
+      else
+        code = code.gsub(/\$\$/) do |match|
+          case @language
+          when "c"
+            "out_token_info->pvalue.v_#{pattern.ptypename}"
+          when "d"
+            "out_token_info.pvalue.v_#{pattern.ptypename}"
+          end
+        end
+        code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
+          mode_name = $1
+          mode_id = @lexer.mode_id(mode_name)
+          unless mode_id
+            raise Error.new("Lexer mode '#{mode_name}' not found")
+          end
+          case @language
+          when "c"
+            "context->mode = #{mode_id}u"
+          when "d"
+            "context.mode = #{mode_id}u"
+          end
+        end
+      end
+      code
+    end
+
+    # Get the parser value type for the start rule.
+    #
+    # @return [Array<String>]
+    #   Start rule parser value type name and type string.
+    def start_rule_type
+      start_rule = @grammar.rules.find do |rule|
+        rule.name == "Start"
+      end
+      [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
+    end
+
+    # Get an unsigned integer type that can hold the given maximum value.
+    #
+    # @param max [Integer]
+    #   Maximum value to store.
+    #
+    # @return [String]
+    #   Type.
+    def get_type_for(max)
+      if max <= 0xFF
+        case @language
+        when "c"
+          "uint8_t"
+        when "d"
+          "ubyte"
+        end
+      elsif max <= 0xFFFF
+        case @language
+        when "c"
+          "uint16_t"
+        when "d"
+          "ushort"
+        end
+      else
+        case @language
+        when "c"
+          "uint32_t"
+        else
+          "uint"
+        end
+      end
+    end
+
+  end
+
+end
@@ -0,0 +1,256 @@
+class Propane
+
+  class Grammar
+
+    # Reserve identifiers beginning with a double-underscore for internal use.
+    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
+
+    attr_reader :modulename
+    attr_reader :patterns
+    attr_reader :rules
+    attr_reader :tokens
+    attr_reader :code_blocks
+    attr_reader :ptypes
+    attr_reader :prefix
+
+    def initialize(input)
+      @patterns = []
+      @tokens = []
+      @rules = []
+      @code_blocks = {}
+      @line_number = 1
+      @next_line_number = @line_number
+      @mode = nil
+      @input = input.gsub("\r\n", "\n")
+      @ptypes = {"default" => "void *"}
+      @prefix = "p_"
+      parse_grammar!
+    end
+
+    def ptype
+      @ptypes["default"]
+    end
+
+    def invalid_token_id
+      @tokens.size
+    end
+
+    private
+
+    def parse_grammar!
+      while @input.size > 0
+        parse_statement!
+      end
+    end
+
+    def parse_statement!
+      if parse_white_space!
+      elsif parse_comment_line!
+      elsif @mode.nil? && parse_mode_label!
+      elsif parse_module_statement!
+      elsif parse_ptype_statement!
+      elsif parse_pattern_statement!
+      elsif parse_token_statement!
+      elsif parse_tokenid_statement!
+      elsif parse_drop_statement!
+      elsif parse_rule_statement!
+      elsif parse_code_block_statement!
+      elsif parse_prefix_statement!
+      else
+        if @input.size > 25
+          @input = @input.slice(0..20) + "..."
+        end
+        raise Error.new("Unexpected grammar input at line #{@line_number}: #{@input.chomp}")
+      end
+    end
+
+    def parse_mode_label!
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
+        @mode = md[1]
+      end
+    end
+
+    def parse_white_space!
+      consume!(/\s+/)
+    end
+
+    def parse_comment_line!
+      consume!(/#.*\n/)
+    end
+
+    def parse_module_statement!
+      if consume!(/module\s+/)
+        md = consume!(/([\w.]+)\s*/, "expected module name")
+        @modulename = md[1]
+        consume!(/;/, "expected `;'")
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_ptype_statement!
+      if consume!(/ptype\s+/)
+        name = "default"
+        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          name = md[1]
+        end
+        md = consume!(/([^;]+);/, "expected parser result type expression")
+        @ptypes[name] = md[1].strip
+      end
+    end
+
+    def parse_token_statement!
+      if consume!(/token\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
+        name = md[1]
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
+        end
+        pattern = parse_pattern! || name
+        consume!(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected pattern or `;' or code block")
+        end
+        token = Token.new(name, ptypename, @line_number)
+        @tokens << token
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @patterns << pattern
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_tokenid_statement!
+      if md = consume!(/tokenid\s+/)
+        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
+        name = md[1]
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
+        end
+        consume!(/;/, "expected `;'");
+        token = Token.new(name, ptypename, @line_number)
+        @tokens << token
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_drop_statement!
+      if md = consume!(/drop\s+/)
+        pattern = parse_pattern!
+        unless pattern
+          raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
+        end
+        consume!(/\s+/)
+        consume!(/;/, "expected `;'")
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_rule_statement!
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
+        rule_name, ptypename = *md[1, 2]
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        components = md[1].strip.split(/\s+/)
+        unless code = parse_code_block!
+          consume!(/;/, "expected pattern or `;' or code block")
+        end
+        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_pattern_statement!
+      if pattern = parse_pattern!
+        consume!(/\s+/)
+        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          ptypename = md[1]
+        end
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block to follow pattern")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_code_block_statement!
+      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+        name, code = md[1..2]
+        if @code_blocks[name]
+          @code_blocks[name] += code
+        else
+          @code_blocks[name] = code
+        end
+        @mode = nil
+        true
+      end
+    end
+
+    def parse_prefix_statement!
+      if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
+        @prefix = md[1]
+        true
+      end
+    end
+
+    def parse_pattern!
+      if md = consume!(%r{/})
+        pattern = ""
+        while !consume!(%r{/})
+          if consume!(%r{\\})
+            pattern += "\\"
+            if md = consume!(%r{(.)})
+              pattern += md[1]
+            else
+              raise Error.new("Line #{@line_number}: unterminated escape sequence")
+            end
+          elsif md = consume!(%r{(.)})
+            pattern += md[1]
+          end
+        end
+        pattern
+      end
+    end
+
+    def parse_code_block!
+      if md = consume!(/<<\n(.*?)^>>\n/m)
+        md[1]
+      end
+    end
+
+    # Check if the input string matches the given regex.
+    #
+    # If so, remove the match from the input string, and update the line
+    # number. If the regex is not matched and an error message is provided,
+    # the error is raised.
+    #
+    # @param regex [Regexp]
+    #   Regex to attempt to match.
+    # @param error_message [String, nil]
+    #   Error message to display if the regex is not matched. If nil and the
+    #   regex is not matched, an error is not raised.
+    #
+    # @return [MatchData, nil]
+    #   MatchData for the given regex if it was matched and removed from the
+    #   input.
+    def consume!(regex, error_message = nil)
+      @line_number = @next_line_number
+      if md = @input.match(/\A#{regex}/)
+        @input.slice!(0, md[0].size)
+        @next_line_number += md[0].count("\n")
+        md
+      elsif error_message
+        raise Error.new("Line #{@line_number}: Error: #{error_message}")
+      else
+        false
+      end
+    end
+
+  end
+
+end
@@ -0,0 +1,68 @@
+class Propane
+  class Lexer
+
+    attr_reader :state_table
+    attr_reader :transition_table
+    attr_reader :mode_table
+
+    def initialize(grammar)
+      @grammar = grammar
+      build_tables!
+    end
+
+    # Get ID for a mode.
+    #
+    # @param mode_name [String]
+    #   Mode name.
+    #
+    # @return [Integer, nil]
+    #   Mode ID.
+    def mode_id(mode_name)
+      if mode_info = @modes[mode_name]
+        mode_info[:id]
+      end
+    end
+
+    private
+
+    def build_tables!
+      @modes = @grammar.patterns.group_by do |pattern|
+        pattern.mode
+      end.transform_values do |patterns|
+        {dfa: DFA.new(patterns)}
+      end
+      @modes.each_with_index do |(mode_name, mode_info), index|
+        mode_info[:id] = index
+      end
+      @state_table = []
+      @transition_table = []
+      @mode_table = []
+      @modes.each do |mode_name, mode_info|
+        state_table_offset = @state_table.size
+        @mode_table << {
+          state_table_offset: state_table_offset,
+        }
+        states = mode_info[:dfa].enumerate
+        states.each do |state, id|
+          token = state.accepts && state.accepts.token && state.accepts.token.id
+          code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
+          @state_table << {
+            transition_table_index: @transition_table.size,
+            n_transitions: state.transitions.size,
+            accepts: !!state.accepts,
+            token: token,
+            code_id: code_id,
+          }
+          state.transitions.each do |transition|
+            @transition_table << {
+              first: transition.code_point_range.first,
+              last: transition.code_point_range.last,
+              destination: states[transition.destination] + state_table_offset,
+            }
+          end
+        end
+      end
+    end
+
+  end
+end
@@ -0,0 +1,89 @@
+class Propane
+  class Lexer
+
+    class DFA < FA
+
+      def initialize(patterns)
+        super()
+        start_nfa = Regex::NFA.new
+        patterns.each do |pattern|
+          start_nfa.start_state.add_transition(nil, pattern.nfa.start_state)
+        end
+        @nfa_state_sets = {}
+        @states = []
+        @to_process = Set.new
+        nil_transition_states = start_nfa.start_state.nil_transition_states
+        register_nfa_state_set(nil_transition_states)
+        while @to_process.size > 0
+          state_set = @to_process.first
+          @to_process.delete(state_set)
+          process_nfa_state_set(state_set)
+        end
+        @start_state = @states[0]
+      end
+
+      private
+
+      def register_nfa_state_set(nfa_state_set)
+        unless @nfa_state_sets.include?(nfa_state_set)
+          state_id = @states.size
+          @nfa_state_sets[nfa_state_set] = state_id
+          @states << State.new
+          @to_process << nfa_state_set
+        end
+      end
+
+      def process_nfa_state_set(nfa_state_set)
+        state_id = @nfa_state_sets[nfa_state_set]
+        state = @states[state_id]
+        if state_id > 0
+          nfa_state_set.each do |nfa_state|
+            if nfa_state.accepts
+              if state.accepts
+                if nfa_state.accepts.line_number < state.accepts.line_number
+                  state.accepts = nfa_state.accepts
+                end
+              else
+                state.accepts = nfa_state.accepts
+              end
+            end
+          end
+        end
+        transitions = transitions_for(nfa_state_set)
+        while transitions.size > 0
+          subrange = CodePointRange.first_subrange(transitions.map(&:code_point_range))
+          dest_nfa_states = transitions.reduce(Set.new) do |result, transition|
+            if transition.code_point_range.include?(subrange)
+              result << transition.destination
+            end
+            result
+          end
+          dest_nfa_states = dest_nfa_states.reduce(Set.new) do |result, dest_nfa_state|
+            result + dest_nfa_state.nil_transition_states
+          end
+          register_nfa_state_set(dest_nfa_states)
+          dest_state = @states[@nfa_state_sets[dest_nfa_states]]
+          state.add_transition(subrange, dest_state)
+          transitions.delete_if do |transition|
+            transition.code_point_range.last <= subrange.last
+          end
+          transitions.map! do |transition|
+            if transition.code_point_range.first <= subrange.last
+              Regex::NFA::State::Transition.new(CodePointRange.new(subrange.last + 1, transition.code_point_range.last), transition.destination)
+            else
+              transition
+            end
+          end
+        end
+      end
+
+      def transitions_for(nfa_state_set)
+        nfa_state_set.reduce([]) do |result, state|
+          result + state.cp_transitions
+        end
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,261 @@
+class Propane
+
+  class Parser
+
+    attr_reader :state_table
+    attr_reader :shift_table
+    attr_reader :reduce_table
+    attr_reader :rule_sets
+
+    def initialize(grammar, rule_sets, log)
+      @grammar = grammar
+      @rule_sets = rule_sets
+      @log = log
+      @item_sets = []
+      @item_sets_set = {}
+      start_item = Item.new(grammar.rules.first, 0)
+      eval_item_sets = Set[ItemSet.new([start_item])]
+
+      while eval_item_sets.size > 0
+        item_set = eval_item_sets.first
+        eval_item_sets.delete(item_set)
+        unless @item_sets_set.include?(item_set)
+          item_set.id = @item_sets.size
+          @item_sets << item_set
+          @item_sets_set[item_set] = item_set
+          item_set.following_symbols.each do |following_symbol|
+            unless following_symbol.name == "$EOF"
+              following_set = item_set.build_following_item_set(following_symbol)
+              eval_item_sets << following_set
+            end
+          end
+        end
+      end
+
+      @item_sets.each do |item_set|
+        process_item_set(item_set)
+      end
+
+      build_reduce_actions!
+      write_log!
+      build_tables!
+    end
+
+    private
+
+    def build_tables!
+      @state_table = []
+      @shift_table = []
+      @reduce_table = []
+      @item_sets.each do |item_set|
+        shift_entries = item_set.following_symbols.map do |following_symbol|
+          state_id =
+            if following_symbol.name == "$EOF"
+              0
+            else
+              item_set.following_item_set[following_symbol].id
+            end
+          {
+            symbol_id: following_symbol.id,
+            state_id: state_id,
+          }
+        end
+        reduce_entries =
+          case ra = item_set.reduce_actions
+          when Rule
+            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
+              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
+          when Hash
+            ra.map do |token, rule|
+              {token_id: token.id, rule_id: rule.id,
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
+            end
+          else
+            []
+          end
+        @state_table << {
+          shift_index: @shift_table.size,
+          n_shifts: shift_entries.size,
+          reduce_index: @reduce_table.size,
+          n_reduces: reduce_entries.size,
+        }
+        @shift_table += shift_entries
+        @reduce_table += reduce_entries
+      end
+    end
+
+    def process_item_set(item_set)
+      item_set.following_symbols.each do |following_symbol|
+        unless following_symbol.name == "$EOF"
+          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
+          item_set.following_item_set[following_symbol] = following_set
+          following_set.in_sets << item_set
+        end
+      end
+    end
+
+    # Build the reduce actions for each ItemSet.
+    #
+    # @return [void]
+    def build_reduce_actions!
+      @item_sets.each do |item_set|
+        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
+      end
+    end
+
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [nil, Rule, Hash]
+    #   If no reduce actions are possible for the given item set, nil.
+    #   If only one reduce action is possible for the given item set, the Rule
+    #   to reduce.
+    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
+    def build_reduce_actions_for_item_set(item_set)
+      # To build the reduce actions, we start by looking at any
+      # "complete" items, i.e., items where the parse position is at the
+      # end of a rule. These are the only rules that are candidates for
+      # reduction in the current ItemSet.
+      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+
+      # If there are no rules to reduce for this ItemSet, we're done here.
+      return nil if reduce_rules.size == 0
+
+      # If there is exactly one rule to reduce for this ItemSet, then do not
+      # figure out the lookaheads; just reduce it.
+      return reduce_rules.first if reduce_rules.size == 1
+
+      # Otherwise, we have more than one possible rule to reduce.
+
+      # We will be looking for all possible tokens that can follow instances of
+      # these rules. Rather than looking through the entire grammar for the
+      # possible following tokens, we will only look in the item sets leading
+      # up to this one. This restriction gives us a more precise lookahead set,
+      # and allows us to parse LALR grammars.
+      item_sets = item_set.leading_item_sets
+      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+        lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
+        lookahead_tokens_for_rule.each do |lookahead_token|
+          if existing_reduce_rule = reduce_actions[lookahead_token]
+            raise Error.new("Error: reduce/reduce conflict between rule #{existing_reduce_rule.id} (#{existing_reduce_rule.name}) and rule #{reduce_rule.id} (#{reduce_rule.name})")
+          end
+          reduce_actions[lookahead_token] = reduce_rule
+        end
+        reduce_actions
+      end
+    end
+
+    # Build the set of lookahead Tokens that should cause the given Rule to be
+    # reduced in the given context of ItemSets.
+    #
+    # @param rule [Rule]
+    #   Rule to reduce.
+    # @param item_sets [Set<ItemSet>]
+    #   ItemSets to consider for the context in which to reduce this Rule.
+    #
+    # @return [Set<Token>]
+    #   Possible lookahead Tokens for the given Rule within the context of the
+    #   given ItemSets.
+    def build_lookahead_tokens_to_reduce(rule, item_sets)
+      # We need to look for possible following tokens for this reduce rule. We
+      # do this by looking for tokens that follow the reduce rule, or the
+      # start token set for any other rule that follows the reduce rule.
+      # While doing this, the following situations could arise:
+      # 1. We may come across a following rule that could be empty. In this
+      #    case, in addition to the start token set for that rule, we must also
+      #    continue to the next following symbol after the potentially empty
+      #    rule and continue the search for potential following tokens.
+      # 2. We may reach the end of a rule that was not one of the original
+      #    reduce rules. In this case, we must also search for all potential
+      #    following tokens for this rule as well.
+      lookahead_tokens = Set.new
+      rule_sets_to_check_after = [rule.rule_set]
+      checked_rule_sets = Set.new
+      while !rule_sets_to_check_after.empty?
+        rule_set = rule_sets_to_check_after.slice!(0)
+        checked_rule_sets << rule_set
+        # For each RuleSet we're checking, we're going to look through all
+        # items in the item sets of interest and gather all possible following
+        # tokens to form the lookahead token set.
+        item_sets.each do |item_set|
+          item_set.items.each do |item|
+            if item.following_symbol == rule_set
+              (1..).each do |offset|
+                case symbol = item.following_symbol(offset)
+                when nil
+                  rule_set = item.rule.rule_set
+                  unless checked_rule_sets.include?(rule_set)
+                    rule_sets_to_check_after << rule_set
+                  end
+                  break
+                when Token
+                  lookahead_tokens << symbol
+                  break
+                when RuleSet
+                  lookahead_tokens += symbol.start_token_set
+                  unless symbol.could_be_empty?
+                    break
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      lookahead_tokens
+    end
+
+    def write_log!
+      @log.puts Util.banner("Parser Rules")
+      @grammar.rules.each do |rule|
+        @log.puts
+        @log.puts "Rule #{rule.id}:"
+        @log.puts "  #{rule}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser Tokens")
+      @log.puts
+      @grammar.tokens.each do |token|
+        @log.puts "Token #{token.id}: #{token.name}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser Rule Sets")
+      @rule_sets.each do |rule_set_name, rule_set|
+        @log.puts
+        @log.puts "Rule Set #{rule_set.id}: #{rule_set_name}"
+        @log.puts "  Start token set: #{rule_set.start_token_set.map(&:name).join(", ")}"
+      end
+
+      @log.puts
+      @log.puts Util.banner("Parser States")
+      @item_sets.each do |item_set|
+        @log.puts
+        @log.puts "State #{item_set.id}:"
+        @log.puts item_set.to_s.gsub(/^/, "  ")
+        incoming_ids = item_set.in_sets.map(&:id)
+        @log.puts
+        @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
+        @log.puts "  Outgoing states:"
+        item_set.following_item_set.each do |following_symbol, following_item_set|
+          @log.puts "    #{following_symbol.name} => #{following_item_set.id}"
+        end
+        @log.puts
+        @log.puts "  Reduce actions:"
+        case item_set.reduce_actions
+        when Rule
+          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
+        when Hash
+          item_set.reduce_actions.each do |token, rule|
+            @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
+          end
+        end
+      end
+    end
+
+  end
+
+end
@@ -0,0 +1,146 @@
+class Propane
+  class Parser
+
+    # Represent a parser "item", which is a position in a Rule that the parser
+    # could potentially be at.
+    class Item
+
+      # @return [Rule]
+      #   The Rule corresponding to this Item.
+      attr_reader :rule
+
+      # @return [Integer]
+      #   The parse position in this item.
+      attr_reader :position
+
+      # Construct an Item.
+      #
+      # @param rule [Rule]
+      #   The Rule corresponding to this Item.
+      # @param position [Integer]
+      #   The parse position in this Item.
+      def initialize(rule, position)
+        @rule = rule
+        @position = position
+      end
+
+      # Hash function.
+      #
+      # @return [Integer]
+      #   Hash code.
+      def hash
+        [@rule, @position].hash
+      end
+
+      # Compare Item objects.
+      #
+      # @param other [Item]
+      #   Item to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the Items are equal.
+      def ==(other)
+        @rule == other.rule && @position == other.position
+      end
+
+      # Compare Item objects.
+      #
+      # @param other [Item]
+      #   Item to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the Items are equal.
+      def eql?(other)
+        self == other
+      end
+
+      # Return the set of Items obtained by "closing" the current item.
+      #
+      # If the following symbol for the current item is another Rule name, then
+      # this method will return all Items for that Rule with a position of 0.
+      # Otherwise, an empty Array is returned.
+      #
+      # @return [Array<Item>]
+      #   Items obtained by "closing" the current item.
+      def closed_items
+        if @rule.components[@position].is_a?(RuleSet)
+          @rule.components[@position].rules.map do |rule|
+            Item.new(rule, 0)
+          end
+        else
+          []
+        end
+      end
+
+      # Return whether the item is "complete", meaning that the parse position
+      # marker is at the end of the rule.
+      #
+      # @return [Boolean]
+      #   Whether the item is "complete".
+      def complete?
+        @position == @rule.components.size
+      end
+
+      # Get the following symbol for the Item.
+      #
+      # That is, the symbol which follows the parse position marker in the
+      # current Item.
+      #
+      # @param offset [Integer]
+      #   Offset from current parse position to examine.
+      #
+      # @return [Token, RuleSet, nil]
+      #   Following symbol for the Item.
+      def following_symbol(offset = 0)
+        @rule.components[@position + offset]
+      end
+
+      # Get the previous symbol for the Item.
+      #
+      # That is, the symbol which precedes the parse position marker in the
+      # current Item.
+      #
+      # @return [Token, RuleSet, nil]
+      #   Previous symbol for this Item.
+      def previous_symbol
+        if @position > 0
+          @rule.components[@position - 1]
+        end
+      end
+
+      # Get whether this Item is followed by the provided symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Symbol to query.
+      #
+      # @return [Boolean]
+      #   Whether this Item is followed by the provided symbol.
+      def followed_by?(symbol)
+        following_symbol == symbol
+      end
+
+      # Get the following item for this Item.
+      #
+      # That is, the Item formed by moving the parse position marker one place
+      # forward from its position in this Item.
+      #
+      # @return [Item]
+      #   The following item for this Item.
+      def following_item
+        Item.new(@rule, @position + 1)
+      end
+
+      # Represent the Item as a String.
+      #
+      # @return [String]
+      #   The Item represented as a String.
+      def to_s
+        parts = @rule.components.map(&:name)
+        parts[@position, 0] = "."
+        "#{@rule.name} -> #{parts.join(" ")}"
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,146 @@
+class Propane
+  class Parser
+
+    # Represent a parser "item set", which is a set of possible items that the
+    # parser could currently be parsing.
+    class ItemSet
+
+      # @return [Set<Item>]
+      #   Items in this ItemSet.
+      attr_reader :items
+
+      # @return [Integer]
+      #   ID of this ItemSet.
+      attr_accessor :id
+
+      # @return [Hash]
+      #   Maps a following symbol to its ItemSet.
+      attr_reader :following_item_set
+
+      # @return [Set<ItemSet>]
+      #   ItemSets leading to this item set.
+      attr_reader :in_sets
+
+      # @return [nil, Rule, Hash]
+      #   Reduce actions, mapping lookahead tokens to rules.
+      attr_accessor :reduce_actions
+
+      # Build an ItemSet.
+      #
+      # @param items [Array<Item>]
+      #   Items in this ItemSet.
+      def initialize(items)
+        @items = Set.new(items)
+        @following_item_set = {}
+        @in_sets = Set.new
+        close!
+      end
+
+      # Get the set of following symbols for all Items in this ItemSet.
+      #
+      # @return [Set<Token, RuleSet>]
+      #   Set of following symbols for all Items in this ItemSet.
+      def following_symbols
+        Set.new(@items.map(&:following_symbol).compact)
+      end
+
+      # Build a following ItemSet for the given following symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Following symbol to build the following ItemSet for.
+      #
+      # @return [ItemSet]
+      #   Following ItemSet for the given following symbol.
+      def build_following_item_set(symbol)
+        ItemSet.new(items_followed_by(symbol).map(&:following_item))
+      end
+
+      # Hash function.
+      #
+      # @return [Integer]
+      #   Hash code.
+      def hash
+        @items.hash
+      end
+
+      # Compare ItemSet objects.
+      #
+      # @param other [ItemSet]
+      #   ItemSet to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the ItemSets are equal.
+      def ==(other)
+        @items.eql?(other.items)
+      end
+
+      # Compare ItemSet objects.
+      #
+      # @param other [ItemSet]
+      #   ItemSet to compare to.
+      #
+      # @return [Boolean]
+      #   Whether the ItemSets are equal.
+      def eql?(other)
+        self == other
+      end
+
+      # Set of ItemSets that lead to this ItemSet.
+      #
+      # This set includes this ItemSet.
+      #
+      # @return [Set<ItemSet>]
+      #   Set of all ItemSets that lead up to this ItemSet.
+      def leading_item_sets
+        @in_sets.reduce(Set[self]) do |result, item_set|
+          result + item_set.leading_item_sets
+        end
+      end
+
+      # Represent the ItemSet as a String.
+      #
+      # @return [String]
+      #   The ItemSet represented as a String.
+      def to_s
+        @items.map(&:to_s).join("\n")
+      end
+
+      private
+
+      # Close the ItemSet.
+      #
+      # This is done by recursively adding the closed Items for each Item in
+      # the ItemSet.
+      #
+      # @return [void]
+      def close!
+        eval_items = @items.dup
+        while eval_items.size > 0
+          item = eval_items.first
+          eval_items.delete(item)
+          item.closed_items.each do |new_item|
+            unless @items.include?(new_item)
+              eval_items << new_item
+            end
+          end
+          @items += eval_items
+        end
+      end
+
+      # Get the Items followed by the given following symbol.
+      #
+      # @param symbol [Token, RuleSet]
+      #   Following symbol.
+      #
+      # @return [Array<Item>]
+      #   Items followed by the given following symbol.
+      def items_followed_by(symbol)
+        @items.select do |item|
+          item.followed_by?(symbol)
+        end
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,65 @@
+class Propane
+
+  class Pattern
+
+    # @return [String, nil]
+    #   Code block to execute when the pattern is matched.
+    attr_reader :code
+
+    # @option options [Integer, nil] :code_id
+    #   Code block ID.
+    attr_accessor :code_id
+
+    # @return [String, nil]
+    #   Pattern.
+    attr_reader :pattern
+
+    # @return [Token, nil]
+    #   Token to be returned by this pattern.
+    attr_reader :token
+
+    # @return [Integer, nil]
+    #   Line number where the pattern was defined in the input grammar.
+    attr_reader :line_number
+
+    # @return [Regex::NFA, nil]
+    #   Regex NFA for matching the pattern.
+    attr_reader :nfa
+
+    # @return [String, nil]
+    #   Lexer mode for this pattern.
+    attr_accessor :mode
+
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
+
+    # Construct a Pattern.
+    #
+    # @param options [Hash]
+    #   Optional parameters.
+    # @option options [String, nil] :code
+    #   Code block to execute when the pattern is matched.
+    # @option options [String, nil] :pattern
+    #   Pattern.
+    # @option options [Token, nil] :token
+    #   Token to be returned by this pattern.
+    # @option options [Integer, nil] :line_number
+    #   Line number where the token was defined in the input grammar.
+    # @option options [String, nil] :mode
+    #   Lexer mode for this pattern.
+    def initialize(options)
+      @code = options[:code]
+      @pattern = options[:pattern]
+      @token = options[:token]
+      @line_number = options[:line_number]
+      @mode = options[:mode]
+      @ptypename = options[:ptypename]
+      regex = Regex.new(@pattern)
+      regex.nfa.end_state.accepts = self
+      @nfa = regex.nfa
+    end
+
+  end
+
+end
@@ -0,0 +1,162 @@
+class Propane
+  class Regex
+
+    attr_reader :unit
+    attr_reader :nfa
+
+    def initialize(pattern)
+      @pattern = pattern.dup
+      @unit = parse_alternates
+      @nfa = @unit.to_nfa
+      if @pattern != ""
+        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
+      end
+    end
+
+    private
+
+    def parse_alternates
+      au = AlternatesUnit.new
+      while @pattern != ""
+        c = @pattern[0]
+        return au if c == ")"
+        @pattern.slice!(0)
+        case c
+        when "["
+          au << parse_character_class
+        when "("
+          au << parse_group
+        when "*", "+", "?", "{"
+          if last_unit = au.last_unit
+            case c
+            when "*"
+              min_count, max_count = 0, nil
+            when "+"
+              min_count, max_count = 1, nil
+            when "?"
+              min_count, max_count = 0, 1
+            when "{"
+              min_count, max_count = parse_curly_count
+            end
+            mu = MultiplicityUnit.new(last_unit, min_count, max_count)
+            au.replace_last!(mu)
+          else
+            raise Error.new("#{c} follows nothing")
+          end
+        when "|"
+          au.new_alternate!
+        when "\\"
+          au << parse_backslash
+        when "."
+          au << period_character_class
+        else
+          au << CharacterRangeUnit.new(c)
+        end
+      end
+      au
+    end
+
+    def parse_group
+      au = parse_alternates
+      if @pattern[0] != ")"
+        raise Error.new("Unterminated group in pattern")
+      end
+      @pattern.slice!(0)
+      au
+    end
+
+    def parse_character_class
+      ccu = CharacterClassUnit.new
+      index = 0
+      loop do
+        if @pattern == ""
+          raise Error.new("Unterminated character class")
+        end
+        c = @pattern.slice!(0)
+        if c == "]"
+          break
+        elsif c == "^" && index == 0
+          ccu.negate = true
+        elsif c == "-" && (ccu.size == 0 || @pattern[0] == "]")
+          ccu << CharacterRangeUnit.new(c)
+        elsif c == "\\"
+          ccu << parse_backslash
+        elsif c == "-" && @pattern[0] != "]"
+          begin_cu = ccu.last_unit
+          unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
+            raise Error.new("Character range must be between single characters")
+          end
+          if @pattern[0] == "\\"
+            @pattern.slice!(0)
+            end_cu = parse_backslash
+            unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
+              raise Error.new("Character range must be between single characters")
+            end
+            max_code_point = end_cu.code_point
+          else
+            max_code_point = @pattern[0].ord
+            @pattern.slice!(0)
+          end
+          cru = CharacterRangeUnit.new(begin_cu.first, max_code_point)
+          ccu.replace_last!(cru)
+        else
+          ccu << CharacterRangeUnit.new(c)
+        end
+        index += 1
+      end
+      ccu
+    end
+
+    def parse_curly_count
+      if @pattern =~ /^(\d+)(?:(,)(\d*))?\}(.*)$/
+        min_count, comma, max_count, pattern = $1, $2, $3, $4
+        min_count = min_count.to_i
+        if comma.to_s == ""
+          max_count = min_count
+        elsif max_count.to_s != ""
+          max_count = max_count.to_i
+          if max_count < min_count
+            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
+          end
+        else
+          max_count = nil
+        end
+        @pattern = pattern
+        [min_count, max_count]
+      else
+        raise Error.new("Unexpected match count at #{@pattern}")
+      end
+    end
+
+    def parse_backslash
+      if @pattern == ""
+        raise Error.new("Error: unfollowed \\")
+      else
+        c = @pattern.slice!(0)
+        case c
+        when "d"
+          CharacterRangeUnit.new("0", "9")
+        when "s"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new(" ")
+          ccu << CharacterRangeUnit.new("\t")
+          ccu << CharacterRangeUnit.new("\r")
+          ccu << CharacterRangeUnit.new("\n")
+          ccu << CharacterRangeUnit.new("\f")
+          ccu << CharacterRangeUnit.new("\v")
+          ccu
+        else
+          CharacterRangeUnit.new(c)
+        end
+      end
+    end
+
+    def period_character_class
+      ccu = CharacterClassUnit.new
+      ccu << CharacterRangeUnit.new(0, "\n".ord - 1)
+      ccu << CharacterRangeUnit.new("\n".ord + 1, 0xFFFFFFFF)
+      ccu
+    end
+
+  end
+end
@@ -0,0 +1,26 @@
+class Propane
+  class Regex
+
+    class NFA < FA
+
+      attr_reader :end_state
+
+      def initialize
+        super()
+        @end_state = State.new
+      end
+
+      class << self
+
+        def empty
+          nfa = NFA.new
+          nfa.start_state.add_transition(nil, nfa.end_state)
+          nfa
+        end
+
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,172 @@
+class Propane
+  class Regex
+
+    class Unit
+    end
+
+    class SequenceUnit < Unit
+      attr_accessor :units
+      def initialize
+        @units = []
+      end
+      def method_missing(*args)
+        @units.__send__(*args)
+      end
+      def to_nfa
+        if @units.empty?
+          NFA.empty
+        else
+          nfa = NFA.new
+          unit_nfas = @units.map do |unit|
+            unit.to_nfa
+          end
+          nfa.start_state.add_transition(nil, unit_nfas[0].start_state)
+          unit_nfas.reduce do |prev_nfa, next_nfa|
+            prev_nfa.end_state.add_transition(nil, next_nfa.start_state)
+            next_nfa
+          end.end_state.add_transition(nil, nfa.end_state)
+          nfa
+        end
+      end
+    end
+
+    class AlternatesUnit < Unit
+      attr_accessor :alternates
+      def initialize
+        @alternates = []
+        new_alternate!
+      end
+      def new_alternate!
+        @alternates << SequenceUnit.new
+      end
+      def <<(unit)
+        @alternates[-1] << unit
+      end
+      def last_unit
+        @alternates[-1][-1]
+      end
+      def replace_last!(new_unit)
+        @alternates[-1][-1] = new_unit
+      end
+      def to_nfa
+        if @alternates.size == 0
+          NFA.empty
+        elsif @alternates.size == 1
+          @alternates[0].to_nfa
+        else
+          nfa = NFA.new
+          alternate_nfas = @alternates.map do |alternate|
+            alternate.to_nfa
+          end
+          alternate_nfas.each do |alternate_nfa|
+            nfa.start_state.add_transition(nil, alternate_nfa.start_state)
+            alternate_nfa.end_state.add_transition(nil, nfa.end_state)
+          end
+          nfa
+        end
+      end
+    end
+
+    class CharacterRangeUnit < Unit
+      attr_reader :code_point_range
+      def initialize(c1, c2 = nil)
+        @code_point_range = CodePointRange.new(c1, c2)
+      end
+      def first
+        @code_point_range.first
+      end
+      def last
+        @code_point_range.last
+      end
+      def to_nfa
+        nfa = NFA.new
+        nfa.start_state.add_transition(@code_point_range, nfa.end_state)
+        nfa
+      end
+    end
+
+    class CharacterClassUnit < Unit
+      attr_accessor :units
+      attr_accessor :negate
+      def initialize
+        @units = []
+        @negate = false
+      end
+      def initialize
+        @units = []
+      end
+      def method_missing(*args)
+        @units.__send__(*args)
+      end
+      def <<(thing)
+        if thing.is_a?(CharacterClassUnit)
+          thing.each do |ccu_unit|
+            @units << ccu_unit
+          end
+        else
+          @units << thing
+        end
+      end
+      def last_unit
+        @units[-1]
+      end
+      def replace_last!(new_unit)
+        @units[-1] = new_unit
+      end
+      def to_nfa
+        nfa = NFA.new
+        if @units.empty?
+          nfa.start_state.add_transition(nil, nfa.end_state)
+        else
+          code_point_ranges = @units.map(&:code_point_range)
+          if @negate
+            code_point_ranges = CodePointRange.invert_ranges(code_point_ranges)
+          end
+          code_point_ranges.each do |code_point_range|
+            nfa.start_state.add_transition(code_point_range, nfa.end_state)
+          end
+        end
+        nfa
+      end
+    end
+
+    class MultiplicityUnit < Unit
+      attr_accessor :unit
+      attr_accessor :min_count
+      attr_accessor :max_count
+      def initialize(unit, min_count, max_count)
+        @unit = unit
+        @min_count = min_count
+        @max_count = max_count
+      end
+      def to_nfa
+        nfa = NFA.new
+        last_state = nfa.start_state
+        unit_nfa = nil
+        @min_count.times do
+          unit_nfa = @unit.to_nfa
+          last_state.add_transition(nil, unit_nfa.start_state)
+          last_state = unit_nfa.end_state
+        end
+        last_state.add_transition(nil, nfa.end_state)
+        if @max_count.nil?
+          if @min_count == 0
+            unit_nfa = @unit.to_nfa
+            last_state.add_transition(nil, unit_nfa.start_state)
+          end
+          unit_nfa.end_state.add_transition(nil, unit_nfa.start_state)
+          unit_nfa.end_state.add_transition(nil, nfa.end_state)
+        else
+          (@max_count - @min_count).times do
+            unit_nfa = @unit.to_nfa
+            last_state.add_transition(nil, unit_nfa.start_state)
+            unit_nfa.end_state.add_transition(nil, nfa.end_state)
+            last_state = unit_nfa.end_state
+          end
+        end
+        nfa
+      end
+    end
+
+  end
+end
@@ -0,0 +1,73 @@
+class Propane
+
+  class Rule
+
+    # @return [Array<Token, RuleSet>]
+    #   Rule components.
+    attr_reader :components
+
+    # @return [String]
+    #   User code associated with the rule.
+    attr_reader :code
+
+    # @return [Integer]
+    #   Rule ID.
+    attr_accessor :id
+
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
+
+    # @return [Integer]
+    #   Line number where the rule was defined in the input grammar.
+    attr_reader :line_number
+
+    # @return [String]
+    #   Rule name.
+    attr_reader :name
+
+    # @return [RuleSet]
+    #   The RuleSet that this Rule is a part of.
+    attr_accessor :rule_set
+
+    # Construct a Rule.
+    #
+    # @param name [String]
+    #   Rule name.
+    # @param components [Array<String>]
+    #   Rule components.
+    # @param code [String]
+    #   User code associated with the rule.
+    # @param ptypename [String, nil]
+    #   Parser type name.
+    # @param line_number [Integer]
+    #   Line number where the rule was defined in the input grammar.
+    def initialize(name, components, code, ptypename, line_number)
+      @name = name
+      @components = components
+      @code = code
+      @ptypename = ptypename
+      @line_number = line_number
+    end
+
+    # Return whether the Rule is empty.
+    #
+    # A Rule is empty if it has no components.
+    #
+    # @return [Boolean]
+    #   Whether the Rule is empty.
+    def empty?
+      @components.empty?
+    end
+
+    # Represent the Rule as a String.
+    #
+    # @return [String]
+    #   Rule represented as a String.
+    def to_s
+      "#{@name} -> #{@components.map(&:name).join(" ")}"
+    end
+
+  end
+
+end
@@ -0,0 +1,80 @@
+class Propane
+
+  class RuleSet
+
+    # @return [Integer]
+    #   ID of the RuleSet.
+    attr_reader :id
+
+    # @return [String]
+    #   Name of the RuleSet.
+    attr_reader :name
+
+    # @return [String, nil]
+    #   Parser type name.
+    attr_accessor :ptypename
+
+    # @return [Array<Rule>]
+    #   Rules in the RuleSet.
+    attr_reader :rules
+
+    # @return [Boolean]
+    #   Whether the RuleSet could expand to an empty sequence.
+    attr_writer :could_be_empty
+
+    # Construct a RuleSet.
+    #
+    # @param name [String]
+    #   Name of the RuleSet.
+    # @param id [Integer]
+    #   ID of the RuleSet.
+    def initialize(name, id)
+      @id = id
+      @name = name
+      @rules = []
+      @could_be_empty = false
+    end
+
+    # Add a Rule to the RuleSet.
+    #
+    # @param rule [Rule]
+    #   Rule to add.
+    def <<(rule)
+      @rules << rule
+    end
+
+    # Return whether any Rule in the RuleSet is empty.
+    #
+    # @return [Boolean]
+    #   Whether any rule in the RuleSet is empty.
+    def could_be_empty?
+      @could_be_empty
+    end
+
+    # Build the start token set for the RuleSet.
+    #
+    # @return [Set<Token>]
+    #   Start token set for the RuleSet.
+    def start_token_set
+      if @_start_token_set.nil?
+        @_start_token_set = Set.new
+        @rules.each do |rule|
+          rule.components.each do |component|
+            if component.is_a?(Token)
+              @_start_token_set << component
+              break
+            else
+              @_start_token_set += component.start_token_set
+              unless component.could_be_empty?
+                break
+              end
+            end
+          end
+        end
+      end
+      @_start_token_set
+    end
+
+  end
+
+end
@@ -0,0 +1,61 @@
+class Propane
+
+  class Token
+
+    class << self
+
+      # Name of the token to use in code (special characters replaced).
+      #
+      # @return [String]
+      #   Name of the token to use in code (special characters replaced).
+      def code_name(name)
+        name.sub(/^\$/, "__")
+      end
+
+    end
+
+    # @return [String, nil]
+    #   Token name.
+    attr_reader :name
+
+    # @return [String, nil]
+    #   Parser value type name.
+    attr_accessor :ptypename
+
+    # @return [Integer, nil]
+    #   Token ID.
+    attr_accessor :id
+
+    # @return [Integer, nil]
+    #   Line number where the token was defined in the input grammar.
+    attr_reader :line_number
+
+    # Construct a Token.
+    #
+    # @param name [String, nil]
+    #   Token name.
+    # @param ptypename [String, nil]
+    #   Parser value type for this token.
+    # @param line_number [Integer, nil]
+    #   Line number where the token was defined in the input grammar.
+    def initialize(name, ptypename, line_number)
+      @name = name
+      @ptypename = ptypename
+      @line_number = line_number
+    end
+
+    # Name of the token to use in code (special characters replaced).
+    #
+    # @return [String]
+    #   Name of the token to use in code (special characters replaced).
+    def code_name
+      self.class.code_name(@name)
+    end
+
+    def to_s
+      @name
+    end
+
+  end
+
+end
@@ -0,0 +1,17 @@
+class Propane
+
+  # Utility methods.
+  module Util
+
+    class << self
+
+      def banner(message)
+        s = "*" * (message.size + 4)
+        "#{s}\n* #{message} *\n#{s}\n"
+      end
+
+    end
+
+  end
+
+end
@@ -0,0 +1,3 @@
+class Propane
+  VERSION = "1.0.0"
+end
@@ -0,0 +1,68 @@
+<html>
+  <head>
+    <title>Propane User Guide<%= subpage_title %> - Version <%= Propane::VERSION %></title>
+    <style>
+      body {
+        background-color: #CCC;
+        margin: 0px;
+        padding: 0px;
+      }
+      #body-content {
+        margin-left: auto;
+        margin-right: auto;
+        margin-top: 1em;
+        margin-bottom: 1em;
+        border: 1px solid black;
+        background-color: #FFF;
+        width: 120ex;
+        padding: 2ex;
+      }
+      a {
+        text-decoration: none;
+      }
+      .separator {
+        height: 2em;
+      }
+      .img_block_center {
+        display: block;
+        margin-left: auto;
+        margin-right: auto;
+      }
+      .page_nav {
+        width: 100%;
+        table-layout: fixed;
+      }
+      .page_nav_toc {
+        text-align: center;
+      }
+      .page_nav_next {
+        text-align: right;
+      }
+      .code {
+        padding-left: 2ex;
+        width: 116ex;
+        overflow-x: auto;
+      }
+      .ruby_code .normal {}
+      .ruby_code .comment { color: #005; font-style: italic; }
+      .ruby_code .keyword { color: #A00; font-weight: bold; }
+      .ruby_code .method { color: #077; }
+      .ruby_code .class { color: #074; }
+      .ruby_code .module { color: #050; }
+      .ruby_code .punct { color: #447; font-weight: bold; }
+      .ruby_code .symbol { color: #099; }
+      .ruby_code .string { color: #090; }
+      .ruby_code .char { color: #F07; }
+      .ruby_code .ident { color: #004; }
+      .ruby_code .constant { color: #07F; }
+      .ruby_code .regex { color: #B66; }
+      .ruby_code .number { color: #D55; }
+      .ruby_code .attribute { color: #377; }
+      .ruby_code .global { color: #3B7; }
+      .ruby_code .expr { color: #227; }
+    </style>
+  </head>
+  <body>
+    <div id="body-content"><%= content %></div>
+  </body>
+</html>
@@ -0,0 +1,129 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "fileutils"
+require "digest/md5"
+
+if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
+  VERSION = $1
+else
+  raise "Could not determine version."
+end
+PROG_NAME = "propane"
+START_FILE = "bin/#{PROG_NAME}"
+LIB_DIR = "lib"
+DIST = "dist"
+
+ASSETS_TEMPLATE = <<EOF
+class Propane
+  module Assets
+    class << self
+      def get(name)
+        case name
+<% Dir.glob("assets/*").each  do |asset_file| %>
+        when <%= File.basename(asset_file).inspect %>
+          <%= File.binread(asset_file).inspect %>
+<% end %>
+        end
+      end
+    end
+  end
+end
+EOF
+
+assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
+files_processed = {}
+combined_file = []
+
+combine_files = lambda do |file|
+  File.read(file, mode: "rb").each_line do |line|
+    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
+      require_name = $1
+      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
+        path = "#{LIB_DIR}/#{require_name}.rb"
+        if File.exist?(path)
+          unless files_processed[path]
+            files_processed[path] = true
+            if require_name == "propane/assets"
+              combined_file << assets_module
+            else
+              combine_files[path]
+            end
+          end
+        else
+          raise "require path #{path.inspect} not found"
+        end
+      else
+        combined_file << line
+      end
+    else
+      combined_file << line
+    end
+  end
+end
+
+combine_files[START_FILE]
+
+# Strip Ruby comment lines and empty lines to save some space, but do not
+# remove lines that are in heredoc sections. This isn't terribly robust to be
+# used in the wild, but works for the heredoc instances for this project.
+stripped = []
+heredoc_end = nil
+combined_file.each do |line|
+  if line =~ /<<-?([A-Z]+)/
+    heredoc_end = $1
+  end
+  if heredoc_end and line =~ /^\s*#{heredoc_end}/
+    heredoc_end = nil
+  end
+  if line !~ /#\sspecs/
+    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
+      stripped << line
+    end
+  end
+end
+
+license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
+  if line.size > 0
+    "# #{line}"
+  else
+    "#"
+  end
+end
+
+require "zlib"
+require "base64"
+compressed_script = Zlib::Deflate.deflate(stripped.join)
+encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
+hash = Digest::MD5.hexdigest(encoded_compressed_script)
+
+FileUtils.rm_rf(DIST)
+FileUtils.mkdir_p(DIST)
+File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
+  fh.write(<<EOF)
+#!/usr/bin/env ruby
+
+#{license}
+
+script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
+unless File.exist?(script)
+  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
+    require "zlib"
+    require "base64"
+    encoded_compressed = $1
+    unescaped_compressed = Base64.decode64(encoded_compressed)
+    inflated = Zlib::Inflate.inflate(unescaped_compressed)
+    File.open(script, "wb") do |fh|
+      fh.write(inflated)
+    end
+  else
+    raise "Could not decompress."
+  end
+end
+load script
+if __FILE__ == $0
+  Propane::CLI.new.run(ARGV)
+end
+#==>#{encoded_compressed_script}
+EOF
+end
@@ -0,0 +1,227 @@
+#!/usr/bin/env ruby
+
+require "erb"
+require "fileutils"
+require "redcarpet"
+require "syntax"
+require "syntax/convertors/html"
+require "propane/version"
+
+def load_file(file_name)
+  contents = File.read(file_name)
+  contents.gsub(/\$\{include (.*?)\}/) do |match|
+    include_file_name = $1
+    File.read(include_file_name)
+  end
+end
+
+class Generator
+  class Section
+    attr_reader :number
+    attr_reader :title
+    attr_reader :page
+    attr_reader :contents
+    attr_reader :anchor
+    def initialize(number, title, page, anchor)
+      @number = number
+      @title = title.gsub(/[`]/, "")
+      @page = page
+      @anchor = anchor
+      @contents = ""
+    end
+    def append(contents)
+      @contents += contents
+    end
+  end
+
+  class Page
+    attr_reader :name
+    attr_reader :title
+    attr_accessor :contents
+    def initialize(name, title, contents = "")
+      @name = name
+      @title = title
+      @contents = contents
+    end
+  end
+
+  def initialize(input, output_file, multi_page)
+    current_page =
+      if multi_page
+        nil
+      else
+        File.basename(output_file).sub(/\.html$/, "")
+      end
+    @sections = []
+    @current_section_number = [0]
+    @lines = input.lines
+    while @lines.size > 0
+      line = @lines.slice!(0)
+      if line =~ /^```(.*)$/
+        @sections.last.append(gather_code_section($1))
+      elsif line.chomp == "${remove}"
+        remove_section
+      elsif line =~ /^(#+)(>)?\s*(.*)$/
+        level_text, new_page_text, title_text = $1, $2, $3
+        level = $1.size
+        new_page = !new_page_text.nil?
+        section_number = get_next_section_number(level)
+        anchor = make_anchor(section_number, title_text)
+        if multi_page and (new_page or current_page.nil?)
+          current_page = anchor
+        end
+        @sections << Section.new(section_number, title_text, current_page, anchor)
+        @sections.last.append("#{level_text} #{section_number} #{title_text}")
+      elsif @sections.size > 0
+        @sections.last.append(line)
+      end
+    end
+
+    renderer = Redcarpet::Render::HTML.new
+    @markdown_renderer = Redcarpet::Markdown.new(renderer)
+    changelog = @markdown_renderer.render(File.read("CHANGELOG.md"))
+
+    @pages = [Page.new("toc", "Table of Contents", render_toc)]
+    @sections.each do |section|
+      unless @pages.last.name == section.page
+        @pages << Page.new(section.page, "#{section.number} #{section.title}")
+      end
+      @pages.last.contents += render_section(section)
+    end
+
+    @pages.each do |page|
+      page.contents.gsub!("${changelog}", changelog)
+      page.contents.gsub!(%r{\$\{#(.+?)\}}) do |match|
+        section_name = $1
+        href = get_link_to_section(section_name)
+        %[<a href="#{href}">#{section_name}</a>]
+      end
+    end
+
+    template = File.read("rb/assets/user_guide.html.erb")
+    erb = ERB.new(template, trim_mode: "<>")
+
+    if multi_page
+      @pages.each_with_index do |page, page_index|
+        subpage_title = " - #{page.title}"
+        page_nav_bar = render_page_nav_bar(page_index)
+        content = page_nav_bar + separator + page.contents + separator + page_nav_bar
+        html_result = erb.result(binding.clone)
+        File.open(File.join(output_file, "#{page.name}.html"), "w") do |fh|
+          fh.write(html_result)
+        end
+      end
+    else
+      subpage_title = ""
+      content = @pages.reduce("") do |result, page|
+        result + page.contents
+      end
+      html_result = erb.result(binding.clone)
+      File.open(output_file, "w") do |fh|
+        fh.write(html_result)
+      end
+    end
+  end
+
+  def separator
+    %[<div class="separator"></div>]
+  end
+
+  def render_page_nav_bar(page_index)
+    page_nav_prev =
+      if page_index > 1
+        %[<a href="#{@pages[page_index - 1].name}.html">&laquo; Prev<br/>#{@pages[page_index - 1].title}</a>]
+      else
+        ""
+      end
+    page_nav_toc =
+      if page_index > 0
+        %[<a href="toc.html">Table of Contents</a>]
+      else
+        ""
+      end
+    page_nav_next =
+      if page_index < @pages.size - 1
+        %[<a href="#{@pages[page_index + 1].name}.html">Next &raquo<br/>#{@pages[page_index + 1].title}</a>]
+      else
+        ""
+      end
+    %[<table class="page_nav"><tr>] + \
+    %[<td class="page_nav_prev">#{page_nav_prev}</td>] + \
+    %[<td class="page_nav_toc">#{page_nav_toc}</td>] + \
+    %[<td class="page_nav_next">#{page_nav_next}</td>] + \
+    %[</tr></table>]
+  end
+
+  def render_toc
+    toc_content = %[<h1>Table of Contents</h1>\n]
+    @sections.each do |section|
+      indent = section.number.split(".").size - 1
+      toc_content += %[<span style="padding-left: #{4 * indent}ex;">]
+      toc_content += %[<a href="#{section.page}.html##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
+      toc_content += %[</span>]
+    end
+    toc_content
+  end
+
+  def render_section(section)
+    %[<a name="#{section.anchor}" />] + \
+      @markdown_renderer.render(section.contents)
+  end
+
+  def make_anchor(section_number, section_title)
+    "s" + ("#{section_number} #{section_title}").gsub(/[^a-zA-Z0-9]/, "_")
+  end
+
+  def get_link_to_section(section_name)
+    section = @sections.find do |section|
+      section.title == section_name
+    end
+    raise "Could not find section #{section_name}" unless section
+    "#{section.page}.html##{section.anchor}"
+  end
+
+  def gather_code_section(syntax)
+    code = ""
+    loop do
+      line = @lines.slice!(0)
+      if line =~ /^```/
+        break
+      end
+      code += line
+    end
+    if syntax != ""
+      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
+      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
+    else
+      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
+    end
+  end
+
+  def remove_section
+    loop do
+      line = @lines.slice!(0)
+      if line.chomp == "${/remove}"
+        break
+      end
+    end
+  end
+
+  def get_next_section_number(level)
+    if @current_section_number.size == level - 1
+      @current_section_number << 1
+    elsif @current_section_number.size >= level
+      @current_section_number[level - 1] += 1
+      @current_section_number.slice!(level, @current_section_number.size)
+    else
+      raise "Section level change from #{@current_section_number.size} to #{level}"
+    end
+    @current_section_number.join(".")
+  end
+end
+
+input = load_file("doc/user_guide.md")
+FileUtils.rm_rf("gen/user_guide")
+FileUtils.mkdir_p("gen/user_guide")
+Generator.new(input, "gen/user_guide/user_guide.html", false)
+Generator.new(input, "gen/user_guide", true)
@@ -0,0 +1,183 @@
+<<header
+#include "json_types.h"
+#include "testutils.h"
+>>
+<<
+#include "math.h"
+#include <stdbool.h>
+static str_t string_value;
+>>
+
+ptype JSONValue *;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n = 0.0;
+    bool negative = false;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative = false;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp = 0.0;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = JSONValue_new(JSON_NUMBER);
+    $$->number = n;
+>>
+token true <<
+  $$ = JSONValue_new(JSON_TRUE);
+>>
+token false <<
+  $$ = JSONValue_new(JSON_FALSE);
+>>
+token null <<
+  $$ = JSONValue_new(JSON_NULL);
+>>
+/"/ <<
+  $mode(string);
+  str_init(&string_value, "");
+>>
+string: token string /"/ <<
+  $$ = JSONValue_new(JSON_STRING);
+  $$->string = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  str_append(&string_value, "\"");
+>>
+string: /\\\\/ <<
+  str_append(&string_value, "\\");
+>>
+string: /\\\// <<
+  str_append(&string_value, "/");
+>>
+string: /\\b/ <<
+  str_append(&string_value, "\b");
+>>
+string: /\\f/ <<
+  str_append(&string_value, "\f");
+>>
+string: /\\n/ <<
+  str_append(&string_value, "\n");
+>>
+string: /\\r/ <<
+  str_append(&string_value, "\r");
+>>
+string: /\\t/ <<
+  str_append(&string_value, "\t");
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  str_append(&string_value, s);
+>>
+string: /[^\\]/ <<
+  char s[] = {match[0], 0};
+  str_append(&string_value, s);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONObject_new();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
+  $$ = $1;
+>>
+KeyValue -> string colon Value <<
+  $$ = JSONObject_new();
+  JSONObject_append($$, str_cstr(&$1->string), $3);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONArray_new();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  JSONArray_append($1, $3);
+  $$ = $1;
+>>
@@ -0,0 +1,179 @@
+<<
+import std.math;
+import json_types;
+private string string_value;
+>>
+
+ptype JSONValue;
+ptype array = JSONValue[];
+ptype dict = JSONValue[string];
+ptype string = string;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n;
+    bool negative;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = new JSONNumber(n);
+>>
+token true <<
+  $$ = new JSONTrue();
+>>
+token false <<
+  $$ = new JSONFalse();
+>>
+token null <<
+  $$ = new JSONNull();
+>>
+/"/ <<
+  $mode(string);
+  string_value = "";
+>>
+string: token string (string) /"/ <<
+  $$ = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  string_value ~= "\"";
+>>
+string: /\\\\/ <<
+  string_value ~= "\\";
+>>
+string: /\\\// <<
+  string_value ~= "/";
+>>
+string: /\\b/ <<
+  string_value ~= "\b";
+>>
+string: /\\f/ <<
+  string_value ~= "\f";
+>>
+string: /\\n/ <<
+  string_value ~= "\n";
+>>
+string: /\\r/ <<
+  string_value ~= "\r";
+>>
+string: /\\t/ <<
+  string_value ~= "\t";
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  string_value ~= "{" ~ match[2..6] ~ "}";
+>>
+string: /[^\\]/ <<
+  string_value ~= match;
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = new JSONString($1);
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = new JSONObject();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = new JSONObject($2);
+>>
+KeyValues (dict) -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  foreach (key, value; $3)
+  {
+    $1[key] = value;
+  }
+  $$ = $1;
+>>
+KeyValue (dict) -> string colon Value <<
+  $$ = [$1: $3];
+>>
+Array -> lbracket rbracket <<
+  $$ = new JSONArray();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = new JSONArray($2);
+>>
+Values (array) -> Value <<
+  $$ = [$1];
+>>
+Values -> Values comma Value <<
+  $$ = $1 ~ [$3];
+>>
@@ -0,0 +1,64 @@
+#include "json_types.h"
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+JSONValue * JSONValue_new(size_t id)
+{
+    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    jv->id = id;
+    return jv;
+}
+
+JSONValue * JSONObject_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_OBJECT);
+    jv->object.size = 0u;
+    return jv;
+}
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
+{
+    size_t const size = object->object.size;
+    for (size_t i = 0u; i < size; i++)
+    {
+        if (strcmp(name, object->object.entries[i].name) == 0)
+        {
+            object->object.entries[i].value = value;
+            return;
+        }
+    }
+    size_t const new_size = size + 1;
+    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    if (size > 0)
+    {
+        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
+        free(object->object.entries);
+    }
+    object->object.entries = new_entries;
+    object->object.entries[size].name = name;
+    object->object.entries[size].value = value;
+    object->object.size = new_size;
+}
+
+JSONValue * JSONArray_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_ARRAY);
+    jv->array.size = 0u;
+    return jv;
+}
+
+void JSONArray_append(JSONValue * array, JSONValue * value)
+{
+    size_t const size = array->array.size;
+    size_t const new_size = size + 1;
+    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    if (array->array.size > 0)
+    {
+        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
+        free(array->array.entries);
+    }
+    array->array.entries = new_entries;
+    array->array.entries[size] = value;
+    array->array.size = new_size;
+}
@@ -0,0 +1,63 @@
+class JSONValue
+{
+}
+
+class JSONObject : JSONValue
+{
+    JSONValue[string] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[string] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONArray : JSONValue
+{
+    JSONValue[] value;
+
+    this()
+    {
+    }
+
+    this(JSONValue[] value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONNumber : JSONValue
+{
+    double value;
+
+    this(double value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONString : JSONValue
+{
+    string value;
+
+    this(string value)
+    {
+        this.value = value;
+    }
+}
+
+class JSONTrue : JSONValue
+{
+}
+
+class JSONFalse : JSONValue
+{
+}
+
+class JSONNull : JSONValue
+{
+}
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <stddef.h>
+#include "testutils.h"
+
+#define JSON_OBJECT 0u
+#define JSON_ARRAY 1u
+#define JSON_NUMBER 2u
+#define JSON_STRING 3u
+#define JSON_TRUE 4u
+#define JSON_FALSE 5u
+#define JSON_NULL 6u
+
+typedef struct JSONValue_s
+{
+    size_t id;
+    union
+    {
+        struct
+        {
+            size_t size;
+            struct
+            {
+                char const * name;
+                struct JSONValue_s * value;
+            } * entries;
+        } object;
+        struct
+        {
+            size_t size;
+            struct JSONValue_s ** entries;
+        } array;
+        double number;
+        str_t string;
+    };
+} JSONValue;
+
+JSONValue * JSONValue_new(size_t id);
+
+JSONValue * JSONObject_new(void);
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
+
+JSONValue * JSONArray_new(void);
+
+void JSONArray_append(JSONValue * array, JSONValue * value);
@@ -0,0 +1,87 @@
+class Propane
+  describe CodePointRange do
+
+    describe "#<=>" do
+      it "sorts ranges" do
+        arr = [
+          CodePointRange.new(100,102),
+          CodePointRange.new(65, 68),
+          CodePointRange.new(65, 65),
+          CodePointRange.new(100, 100),
+          CodePointRange.new(68, 70),
+        ]
+        arr.sort!
+        expect(arr[0]).to eq CodePointRange.new(65, 65)
+        expect(arr[1]).to eq CodePointRange.new(65, 68)
+        expect(arr[2]).to eq CodePointRange.new(68, 70)
+        expect(arr[3]).to eq CodePointRange.new(100, 100)
+        expect(arr[4]).to eq CodePointRange.new(100, 102)
+      end
+    end
+
+    describe "#include?" do
+      it "returns whether the code point is included in the range" do
+        expect(CodePointRange.new(100).include?(100)).to be_truthy
+        expect(CodePointRange.new(100, 100).include?(99)).to be_falsey
+        expect(CodePointRange.new(100, 100).include?(101)).to be_falsey
+        expect(CodePointRange.new(100, 120).include?(99)).to be_falsey
+        expect(CodePointRange.new(100, 120).include?(100)).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(110)).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(120)).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(121)).to be_falsey
+      end
+
+      it "returns whether the range is included in the range" do
+        expect(CodePointRange.new(100).include?(CodePointRange.new(100))).to be_truthy
+        expect(CodePointRange.new(100, 100).include?(CodePointRange.new(99))).to be_falsey
+        expect(CodePointRange.new(100, 100).include?(CodePointRange.new(99, 100))).to be_falsey
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(90, 110))).to be_falsey
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(110, 130))).to be_falsey
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(100, 120))).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(100, 110))).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(110, 120))).to be_truthy
+        expect(CodePointRange.new(100, 120).include?(CodePointRange.new(102, 118))).to be_truthy
+      end
+    end
+
+    describe ".invert_ranges" do
+      it "inverts ranges" do
+        expect(CodePointRange.invert_ranges(
+          [CodePointRange.new(60, 90),
+           CodePointRange.new(80, 85),
+           CodePointRange.new(80, 100),
+           CodePointRange.new(101),
+           CodePointRange.new(200, 300)])).to eq [
+             CodePointRange.new(0, 59),
+             CodePointRange.new(102, 199),
+             CodePointRange.new(301, 0x7FFFFFFF)]
+        expect(CodePointRange.invert_ranges(
+          [CodePointRange.new(0, 500),
+           CodePointRange.new(7000, 0x7FFFFFFF)])).to eq [
+             CodePointRange.new(501, 6999)]
+      end
+    end
+
+    describe ".first_subrange" do
+      it "returns the first subrange to split" do
+        expect(CodePointRange.first_subrange(
+          [CodePointRange.new(65, 90),
+           CodePointRange.new(66, 66),
+           CodePointRange.new(80, 90)])).to eq CodePointRange.new(65)
+        expect(CodePointRange.first_subrange(
+          [CodePointRange.new(65, 90)])).to eq CodePointRange.new(65, 90)
+        expect(CodePointRange.first_subrange(
+          [CodePointRange.new(65, 90),
+           CodePointRange.new(80, 90)])).to eq CodePointRange.new(65, 79)
+        expect(CodePointRange.first_subrange(
+          [CodePointRange.new(65, 90),
+           CodePointRange.new(65, 100),
+           CodePointRange.new(65, 95)])).to eq CodePointRange.new(65, 90)
+        expect(CodePointRange.first_subrange(
+          [CodePointRange.new(100, 120),
+           CodePointRange.new(70, 90)])).to eq CodePointRange.new(70, 90)
+      end
+    end
+
+  end
+end
@@ -0,0 +1,231 @@
+class Propane
+  describe Grammar do
+    it "parses a user grammar" do
+      input = <<EOF
+# Comment line
+
+module a.b;
+ptype   XYZ  *  ;
+
+token while;
+
+token id
+  /[a-zA-Z_][a-zA-Z_0-9]*/;
+
+token token_with_code <<
+Code for the token
+>>
+
+tokenid token_with_no_pattern;
+
+drop /\\s+/;
+
+A -> B <<
+  a = 42;
+>>
+B -> C while id;
+B -> <<
+  b = 0;
+>>
+EOF
+      grammar = Grammar.new(input)
+      expect(grammar.modulename).to eq "a.b"
+      expect(grammar.ptype).to eq "XYZ  *"
+      expect(grammar.ptypes).to eq("default" => "XYZ  *")
+      expect(grammar.prefix).to eq "p_"
+
+      o = grammar.tokens.find {|token| token.name == "while"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 6
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "while"
+      expect(o.line_number).to eq 6
+      expect(o.code).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "id"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 9
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
+      expect(o.line_number).to eq 9
+      expect(o.code).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "token_with_code"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 11
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.pattern).to eq "token_with_code"
+      expect(o.line_number).to eq 11
+      expect(o.code).to eq "Code for the token\n"
+
+      o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 15
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 17
+      expect(o.token).to be_nil
+      expect(o.code).to be_nil
+
+      expect(grammar.rules.size).to eq 3
+
+      o = grammar.rules[0]
+      expect(o.name).to eq "A"
+      expect(o.components).to eq %w[B]
+      expect(o.line_number).to eq 19
+      expect(o.code).to eq "  a = 42;\n"
+
+      o = grammar.rules[1]
+      expect(o.name).to eq "B"
+      expect(o.components).to eq %w[C while id]
+      expect(o.line_number).to eq 22
+      expect(o.code).to be_nil
+
+      o = grammar.rules[2]
+      expect(o.name).to eq "B"
+      expect(o.components).to eq []
+      expect(o.line_number).to eq 23
+      expect(o.code).to eq "  b = 0;\n"
+    end
+
+    it "parses code segments with semicolons" do
+      input = <<EOF
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input)
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 1
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq "  a = b;\n  return c;\n"
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+      expect(o.line_number).to eq 6
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[  writeln("Hello there");\n]
+    end
+
+    it "supports mode labels" do
+      input = <<EOF
+token a;
+m1: token b;
+/foo/ <<
+>>
+m2: /bar/ <<
+>>
+drop /q/;
+m3: drop /r/;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "a"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "b"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m1"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m2"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
+      expect(o).to_not be_nil
+      expect(o.mode).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
+      expect(o).to_not be_nil
+      expect(o.mode).to eq "m3"
+    end
+
+    it "allows assigning ptypes to tokens and rules" do
+      input = <<EOF
+ptype Subnode *;
+ptype string = char *;
+ptype integer = int;
+ptype node = Node *;
+
+token abc(string);
+token bar;
+tokenid int(integer);
+
+/xyz/ (string) <<
+>>
+
+/z28/ <<
+>>
+
+Start (node) -> R;
+R -> abc int;
+EOF
+      grammar = Grammar.new(input)
+
+      o = grammar.tokens.find {|token| token.name == "abc"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.tokens.find {|token| token.name == "bar"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.tokens.find {|token| token.name == "int"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "integer"
+
+      o = grammar.rules.find {|rule| rule.name == "Start"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "node"
+
+      o = grammar.rules.find {|rule| rule.name == "R"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "xyz"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to eq "string"
+
+      o = grammar.patterns.find {|pattern| pattern.pattern == "z28"}
+      expect(o).to_not be_nil
+      expect(o.ptypename).to be_nil
+    end
+  end
+end
@@ -0,0 +1,131 @@
+class TestLexer
+  def initialize(token_dfa)
+    @token_dfa = token_dfa
+  end
+
+  def lex(input)
+    input_chars = input.chars
+    output = []
+    while lexed_token = lex_token(input_chars)
+      output << lexed_token
+      input_chars.slice!(0, lexed_token[1].size)
+    end
+    unless input_chars.empty?
+      raise "Unmatched input #{input_chars.join(" ")}"
+    end
+    output
+  end
+
+  def lex_token(input_chars)
+    return nil if input_chars.empty?
+    s = ""
+    current_state = @token_dfa.start_state
+    last_accepts = nil
+    last_s = nil
+    input_chars.each_with_index do |input_char, index|
+      if next_state = transition(current_state, input_char)
+        s += input_char
+        current_state = next_state
+        if current_state.accepts
+          last_accepts = current_state.accepts
+          last_s = s
+        end
+      else
+        break
+      end
+    end
+    if last_accepts
+      name = last_accepts.token ? last_accepts.token.name : nil
+      [name, last_s]
+    end
+  end
+
+  def transition(state, input_char)
+    state.transitions.each do |transition|
+      if transition.code_point_range.include?(input_char.ord)
+        return transition.destination
+      end
+    end
+    nil
+  end
+end
+
+def run(grammar, input)
+  grammar = Propane::Grammar.new(grammar)
+  token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
+  test_lexer = TestLexer.new(token_dfa)
+  test_lexer.lex(input)
+end
+
+describe Propane::Lexer::DFA do
+  it "lexes a simple token" do
+    expect(run(<<EOF, "foo")).to eq [["foo", "foo"]]
+token foo;
+EOF
+  end
+
+  it "lexes two tokens" do
+    expected = [
+      ["foo", "foo"],
+      ["bar", "bar"],
+    ]
+    expect(run(<<EOF, "foobar")).to eq expected
+token foo;
+token bar;
+EOF
+  end
+
+  it "lexes the longer of multiple options" do
+    expected = [
+      ["identifier", "foobar"],
+    ]
+    expect(run(<<EOF, "foobar")).to eq expected
+token foo;
+token bar;
+token identifier /[a-z]+/;
+EOF
+    expected = [
+      ["plusplus", "++"],
+      ["plus", "+"],
+    ]
+    expect(run(<<EOF, "+++")).to eq expected
+token plus /\\+/;
+token plusplus /\\+\\+/;
+EOF
+  end
+
+  it "lexes whitespace" do
+    expected = [
+      ["foo", "foo"],
+      ["WS", " \t"],
+      ["bar", "bar"],
+    ]
+    expect(run(<<EOF, "foo \tbar")).to eq expected
+token foo;
+token bar;
+token WS /\\s+/;
+EOF
+  end
+
+  it "allows dropping a matched pattern" do
+    expected = [
+      ["foo", "foo"],
+      [nil, " \t"],
+      ["bar", "bar"],
+    ]
+    expect(run(<<EOF, "foo \tbar")).to eq expected
+token foo;
+token bar;
+drop /\\s+/;
+EOF
+  end
+
+  it "matches a semicolon" do
+    expected = [
+      ["semicolon", ";"],
+    ]
+    expect(run(<<EOF, ";")).to eq expected
+token semicolon /;/;
+EOF
+  end
+end
@@ -0,0 +1,19 @@
+class Propane
+  class Parser
+
+    describe Item do
+
+      it "operates properly with a set" do
+        rule = Object.new
+        item1 = Item.new(rule, 2)
+        item2 = Item.new(rule, 2)
+        expect(item1).to eq item2
+        expect(item1.eql?(item2)).to be_truthy
+        set = Set.new([item1, item2])
+        expect(set.size).to eq 1
+      end
+
+    end
+
+  end
+end
@@ -0,0 +1,333 @@
+class Propane
+  RSpec.describe Regex do
+
+    it "parses an empty expression" do
+      regex = Regex.new("")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0].size).to eq 0
+    end
+
+    it "parses a single character unit expression" do
+      regex = Regex.new("a")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a group with a single character unit expression" do
+      regex = Regex.new("(a)")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::AlternatesUnit
+      alt_unit = seq_unit[0]
+      expect(alt_unit.alternates.size).to eq 1
+      expect(alt_unit.alternates[0]).to be_a Regex::SequenceUnit
+      expect(alt_unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a *" do
+      regex = Regex.new("a*")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 0
+      expect(m_unit.max_count).to be_nil
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a +" do
+      regex = Regex.new("a+")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 1
+      expect(m_unit.max_count).to be_nil
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a ?" do
+      regex = Regex.new("a?")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 0
+      expect(m_unit.max_count).to eq 1
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a multiplicity count" do
+      regex = Regex.new("a{5}")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 5
+      expect(m_unit.max_count).to eq 5
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a minimum-only multiplicity count" do
+      regex = Regex.new("a{5,}")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 5
+      expect(m_unit.max_count).to be_nil
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses a minimum and maximum multiplicity count" do
+      regex = Regex.new("a{5,8}")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
+      m_unit = seq_unit[0]
+      expect(m_unit.min_count).to eq 5
+      expect(m_unit.max_count).to eq 8
+      expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
+      expect(m_unit.unit.first).to eq "a".ord
+    end
+
+    it "parses an escaped *" do
+      regex = Regex.new("a\\*")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 2
+      expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[0].first).to eq "a".ord
+      expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[1].first).to eq "*".ord
+    end
+
+    it "parses an escaped +" do
+      regex = Regex.new("a\\+")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 2
+      expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[0].first).to eq "a".ord
+      expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[1].first).to eq "+".ord
+    end
+
+    it "parses an escaped \\" do
+      regex = Regex.new("\\\\d")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 2
+      expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[0].first).to eq "\\".ord
+      expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
+      expect(seq_unit[1].first).to eq "d".ord
+    end
+
+    it "parses a character class" do
+      regex = Regex.new("[a-z_]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_falsey
+      expect(ccu.size).to eq 2
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "a".ord
+      expect(ccu[0].last).to eq "z".ord
+      expect(ccu[1]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[1].first).to eq "_".ord
+    end
+
+    it "parses a negated character class" do
+      regex = Regex.new("[^xyz]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_truthy
+      expect(ccu.size).to eq 3
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "x".ord
+    end
+
+    it "parses - as a plain character at beginning of a character class" do
+      regex = Regex.new("[-9]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.size).to eq 2
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "-".ord
+    end
+
+    it "parses - as a plain character at end of a character class" do
+      regex = Regex.new("[0-]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.size).to eq 2
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "0".ord
+      expect(ccu[1]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[1].first).to eq "-".ord
+    end
+
+    it "parses - as a plain character at beginning of a negated character class" do
+      regex = Regex.new("[^-9]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_truthy
+      expect(ccu.size).to eq 2
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "-".ord
+    end
+
+    it "parses . as a plain character in a character class" do
+      regex = Regex.new("[.]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_falsey
+      expect(ccu.size).to eq 1
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq ".".ord
+    end
+
+    it "parses - as a plain character when escaped in middle of character class" do
+      regex = Regex.new("[0\\-9]")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_falsey
+      expect(ccu.size).to eq 3
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "0".ord
+      expect(ccu[1]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[1].first).to eq "-".ord
+      expect(ccu[2]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[2].first).to eq "9".ord
+    end
+
+    it "parses alternates" do
+      regex = Regex.new("ab|c")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 2
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[1]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[0].size).to eq 2
+      expect(regex.unit.alternates[1].size).to eq 1
+    end
+
+    it "parses a ." do
+      regex = Regex.new("a.b")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[0][1]).to be_a Regex::CharacterClassUnit
+      expect(regex.unit.alternates[0][1].units.size).to eq 2
+      expect(regex.unit.alternates[0][2]).to be_a Regex::CharacterRangeUnit
+    end
+
+    it "parses something complex" do
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 4
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[0].size).to eq 1
+      expect(regex.unit.alternates[0][0]).to be_a Regex::MultiplicityUnit
+      expect(regex.unit.alternates[0][0].min_count).to eq 0
+      expect(regex.unit.alternates[0][0].max_count).to be_nil
+      expect(regex.unit.alternates[0][0].unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates[0][0].unit.alternates.size).to eq 2
+      expect(regex.unit.alternates[0][0].unit.alternates[0]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[0][0].unit.alternates[0].size).to eq 1
+      expect(regex.unit.alternates[0][0].unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[0][0].unit.alternates[1]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[0][0].unit.alternates[1].size).to eq 0
+      expect(regex.unit.alternates[1]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[1].size).to eq 1
+      expect(regex.unit.alternates[1][0]).to be_a Regex::CharacterClassUnit
+      expect(regex.unit.alternates[1][0].negate).to be_truthy
+      expect(regex.unit.alternates[1][0].size).to eq 1
+      expect(regex.unit.alternates[1][0][0]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[2]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[2].size).to eq 2
+      expect(regex.unit.alternates[2][0]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[2][0].first).to eq "|".ord
+      expect(regex.unit.alternates[2][1]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[2][1].first).to eq "v".ord
+      expect(regex.unit.alternates[3]).to be_a Regex::SequenceUnit
+      expect(regex.unit.alternates[3].size).to eq 1
+      expect(regex.unit.alternates[3][0]).to be_a Regex::MultiplicityUnit
+      expect(regex.unit.alternates[3][0].min_count).to eq 1
+      expect(regex.unit.alternates[3][0].max_count).to be_nil
+      expect(regex.unit.alternates[3][0].unit).to be_a Regex::CharacterClassUnit
+      expect(regex.unit.alternates[3][0].unit.size).to eq 1
+      expect(regex.unit.alternates[3][0].unit[0]).to be_a Regex::CharacterRangeUnit
+      expect(regex.unit.alternates[3][0].unit[0].first).to eq "x".ord
+      expect(regex.unit.alternates[3][0].unit[0].last).to eq "y".ord
+    end
+
+  end
+end
@@ -0,0 +1,734 @@
+require "fileutils"
+require "open3"
+
+Results = Struct.new(:stdout, :stderr, :status)
+
+describe Propane do
+  before(:all) do
+    @statics = {}
+  end
+
+  def write_grammar(grammar, options = {})
+    options[:name] ||= ""
+    File.write("spec/run/testparser#{options[:name]}.propane", grammar)
+  end
+
+  def build_parser(options = {})
+    @statics[:build_test_id] ||= 0
+    @statics[:build_test_id] += 1
+    if ENV["dist_specs"]
+      command = %W[dist/propane]
+    else
+      command = %W[ruby -I spec/run -r _simplecov_setup -I lib bin/propane]
+      command_prefix =
+        if ENV["partial_specs"]
+          "p"
+        else
+          "b"
+        end
+      command_name = "#{command_prefix}#{@statics[:build_test_id]}"
+      File.open("spec/run/_simplecov_setup.rb", "w") do |fh|
+        fh.puts <<EOF
+require "bundler"
+Bundler.setup
+require "simplecov"
+class MyFormatter
+  def format(*args)
+  end
+end
+SimpleCov.start do
+  command_name(#{command_name.inspect})
+  filters.clear
+  add_filter do |src|
+    !(src.filename[SimpleCov.root])
+  end
+  formatter(MyFormatter)
+end
+# force color off
+ENV["TERM"] = nil
+EOF
+      end
+    end
+    command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+    if (options[:capture])
+      stdout, stderr, status = Open3.capture3(*command)
+      Results.new(stdout, stderr, status)
+    else
+      result = system(*command)
+      expect(result).to be_truthy
+    end
+  end
+
+  def compile(test_files, options = {})
+    test_files = Array(test_files)
+    options[:parsers] ||= [""]
+    parsers = options[:parsers].map do |name|
+      "spec/run/testparser#{name}.#{options[:language]}"
+    end
+    case options[:language]
+    when "c"
+      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
+    when "d"
+      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+    end
+    expect(result).to be_truthy
+  end
+
+  def run
+    stdout, stderr, status = Open3.capture3("spec/run/testparser")
+    File.binwrite("spec/run/.stderr", stderr)
+    File.binwrite("spec/run/.stdout", stdout)
+    stderr.sub!(/^.*modules passed unittests\n/, "")
+    Results.new(stdout, stderr, status)
+  end
+
+  def lines(str)
+    str.lines.map(&:chomp)
+  end
+
+  def verify_lines(lines, patterns)
+    if lines.is_a?(String)
+      lines = lines.lines.map(&:chomp)
+    end
+    patterns.each_with_index do |pattern, i|
+      found_index =
+        if pattern.is_a?(Regexp)
+          lines.find_index {|line| line =~ pattern}
+        else
+          lines.find_index do |line|
+            line.chomp == pattern.chomp
+          end
+        end
+      unless found_index
+        $stderr.puts "Lines:"
+        $stderr.puts lines
+        raise "A line matching #{pattern.inspect} (index #{i}) was not found."
+      end
+    end
+  end
+
+  before(:each) do
+    FileUtils.rm_rf("spec/run")
+    FileUtils.mkdir_p("spec/run")
+  end
+
+  %w[d c].each do |language|
+
+    context "#{language.upcase} language" do
+
+      it "generates a lexer" do
+        write_grammar <<EOF
+token int /\\d+/;
+token plus /\\+/;
+token times /\\*/;
+drop /\\s+/;
+Start -> Foo;
+Foo -> int <<
+>>
+Foo -> plus <<
+>>
+EOF
+        build_parser(language: language)
+        compile("spec/test_lexer.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "detects a lexer error when an unknown character is seen" do
+        case language
+        when "c"
+          write_grammar <<EOF
+ptype int;
+token int /\\d+/ <<
+  int v = 0;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+Start -> int <<
+  $$ = $1;
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+ptype int;
+token int /\\d+/ <<
+  int v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+Start -> int <<
+  $$ = $1;
+>>
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_lexer_unknown_character.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "generates a parser" do
+        write_grammar <<EOF
+token plus /\\+/;
+token times /\\*/;
+token zero /0/;
+token one /1/;
+Start -> E;
+E -> E times B;
+E -> E plus B;
+E -> B;
+B -> zero;
+B -> one;
+EOF
+        build_parser(language: language)
+      end
+
+      it "generates a parser that does basic math - user guide example" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <math.h>
+>>
+
+ptype size_t;
+
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  size_t v = 0u;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = (size_t)pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.math;
+>>
+
+ptype ulong;
+
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> E1 <<
+  $$ = $1;
+>>
+E1 -> E2 <<
+  $$ = $1;
+>>
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+E2 -> E3 <<
+  $$ = $1;
+>>
+E2 -> E2 times E3 <<
+  $$ = $1 * $3;
+>>
+E3 -> E4 <<
+  $$ = $1;
+>>
+E3 -> E3 power E4 <<
+  $$ = pow($1, $3);
+>>
+E4 -> integer <<
+  $$ = $1;
+>>
+E4 -> lparen E1 rparen <<
+  $$ = $2;
+>>
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_basic_math_grammar.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "generates an SLR parser" do
+        write_grammar <<EOF
+token one /1/;
+Start -> E;
+E -> one E;
+E -> one;
+EOF
+        build_parser(language: language)
+      end
+
+      it "distinguishes between multiple identical rules with lookahead symbol" do
+        write_grammar <<EOF
+token a;
+token b;
+Start -> R1 a;
+Start -> R2 b;
+R1 -> a b;
+R2 -> a b;
+EOF
+        build_parser(language: language)
+        compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+      end
+
+      it "handles reducing a rule that could be arrived at from multiple states" do
+        write_grammar <<EOF
+token a;
+token b;
+drop /\\s+/;
+Start -> a R1;
+Start -> b R1;
+R1 -> b;
+EOF
+        build_parser(language: language)
+        compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+      end
+
+      it "executes user code when matching lexer token" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc <<
+  printf("abc!\\n");
+>>
+token def;
+Start -> Abcs def;
+Abcs -> ;
+Abcs -> abc Abcs;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token abc <<
+  writeln("abc!");
+>>
+token def;
+Start -> Abcs def;
+Abcs -> ;
+Abcs -> abc Abcs;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_user_code.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "abc!",
+          "pass1",
+          "abc!",
+          "abc!",
+          "pass2",
+        ])
+      end
+
+      it "supports a pattern statement" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+/def/ <<
+  printf("def!\\n");
+>>
+Start -> abc;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token abc;
+/def/ <<
+  writeln("def!");
+>>
+Start -> abc;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_pattern.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "def!",
+          "pass1",
+          "def!",
+          "def!",
+          "pass2",
+        ])
+      end
+
+      it "supports returning tokens from pattern code blocks" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+/def/ <<
+  printf("def!\\n");
+>>
+/ghi/ <<
+  printf("ghi!\\n");
+  return $token(abc);
+>>
+Start -> abc;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token abc;
+/def/ <<
+  writeln("def!");
+>>
+/ghi/ <<
+  writeln("ghi!");
+  return $token(abc);
+>>
+Start -> abc;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_return_token_from_pattern.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "def!",
+          "ghi!",
+          "def!",
+        ])
+      end
+
+      it "supports lexer modes" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  printf("begin string mode\\n");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  printf("captured string\\n");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  writeln("begin string mode");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  writeln("captured string");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_lexer_modes.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "begin string mode",
+          "captured string",
+          "pass1",
+          "begin string mode",
+          "captured string",
+          "pass2",
+        ])
+      end
+
+      it "executes user code associated with a parser rule" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+token a;
+token b;
+Start -> A B <<
+  printf("Start!\\n");
+>>
+A -> a <<
+  printf("A!\\n");
+>>
+B -> b <<
+  printf("B!\\n");
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token a;
+token b;
+Start -> A B <<
+  writeln("Start!");
+>>
+A -> a <<
+  writeln("A!");
+>>
+B -> b <<
+  writeln("B!");
+>>
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_parser_rule_user_code.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "A!",
+          "B!",
+          "Start!",
+        ])
+      end
+
+      it "parses lists" do
+        write_grammar <<EOF
+ptype #{language == "c" ? "uint32_t" : "uint"};
+token a;
+Start -> As <<
+  $$ = $1;
+>>
+As -> <<
+  $$ = 0u;
+>>
+As -> As a <<
+  $$ = $1 + 1u;
+>>
+EOF
+        build_parser(language: language)
+        compile("spec/test_parsing_lists.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        expect(results.stderr).to eq ""
+      end
+
+      it "fails to generate a parser for a LR(1) grammar that is not LALR" do
+        write_grammar <<EOF
+token a;
+token b;
+token c;
+token d;
+token e;
+Start -> a E c;
+Start -> a F d;
+Start -> b F c;
+Start -> b E d;
+E -> e;
+F -> e;
+EOF
+        results = build_parser(capture: true, language: language)
+        expect(results.status).to_not eq 0
+        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+      end
+
+      it "provides matched text to user code blocks" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <stdlib.h>
+>>
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * t = malloc(match_length + 1);
+  strncpy(t, (char *)match, match_length);
+  printf("Matched token is %s\\n", t);
+  free(t);
+>>
+Start -> id;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  writeln("Matched token is ", match);
+>>
+Start -> id;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_lexer_match_text.#{language}", language: language)
+        results = run
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "Matched token is identifier_123",
+          "pass1",
+        ])
+      end
+
+      it "allows storing a result value for the lexer" do
+        case language
+        when "c"
+          write_grammar <<EOF
+ptype uint64_t;
+token word /[a-z]+/ <<
+  $$ = match_length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+ptype ulong;
+token word /[a-z]+/ <<
+  $$ = match.length;
+>>
+Start -> word <<
+  $$ = $1;
+>>
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_lexer_result_value.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "tracks position of parser errors" do
+        write_grammar <<EOF
+token a;
+token num /\\d+/;
+drop /\\s+/;
+Start -> a num Start;
+Start -> a num;
+EOF
+        build_parser(language: language)
+        compile("spec/test_error_positions.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows creating a JSON parser" do
+        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        build_parser(language: language)
+        compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
+      end
+
+      it "allows generating multiple parsers in the same program" do
+        write_grammar(<<EOF, name: "myp1")
+prefix myp1_;
+token a;
+token num /\\d+/;
+drop /\\s+/;
+Start -> a num;
+EOF
+        build_parser(name: "myp1", language: language)
+        write_grammar(<<EOF, name: "myp2")
+prefix myp2_;
+token b;
+token c;
+Start -> b c b;
+EOF
+        build_parser(name: "myp2", language: language)
+        compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+    end
+  end
+end
@@ -0,0 +1,27 @@
+unless ENV["dist_specs"]
+  require "bundler/setup"
+  require "simplecov"
+
+  SimpleCov.start do
+    add_filter "/spec/"
+    add_filter "/.bundle/"
+    if ENV["partial_specs"]
+      command_name "RSpec-partial"
+    else
+      command_name "RSpec"
+    end
+    project_name "Propane"
+    merge_timeout 3600
+  end
+
+  RSpec.configure do |config|
+    # Enable flags like --only-failures and --next-failure
+    config.example_status_persistence_file_path = ".rspec_status"
+
+    config.expect_with :rspec do |c|
+      c.syntax = :expect
+    end
+  end
+end
+
+require "propane"
@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "1 + 2 * 3 + 4";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(11, p_result(&context));
+
+    input = "1 * 2 ** 4 * 3";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(48, p_result(&context));
+
+    input = "(1 + 2) * 3 + 4";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(13, p_result(&context));
+
+    input = "(2 * 2) ** 3 + 4 + 5";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(73, p_result(&context));
+
+    return 0;
+}
@@ -0,0 +1,32 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "1 + 2 * 3 + 4";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(11, p_result(&context));
+
+    input = "1 * 2 ** 4 * 3";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(48, p_result(&context));
+
+    input = "(1 + 2) * 3 + 4";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(13, p_result(&context));
+
+    input = "(2 * 2) ** 3 + 4 + 5";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    assert_eq(73, p_result(&context));
+}
@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a 42";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "a\n123\na  a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 3);
+    assert(context.token == TOKEN_a);
+
+    input = "12";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 0);
+    assert(context.token == TOKEN_num);
+
+    input = "a 12\n\nab";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    assert(p_position(&context).row == 2);
+    assert(p_position(&context).col == 1);
+
+    input = "a 12\n\na\n\n77\na   \xAA";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_DECODE_ERROR);
+    assert(p_position(&context).row == 5);
+    assert(p_position(&context).col == 4);
+
+    return 0;
+}
@@ -0,0 +1,37 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a 42";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "a\n123\na  a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(2, 3));
+    assert(context.token == TOKEN_a);
+
+    input = "12";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(0, 0));
+    assert(context.token == TOKEN_num);
+
+    input = "a 12\n\nab";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    assert(p_position(&context) == p_position_t(2, 1));
+
+    input = "a 12\n\na\n\n77\na   \xAA";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_DECODE_ERROR);
+    assert(p_position(&context) == p_position_t(5, 4));
+}
@@ -0,0 +1,92 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    size_t result;
+    p_code_point_t code_point;
+    uint8_t code_point_length;
+
+    result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == '5');
+    assert(code_point_length == 1u);
+
+    result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
+    assert(result == P_EOF);
+
+    result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0xA9u);
+    assert(code_point_length == 2u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0x1F9E1u);
+    assert(code_point_length == 4u);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+
+    p_token_info_t token_info;
+    char const * input = "5 + 4 * \n677 + 567";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 2u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_times);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 4u);
+    assert(token_info.length == 1u);
+    assert(token_info.token == TOKEN_plus);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 6u);
+    assert(token_info.length == 3u);
+    assert(token_info.token == TOKEN_int);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 9u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    p_context_init(&context, (uint8_t const *)"", 0u);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
+    assert(token_info.length == 0u);
+    assert(token_info.token == TOKEN___EOF);
+
+    return 0;
+}
@@ -0,0 +1,69 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    size_t result;
+    p_code_point_t code_point;
+    ubyte code_point_length;
+
+    result = p_decode_code_point("5", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == '5');
+    assert(code_point_length == 1u);
+
+    result = p_decode_code_point("", &code_point, &code_point_length);
+    assert(result == P_EOF);
+
+    result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0xA9u);
+    assert(code_point_length == 2u);
+
+    result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+    assert(result == P_SUCCESS);
+    assert(code_point == 0x1F9E1u);
+    assert(code_point_length == 4u);
+
+    result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+
+    result = p_decode_code_point("\xfe", &code_point, &code_point_length);
+    assert(result == P_DECODE_ERROR);
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "5 + 4 * \n677 + 567";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
+
+    p_context_init(&context, "");
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
+}
@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "identifier_123";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    return 0;
+}
@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `identifier_123`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+}
@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc \"a string\" def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc \"abc def\" def";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}
@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc "a string" def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc "abc def" def`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}
@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "fabulous";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 8u);
+
+    return 0;
+}
@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `x`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = `fabulous`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 8u);
+}
@@ -0,0 +1,18 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "x";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+
+    input = "123";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 123u);
+
+    return 0;
+}
@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `x`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+
+    input = `123`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 123u);
+}
@@ -0,0 +1,19 @@
+#include "testparsermyp1.h"
+#include "testparsermyp2.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input1 = "a\n1";
+    myp1_context_t context1;
+    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+
+    char const * input2 = "bcb";
+    myp2_context_t context2;
+    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,21 @@
+import testparsermyp1;
+import testparsermyp2;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input1 = "a\n1";
+    myp1_context_t context1;
+    myp1_context_init(&context1, input1);
+    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+
+    string input2 = "bcb";
+    myp2_context_t context2;
+    myp2_context_init(&context2, input2);
+    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+}
@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "aba";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aba";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}
@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context).row == 0);
+    assert(p_position(&context).col == 1);
+    assert(context.token == TOKEN___EOF);
+
+    input = "a b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "bb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,25 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(&context) == p_position_t(0, 1));
+    assert(context.token == TOKEN___EOF);
+
+    input = "a b";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "bb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}
@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "ab";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}
@@ -0,0 +1,56 @@
+#include "testparser.h"
+#include "json_types.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "{}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_OBJECT);
+
+    input = "[]";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_ARRAY);
+
+    input = "-45.6";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == -45.6);
+
+    input = "2E-2";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == 0.02);
+
+    input = "{\"hi\":true}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    JSONValue * o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(1, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "hi") == 0);
+    assert(o->object.entries[0].value->id == JSON_TRUE);
+
+    input = "{\"ff\": false, \"nn\": null}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(2, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "ff") == 0);
+    assert(o->object.entries[0].value->id == JSON_FALSE);
+    assert(strcmp(o->object.entries[1].name, "nn") == 0);
+    assert(o->object.entries[1].value->id == JSON_NULL);
+
+    return 0;
+}
@@ -0,0 +1,56 @@
+import testparser;
+import std.stdio;
+import json_types;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = ``;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = `{}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+
+    input = `[]`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONArray)p_result(&context));
+
+    input = `-45.6`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(&context));
+    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+
+    input = `2E-2`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(&context));
+    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+
+    input = `{"hi":true}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+    JSONObject o = cast(JSONObject)p_result(&context);
+    assert(o.value["hi"]);
+    assert(cast(JSONTrue)o.value["hi"]);
+
+    input = `{"ff": false, "nn": null}`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(&context));
+    o = cast(JSONObject)p_result(&context);
+    assert(o.value["ff"]);
+    assert(cast(JSONFalse)o.value["ff"]);
+    assert(o.value["nn"]);
+    assert(cast(JSONNull)o.value["nn"]);
+}
@@ -0,0 +1,24 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 16u);
+
+    return 0;
+}
@@ -0,0 +1,26 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 1u);
+
+    input = "";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 0u);
+
+    input = "aaaaaaaaaaaaaaaa";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context) == 16u);
+}
@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "defabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}
@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = "defabcdef";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}
@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "defghidef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "defghidef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}
@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abcabcdef";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}
@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abcdef";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = "abcabcdef";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}
@@ -0,0 +1,38 @@
+#include <stdio.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include "testutils.h"
+
+void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
+{
+    if (expected != actual)
+    {
+        fprintf(stderr, "%s:%lu: expected %lu, got %lu\n", file, line, expected, actual);
+        assert(false);
+    }
+}
+
+void str_init(str_t * str, char const * cs)
+{
+    size_t length = strlen(cs);
+    str->cs = malloc(length + 1u);
+    strcpy(str->cs, cs);
+}
+
+void str_append(str_t * str, char const * cs)
+{
+    size_t length = strlen(str->cs);
+    size_t length2 = strlen(cs);
+    char * new_cs = malloc(length + length2 + 1u);
+    memcpy(new_cs, str->cs, length);
+    strcpy(&new_cs[length], cs);
+    free(str->cs);
+    str->cs = new_cs;
+}
+
+void str_free(str_t * str)
+{
+    free(str->cs);
+}
@@ -0,0 +1,10 @@
+import std.stdio;
+
+void assert_eq(T)(T expected, T actual, string file = __FILE__, size_t line = __LINE__)
+{
+    if (expected != actual)
+    {
+        stderr.writeln(file, ":", line, ": expected ", expected, ", got ", actual);
+        assert(false);
+    }
+}
@@ -0,0 +1,19 @@
+#pragma once
+
+void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
+
+#define assert_eq(expected, actual) \
+    assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
+
+typedef struct
+{
+    char * cs;
+} str_t;
+
+void str_init(str_t * str, char const * cs);
+void str_append(str_t * str, char const * cs);
+void str_free(str_t * str);
+static inline char * str_cstr(str_t * str)
+{
+    return str->cs;
+}
@@ -1,14 +0,0 @@
-
-all:
-	for d in *; do \
-		if [ -d $$d ]; then \
-			make -C $$d; \
-		fi; \
-	done
-
-clean:
-	for d in *; do \
-		if [ -d $$d ]; then \
-			make -C $$d clean; \
-		fi; \
-	done
@@ -1,15 +0,0 @@
-
-TARGET   := test
-I_SOURCE := itest
-CXXFLAGS := -O2
-LDFLAGS  := -lpcre
-
-all: $(TARGET)
-	./$(TARGET)
-
-$(TARGET): $(shell which imbecile) $(I_SOURCE).I $(wildcard *.cc)
-	imbecile $(I_SOURCE).I
-	$(CXX) -o $@ *.cc $(LDFLAGS)
-
-clean:
-	-rm -f $(TARGET) *.o $(I_SOURCE).cc $(I_SOURCE).h
@@ -1,37 +0,0 @@
-
-[tokens]
-
-AND         and
-OR          or
-NOT         not
-LPAREN      \(
-RPAREN      \)
-WS          \s+
-EQUALS      = %{ cout << "Saw '='" << endl; %}
-IDENTIFIER  [a-zA-Z_][a-zA-Z_0-9]* %{
-    cout << "Identify: '" << matches[0] << "'" << endl;
-%}
-
-DEC_INT     [1-9]\d*\b
-${
-    uint64_t value;
-$}
-%{
-    sscanf(matches[0].c_str(), "%lld", &value);
-    cout << "value: " << value << endl;
-%}
-
-HEX_INT     0x([0-9a-fA-F]+)\b ${ uint64_t value; $} %{
-    sscanf(matches[1].c_str(), "%llx", &value);
-    cout << "value: " << value << endl;
-%}
-
-OCT_INT     0([0-7]*)\b
-BIN_INT     0b([01]+)\b
-
-[rules]
-
-Assignment := IDENTIFIER ASSIGN Expression
-
-Expression := IDENTIFIER \
-            | Assignment