v4.6.0

Allow user code to access specific pvalue values, especially when
multiple ptypes are in use.

.github/workflows/run-tests.yml

@@ -0,0 +1,38 @@
+name: Run Propane Tests
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ruby-version: ['3.4']
+
+    steps:
+    - name: Install dependencies (Linux)
+      if: runner.os == 'Linux'
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+
+    - name: Install dependencies (macOS)
+      if: runner.os == 'macOS'
+      run: brew install gcc ldc
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests
+      run: rake all

.rspec

@@ -1,3 +1,2 @@
---format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,3 +1,206 @@
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
+## v4.0.0
+
+### New Features
+
+- Add `context_user_fields` statement to allow custom context user fields.
+- Add `token_user_fields` statement to allow custom token user fields.
+- Add `on_token_node` statement to allow custom code when constructing token nodes.
+- Add `free_token_node` statement to allow custom code when freeing token nodes.
+- Add `p_context_delete()`.
+- Allow `drop` patterns to execute lexer user code blocks.
+
+### Breaking Changes
+
+- Replace `p_context_init()` with `p_context_new()` and `p_context_delete()`.
+- Renamed `p_free_tree()` to `p_tree_delete()`.
+- The `free_token_node` statement now takes a user code block instead of a
+  function name parameter.
+
+## v3.0.0
+
+### New Features
+
+- Add support for multiple starting rules (#38)
+- Add `p_free_tree()` functions to reclaim generated tree memory
+- Add `free_token_node` grammar statement to reclaim user-allocated memory stored in a Token tree node `pvalue` field
+- Add valgrind memory leak tests to unit tests
+- Fix build issues for C++ to officially support C++ target output
+
+### Improvements
+
+- Document `p_lex()` and `p_token_info_t` in user guide (#37)
+
+### Breaking Changes
+
+- Rename AST generation mode to tree generation mode (see [UPGRADING.md](UPGRADING.md))
+
+## v2.3.0
+
+### New Features
+
+- Add \D, \S, \w, \W special character classes
+
+### Improvements
+
+- Include line numbers for pattern errors
+- Improve performance in a few places
+- Parallelize parser table generation on Linux hosts
+- Add github workflow to run unit tests
+
+### Fixes
+
+- Fix a couple clang warnings for C backend
+- Fix C backend not fully initializing pvalues when multiple ptypes are used with different sizes.
+- Fix some user guide examples
+
+## v2.2.1
+
+### Fixes
+
+- Fix GC issue for D backend when AST is enabled (#36)
+
+## v2.2.0
+
+### Improvements
+
+- Allow multiple lexer modes to be specified for a lexer pattern (#35)
+- Document p_decode_code_point() API function (#34)
+
+## v2.1.1
+
+### Fixes
+
+- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
+
+## v2.1.0
+
+### Improvements
+
+- Report rule name and line number for conflicting AST node field positions errors (#32)
+
+## v2.0.0
+
+### Improvements
+
+- Log conflicting rules on reduce/reduce conflict (#31)
+- Use 1-based row and column values for position values (#30)
+
+### Fixes
+
+- Fix named optional rules (#29)
+
+### Upgrading
+
+- Adjust all uses of p_position_t row and col values to expect 1-based instead
+of 0-based values.
+
+## v1.5.1
+
+### Improvements
+
+- Improve performance (#28)
+
+## v1.5.0
+
+### New Features
+
+- Track start and end text positions for tokens and rules in AST node structures (#27)
+- Add warnings for shift/reduce conflicts to log file (#25)
+- Add -w command line switch to treat warnings as errors and output to stderr (#26)
+- Add rule field aliases (#24)
+
+### Improvements
+
+- Show line numbers of rules on conflict (#23)
+
+## v1.4.0
+
+### New Features
+
+- Allow user to specify AST node name prefix or suffix
+- Allow specifying the start rule name
+- Allow rule terms to be marked as optional
+
+### Improvements
+
+- Give a better error message when a referenced ptype has not been declared
+
+## v1.3.0
+
+### New Features
+
+- Add AST generation (#22)
+
+## v1.2.0
+
+### New Features
+
+- Allow one line user code blocks (#21)
+- Add backslash escape codes (#19)
+- Add API to access unexpected token found (#18)
+- Add token_names API (#17)
+- Add D example to user guide for p_context_init() (#16)
+- Allow user termination from lexer code blocks (#15)
+
+### Fixes
+
+- Fix generator hang when state transition cycle is present (#20)
+
+## v1.1.0
+
+### New Features
+
+- Add user parser terminations (#13)
+- Document generated parser API in user guide (#14)
+
 ## v1.0.0
 
 - Initial release

Gemfile

@@ -1,5 +1,6 @@
 source "https://rubygems.org"
 
+gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"

Gemfile.lock

@@ -1,40 +1,48 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.5.0)
-    docile (1.4.0)
-    psych (5.1.0)
+    base64 (0.3.0)
+    date (3.5.1)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    erb (6.0.4)
+    psych (5.4.0)
+      date
       stringio
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.4.2)
+    rdoc (7.2.0)
+      erb
       psych (>= 4.0.0)
-    redcarpet (3.6.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+      tsort
+    redcarpet (3.6.1)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.7)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    stringio (3.0.7)
+    stringio (3.2.0)
     syntax (1.2.2)
+    tsort (0.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  base64
   rake
   rdoc
   redcarpet
@@ -43,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.14

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2023 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -6,7 +6,10 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * target C or D language outputs
+  * targets C, C++, or D language outputs
+  * optionally supports automatic full parse tree generation
+  * supports starting parsing from multiple start rules
+  * tracks input text start and end positions for all matched tokens/rules
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
@@ -17,6 +20,7 @@ can be copied into and versioned in a project's source tree.
 The only requirement to run Propane is that the system has a Ruby interpreter
 installed.
 The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+
 Simply copy the `propane` executable script into the desired location within
 the project to be built (typically the root of the repository) and mark it
 executable.
@@ -29,9 +33,14 @@ Propane is typically invoked from the command-line as `./propane`.
 
     Usage: ./propane [options] <input-file> <output-file>
     Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
+      -h, --help  Show this usage and exit.
+      --log LOG   Write log file. This will show all parser states and their
+                  associated shifts and reduces. It can be helpful when
+                  debugging a grammar.
+      --version   Show program version and exit.
+      -w          Treat warnings as errors. This option will treat shift/reduce
+                  conflicts as fatal errors and will print them to stderr in
+                  addition to the log file.
 
 The user must specify the path to a Propane input grammar file and a path to an
 output file.
@@ -55,10 +64,10 @@ import std.math;
 ptype ulong;
 
 # A few basic arithmetic operators.
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
+token plus /\+/;
+token times /\*/;
+token power /\*\*/;
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -67,38 +76,22 @@ token integer /\\d+/ <<
   }
   $$ = v;
 >>
-token lparen /\\(/;
-token rparen /\\)/;
+token lparen /\(/;
+token rparen /\)/;
 # Drop whitespace.
-drop /\\s+/;
+drop /\s+/;
 
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
 E3 -> E3 power E4 <<
   $$ = pow($1, $3);
 >>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
 Grammar files can contain comment lines beginning with `#` which are ignored.

Rakefile.rb

@@ -1,5 +1,7 @@
 require "rake/clean"
 require "rspec/core/rake_task"
+require "simplecov"
+require "stringio"
 
 CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
 
@@ -12,6 +14,18 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
   end
 end
+task :spec do |task, args|
+  unless ENV["dist_specs"]
+    original_stdout = $stdout
+    sio = StringIO.new
+    $stdout = sio
+    SimpleCov.collate Dir["coverage/.resultset.json"]
+    $stdout = original_stdout
+    sio.string.lines.each do |line|
+      $stdout.write(line) unless line =~ /Coverage report generated for/
+    end
+  end
+end
 
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.

UPGRADING.md

@@ -0,0 +1,21 @@
+## v4.0.0
+
+### API Changes
+
+- Replace any calls to `p_context_init()` with `p_context_new()`.
+- Replace any references to the address of a statically allocated context
+  structure with the pointer returned from `p_context_init()` (e.g. `&context`
+  -> `context`).
+- Add a call to `p_context_delete()` (for C or C++) after lexing/parsing to
+  reclaim context memory.
+- Rename `p_free_tree()` calls to `p_tree_delete()`.
+- Change `free_token_node` statement calls from taking a function name argument
+  to taking a user code block.
+
+## v3.0.0
+
+### Grammar Changes
+
+- Rename `ast;` statement to `tree;`.
+- Rename `ast_prefix;` statement to `tree_prefix;`.
+- Rename `ast_suffix;` statement to `tree_suffix;`.

assets/parser.c.erb

@@ -3,6 +3,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+const char * <%= @grammar.prefix %>token_names[] = {
+<% @grammar.tokens.each_with_index do |token, index| %>
+    "<%= token.name %>",
+<% end %>
+};
+
 /**************************************************************************
  * User code blocks
  *************************************************************************/
@@ -21,6 +32,7 @@
 #define P_UNEXPECTED_TOKEN 3u
 #define P_DROP 4u
 #define P_EOF 5u
+#define P_USER_TERMINATED 6u
 <% end %>
 
 /* An invalid ID value. */
@@ -31,27 +43,52 @@
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
  * @param input_length
  *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length)
 {
-    /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext = {0};
+<% if @cpp %>
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t();
+<% else %>
+    <%= @grammar.prefix %>context_t * context = (<%= @grammar.prefix %>context_t *)calloc(1, sizeof(<%= @grammar.prefix %>context_t));
+<% end %>
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.input_length = input_length;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context->input = input;
+    context->input_length = input_length;
+    context->text_position.row = 1u;
+    context->text_position.col = 1u;
+    context->mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * For C++, destructors will be called for any context user fields. However, if
+ * pointers are used to store allocated resources, the user should free them
+ * before calling this function.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+<% if @cpp %>
+    delete context;
+<% else %>
+    free(context);
+<% end %>
 }
 
 /**************************************************************************
@@ -214,7 +251,10 @@ typedef struct
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta. */
+    /** Input text position delta to end of token. */
+    <%= @grammar.prefix %>position_t end_delta_position;
+
+    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -308,9 +348,12 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
  *
  * @param context
  *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
  *
  * @reval P_SUCCESS
  *   A token was successfully lexed.
@@ -324,8 +367,10 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
     lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
 {
-    lexer_match_info_t longest_match = {0};
-    lexer_match_info_t attempt_match = {0};
+    lexer_match_info_t longest_match;
+    memset(&longest_match, 0, sizeof(longest_match));
+    lexer_match_info_t attempt_match;
+    memset(&attempt_match, 0, sizeof(attempt_match));
     *out_match_info = longest_match;
     uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
     for (;;)
@@ -339,14 +384,16 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
         switch (result)
         {
         case P_SUCCESS:
+            {
             lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
+                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -369,6 +416,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 *out_unexpected_input_length = attempt_match.length + code_point_length;
                 return P_UNEXPECTED_INPUT;
             }
+            }
             break;
 
         case P_EOF:
@@ -390,7 +438,6 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
-            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -422,25 +469,50 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
-    <%= @grammar.prefix %>token_info_t token_info = {0};
+    <%= @grammar.prefix %>token_info_t token_info;
+    memset(&token_info, 0, sizeof(token_info));
     token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
     switch (result)
     {
     case P_SUCCESS:
+        {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state->code_id, match, match_info.length, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -468,12 +540,13 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
         *out_token_info = token_info;
+        }
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
+        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -511,6 +584,8 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -528,6 +603,9 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  * Parser
  *************************************************************************/
 
+/** Invalid position value. */
+#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
+
 /** Reduce ID type. */
 typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
 
@@ -587,6 +665,25 @@ typedef struct
      * reduce action.
      */
     parser_state_id_t n_states;
+<% if @grammar.tree %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    uint16_t const * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set tree node fields.
+     */
+    uint16_t rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
+<% end %>
 } reduce_t;
 
 /** Parser state entry. */
@@ -615,21 +712,63 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
+<% if @grammar.tree %>
+    /** tree node. */
+    void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
+<% end %>
 } state_value_t;
 
+<% if @grammar.tree %>
+/** Common tree node structure. */
+typedef struct TreeNode_s
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct TreeNode_s * fields[];
+} TreeNode;
+<% end %>
+
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
-    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
+    {<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u},
 <%   end %>
 };
 
+<% if @grammar.tree %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
+    {
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
+<%     if @grammar.tree %>
+        <%= reduce[:n_states] %>u, /* Number of states */
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+        NULL, /* No rule set node field index map (flat map) */
+<%       else %>
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
+<%       end %>
+        <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
+        <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u},
+<%     end %>
 <%   end %>
 };
 
@@ -699,7 +838,7 @@ static void state_values_stack_push(state_values_stack_t * stack)
     if (current_length >= current_capacity)
     {
         size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
+        state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
         memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
@@ -733,17 +872,70 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
+<% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
  * @param rule The ID of the rule.
  *
- * @return Parse value.
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
-static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
+static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
-    <%= @grammar.prefix %>value_t _pvalue = {0};
-
     switch (rule)
     {
 <%   @grammar.rules.each do |rule| %>
@@ -756,8 +948,9 @@ static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_value
     default: break;
     }
 
-    return _pvalue;
+    return P_SUCCESS;
 }
+<% end %>
 
 /**
  * Check if the parser should shift to a new state.
@@ -813,33 +1006,42 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
+ * @start_state_id
+ *   ID of the state in which to start.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context->token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.tree %>
+    void * reduced_parser_node;
+<% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
     state_values_stack_init(&statevalues);
     state_values_stack_push(&statevalues);
+    state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
     size_t result;
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -858,7 +1060,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
+<% if @grammar.tree %>
+                context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+<% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
                 result = P_SUCCESS;
                 break;
             }
@@ -867,19 +1073,45 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         {
             /* We have something to shift. */
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
+<% if @grammar.tree %>
+<%   if @cpp %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
+<%   else %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
+<%   end %>
+                token_tree_node->position = token_info.position;
+                token_tree_node->end_position = token_info.end_position;
+                token_tree_node->n_fields = 0u;
+                token_tree_node->is_token = 1u;
+                token_tree_node->token = token;
+                token_tree_node->pvalue = token_info.pvalue;
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
+                new_state_info->tree_node = token_tree_node;
+<% else %>
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
+<% end %>
                 token = INVALID_TOKEN_ID;
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
             }
             else
             {
                 /* We shifted a RuleSet. */
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
-                <%= @grammar.prefix %>value_t new_parse_result = {0};
+<% if @grammar.tree %>
+                new_state_info->tree_node = reduced_parser_node;
+<% else %>
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
+                <%= @grammar.prefix %>value_t new_parse_result;
+                memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
+<% end %>
                 reduced_rule_set = INVALID_ID;
             }
             continue;
@@ -889,7 +1121,74 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
+<% if @grammar.tree %>
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
+                TreeNode * node = (TreeNode *)malloc(bytes);
+                memset(node, 0, bytes);
+                node->position = INVALID_POSITION;
+                node->end_position = INVALID_POSITION;
+                node->n_fields = n_fields;
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                    }
+                }
+                else
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
+                    }
+                }
+                bool position_found = false;
+                for (size_t i = 0; i < n_fields; i++)
+                {
+                    TreeNode * child = node->fields[i];
+                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
+                    {
+                        if (!position_found)
+                        {
+                            node->position = child->position;
+                            position_found = true;
+                        }
+                        node->end_position = child->end_position;
+                    }
+                }
+                reduced_parser_node = node;
+            }
+            else
+            {
+                reduced_parser_node = NULL;
+            }
+<% else %>
+            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
+            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                state_values_stack_free(&statevalues);
+                return P_USER_TERMINATED;
+            }
+            reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
+<% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
@@ -909,6 +1208,19 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
     return result;
 }
 
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, 0u);
+}
+
+<% @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, <%= i %>u);
+}
+<% end %>
+
 /**
  * Get the parse result value.
  *
@@ -917,10 +1229,29 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  *
  * @return Parse result value.
  */
+<% if @grammar.tree %>
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
+}
+<%   end %>
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
     return context->parse_result.v_<%= start_rule_type[0] %>;
 }
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return context->parse_result.v_<%= start_rule_type(i)[0] %>;
+}
+<%   end %>
+<% end %>
 
 /**
  * Get the current text input position.
@@ -934,3 +1265,71 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
     return context->text_position;
 }
+
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
+size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
+{
+    return context->user_terminate_code;
+}
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context->token;
+}
+<% if @grammar.tree %>
+
+static void tree_delete(TreeNode * node)
+{
+    if (node->is_token)
+    {
+        <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
+<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
+<%   if @cpp %>
+        delete token_tree_node;
+<%   else %>
+        free(token_tree_node);
+<%   end %>
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                tree_delete(node->fields[i]);
+            }
+        }
+        free(node);
+    }
+}
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+/**
+ * Free all tree node memory.
+ */
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete((TreeNode *)tree);
+}
+<%   end %>
+<% end %>

assets/parser.d.erb

@@ -8,6 +8,9 @@
 module <%= @grammar.modulename %>;
 <% end %>
 
+import core.memory;
+import core.stdc.stdlib : malloc, free;
+
 /**************************************************************************
  * User code blocks
  *************************************************************************/
@@ -27,10 +30,11 @@ public enum : size_t
     <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN,
     <%= @grammar.prefix.upcase %>DROP,
     <%= @grammar.prefix.upcase %>EOF,
+    <%= @grammar.prefix.upcase %>USER_TERMINATED,
 }
 
 /** Token type. */
-public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
 
 /** Token IDs. */
 public enum : <%= @grammar.prefix %>token_t
@@ -42,11 +46,39 @@ public enum : <%= @grammar.prefix %>token_t
 <%   end %>
 <% end %>
     INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
+    TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
 }
 
 /** Code point type. */
 public alias <%= @grammar.prefix %>code_point_t = uint;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+public struct <%= @grammar.prefix %>position_t
+{
+    /** Input text row (1-based). */
+    uint row;
+
+    /** Input text column (1-based). */
+    uint col;
+
+    /** Invalid position value. */
+    enum INVALID = <%= @grammar.prefix %>position_t(0u, 0u);
+
+    /** Return whether the position is valid. */
+    public @property bool valid()
+    {
+        return row != 0u;
+    }
+}
+
+<% if @grammar.tree %>
+/** Parser values type. */
+public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
+<% else %>
 /** Parser values type(s). */
 public union <%= @grammar.prefix %>value_t
 {
@@ -55,26 +87,79 @@ public union <%= @grammar.prefix %>value_t
 <%   end %>
 }
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-public struct <%= @grammar.prefix %>position_t
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
 {
-    /** Input text row (0-based). */
-    uint row;
-
-    /** Input text column (0-based). */
-    uint col;
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
 }
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+/** Common tree node structure. */
+private struct TreeNode
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
+    void *[0] fields;
+}
+
+/** Tree node types. @{ */
+public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+{
+    /* TreeNode fields must be present in the same order here. */
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+<%= @grammar.token_user_fields %>
+}
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
+<%     rule_set.tree_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        <%= type %> * <%= field_name %>;
+<%       end %>
+    }
+<%     end %>
+}
+
+<%   end %>
+/** @} */
+<% end %>
 
 /** Lexed token information. */
 public struct <%= @grammar.prefix %>token_info_t
 {
-    /** Text position where the token was found. */
+    /** Text position of first code point in token. */
     <%= @grammar.prefix %>position_t position;
 
+    /** Text position of last code point in token. */
+    <%= @grammar.prefix %>position_t end_position;
+
     /** Number of input bytes used by the token. */
     size_t length;
 
@@ -110,10 +195,19 @@ public struct <%= @grammar.prefix %>context_t
     /* Parser context data. */
 
     /** Parse result value. */
+<% if @grammar.tree %>
+    void * parse_result;
+<% else %>
     <%= @grammar.prefix %>value_t parse_result;
+<% end %>
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
+
+    /** User terminate code. */
+    size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 }
 
 /**************************************************************************
@@ -141,6 +235,7 @@ private enum : size_t
     P_UNEXPECTED_TOKEN,
     P_DROP,
     P_EOF,
+    P_USER_TERMINATED,
 }
 <% end %>
 
@@ -152,24 +247,39 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
- * Initialize lexer/parser context structure.
+ * Allocate and initialize lexer/parser context structure.
+ *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
- * @param[out] context
- *   Lexer/parser context structure.
  * @param input
  *   Text input.
+ * @param input_length
+ *   Text input length.
+ *
+ * @return Context structure for lexer/parser.
  */
-public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, string input)
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(string input)
 {
     /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext;
+    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t;
 
     /* Lexer initialization. */
-    newcontext.input = input;
-    newcontext.mode = <%= @lexer.mode_id("default") %>;
+    context.input = input;
+    context.text_position.row = 1u;
+    context.text_position.col = 1u;
+    context.mode = <%= @lexer.mode_id("default") %>;
 
-    /* Copy to the user's context structure. */
-    *context = newcontext;
+    return context;
+}
+
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
 }
 
 /**************************************************************************
@@ -330,7 +440,10 @@ private struct lexer_match_info_t
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta. */
+    /** Input text position delta to end of token. */
+    <%= @grammar.prefix %>position_t end_delta_position;
+
+    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -422,9 +535,12 @@ private lexer_state_id_t check_lexer_transition(uint current_state, uint code_po
  *
  * @param context
  *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
  *
  * @reval P_SUCCESS
  *   A token was successfully lexed.
@@ -455,10 +571,11 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
+                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -502,7 +619,6 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
-            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -534,13 +650,14 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     token_info.position = context.text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
@@ -548,11 +665,33 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state.code_id, match, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -580,12 +719,12 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
         *out_token_info = token_info;
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
+        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -623,6 +762,8 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -699,6 +840,25 @@ private struct reduce_t
      * reduce action.
      */
     parser_state_id_t n_states;
+<% if @grammar.tree %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    immutable(ushort) * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set tree node fields.
+     */
+    ushort rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
+<% end %>
 }
 
 /** Parser state entry. */
@@ -727,8 +887,15 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
+<% if @grammar.tree %>
+    /** Tree node. */
+    void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
+<% end %>
 
     this(size_t state_id)
     {
@@ -739,14 +906,37 @@ private struct state_value_t
 /** Parser shift table. */
 private immutable shift_t[] parser_shift_table = [
 <%   @parser.shift_table.each do |shift| %>
-    shift_t(<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u),
+    shift_t(<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u),
 <%   end %>
 ];
 
+<% if @grammar.tree %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
+    reduce_t(
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
+<%     if @grammar.tree %>
+        <%= reduce[:n_states] %>u, /* Number of states */
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+        null, /* No rule set node field index map (flat map) */
+<%       else %>
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
+<%       end %>
+        <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
+        <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u), /* Number of states */
+<%     end %>
 <%   end %>
 ];
 
@@ -757,17 +947,66 @@ private immutable parser_state_t[] parser_state_table = [
 <%   end %>
 ];
 
+<% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
  * @param rule The ID of the rule.
  *
- * @return Parse value.
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
-private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[] statevalues, uint n_states)
+private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
-    <%= @grammar.prefix %>value_t _pvalue;
-
     switch (rule)
     {
 <%   @grammar.rules.each do |rule| %>
@@ -780,8 +1019,9 @@ private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[
     default: break;
     }
 
-    return _pvalue;
+    return P_SUCCESS;
 }
+<% end %>
 
 /**
  * Check if the parser should shift to a new state.
@@ -837,30 +1077,39 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
+ * @start_state_id
+ *   ID of the state in which to start.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context.token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
+    statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.tree %>
+    void * reduced_parser_node;
+<% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
@@ -878,7 +1127,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
+<% if @grammar.tree %>
+                context.parse_result = statevalues[$-1].tree_node;
+<% else %>
                 context.parse_result = statevalues[$-1].pvalue;
+<% end %>
                 return P_SUCCESS;
             }
         }
@@ -889,15 +1142,29 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
+<% if @grammar.tree %>
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
+<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
+                statevalues[$-1].tree_node = token_tree_node;
+<% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
+<% end %>
+                token = INVALID_TOKEN_ID;
             }
             else
             {
                 /* We shifted a RuleSet. */
+<% if @grammar.tree %>
+                statevalues[$-1].tree_node = reduced_parser_node;
+<% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
+<% end %>
                 reduced_rule_set = INVALID_ID;
             }
             continue;
@@ -907,7 +1174,77 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states);
+<% if @grammar.tree %>
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = statevalues[$ - 1].tree_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
+                TreeNode * node = cast(TreeNode *)malloc(node_size);
+                GC.addRange(node, node_size);
+                node.position = <%= @grammar.prefix %>position_t.INVALID;
+                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
+                foreach (i; 0..n_fields)
+                {
+                    node.fields[i] = null;
+                }
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                    }
+                }
+                else
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
+                    }
+                }
+                bool position_found = false;
+                foreach (i; 0..n_fields)
+                {
+                    TreeNode * child = cast(TreeNode *)node.fields[i];
+                    if (child && child.position.valid)
+                    {
+                        if (!position_found)
+                        {
+                            node.position = child.position;
+                            position_found = true;
+                        }
+                        node.end_position = child.end_position;
+                    }
+                }
+                reduced_parser_node = node;
+            }
+            else
+            {
+                reduced_parser_node = null;
+            }
+<% else %>
+            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+            reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
+<% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
             continue;
@@ -924,6 +1261,19 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
     }
 }
 
+public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, 0u);
+}
+
+<% @grammar.start_rules.each_with_index do |start_rule, i| %>
+
+public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return parse_from(context, <%= i %>u);
+}
+<% end %>
+
 /**
  * Get the parse result value.
  *
@@ -932,10 +1282,58 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  *
  * @return Parse result value.
  */
+<% if @grammar.tree %>
+public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+{
+    return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
+}
+<%   end %>
+<% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
     return context.parse_result.v_<%= start_rule_type[0] %>;
 }
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
+{
+    return context.parse_result.v_<%= start_rule_type(i)[0] %>;
+}
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
 
 /**
  * Get the current text input position.
@@ -949,3 +1347,26 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
 {
     return context.text_position;
 }
+
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
+public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
+{
+    return context.user_terminate_code;
+}
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context.token;
+}

assets/parser.h.erb

@@ -20,9 +20,10 @@
 #define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
 #define <%= @grammar.prefix.upcase %>DROP 4u
 #define <%= @grammar.prefix.upcase %>EOF 5u
+#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
 
 /** Token type. */
-typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
 
 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -32,13 +33,35 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
 
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+typedef struct
+{
+    /** Input text row (1-based). */
+    uint32_t row;
+
+    /** Input text column (1-based). */
+    uint32_t col;
+} <%= @grammar.prefix %>position_t;
+
+/** Return whether the position is valid. */
+#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
+
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>
 
+<% if @grammar.tree %>
+/** Parser values type. */
+typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
+<% else %>
 /** Parser values type(s). */
 typedef union
 {
@@ -47,26 +70,76 @@ typedef union
 <%   end %>
 } <%= @grammar.prefix %>value_t;
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
 {
-    /** Input text row (0-based). */
-    uint32_t row;
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
 
-    /** Input text column (0-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+/** Tree node types. @{ */
+typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
+{
+    <% # TreeNode fields must be present in the same order here. # %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+<%= @grammar.token_user_fields %>
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+struct <%= name %>;
+<%   end %>
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
+{
+    <% # TreeNode fields must be present in the same order here. # %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+<%     rule_set.tree_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        struct <%= type %> * <%= field_name %>;
+<%       end %>
+    };
+<%     end %>
+} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
+
+<%   end %>
+/** @} */
+<% end %>
 
 /** Lexed token information. */
 typedef struct
 {
-    /** Text position where the token was found. */
+    /** Text position of first code point in token. */
     <%= @grammar.prefix %>position_t position;
 
+    /** Text position of last code point in token. */
+    <%= @grammar.prefix %>position_t end_position;
+
     /** Number of input bytes used by the token. */
     size_t length;
 
@@ -105,13 +178,31 @@ typedef struct
     /* Parser context data. */
 
     /** Parse result value. */
+<% if @grammar.tree %>
+    void * parse_result;
+<% else %>
     <%= @grammar.prefix %>value_t parse_result;
+<% end %>
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
+
+    /** User terminate code. */
+    size_t user_terminate_code;
+
+<%= @grammar.context_user_fields %>
 } <%= @grammar.prefix %>context_t;
 
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+extern const char * <%= @grammar.prefix %>token_names[];
+
+<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length);
+
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
         <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
@@ -119,7 +210,31 @@ size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t inp
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
 
+<% if @grammar.tree %>
+<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+<%   end %>
+<% end %>
+
+<% if @grammar.tree %>
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
+<%   end %>
+<% end %>
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
+
+size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

extra/vim/ftdetect/propane.vim

@@ -0,0 +1 @@
+au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -0,0 +1,44 @@
+" Vim syntax file for Propane
+" Language: propane
+" Maintainer: Josh Holtrop
+" URL: https://github.com/holtrop/propane
+
+if exists("b:current_syntax")
+  finish
+endif
+
+if !exists("b:propane_subtype")
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
+endif
+
+exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
+
+syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
+
+syn match propaneComment "#.*"
+syn match propaneOperator "->"
+syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
+syn match propaneFieldOperator ":" contained
+syn match propaneOperator "?"
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
+
+syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
+
+hi def link propaneComment Comment
+hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
+hi def link propaneRegex String
+hi def link propaneOperator Operator
+hi def link propaneFieldOperator Operator
+hi def link propaneDelimiter Delimiter
+hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -31,10 +31,10 @@ class Propane
 
   class << self
 
-    def run(input_file, output_file, log_file)
+    def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
-        generator = Generator.new(grammar, output_file, log_file)
+        grammar = Grammar.new(File.read(input_file), input_file)
+        generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e
         $stderr.puts e.message

lib/propane/cli.rb

@@ -4,15 +4,21 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  --log LOG   Write log file
-  --version   Show program version and exit
-  -h, --help  Show this usage and exit
+  -h, --help  Show this usage and exit.
+  --log LOG   Write log file. This will show all parser states and their
+              associated shifts and reduces. It can be helpful when
+              debugging a grammar.
+  --version   Show program version and exit.
+  -w          Treat warnings as errors. This option will treat shift/reduce
+              conflicts as fatal errors and will print them to stderr in
+              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
+        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -24,11 +30,13 @@ EOF
               log_file = args[i]
             end
           when "--version"
-            puts "propane v#{VERSION}"
+            puts "propane version #{VERSION}"
             return 0
           when "-h", "--help"
             puts USAGE
             return 0
+          when "-w"
+            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -45,7 +53,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file)
+        Propane.run(*params, log_file, options)
       end
 
     end

lib/propane/generator.rb

@@ -2,7 +2,7 @@ class Propane
 
   class Generator
 
-    def initialize(grammar, output_file, log_file)
+    def initialize(grammar, output_file, log_file, options)
       @grammar = grammar
       @output_file = output_file
       if log_file
@@ -11,24 +11,40 @@ class Propane
         @log = StringIO.new
       end
       @language =
-        if output_file =~ /\.([a-z]+)$/
-          $1
-        else
+        if output_file.end_with?(".d")
           "d"
+        elsif output_file.end_with?(".c")
+          "c"
+        elsif output_file =~ %r{\.(cc|cpp|cxx)$}
+          @cpp = true
+          "c"
+        else
+          raise Error.new("Could not determine target language from output file name (#{output_file})")
         end
+      @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [@language]
+      extensions = [nil]
       if @language == "c"
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension}.erb")
-        erb = ERB.new(template, trim_mode: "<>")
+        template = Assets.get("parser.#{extension || @language}.erb")
+        if extension
           output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
-        result = erb.result(binding.clone)
+        else
+          output_file = @output_file
+        end
+        erb = ERB.new(template, trim_mode: "<>")
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            %[#line #{i + 2} "#{output_file}"\n]
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -42,8 +58,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.mode.nil?
-          pattern.mode = "default"
+        if pattern.modes.empty?
+          pattern.modes << "default"
           found_default = true
         end
         pattern.ptypename ||= "default"
@@ -51,6 +67,7 @@ class Propane
       unless found_default
         raise Error.new("No patterns found for default mode")
       end
+      check_ptypes!
       # Add EOF token.
       @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
@@ -65,12 +82,18 @@ class Propane
         end
         tokens_by_name[token.name] = token
       end
-      # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == "Start"}
-        raise Error.new("Start rule not found")
+      # Create real start rule(s).
+      real_start_rules = @grammar.start_rules.map do |start_rule|
+        unless @grammar.rules.find {|rule| rule.name == start_rule}
+          raise Error.new("Start rule `#{start_rule}` not found")
         end
-      # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
+        Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil)
+      end
+      # Add real start rules before user-given rules.
+      @grammar.rules = real_start_rules + @grammar.rules
+      # Generate and add rules for optional components.
+      generate_optional_component_rules!(tokens_by_name)
+      # Build rule sets.
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|
@@ -119,10 +142,55 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
+      rule_sets.each do |name, rule_set|
+        rule_set.finalize(@grammar)
+      end
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, @log)
+      @parser = Parser.new(@grammar, rule_sets, @log, @options)
+    end
+
+    # Check that any referenced ptypes have been defined.
+    def check_ptypes!
+      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
+        if potor.ptypename
+          unless @grammar.ptypes.include?(potor.ptypename)
+            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
+          end
+        end
+      end
+    end
+
+    # Generate and add rules for any optional components.
+    def generate_optional_component_rules!(tokens_by_name)
+      optional_rules_added = Set.new
+      @grammar.rules.each do |rule|
+        rule.components.each do |component|
+          if component =~ /^(.*)\?$/
+            c = $1
+            unless optional_rules_added.include?(component)
+              # Create two rules for the optional component: one empty and
+              # one just matching the component.
+              # We need to find the ptypename for the optional component in
+              # order to copy it to the generated rules.
+              if tokens_by_name[c]
+                # The optional component is a token.
+                ptypename = tokens_by_name[c].ptypename
+              else
+                # The optional component must be a rule, so find any instance
+                # of that rule that specifies a ptypename.
+                ptypename = @grammar.rules.reduce(nil) do |result, rule|
+                  rule.name == c && rule.ptypename ? rule.ptypename : result
+                end
+              end
+              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
+              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optional_rules_added << component
+            end
+          end
+        end
+      end
     end
 
     # Determine which grammar rules could expand to empty sequences.
@@ -198,10 +266,43 @@ class Propane
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
       end
+      code = code.gsub(/\$terminate\((.*)\);/) do |match|
+        user_terminate_code = $1
+        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
+        case @language
+        when "c"
+          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "d"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        end
+      end
+      code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "context->#{fieldname}"
+        when "d"
+          "context.#{fieldname}"
+        end
+      end
+      code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "token_tree_node->#{fieldname}"
+        when "d"
+          "token_tree_node.#{fieldname}"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
+          case @language
+          when "c"
+            "_pvalue->v_#{rule.ptypename}"
+          when "d"
             "_pvalue.v_#{rule.ptypename}"
           end
+        end
         code = code.gsub(/\$(\d+)/) do |match|
           index = $1.to_i
           case @language
@@ -211,8 +312,37 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
+        end
+        code = code.gsub(/\$\{(\w+)\}/) do |match|
+          aliasname = $1
+          if index = rule.aliases[aliasname]
+            case @language
+            when "c"
+              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
+            when "d"
+              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
+            end
+          else
+            raise Error.new("Field alias '#{aliasname}' not found")
+          end
+        end
       else
         code = code.gsub(/\$\$/) do |match|
+          if @grammar.tree
+            case @language
+            when "c"
+              "out_token_info->pvalue"
+            when "d"
+              "out_token_info.pvalue"
+            end
+          else
             case @language
             when "c"
               "out_token_info->pvalue.v_#{pattern.ptypename}"
@@ -220,6 +350,23 @@ class Propane
               "out_token_info.pvalue.v_#{pattern.ptypename}"
             end
           end
+        end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)
@@ -237,13 +384,21 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]
     #   Start rule parser value type name and type string.
-    def start_rule_type
+    def start_rule_type(start_rule_index = 0)
       start_rule = @grammar.rules.find do |rule|
-        rule.name == "Start"
+        rule.name == @grammar.start_rules[start_rule_index]
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -5,26 +5,45 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :context_user_fields
+    attr_reader :lex_fn
+    attr_reader :tree
+    attr_reader :tree_prefix
+    attr_reader :tree_suffix
+    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
-    attr_reader :rules
+    attr_accessor :rules
+    attr_reader :start_rules
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
     attr_reader :prefix
+    attr_reader :on_token_node
+    attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
+      @start_rules = []
       @tokens = []
       @rules = []
       @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
-      @mode = nil
+      @modeline = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
+      @tree = false
+      @tree_prefix = ""
+      @tree_suffix = ""
+      @free_token_node = ""
+      @context_user_fields = nil
+      @on_token_node = ""
+      @token_user_fields = nil
       parse_grammar!
+      @start_rules << "Start" if @start_rules.empty?
     end
 
     def ptype
@@ -35,6 +54,10 @@ class Propane
       @tokens.size
     end
 
+    def terminate_token_id
+      @tokens.size + 1
+    end
+
     private
 
     def parse_grammar!
@@ -46,16 +69,26 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @mode.nil? && parse_mode_label!
+      elsif @modeline.nil? && parse_mode_label!
+      elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
+      elsif parse_tree_statement!
+      elsif parse_tree_prefix_statement!
+      elsif parse_tree_suffix_statement!
+      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
+      elsif parse_on_token_node_statement!
+      elsif parse_token_user_fields_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
+      elsif parse_start_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -65,8 +98,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
-        @mode = md[1]
+      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
+        @modeline = md[1]
       end
     end
 
@@ -78,20 +111,85 @@ class Propane
       consume!(/#.*\n/)
     end
 
+    def parse_context_user_fields_statement!
+      if md = consume!(/context_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @context_user_fields ||= ""
+        @context_user_fields += code
+      end
+    end
+
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
+    def parse_tree_statement!
+      if consume!(/tree\s*;/)
+        @tree = true
+      end
+    end
+
+    def parse_tree_prefix_statement!
+      if md = consume!(/tree_prefix\s+(\w+)\s*;/)
+        @tree_prefix = md[1]
+      end
+    end
+
+    def parse_tree_suffix_statement!
+      if md = consume!(/tree_suffix\s+(\w+)\s*;/)
+        @tree_suffix = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @mode = nil
+        @modeline = nil
         true
       end
     end
 
+    def parse_on_token_node_statement!
+      if md = consume!(/on_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @on_token_node += code
+      end
+    end
+
+    def parse_token_user_fields_statement!
+      if md = consume!(/token_user_fields\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @token_user_fields ||= ""
+        @token_user_fields += code
+      end
+    end
+
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @free_token_node += code
+      end
+    end
+
     def parse_ptype_statement!
       if consume!(/ptype\s+/)
         name = "default"
         if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
+          end
           name = md[1]
         end
         md = consume!(/([^;]+);/, "expected parser result type expression")
@@ -104,18 +202,21 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
+          end
           ptypename = md[1]
         end
         pattern = parse_pattern! || name
         consume!(/\s+/)
         unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+          consume!(/;/, "expected `;' or code block")
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
         @patterns << pattern
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -125,12 +226,15 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
+          end
           ptypename = md[1]
         end
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -142,9 +246,11 @@ class Propane
           raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
         end
         consume!(/\s+/)
-        consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
-        @mode = nil
+        unless code = parse_code_block!
+          consume!(/;/, "expected `;' or code block")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline)
+        @modeline = nil
         true
       end
     end
@@ -152,13 +258,20 @@ class Propane
     def parse_rule_statement!
       if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
         rule_name, ptypename = *md[1, 2]
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        if @tree && ptypename
+          raise Error.new("Multiple ptypes are unsupported in tree mode")
+        end
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
+        if @tree
+          consume!(/;/, "expected `;'")
+        else
           unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+            consume!(/;/, "expected `;' or code block")
+          end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -167,26 +280,46 @@ class Propane
       if pattern = parse_pattern!
         consume!(/\s+/)
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @tree
+            raise Error.new("Multiple ptypes are unsupported in tree mode")
+          end
           ptypename = md[1]
         end
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
-        @mode = nil
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
+        @modeline = nil
         true
       end
     end
 
+    def parse_start_statement!
+      if md = consume!(/start\s+([\w\s]*);/)
+        start_rules = md[1].split(/\s+/).map(&:strip)
+        start_rules.each do |start_rule|
+          @start_rules << start_rule unless @start_rules.include?(start_rule)
+        end
+      end
+    end
+
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
+        code = code.chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
           @code_blocks[name] = code
         end
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -198,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -211,6 +351,8 @@ class Propane
             end
           elsif md = consume!(%r{(.)})
             pattern += md[1]
+          elsif @input == "" || @input.start_with?("\n")
+            raise Error.new("Line #{@line_number}: Unterminated pattern; expected `/`")
           end
         end
         pattern
@@ -218,8 +360,16 @@ class Propane
     end
 
     def parse_code_block!
-      if md = consume!(/<<\n(.*?)^>>\n/m)
-        md[1]
+      if md = consume!(/<<(.*?)>>\n/m)
+        code = md[1].chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
+        code
       end
     end
 
@@ -251,6 +401,14 @@ class Propane
       end
     end
 
+    def get_modes_from_modeline
+      if @modeline
+        Set[*@modeline.split(",").map(&:strip)]
+      else
+        Set.new
+      end
+    end
+
   end
 
 end

lib/propane/lexer.rb

@@ -26,8 +26,14 @@ class Propane
     private
 
     def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
-        pattern.mode
+      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
+        result + pattern.modes
+      end
+      @modes = modenames.reduce({}) do |result, modename|
+        result[modename] = @grammar.patterns.select do |pattern|
+          pattern.modes.include?(modename)
+        end
+        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -7,26 +7,38 @@ class Propane
     attr_reader :reduce_table
     attr_reader :rule_sets
 
-    def initialize(grammar, rule_sets, log)
+    def initialize(grammar, rule_sets, log, options)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
       @item_sets = []
       @item_sets_set = {}
-      start_item = Item.new(grammar.rules.first, 0)
-      eval_item_sets = Set[ItemSet.new([start_item])]
+      @warnings = Set.new
+      @errors = Set.new
+      @options = options
+      start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule|
+        Item.new(start_rule, 0)
+      end
+      start_item_sets = start_items.map {|item| ItemSet.new([item])}
+      eval_item_sets = Set[*start_item_sets]
 
       while eval_item_sets.size > 0
-        item_set = eval_item_sets.first
+        item_set =
+          if start_item_sets.size > 0
+            # Ensure we evaluate start_item_sets first in order
+            start_item_sets.slice!(0)
+          else
+            eval_item_sets.first
+          end
         eval_item_sets.delete(item_set)
         unless @item_sets_set.include?(item_set)
           item_set.id = @item_sets.size
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
-          item_set.following_symbols.each do |following_symbol|
-            unless following_symbol.name == "$EOF"
-              following_set = item_set.build_following_item_set(following_symbol)
-              eval_item_sets << following_set
+          item_set.next_symbols.each do |next_symbol|
+            unless next_symbol.name == "$EOF"
+              next_item_set = item_set.build_next_item_set(next_symbol)
+              eval_item_sets << next_item_set
             end
           end
         end
@@ -37,8 +49,21 @@ class Propane
       end
 
       build_reduce_actions!
-      write_log!
       build_tables!
+      write_log!
+      errormessage = ""
+      if @errors.size > 0
+        errormessage += @errors.join("\n")
+      end
+      if @warnings.size > 0 && @options[:warnings_as_errors]
+        if errormessage != ""
+          errormessage += "\n"
+        end
+        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
+      end
+      if errormessage != ""
+        raise Error.new(errormessage)
+      end
     end
 
     private
@@ -48,48 +73,47 @@ class Propane
       @shift_table = []
       @reduce_table = []
       @item_sets.each do |item_set|
-        shift_entries = item_set.following_symbols.map do |following_symbol|
-          state_id =
-            if following_symbol.name == "$EOF"
-              0
-            else
-              item_set.following_item_set[following_symbol].id
+        unless item_set.reduce_rules.empty?
+          item_set.shift_entries.each do |shift_entry|
+            token = shift_entry[:symbol]
+            if item_set.reduce_actions
+              if rule = item_set.reduce_actions[token]
+                @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+              end
+            end
           end
-          {
-            symbol_id: following_symbol.id,
-            state_id: state_id,
-          }
         end
         reduce_entries =
-          case ra = item_set.reduce_actions
-          when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
-          when Hash
-            ra.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
+          if rule = item_set.reduce_rule
+            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
+              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+              propagate_optional_target: rule.optional? && rule.components.size == 1}]
+          elsif reduce_actions = item_set.reduce_actions
+            reduce_actions.map do |token, rule|
+              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
           else
             []
           end
         @state_table << {
           shift_index: @shift_table.size,
-          n_shifts: shift_entries.size,
+          n_shifts: item_set.shift_entries.size,
           reduce_index: @reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += shift_entries
+        @shift_table += item_set.shift_entries
         @reduce_table += reduce_entries
       end
     end
 
     def process_item_set(item_set)
-      item_set.following_symbols.each do |following_symbol|
-        unless following_symbol.name == "$EOF"
-          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
-          item_set.following_item_set[following_symbol] = following_set
-          following_set.in_sets << item_set
+      item_set.next_symbols.each do |next_symbol|
+        unless next_symbol.name == "$EOF"
+          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
+          item_set.next_item_set[next_symbol] = next_item_set
+          next_item_set.in_sets << item_set
         end
       end
     end
@@ -99,7 +123,109 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
-        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
+        build_shift_entries(item_set)
+        build_reduce_actions_for_item_set(item_set)
+      end
+      item_sets_to_process = @item_sets.select do |item_set|
+        # We need lookahead reduce actions if:
+        # 1) There is more than one possible rule to reduce. In this case the
+        #    lookahead token can help choose which rule to reduce.
+        # 2) There is at least one shift action and one reduce action for
+        #    this item set. In this case the lookahead reduce actions are
+        #    needed to test for a Shift/Reduce conflict.
+        item_set.reduce_rules.size > 1 ||
+          (item_set.reduce_rules.size > 0 && item_set.shift_entries.size > 0)
+      end
+      if RbConfig::CONFIG["host_os"] =~ /linux/
+        item_sets_by_id = {}
+        item_sets_to_process.each do |item_set|
+          item_sets_by_id[item_set.object_id] = item_set
+        end
+        tokens_by_id = {}
+        @grammar.tokens.each do |token|
+          tokens_by_id[token.object_id] = token
+        end
+        rules_by_id = {}
+        @grammar.rules.each do |rule|
+          rules_by_id[rule.object_id] = rule
+        end
+        n_threads = Util.determine_n_threads
+        semaphore = Mutex.new
+        queue = Queue.new
+        threads = {}
+        n_threads.times do
+          piper, pipew = IO.pipe
+          thread = Thread.new do
+            loop do
+              item_set = nil
+              semaphore.synchronize do
+                item_set = item_sets_to_process.slice!(0)
+              end
+              break if item_set.nil?
+              fork do
+                piper.close
+                build_lookahead_reduce_actions_for_item_set(item_set, pipew)
+              end
+            end
+            queue.push(Thread.current)
+          end
+          threads[thread] = [piper, pipew]
+        end
+        until threads.empty?
+          thread = queue.pop
+          piper, pipew = threads[thread]
+          pipew.close
+          thread_txt = piper.read
+          thread_txt.each_line do |line|
+            if line.start_with?("RA,")
+              parts = line.split(",")
+              item_set_id, token_id, rule_id = parts[1..3].map(&:to_i)
+              item_set = item_sets_by_id[item_set_id]
+              unless item_set
+                raise "Internal error: could not find item set from thread"
+              end
+              token = tokens_by_id[token_id]
+              unless item_set
+                raise "Internal error: could not find token from thread"
+              end
+              rule = rules_by_id[rule_id]
+              unless item_set
+                raise "Internal error: could not find rule from thread"
+              end
+              item_set.reduce_actions ||= {}
+              item_set.reduce_actions[token] = rule
+            elsif line.start_with?("Error: ")
+              @errors << line.chomp
+            else
+              raise "Internal error: unhandled thread line #{line}"
+            end
+          end
+          thread.join
+          threads.delete(thread)
+        end
+      else
+        # Fall back to single threaded algorithm.
+        item_sets_to_process.each do |item_set|
+          item_set.reduce_actions = build_lookahead_reduce_actions_for_item_set(item_set)
+        end
+      end
+    end
+
+    # Build the shift entries for a single item set.
+    #
+    # @return [void]
+    def build_shift_entries(item_set)
+      item_set.shift_entries = item_set.next_symbols.map do |next_symbol|
+        state_id =
+          if next_symbol.name == "$EOF"
+            0
+          else
+            item_set.next_item_set[next_symbol].id
+          end
+        {
+          symbol: next_symbol,
+          state_id: state_id,
+        }
       end
     end
 
@@ -108,40 +234,45 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [nil, Rule, Hash]
-    #   If no reduce actions are possible for the given item set, nil.
-    #   If only one reduce action is possible for the given item set, the Rule
-    #   to reduce.
-    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
+    # @return [void]
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
       # "complete" items, i.e., items where the parse position is at the
       # end of a rule. These are the only rules that are candidates for
       # reduction in the current ItemSet.
-      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      # If there are no rules to reduce for this ItemSet, we're done here.
-      return nil if reduce_rules.size == 0
-
-      # If there is exactly one rule to reduce for this ItemSet, then do not
-      # figure out the lookaheads; just reduce it.
-      return reduce_rules.first if reduce_rules.size == 1
-
-      # Otherwise, we have more than one possible rule to reduce.
+      if item_set.reduce_rules.size == 1
+        item_set.reduce_rule = item_set.reduce_rules.first
+      end
+    end
 
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    # @param fh [File]
+    #   Output file handle for multiprocessing mode.
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def build_lookahead_reduce_actions_for_item_set(item_set, fh = nil)
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
-      item_sets = item_set.leading_item_sets
-      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+      item_sets = Set[item_set] + item_set.leading_item_sets
+      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            raise Error.new("Error: reduce/reduce conflict between rule #{existing_reduce_rule.id} (#{existing_reduce_rule.name}) and rule #{reduce_rule.id} (#{reduce_rule.name})")
+            error = "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
+            @errors << error
+            fh.puts(error) if fh
           end
           reduce_actions[lookahead_token] = reduce_rule
+          fh.puts "RA,#{item_set.object_id},#{lookahead_token.object_id},#{reduce_rule.object_id}" if fh
         end
         reduce_actions
       end
@@ -181,13 +312,14 @@ class Propane
         # tokens to form the lookahead token set.
         item_sets.each do |item_set|
           item_set.items.each do |item|
-            if item.following_symbol == rule_set
+            if item.next_symbol == rule_set
               (1..).each do |offset|
-                case symbol = item.following_symbol(offset)
+                case symbol = item.next_symbol(offset)
                 when nil
                   rule_set = item.rule.rule_set
                   unless checked_rule_sets.include?(rule_set)
                     rule_sets_to_check_after << rule_set
+                    checked_rule_sets << rule_set
                   end
                   break
                 when Token
@@ -240,20 +372,26 @@ class Propane
         @log.puts
         @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
         @log.puts "  Outgoing states:"
-        item_set.following_item_set.each do |following_symbol, following_item_set|
-          @log.puts "    #{following_symbol.name} => #{following_item_set.id}"
+        item_set.next_item_set.each do |next_symbol, next_item_set|
+          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
         end
         @log.puts
         @log.puts "  Reduce actions:"
-        case item_set.reduce_actions
-        when Rule
-          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
-        when Hash
+        if item_set.reduce_rule
+          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
+        elsif item_set.reduce_actions
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
           end
         end
       end
+      if @warnings.size > 0
+        @log.puts
+        @log.puts "Warnings:"
+        @warnings.each do |warning|
+          @log.puts "  #{warning}"
+        end
+      end
     end
 
   end

lib/propane/parser/item.rb

@@ -22,6 +22,7 @@ class Propane
       def initialize(rule, position)
         @rule = rule
         @position = position
+        @_hash = [@rule, @position].hash
       end
 
       # Hash function.
@@ -29,7 +30,7 @@ class Propane
       # @return [Integer]
       #   Hash code.
       def hash
-        [@rule, @position].hash
+        @_hash
       end
 
       # Compare Item objects.
@@ -56,7 +57,7 @@ class Propane
 
       # Return the set of Items obtained by "closing" the current item.
       #
-      # If the following symbol for the current item is another Rule name, then
+      # If the next symbol for the current item is another Rule name, then
       # this method will return all Items for that Rule with a position of 0.
       # Otherwise, an empty Array is returned.
       #
@@ -81,17 +82,17 @@ class Propane
         @position == @rule.components.size
       end
 
-      # Get the following symbol for the Item.
+      # Get the next symbol for the Item.
       #
-      # That is, the symbol which follows the parse position marker in the
+      # That is, the symbol which is after the parse position marker in the
       # current Item.
       #
       # @param offset [Integer]
       #   Offset from current parse position to examine.
       #
       # @return [Token, RuleSet, nil]
-      #   Following symbol for the Item.
-      def following_symbol(offset = 0)
+      #   Next symbol for the Item.
+      def next_symbol(offset = 0)
         @rule.components[@position + offset]
       end
 
@@ -108,25 +109,25 @@ class Propane
         end
       end
 
-      # Get whether this Item is followed by the provided symbol.
+      # Get whether this Item's next symbol is the given symbol.
       #
       # @param symbol [Token, RuleSet]
       #   Symbol to query.
       #
       # @return [Boolean]
-      #   Whether this Item is followed by the provided symbol.
-      def followed_by?(symbol)
-        following_symbol == symbol
+      #   Whether this Item's next symbol is the given symbol.
+      def next_symbol?(symbol)
+        next_symbol == symbol
       end
 
-      # Get the following item for this Item.
+      # Get the next item for this Item.
       #
       # That is, the Item formed by moving the parse position marker one place
       # forward from its position in this Item.
       #
       # @return [Item]
-      #   The following item for this Item.
-      def following_item
+      #   The next item for this Item.
+      def next_item
         Item.new(@rule, @position + 1)
       end
 

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing.
+    # parser could currently be parsing. This is equivalent to a parser state.
     class ItemSet
 
       # @return [Set<Item>]
@@ -14,45 +14,58 @@ class Propane
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a following symbol to its ItemSet.
-      attr_reader :following_item_set
+      #   Maps a next symbol to its ItemSet.
+      attr_reader :next_item_set
 
       # @return [Set<ItemSet>]
       #   ItemSets leading to this item set.
       attr_reader :in_sets
 
-      # @return [nil, Rule, Hash]
-      #   Reduce actions, mapping lookahead tokens to rules.
+      # @return [nil, Rule]
+      #   Rule to reduce if there is only one possibility.
+      attr_accessor :reduce_rule
+
+      # @return [Set<Rule>]
+      #   Set of rules that could be reduced in this parser state.
+      attr_accessor :reduce_rules
+
+      # @return [nil, Hash]
+      #   Reduce actions, mapping lookahead tokens to rules, if there is
+      #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
+      # @return [Array<Hash>]
+      #   Shift table entries.
+      attr_accessor :shift_entries
+
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
       #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @following_item_set = {}
+        @next_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      # Get the set of following symbols for all Items in this ItemSet.
+      # Get the set of next symbols for all Items in this ItemSet.
       #
       # @return [Set<Token, RuleSet>]
-      #   Set of following symbols for all Items in this ItemSet.
-      def following_symbols
-        Set.new(@items.map(&:following_symbol).compact)
+      #   Set of next symbols for all Items in this ItemSet.
+      def next_symbols
+        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
       end
 
-      # Build a following ItemSet for the given following symbol.
+      # Build a next ItemSet for the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol to build the following ItemSet for.
+      #   Next symbol to build the next ItemSet for.
       #
       # @return [ItemSet]
-      #   Following ItemSet for the given following symbol.
-      def build_following_item_set(symbol)
-        ItemSet.new(items_followed_by(symbol).map(&:following_item))
+      #   Next ItemSet for the given next symbol.
+      def build_next_item_set(symbol)
+        ItemSet.new(items_with_next(symbol).map(&:next_item))
       end
 
       # Hash function.
@@ -87,13 +100,26 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
-      # This set includes this ItemSet.
-      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @in_sets.reduce(Set[self]) do |result, item_set|
-          result + item_set.leading_item_sets
+        @_leading_item_sets ||=
+          begin
+            result = Set.new
+            eval_sets = Set[self]
+            evaled = Set.new
+            while eval_sets.size > 0
+              eval_set = eval_sets.first
+              eval_sets.delete(eval_set)
+              evaled << eval_set
+              eval_set.in_sets.each do |in_set|
+                result << in_set
+                unless evaled.include?(in_set)
+                  eval_sets << in_set
+                end
+              end
+            end
+            result
           end
       end
 
@@ -127,16 +153,16 @@ class Propane
         end
       end
 
-      # Get the Items followed by the given following symbol.
+      # Get the Items with the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol.
+      #   Next symbol.
       #
       # @return [Array<Item>]
-      #   Items followed by the given following symbol.
-      def items_followed_by(symbol)
+      #   Items with the given next symbol.
+      def items_with_next(symbol)
         @items.select do |item|
-          item.followed_by?(symbol)
+          item.next_symbol?(symbol)
         end
       end
 

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [String, nil]
-    #   Lexer mode for this pattern.
-    attr_accessor :mode
+    # @return [Set]
+    #   Lexer modes for this pattern.
+    attr_accessor :modes
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,16 +46,16 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :mode
-    #   Lexer mode for this pattern.
+    # @option options [String, nil] :modes
+    #   Lexer modes for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @mode = options[:mode]
+      @modes = options[:modes]
       @ptypename = options[:ptypename]
-      regex = Regex.new(@pattern)
+      regex = Regex.new(@pattern, @line_number)
       regex.nfa.end_state.accepts = self
       @nfa = regex.nfa
     end

lib/propane/regex.rb

@@ -4,12 +4,13 @@ class Propane
     attr_reader :unit
     attr_reader :nfa
 
-    def initialize(pattern)
+    def initialize(pattern, line_number)
       @pattern = pattern.dup
+      @line_number = line_number
       @unit = parse_alternates
       @nfa = @unit.to_nfa
       if @pattern != ""
-        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
+        raise Error.new(%[Line #{@line_number}: unexpected "#{@pattern}" in pattern])
       end
     end
 
@@ -41,7 +42,7 @@ class Propane
             mu = MultiplicityUnit.new(last_unit, min_count, max_count)
             au.replace_last!(mu)
           else
-            raise Error.new("#{c} follows nothing")
+            raise Error.new("Line #{@line_number}: #{c} follows nothing")
           end
         when "|"
           au.new_alternate!
@@ -59,7 +60,7 @@ class Propane
     def parse_group
       au = parse_alternates
       if @pattern[0] != ")"
-        raise Error.new("Unterminated group in pattern")
+        raise Error.new("Line #{@line_number}: unterminated group in pattern")
       end
       @pattern.slice!(0)
       au
@@ -70,7 +71,7 @@ class Propane
       index = 0
       loop do
         if @pattern == ""
-          raise Error.new("Unterminated character class")
+          raise Error.new("Line #{@line_number}: unterminated character class")
         end
         c = @pattern.slice!(0)
         if c == "]"
@@ -84,13 +85,13 @@ class Propane
         elsif c == "-" && @pattern[0] != "]"
           begin_cu = ccu.last_unit
           unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
-            raise Error.new("Character range must be between single characters")
+            raise Error.new("Line #{@line_number}: character range must be between single characters")
           end
           if @pattern[0] == "\\"
             @pattern.slice!(0)
             end_cu = parse_backslash
             unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
-              raise Error.new("Character range must be between single characters")
+              raise Error.new("Line #{@line_number}: character range must be between single characters")
             end
             max_code_point = end_cu.code_point
           else
@@ -116,7 +117,7 @@ class Propane
         elsif max_count.to_s != ""
           max_count = max_count.to_i
           if max_count < min_count
-            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
+            raise Error.new("Line #{@line_number}: maximum repetition count cannot be less than minimum repetition count")
           end
         else
           max_count = nil
@@ -124,18 +125,33 @@ class Propane
         @pattern = pattern
         [min_count, max_count]
       else
-        raise Error.new("Unexpected match count at #{@pattern}")
+        raise Error.new("Line #{@line_number}: unexpected match count following {")
       end
     end
 
     def parse_backslash
       if @pattern == ""
-        raise Error.new("Error: unfollowed \\")
+        raise Error.new("Line #{@line_number}: error: unfollowed \\")
       else
         c = @pattern.slice!(0)
         case c
+        when "a"
+          CharacterRangeUnit.new("\a")
+        when "b"
+          CharacterRangeUnit.new("\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "D"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu.negate = true
+          ccu
+        when "f"
+          CharacterRangeUnit.new("\f")
+        when "n"
+          CharacterRangeUnit.new("\n")
+        when "r"
+          CharacterRangeUnit.new("\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -145,6 +161,35 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "S"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new(" ")
+          ccu << CharacterRangeUnit.new("\t")
+          ccu << CharacterRangeUnit.new("\r")
+          ccu << CharacterRangeUnit.new("\n")
+          ccu << CharacterRangeUnit.new("\f")
+          ccu << CharacterRangeUnit.new("\v")
+          ccu.negate = true
+          ccu
+        when "t"
+          CharacterRangeUnit.new("\t")
+        when "v"
+          CharacterRangeUnit.new("\v")
+        when "w"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu
+        when "W"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu.negate = true
+          ccu
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/regex/unit.rb

@@ -92,17 +92,20 @@ class Propane
         @units = []
         @negate = false
       end
-      def initialize
-        @units = []
-      end
-      def method_missing(*args)
-        @units.__send__(*args)
+      def method_missing(*args, &block)
+        @units.__send__(*args, &block)
       end
       def <<(thing)
         if thing.is_a?(CharacterClassUnit)
+          if thing.negate
+            CodePointRange.invert_ranges(thing.map(&:code_point_range)).each do |cpr|
+              CharacterRangeUnit.new(cpr.first, cpr.last)
+            end
+          else
             thing.each do |ccu_unit|
               @units << ccu_unit
             end
+          end
         else
           @units << thing
         end

lib/propane/rule.rb

@@ -6,6 +6,10 @@ class Propane
     #   Rule components.
     attr_reader :components
 
+    # @return [Hash]
+    #   Field aliases.
+    attr_reader :aliases
+
     # @return [String]
     #   User code associated with the rule.
     attr_reader :code
@@ -30,6 +34,11 @@ class Propane
     #   The RuleSet that this Rule is a part of.
     attr_accessor :rule_set
 
+    # @return [Array<Integer>]
+    #   Map this rule's components to their positions in the parent RuleSet's
+    #   node field pointer array. This is used for tree construction.
+    attr_accessor :rule_set_node_field_index_map
+
     # Construct a Rule.
     #
     # @param name [String]
@@ -44,7 +53,20 @@ class Propane
     #   Line number where the rule was defined in the input grammar.
     def initialize(name, components, code, ptypename, line_number)
       @name = name
-      @components = components
+      @aliases = {}
+      @components = components.each_with_index.map do |component, i|
+        if component =~ /(\S+):(\S+)/
+          c, aliasname = $1, $2
+          if @aliases[aliasname]
+            raise Error.new("Error: duplicate field alias `#{aliasname}` for rule #{name} defined on line #{line_number}")
+          end
+          @aliases[aliasname] = i
+          c
+        else
+          component
+        end
+      end
+      @rule_set_node_field_index_map = components.map {0}
       @code = code
       @ptypename = ptypename
       @line_number = line_number
@@ -60,6 +82,14 @@ class Propane
       @components.empty?
     end
 
+    # Return whether this is an optional Rule.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional Rule.
+    def optional?
+      @name.end_with?("?")
+    end
+
     # Represent the Rule as a String.
     #
     # @return [String]
@@ -68,6 +98,17 @@ class Propane
       "#{@name} -> #{@components.map(&:name).join(" ")}"
     end
 
+    # Check whether the rule set node field index map is just a 1:1 mapping.
+    #
+    # @return [Boolean]
+    #   Boolean indicating whether the rule set node field index map is just a
+    #   1:1 mapping.
+    def flat_rule_set_node_field_index_map?
+      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
+        v == i
+      end
+    end
+
   end
 
 end

lib/propane/rule_set.rb

@@ -1,7 +1,12 @@
 class Propane
 
+  # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
+    # @return [Array<Hash>]
+    #   tree fields.
+    attr_reader :tree_fields
+
     # @return [Integer]
     #   ID of the RuleSet.
     attr_reader :id
@@ -51,6 +56,24 @@ class Propane
       @could_be_empty
     end
 
+    # Return whether this is an optional RuleSet.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional RuleSet.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # For optional rule sets, return the underlying component that is optional.
+    def option_target
+      @rules.each do |rule|
+        if rule.components.size > 0
+          return rule.components[0]
+        end
+      end
+      raise "Optional rule target not found"
+    end
+
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -75,6 +98,72 @@ class Propane
       @_start_token_set
     end
 
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize(grammar)
+      if grammar.tree
+        build_tree_fields(grammar)
+      end
+    end
+
+    private
+
+    # Build the set of tree fields for this RuleSet.
+    #
+    # This is an Array of Hashes. Each entry in the Array corresponds to a
+    # field location in the tree node. The entry is a Hash. It could have one or
+    # two keys. It will always have the field name with a positional suffix as
+    # a key. It may also have the field name without the positional suffix if
+    # that field only exists in one position across all Rules in the RuleSet.
+    #
+    # @return [void]
+    def build_tree_fields(grammar)
+      field_tree_node_indexes = {}
+      field_indexes_across_all_rules = {}
+      # Stores the index into @tree_fields by field alias name.
+      field_aliases = {}
+      @tree_fields = []
+      @rules.each do |rule|
+        rule.components.each_with_index do |component, i|
+          if component.is_a?(RuleSet) && component.optional?
+            component = component.option_target
+          end
+          if component.is_a?(Token)
+            node_name = "Token"
+          else
+            node_name = component.name
+          end
+          struct_name = "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
+          field_name = "p#{node_name}#{i + 1}"
+          unless field_tree_node_indexes[field_name]
+            field_tree_node_indexes[field_name] = @tree_fields.size
+            @tree_fields << {field_name => struct_name}
+          end
+          rule.aliases.each do |alias_name, index|
+            if index == i
+              alias_tree_fields_index = field_tree_node_indexes[field_name]
+              if field_aliases[alias_name] && field_aliases[alias_name] != alias_tree_fields_index
+                raise Error.new("Error: conflicting tree node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
+              end
+              field_aliases[alias_name] = alias_tree_fields_index
+              @tree_fields[alias_tree_fields_index][alias_name] = @tree_fields[alias_tree_fields_index].first[1]
+            end
+          end
+          field_indexes_across_all_rules[node_name] ||= Set.new
+          field_indexes_across_all_rules[node_name] << field_tree_node_indexes[field_name]
+          rule.rule_set_node_field_index_map[i] = field_tree_node_indexes[field_name]
+        end
+      end
+      field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
+        if indexes_across_all_rules.size == 1
+          # If this field was only seen in one position across all rules,
+          # then add an alias to the positional field name that does not
+          # include the position.
+          @tree_fields[indexes_across_all_rules.first]["p#{node_name}"] =
+            "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
+        end
+      end
+    end
+
   end
 
 end

lib/propane/util.rb

@@ -10,6 +10,32 @@ class Propane
         "#{s}\n* #{message} *\n#{s}\n"
       end
 
+      # Determine the number of threads to use.
+      #
+      # @return [Integer]
+      #   The number of threads to use.
+      def determine_n_threads
+        # Try to figure out how many threads are available on the host hardware.
+        begin
+          case RbConfig::CONFIG["host_os"]
+          when /linux/
+            return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
+          when /mswin|mingw|msys/
+            if `wmic cpu get NumberOfLogicalProcessors -value` =~ /NumberOfLogicalProcessors=(\d+)/
+              return $1.to_i
+            end
+          when /darwin/
+            if `sysctl -n hw.ncpu` =~ /(\d+)/
+              return $1.to_i
+            end
+          end
+        rescue
+        end
+
+        # If we can't figure it out, default to 4.
+        4
+      end
+
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.0.0"
+  VERSION = "4.6.0"
 end

spec/json_parser.c.propane

@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {match[0], 0};
+  char s[] = {(char)match[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_types.c

@@ -5,7 +5,7 @@
 
 JSONValue * JSONValue_new(size_t id)
 {
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
     jv->id = id;
     return jv;
 }
@@ -29,7 +29,7 @@ void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
         }
     }
     size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
     if (size > 0)
     {
         memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
@@ -52,7 +52,7 @@ void JSONArray_append(JSONValue * array, JSONValue * value)
 {
     size_t const size = array->array.size;
     size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
     if (array->array.size > 0)
     {
         memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);

spec/json_types.h

@@ -11,6 +11,12 @@
 #define JSON_FALSE 5u
 #define JSON_NULL 6u
 
+typedef struct JSONObjectEntry_s
+{
+    char const * name;
+    struct JSONValue_s * value;
+} JSONObjectEntry;
+
 typedef struct JSONValue_s
 {
     size_t id;
@@ -19,11 +25,7 @@ typedef struct JSONValue_s
         struct
         {
             size_t size;
-            struct
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
+            JSONObjectEntry * entries;
         } object;
         struct
         {

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,42 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+    end
+
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -144,37 +179,37 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m1"
+      expect(o.modes).to eq Set["m1"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m2"
+      expect(o.modes).to eq Set["m2"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m3"
+      expect(o.modes).to eq Set["m3"]
     end
 
     it "allows assigning ptypes to tokens and rules" do
@@ -197,7 +232,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)
@@ -126,6 +126,74 @@ EOF
     ]
     expect(run(<<EOF, ";")).to eq expected
 token semicolon /;/;
+EOF
+  end
+
+  it "matches a negated character class" do
+    expected = [
+      ["pattern", "/abc/"],
+    ]
+    expect(run(<<EOF, "/abc/")).to eq expected
+token pattern /\\/[^\\s]*\\//;
+EOF
+  end
+
+  it "matches special character classes " do
+    expected = [
+      ["a", "abc123_FOO"],
+    ]
+    expect(run(<<EOF, "abc123_FOO")).to eq expected
+token a /\\w+/;
+EOF
+    expected = [
+      ["b", "FROG*%$#"],
+    ]
+    expect(run(<<EOF, "FROG*%$#")).to eq expected
+token b /FROG\\D{1,4}/;
+EOF
+    expected = [
+      ["c", "$883366"],
+    ]
+    expect(run(<<EOF, "$883366")).to eq expected
+token c /$\\d+/;
+EOF
+    expected = [
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "^&$@")).to eq expected
+token d /^\\W+/;
+EOF
+    expected = [
+      ["a", "abc123_FOO"],
+      [nil, " "],
+      ["b", "FROG*%$#"],
+      [nil, " "],
+      ["c", "$883366"],
+      [nil, " "],
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "abc123_FOO FROG*%$# $883366 ^&$@")).to eq expected
+token a /\\w+/;
+token b /FROG\\D{1,4}/;
+token c /$\\d+/;
+token d /^\\W+/;
+drop /\\s+/;
+EOF
+  end
+
+  it "matches a negated character class with a nested inner negated character class" do
+    expected = [
+      ["t", "$&*"],
+    ]
+    expect(run(<<EOF, "$&*")).to eq expected
+token t /[^%\\W]+/;
+EOF
+  end
+
+  it "\\s matches a newline" do
+    expected = [["s", "\n"]]
+    expect(run(<<EOF, "\n")).to eq expected
+token s /\\s/;
 EOF
   end
 end

spec/propane/regex_spec.rb

@@ -2,14 +2,14 @@ class Propane
   RSpec.describe Regex do
 
     it "parses an empty expression" do
-      regex = Regex.new("")
+      regex = Regex.new("", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0].size).to eq 0
     end
 
     it "parses a single character unit expression" do
-      regex = Regex.new("a")
+      regex = Regex.new("a", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -19,7 +19,7 @@ class Propane
     end
 
     it "parses a group with a single character unit expression" do
-      regex = Regex.new("(a)")
+      regex = Regex.new("(a)", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -33,7 +33,7 @@ class Propane
     end
 
     it "parses a *" do
-      regex = Regex.new("a*")
+      regex = Regex.new("a*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -47,7 +47,7 @@ class Propane
     end
 
     it "parses a +" do
-      regex = Regex.new("a+")
+      regex = Regex.new("a+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -61,7 +61,7 @@ class Propane
     end
 
     it "parses a ?" do
-      regex = Regex.new("a?")
+      regex = Regex.new("a?", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -75,7 +75,7 @@ class Propane
     end
 
     it "parses a multiplicity count" do
-      regex = Regex.new("a{5}")
+      regex = Regex.new("a{5}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -89,7 +89,7 @@ class Propane
     end
 
     it "parses a minimum-only multiplicity count" do
-      regex = Regex.new("a{5,}")
+      regex = Regex.new("a{5,}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -103,7 +103,7 @@ class Propane
     end
 
     it "parses a minimum and maximum multiplicity count" do
-      regex = Regex.new("a{5,8}")
+      regex = Regex.new("a{5,8}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -118,7 +118,7 @@ class Propane
     end
 
     it "parses an escaped *" do
-      regex = Regex.new("a\\*")
+      regex = Regex.new("a\\*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -131,7 +131,7 @@ class Propane
     end
 
     it "parses an escaped +" do
-      regex = Regex.new("a\\+")
+      regex = Regex.new("a\\+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -144,7 +144,7 @@ class Propane
     end
 
     it "parses an escaped \\" do
-      regex = Regex.new("\\\\d")
+      regex = Regex.new("\\\\d", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -157,7 +157,7 @@ class Propane
     end
 
     it "parses a character class" do
-      regex = Regex.new("[a-z_]")
+      regex = Regex.new("[a-z_]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -175,7 +175,7 @@ class Propane
     end
 
     it "parses a negated character class" do
-      regex = Regex.new("[^xyz]")
+      regex = Regex.new("[^xyz]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -189,8 +189,25 @@ class Propane
       expect(ccu[0].first).to eq "x".ord
     end
 
+    it "parses a negated character class with inner character classes" do
+      regex = Regex.new("[^x\\sz]", 1)
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_truthy
+      expect(ccu.size).to eq 8
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "x".ord
+      expect(ccu[1].first).to eq " ".ord
+      expect(ccu[7].first).to eq "z".ord
+    end
+
     it "parses - as a plain character at beginning of a character class" do
-      regex = Regex.new("[-9]")
+      regex = Regex.new("[-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -204,7 +221,7 @@ class Propane
     end
 
     it "parses - as a plain character at end of a character class" do
-      regex = Regex.new("[0-]")
+      regex = Regex.new("[0-]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -220,7 +237,7 @@ class Propane
     end
 
     it "parses - as a plain character at beginning of a negated character class" do
-      regex = Regex.new("[^-9]")
+      regex = Regex.new("[^-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -235,7 +252,7 @@ class Propane
     end
 
     it "parses . as a plain character in a character class" do
-      regex = Regex.new("[.]")
+      regex = Regex.new("[.]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -250,7 +267,7 @@ class Propane
     end
 
     it "parses - as a plain character when escaped in middle of character class" do
-      regex = Regex.new("[0\\-9]")
+      regex = Regex.new("[0\\-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -269,7 +286,7 @@ class Propane
     end
 
     it "parses alternates" do
-      regex = Regex.new("ab|c")
+      regex = Regex.new("ab|c", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 2
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -279,7 +296,7 @@ class Propane
     end
 
     it "parses a ." do
-      regex = Regex.new("a.b")
+      regex = Regex.new("a.b", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -290,7 +307,7 @@ class Propane
     end
 
     it "parses something complex" do
-      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 4
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit

spec/spec_helper.rb

@@ -2,6 +2,10 @@ unless ENV["dist_specs"]
   require "bundler/setup"
   require "simplecov"
 
+  class MyFormatter
+    def format(*args)
+    end
+  end
   SimpleCov.start do
     add_filter "/spec/"
     add_filter "/.bundle/"
@@ -12,6 +16,7 @@ unless ENV["dist_specs"]
     end
     project_name "Propane"
     merge_timeout 3600
+    formatter(MyFormatter)
   end
 
   RSpec.configure do |config|

spec/test_basic_math_grammar.c

@@ -5,25 +5,29 @@
 int main()
 {
     char const * input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
+    p_context_delete(context);
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
+    p_context_delete(context);
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
+    p_context_delete(context);
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
+    p_context_delete(context);
 
     return 0;
 }

spec/test_basic_math_grammar.d

@@ -10,23 +10,23 @@ int main()
 unittest
 {
     string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(11, p_result(context));
 
     input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(48, p_result(context));
 
     input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(13, p_result(context));
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(73, p_result(context));
 }

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_drop_code_block.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_drop_code_block.d

@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_error_positions.c

@@ -5,35 +5,43 @@
 int main()
 {
     char const * input = "a 42";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "a\n123\na  a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 3);
-    assert(context.token == TOKEN_a);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 4);
+    assert(p_token(context) == TOKEN_a);
+    p_context_delete(context);
 
     input = "12";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 0);
-    assert(context.token == TOKEN_num);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 1);
+    assert(p_token(context) == TOKEN_num);
+    p_context_delete(context);
 
     input = "a 12\n\nab";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 1);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context).row == 3);
+    assert(p_position(context).col == 2);
+    p_context_delete(context);
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 5);
-    assert(p_position(&context).col == 4);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context).row == 6);
+    assert(p_position(context).col == 5);
+
+    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
+    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_error_positions.d

@@ -9,29 +9,32 @@ int main()
 unittest
 {
     string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(3, 4));
+    assert(p_token(context) == TOKEN_a);
 
     input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 1));
+    assert(p_token(context) == TOKEN_num);
 
     input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_position(context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(5, 4));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_position(context) == p_position_t(6, 5));
+
+    assert(p_token_names[TOKEN_a] == "a");
+    assert(p_token_names[TOKEN_num] == "num");
 }

spec/test_field_aliases.c

@@ -0,0 +1,14 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "foo1\nbar2";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+    return 0;
+}

spec/test_field_aliases.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "foo1\nbar2";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_lexer.c

@@ -38,55 +38,75 @@ int main()
 
     p_token_info_t token_info;
     char const * input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 2u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 3u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 4u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 6u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 0u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 4u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 6u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 9u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 10u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
-    p_context_init(&context, (uint8_t const *)"", 0u);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
+    context = p_context_new((uint8_t const *)"", 0u);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer.d

@@ -44,26 +44,26 @@ unittest
 {
     p_token_info_t token_info;
     string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
+    context = p_context_new("");
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_lexer_match_text.c

@@ -6,10 +6,11 @@
 int main()
 {
     char const * input = "identifier_123";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_match_text.d

@@ -9,8 +9,8 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 }

spec/test_lexer_modes.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abc \"a string\" def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "abc \"abc def\" def";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_multiple_modes.c

@@ -0,0 +1,22 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc.def";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    printf("pass1\n");
+    p_context_delete(context);
+
+    input = "abc .  abc";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    printf("pass2\n");
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_lexer_multiple_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc.def`;
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc .  abc`;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_lexer_result_value.c

@@ -5,15 +5,17 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "fabulous";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_result_value.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = `fabulous`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 8u);
 }

spec/test_lexer_unknown_character.c

@@ -5,14 +5,16 @@
 int main()
 {
     char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    p_context_delete(context);
 
     input = "123";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_unknown_character.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_INPUT);
 
     input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 123u);
 }

spec/test_match_backslashes.c

@@ -0,0 +1,14 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "\a\b\t\n\v\f\rt";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_match_backslashes.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\a\b\t\n\v\f\rt";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_multiple_parsers.c

@@ -6,14 +6,16 @@
 int main()
 {
     char const * input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new((uint8_t const *)input1, strlen(input1));
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
+    myp1_context_delete(context1);
 
     char const * input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new((uint8_t const *)input2, strlen(input2));
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
+    myp2_context_delete(context2);
 
     return 0;
 }

spec/test_multiple_parsers.d

@@ -10,12 +10,12 @@ int main()
 unittest
 {
     string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
+    myp1_context_t * context1;
+    context1 = myp1_context_new(input1);
+    assert(myp1_parse(context1) == MYP1_SUCCESS);
 
     string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
+    myp2_context_t * context2;
+    context2 = myp2_context_new(input2);
+    assert(myp2_parse(context2) == MYP2_SUCCESS);
 }

spec/test_named_optional_rule_component_tree.c

@@ -0,0 +1,54 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+    assert(start->r == NULL);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "abcd";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start->a != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->r != NULL);
+    assert(start->pR == start->pR3);
+    assert(start->pR == start->r);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "bdc";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->r != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}
+

spec/test_named_optional_rule_component_tree.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+    assert(start.r is null);
+
+    p_tree_delete(start);
+
+    input = "abcd";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.r != null);
+    assert(start.pR == start.pR3);
+    assert(start.pR == start.r);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    p_tree_delete(start);
+
+    input = "bdc";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
+}

spec/test_optional_rule_component.c

@@ -0,0 +1,25 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "b";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abcd";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    input = "abdc";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}
+

spec/test_optional_rule_component.d

@@ -0,0 +1,23 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    input = "abcd";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    input = "abdc";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_optional_rule_component_tree.c

@@ -0,0 +1,51 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "abcd";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start->pToken1 != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->pR == start->pR3);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "bdc";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->pR != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}
+

spec/test_optional_rule_component_tree.d

@@ -0,0 +1,48 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+
+    p_tree_delete(start);
+
+    input = "abcd";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.pR == start.pR3);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    p_tree_delete(start);
+
+    input = "bdc";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    start = p_result(context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
+}

spec/test_parser_identical_rules_lookahead.c

@@ -5,13 +5,15 @@
 int main()
 {
     char const * input = "aba";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "abb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_identical_rules_lookahead.d

@@ -9,11 +9,11 @@ int main()
 unittest
 {
     string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_from_multiple_states.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 1);
-    assert(context.token == TOKEN___EOF);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context).row == 1);
+    assert(p_position(context).col == 2);
+    assert(context->token == TOKEN___EOF);
+    p_context_delete(context);
 
     input = "a b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "bb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_from_multiple_states.d

@@ -9,17 +9,17 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_position(context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parser_rule_user_code.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parser_rule_user_code.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "ab";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_parsing_json.c

@@ -6,51 +6,58 @@
 int main()
 {
     char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_OBJECT);
+    p_context_delete(context);
 
     input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_ARRAY);
+    p_context_delete(context);
 
     input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == -45.6);
+    p_context_delete(context);
 
     input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context)->id == JSON_NUMBER);
+    assert(p_result(context)->number == 0.02);
+    p_context_delete(context);
 
     input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    JSONValue * o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(1, o->object.size);
     assert(strcmp(o->object.entries[0].name, "hi") == 0);
     assert(o->object.entries[0].value->id == JSON_TRUE);
+    p_context_delete(context);
 
     input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    o = p_result(context);
     assert(o->id == JSON_OBJECT);
     assert_eq(2, o->object.size);
     assert(strcmp(o->object.entries[0].name, "ff") == 0);
     assert(o->object.entries[0].value->id == JSON_FALSE);
     assert(strcmp(o->object.entries[1].name, "nn") == 0);
     assert(o->object.entries[1].value->id == JSON_NULL);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_json.d

@@ -10,45 +10,45 @@ int main()
 unittest
 {
     string input = ``;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 
     input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
 
     input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONArray)p_result(context));
 
     input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == -45.6);
 
     input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONNumber)p_result(context));
+    assert((cast(JSONNumber)p_result(context)).value == 0.02);
 
     input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    JSONObject o = cast(JSONObject)p_result(context);
     assert(o.value["hi"]);
     assert(cast(JSONTrue)o.value["hi"]);
 
     input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(cast(JSONObject)p_result(context));
+    o = cast(JSONObject)p_result(context);
     assert(o.value["ff"]);
     assert(cast(JSONFalse)o.value["ff"]);
     assert(o.value["nn"]);

spec/test_parsing_lists.c

@@ -5,20 +5,23 @@
 int main()
 {
     char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
+    p_context_delete(context);
 
     input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
+    p_context_delete(context);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_parsing_lists.d

@@ -9,18 +9,18 @@ int main()
 unittest
 {
     string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 1u);
 
     input = "";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 0u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 0u);
 
     input = "aaaaaaaaaaaaaaaa";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 16u);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    assert(p_result(context) == 16u);
 }

spec/test_pattern.c

@@ -6,15 +6,17 @@
 int main()
 {
     char const * input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass1\n");
+    p_context_delete(context);
 
     input = "defabcdef";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
     printf("pass2\n");
+    p_context_delete(context);
 
     return 0;
 }

spec/test_pattern.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = "abcdef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass1");
 
     input = "defabcdef";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_return_token_from_pattern.c

@@ -5,9 +5,10 @@
 int main()
 {
     char const * input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
 
     return 0;
 }

spec/test_return_token_from_pattern.d

@@ -9,7 +9,7 @@ int main()
 unittest
 {
     string input = "defghidef";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
 }

spec/test_start_rule.c

@@ -0,0 +1,9 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule.d

@@ -0,0 +1,8 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule_tree.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "hi";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
+
+    p_tree_delete(top);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_start_rule_tree.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "hi";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Top * top = p_result(context);
+    assert(top.pToken !is null);
+    assert_eq(TOKEN_hi, top.pToken.token);
+}

spec/test_starting_rules.c

@@ -0,0 +1,30 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "bbbb";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
+    assert_eq(8, result);
+    p_context_delete(context);
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
+    assert_eq(8, result);
+    p_context_delete(context);
+
+    input = "c";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
+    assert_eq(3, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_starting_rules.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "bbbb";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    int result = p_result(context);
+    assert(result == 8);
+
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    result = p_result_Bs(context);
+    assert(result == 8);
+
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    result = p_result_R(context);
+    assert(result == 3);
+}

spec/test_starting_rules_tree.c

@@ -0,0 +1,40 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "bbbb";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert_not_null(start->bs);
+    assert_not_null(start->bs->b);
+    assert_not_null(start->bs->bs->b);
+    assert_not_null(start->bs->bs->bs->b);
+    assert_not_null(start->bs->bs->bs->bs->b);
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
+    assert_not_null(bs->b);
+    assert_not_null(bs->bs->b);
+    assert_not_null(bs->bs->bs->b);
+    assert_not_null(bs->bs->bs->bs->b);
+    p_tree_delete_Bs(bs);
+    p_context_delete(context);
+
+    input = "c";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
+    assert_not_null(r->c);
+    p_tree_delete_R(r);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_starting_rules_tree.d

@@ -0,0 +1,41 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "bbbb";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.bs);
+    assert(start.bs.b);
+    assert(start.bs.bs.b);
+    assert(start.bs.bs.bs.b);
+    assert(start.bs.bs.bs.bs.b);
+
+    p_tree_delete(start);
+
+    context = p_context_new(input);
+    assert(p_parse_Bs(context) == P_SUCCESS);
+    Bs * bs = p_result_Bs(context);
+    assert(bs.b);
+    assert(bs.bs.b);
+    assert(bs.bs.bs.b);
+    assert(bs.bs.bs.bs.b);
+
+    p_tree_delete_Bs(bs);
+
+    input = "c";
+    context = p_context_new(input);
+    assert(p_parse_R(context) == P_SUCCESS);
+    R * r = p_result_R(context);
+    assert(r.c);
+
+    p_tree_delete_R(r);
+}

spec/test_token_user_fields.c

@@ -0,0 +1,46 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input =
+        "# c1\n"
+        "#  c2\n"
+        "\n"
+        "first\n"
+        "\n   \n  \n"
+        "  # s1\n"
+        "   #   s2\n"
+        "second\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start->pIDs);
+    assert(start->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
+#else
+    assert(start->pIDs->id->comments);
+    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
+#endif
+    assert(start->pIDs->pIDs);
+    assert(start->pIDs->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
+#else
+    assert(start->pIDs->pIDs->id->comments);
+    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
+#endif
+
+#ifndef __cplusplus
+    free(context->comments);
+#endif
+    p_context_delete(context);
+    p_tree_delete(start);
+
+    return 0;
+}

spec/test_token_user_fields.d

@@ -0,0 +1,31 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input =
+        "# c1\n" ~
+        "#  c2\n" ~
+        "\n" ~
+        "first\n" ~
+        "\n   \n  \n" ~
+        "  # s1\n" ~
+        "   #   s2\n" ~
+        "second\n";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    Start * start = p_result(context);
+    assert(start.pIDs);
+    assert(start.pIDs.id);
+    assert(start.pIDs.id.comments == "# c1\n#  c2\n");
+    assert(start.pIDs.pIDs);
+    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
+
+    p_tree_delete(start);
+}

spec/test_tree.c

@@ -0,0 +1,64 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    Items * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    ItemsMore * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(start->pItems == NULL);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    input = "2 1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    p_tree_delete(start);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_tree.d

@@ -0,0 +1,62 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    Start * start = p_result(context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    Items * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    ItemsMore * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    p_tree_delete(start);
+
+    input = "";
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(start.pItems is null);
+
+    p_tree_delete(start);
+
+    input = "2 1";
+    context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    start = p_result(context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
+}