wip

This issue manifested when multiple ptypes are present with different sizes.

.github/workflows/run-tests.yml

@@ -0,0 +1,38 @@
+name: Run Propane Tests
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ruby-version: ['3.4']
+
+    steps:
+    - name: Install dependencies (Linux)
+      if: runner.os == 'Linux'
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+
+    - name: Install dependencies (macOS)
+      if: runner.os == 'macOS'
+      run: brew install gcc ldc
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests
+      run: rake all

.rspec

@@ -1,3 +1,2 @@
---format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,3 +1,82 @@
+## v2.3.0
+
+### New Features
+
+- Add \D, \S, \w, \W special character classes
+
+### Improvements
+
+- Include line numbers for pattern errors
+- Improve performance in a few places
+- Parallelize parser table generation on Linux hosts
+- Add github workflow to run unit tests
+
+### Fixes
+
+- Fix a couple clang warnings for C backend
+- Fix C backend not fully initializing pvalues when multiple ptypes are used with different sizes.
+- Fix some user guide examples
+
+## v2.2.1
+
+### Fixes
+
+- Fix GC issue for D backend when AST is enabled (#36)
+
+## v2.2.0
+
+### Improvements
+
+- Allow multiple lexer modes to be specified for a lexer pattern (#35)
+- Document p_decode_code_point() API function (#34)
+
+## v2.1.1
+
+### Fixes
+
+- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
+
+## v2.1.0
+
+### Improvements
+
+- Report rule name and line number for conflicting AST node field positions errors (#32)
+
+## v2.0.0
+
+### Improvements
+
+- Log conflicting rules on reduce/reduce conflict (#31)
+- Use 1-based row and column values for position values (#30)
+
+### Fixes
+
+- Fix named optional rules (#29)
+
+### Upgrading
+
+- Adjust all uses of p_position_t row and col values to expect 1-based instead
+of 0-based values.
+
+## v1.5.1
+
+### Improvements
+
+- Improve performance (#28)
+
+## v1.5.0
+
+### New Features
+
+- Track start and end text positions for tokens and rules in AST node structures (#27)
+- Add warnings for shift/reduce conflicts to log file (#25)
+- Add -w command line switch to treat warnings as errors and output to stderr (#26)
+- Add rule field aliases (#24)
+
+### Improvements
+
+- Show line numbers of rules on conflict (#23)
+
 ## v1.4.0
 
 ### New Features

Gemfile

@@ -1,5 +1,6 @@
 source "https://rubygems.org"
 
+gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"

Gemfile.lock

@@ -1,40 +1,48 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.5.0)
-    docile (1.4.0)
-    psych (5.1.0)
+    base64 (0.3.0)
+    date (3.5.1)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    erb (6.0.1)
+    psych (5.3.1)
+      date
       stringio
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.3.1)
+    rdoc (7.1.0)
+      erb
       psych (>= 4.0.0)
-    redcarpet (3.6.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+      tsort
+    redcarpet (3.6.1)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.6)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    stringio (3.0.7)
+    stringio (3.2.0)
     syntax (1.2.2)
+    tsort (0.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  base64
   rake
   rdoc
   redcarpet

README.md

@@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C or D language outputs
+  * targets C, C++, or D language outputs
   * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script
@@ -31,9 +31,14 @@ Propane is typically invoked from the command-line as `./propane`.
 
     Usage: ./propane [options] <input-file> <output-file>
     Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
+      -h, --help  Show this usage and exit.
+      --log LOG   Write log file. This will show all parser states and their
+                  associated shifts and reduces. It can be helpful when
+                  debugging a grammar.
+      --version   Show program version and exit.
+      -w          Treat warnings as errors. This option will treat shift/reduce
+                  conflicts as fatal errors and will print them to stderr in
+                  addition to the log file.
 
 The user must specify the path to a Propane input grammar file and a path to an
 output file.

Rakefile.rb

@@ -1,5 +1,7 @@
 require "rake/clean"
 require "rspec/core/rake_task"
+require "simplecov"
+require "stringio"
 
 CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
 
@@ -12,6 +14,18 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
   end
 end
+task :spec do |task, args|
+  unless ENV["dist_specs"]
+    original_stdout = $stdout
+    sio = StringIO.new
+    $stdout = sio
+    SimpleCov.collate Dir["coverage/.resultset.json"]
+    $stdout = original_stdout
+    sio.string.lines.each do |line|
+      $stdout.write(line) unless line =~ /Coverage report generated for/
+    end
+  end
+end
 
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.

assets/parser.c.erb

@@ -55,11 +55,14 @@ const char * <%= @grammar.prefix %>token_names[] = {
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
 {
     /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext = {0};
+    <%= @grammar.prefix %>context_t newcontext;
+    memset(&newcontext, 0, sizeof(newcontext));
 
     /* Lexer initialization. */
     newcontext.input = input;
     newcontext.input_length = input_length;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -226,7 +229,10 @@ typedef struct
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta. */
+    /** Input text position delta to end of token. */
+    <%= @grammar.prefix %>position_t end_delta_position;
+
+    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -339,8 +345,10 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
     lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
 {
-    lexer_match_info_t longest_match = {0};
-    lexer_match_info_t attempt_match = {0};
+    lexer_match_info_t longest_match;
+    memset(&longest_match, 0, sizeof(longest_match));
+    lexer_match_info_t attempt_match;
+    memset(&attempt_match, 0, sizeof(attempt_match));
     *out_match_info = longest_match;
     uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
     for (;;)
@@ -354,14 +362,16 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
         switch (result)
         {
         case P_SUCCESS:
+            {
             lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
+                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -384,6 +394,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 *out_unexpected_input_length = attempt_match.length + code_point_length;
                 return P_UNEXPECTED_INPUT;
             }
+            }
             break;
 
         case P_EOF:
@@ -441,16 +452,17 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
-    <%= @grammar.prefix %>token_info_t token_info = {0};
+    <%= @grammar.prefix %>token_info_t token_info;
+    memset(&token_info, 0, sizeof(token_info));
     token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
     switch (result)
     {
     case P_SUCCESS:
+        {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
@@ -491,11 +503,23 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         token_info.token = token_to_accept;
         token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         *out_token_info = token_info;
+        }
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
+        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -552,6 +576,9 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  * Parser
  *************************************************************************/
 
+/** Invalid position value. */
+#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
+
 /** Reduce ID type. */
 typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
 
@@ -667,10 +694,20 @@ typedef struct
 <% end %>
 } state_value_t;
 
+/** Common AST node structure. */
+typedef struct ASTNode_s
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct ASTNode_s * fields[];
+} ASTNode;
+
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
-    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
+    {<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u},
 <%   end %>
 };
 
@@ -685,17 +722,22 @@ const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    {
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , NULL
+        NULL, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
+        <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u},
 <%     end %>
-    },
 <%   end %>
 };
 
@@ -765,7 +807,7 @@ static void state_values_stack_push(state_values_stack_t * stack)
     if (current_length >= current_capacity)
     {
         size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
+        state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
         memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
@@ -932,7 +974,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -949,7 +991,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
+                token_ast_node->position = token_info.position;
+                token_ast_node->end_position = token_info.end_position;
+                token_ast_node->n_fields = 0u;
+                token_ast_node->is_token = 1u;
                 token_ast_node->token = token;
                 token_ast_node->pvalue = token_info.pvalue;
                 state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@@ -965,7 +1011,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                 state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
-                <%= @grammar.prefix %>value_t new_parse_result = {0};
+                <%= @grammar.prefix %>value_t new_parse_result;
+                memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
 <% end %>
                 reduced_rule_set = INVALID_ID;
@@ -984,31 +1031,53 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                void ** node_fields = calloc(parser_reduce_table[reduce_index].rule_set_node_field_array_size, sizeof(void *));
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *);
+                ASTNode * node = (ASTNode *)malloc(bytes);
+                memset(node, 0, bytes);
+                node->position = INVALID_POSITION;
+                node->end_position = INVALID_POSITION;
+                node->n_fields = n_fields;
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node_fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                     }
                 }
-                reduced_parser_node = node_fields;
+                bool position_found = false;
+                for (size_t i = 0; i < n_fields; i++)
+                {
+                    ASTNode * child = node->fields[i];
+                    if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
+                    {
+                        if (!position_found)
+                        {
+                            node->position = child->position;
+                            position_found = true;
+                        }
+                        node->end_position = child->end_position;
+                    }
+                }
+                reduced_parser_node = node;
             }
             else
             {
                 reduced_parser_node = NULL;
             }
 <% else %>
-            <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
+            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
             {
+                state_values_stack_free(&statevalues);
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
@@ -1041,13 +1110,13 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context->parse_result;
+    return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result;
 <% else %>
     return context->parse_result.v_<%= start_rule_type[0] %>;
 <% end %>
@@ -1088,3 +1157,35 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
+<% if @grammar.ast %>
+
+static void free_ast_node(ASTNode * node)
+{
+    if (node->is_token)
+    {
+<%   if @grammar.free_token_node %>
+        <%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
+<%   end %>
+        /* TODO: free value_t */
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                free_ast_node(node->fields[i]);
+            }
+        }
+    }
+    free(node);
+}
+
+/**
+ * Free all AST node memory.
+ */
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast)
+{
+    free_ast_node((ASTNode *)ast);
+}
+<% end %>

assets/parser.d.erb

@@ -8,6 +8,9 @@
 module <%= @grammar.modulename %>;
 <% end %>
 
+import core.memory;
+import core.stdc.stdlib : malloc;
+
 /**************************************************************************
  * User code blocks
  *************************************************************************/
@@ -49,6 +52,29 @@ public enum : <%= @grammar.prefix %>token_t
 /** Code point type. */
 public alias <%= @grammar.prefix %>code_point_t = uint;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+public struct <%= @grammar.prefix %>position_t
+{
+    /** Input text row (0-based). */
+    uint row;
+
+    /** Input text column (0-based). */
+    uint col;
+
+    /** Invalid position value. */
+    enum INVALID = <%= @grammar.prefix %>position_t(0u, 0u);
+
+    /** Return whether the position is valid. */
+    public @property bool valid()
+    {
+        return row != 0u;
+    }
+}
+
 <% if @grammar.ast %>
 /** Parser values type. */
 public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
@@ -63,9 +89,20 @@ public union <%= @grammar.prefix %>value_t
 <% end %>
 
 <% if @grammar.ast %>
+/** Common AST node structure. */
+private struct ASTNode
+{
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    void *[0] fields;
+}
+
 /** AST node types. @{ */
 public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
+    /* ASTNode fields must be present in the same order here. */
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 }
@@ -75,6 +112,8 @@ public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 <%     next if rule_set.optional? %>
 public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
 <%     rule_set.ast_fields.each do |fields| %>
     union
     {
@@ -89,26 +128,15 @@ public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 /** @} */
 <% end %>
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-public struct <%= @grammar.prefix %>position_t
-{
-    /** Input text row (0-based). */
-    uint row;
-
-    /** Input text column (0-based). */
-    uint col;
-}
-
 /** Lexed token information. */
 public struct <%= @grammar.prefix %>token_info_t
 {
-    /** Text position where the token was found. */
+    /** Text position of first code point in token. */
     <%= @grammar.prefix %>position_t position;
 
+    /** Text position of last code point in token. */
+    <%= @grammar.prefix %>position_t end_position;
+
     /** Number of input bytes used by the token. */
     size_t length;
 
@@ -145,7 +173,7 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -208,6 +236,8 @@ public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t *
 
     /* Lexer initialization. */
     newcontext.input = input;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -372,7 +402,10 @@ private struct lexer_match_info_t
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta. */
+    /** Input text position delta to end of token. */
+    <%= @grammar.prefix %>position_t end_delta_position;
+
+    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -500,10 +533,11 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
+                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -586,7 +620,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     <%= @grammar.prefix %>token_info_t token_info;
     token_info.position = context.text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
@@ -633,11 +666,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
         }
         token_info.token = token_to_accept;
         token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
+        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -817,7 +861,7 @@ private struct state_value_t
 /** Parser shift table. */
 private immutable shift_t[] parser_shift_table = [
 <%   @parser.shift_table.each do |shift| %>
-    shift_t(<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u),
+    shift_t(<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u),
 <%   end %>
 ];
 
@@ -832,17 +876,22 @@ immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    reduce_t(
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , null
+        null, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
+        <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u), /* Number of states */
 <%     end %>
-            ),
 <%   end %>
 ];
 
@@ -982,7 +1031,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
+                context.parse_result = statevalues[$-1].ast_node;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -997,7 +1046,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token, token_info.pvalue);
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
                 statevalues[$-1].ast_node = token_ast_node;
 <% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;
@@ -1030,26 +1079,45 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             }
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
-                void *[] node_fields = new void *[parser_reduce_table[reduce_index].rule_set_node_field_array_size];
-                foreach (i; 0..parser_reduce_table[reduce_index].rule_set_node_field_array_size)
+                size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
+                size_t node_size = ASTNode.sizeof + n_fields * (void *).sizeof;
+                ASTNode * node = cast(ASTNode *)malloc(node_size);
+                GC.addRange(node, node_size);
+                node.position = <%= @grammar.prefix %>position_t.INVALID;
+                node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                foreach (i; 0..n_fields)
                 {
-                    node_fields[i] = null;
+                    node.fields[i] = null;
                 }
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node_fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                        node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
                     }
                 }
                 else
                 {
                     foreach (i; 0..parser_reduce_table[reduce_index].n_states)
                     {
-                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                        node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
                     }
                 }
-                reduced_parser_node = node_fields.ptr;
+                bool position_found = false;
+                foreach (i; 0..n_fields)
+                {
+                    ASTNode * child = cast(ASTNode *)node.fields[i];
+                    if (child && child.position.valid)
+                    {
+                        if (!position_found)
+                        {
+                            node.position = child.position;
+                            position_found = true;
+                        }
+                        node.end_position = child.end_position;
+                    }
+                }
+                reduced_parser_node = node;
             }
             else
             {
@@ -1088,13 +1156,13 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context.parse_result;
+    return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result;
 <% else %>
     return context.parse_result.v_<%= start_rule_type[0] %>;
 <% end %>

assets/parser.h.erb

@@ -38,6 +38,23 @@ typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>t
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+typedef struct
+{
+    /** Input text row (0-based). */
+    uint32_t row;
+
+    /** Input text column (0-based). */
+    uint32_t col;
+} <%= @grammar.prefix %>position_t;
+
+/** Return whether the position is valid. */
+#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
+
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>
 
@@ -58,6 +75,11 @@ typedef union
 /** AST node types. @{ */
 typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
+    <% # ASTNode fields must be present in the same order here. # %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
@@ -73,6 +95,11 @@ struct <%= name %>;
 <%     next if rule_set.optional? %>
 typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
+    <% # ASTNode fields must be present in the same order here. # %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
 <%     rule_set.ast_fields.each do |fields| %>
     union
     {
@@ -87,26 +114,15 @@ typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 /** @} */
 <% end %>
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
-{
-    /** Input text row (0-based). */
-    uint32_t row;
-
-    /** Input text column (0-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
-
 /** Lexed token information. */
 typedef struct
 {
-    /** Text position where the token was found. */
+    /** Text position of first code point in token. */
     <%= @grammar.prefix %>position_t position;
 
+    /** Text position of last code point in token. */
+    <%= @grammar.prefix %>position_t end_position;
+
     /** Number of input bytes used by the token. */
     size_t length;
 
@@ -146,7 +162,7 @@ typedef struct
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -175,7 +191,9 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 
 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <% end %>

doc/user_guide.md

@@ -13,8 +13,9 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C or D language outputs
+  * targets C, C++, or D language outputs
   * optionally supports automatic full AST generation
+  * tracks input text start and end positions for all matched tokens/rules
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
@@ -35,9 +36,14 @@ Propane is typically invoked from the command-line as `./propane`.
 
     Usage: ./propane [options] <input-file> <output-file>
     Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
+      -h, --help  Show this usage and exit.
+      --log LOG   Write log file. This will show all parser states and their
+                  associated shifts and reduces. It can be helpful when
+                  debugging a grammar.
+      --version   Show program version and exit.
+      -w          Treat warnings as errors. This option will treat shift/reduce
+                  conflicts as fatal errors and will print them to stderr in
+                  addition to the log file.
 
 The user must specify the path to a Propane input grammar file and a path to an
 output file.
@@ -61,10 +67,10 @@ import std.math;
 ptype ulong;
 
 # A few basic arithmetic operators.
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
+token plus /\+/;
+token times /\*/;
+token power /\*\*/;
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -73,10 +79,10 @@ token integer /\\d+/ <<
   }
   $$ = v;
 >>
-token lparen /\\(/;
-token rparen /\\)/;
+token lparen /\(/;
+token rparen /\)/;
 # Drop whitespace.
-drop /\\s+/;
+drop /\s+/;
 
 Start -> E1 << $$ = $1; >>
 E1 -> E2 << $$ = $1; >>
@@ -149,7 +155,7 @@ Example:
 ```
 ptype ulong;
 
-token integer /\\d+/ <<
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -222,21 +228,21 @@ token two /2/;
 token comma /,/ <<
   $$ = 42;
 >>
-token lparen /\\(/;
-token rparen /\\)/;
-drop /\\s+/;
+token lparen /\(/;
+token rparen /\)/;
+drop /\s+/;
 
 Start -> Items;
 
-Items -> Item ItemsMore;
+Items -> Item:item ItemsMore;
 Items -> ;
 
-ItemsMore -> comma Item ItemsMore;
+ItemsMore -> comma Item:item ItemsMore;
 ItemsMore -> ;
 
 Item -> a;
 Item -> b;
-Item -> lparen Item rparen;
+Item -> lparen Item:item rparen;
 Item -> Dual;
 
 Dual -> One Two;
@@ -257,24 +263,24 @@ Start * start = p_result(&context);
 assert(start.pItems1 !is null);
 assert(start.pItems !is null);
 Items * items = start.pItems;
-assert(items.pItem !is null);
-assert(items.pItem.pToken1 !is null);
-assert_eq(TOKEN_a, items.pItem.pToken1.token);
-assert_eq(11, items.pItem.pToken1.pvalue);
+assert(items.item !is null);
+assert(items.item.pToken1 !is null);
+assert_eq(TOKEN_a, items.item.pToken1.token);
+assert_eq(11, items.item.pToken1.pvalue);
 assert(items.pItemsMore !is null);
 ItemsMore * itemsmore = items.pItemsMore;
-assert(itemsmore.pItem !is null);
-assert(itemsmore.pItem.pItem !is null);
-assert(itemsmore.pItem.pItem.pItem !is null);
-assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
-assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
-assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+assert(itemsmore.item !is null);
+assert(itemsmore.item.item !is null);
+assert(itemsmore.item.item.item !is null);
+assert(itemsmore.item.item.item.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.item.item.item.pToken1.token);
+assert_eq(22, itemsmore.item.item.item.pToken1.pvalue);
 assert(itemsmore.pItemsMore !is null);
 itemsmore = itemsmore.pItemsMore;
-assert(itemsmore.pItem !is null);
-assert(itemsmore.pItem.pToken1 !is null);
-assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
-assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+assert(itemsmore.item !is null);
+assert(itemsmore.item.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.item.pToken1.token);
+assert_eq(22, itemsmore.item.pToken1.pvalue);
 assert(itemsmore.pItemsMore is null);
 ```
 
@@ -320,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null);
 assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 ```
 
+## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
+
+If user lexer code block allocates memory to store in a token node's `pvalue`,
+the `free_token_node` grammar statement can be used to specify the name of a
+function which will be called during the `p_free_ast()` call to free the memory
+associated with a token node.
+
+Example:
+
+```
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+```
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -426,10 +462,10 @@ A regular expression begins and ends with a `/` character.
 Example:
 
 ```
-/#.*$/
+/#.*/
 ```
 
-Regular expressions can include many special characters:
+Regular expressions can include many special characters/sequences:
 
   * The `.` character matches any input character other than a newline.
   * The `*` character matches any number of the previous regex element.
@@ -441,14 +477,17 @@ Regular expressions can include many special characters:
   * The `\` character escapes the following character and changes its meaning:
     * The `\a` sequence matches an ASCII bell character (0x07).
     * The `\b` sequence matches an ASCII backspace character (0x08).
-    * The `\d` sequence matches any character `0` through `9`.
+    * The `\d` sequence is shorthand for the `[0-9]` character class.
+    * The `\D` sequence matches every code point not matched by `\d`.
     * The `\f` sequence matches an ASCII form feed character (0x0C).
     * The `\n` sequence matches an ASCII new line character (0x0A).
     * The `\r` sequence matches an ASCII carriage return character (0x0D).
-    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
-    `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * The `\s` sequence is shorthand for the `[ \t\r\n\f\v]` character class.
+    * The `\S` sequence matches every code point not matched by `\s`.
     * The `\t` sequence matches an ASCII tab character (0x09).
     * The `\v` sequence matches an ASCII vertical tab character (0x0B).
+    * The `\w` sequence is shorthand for the `[a-zA-Z0-9_]` character class.
+    * The `\W` sequence matches every code point not matched by `\w`.
     * Any other character matches itself.
   * The `|` character creates an alternate match.
 
@@ -502,7 +541,7 @@ tokenid str;
   mystringvalue = "";
   $mode(string);
 >>
-string: /[^"]+/ << mystringvalue += match; >>
+string: /[^"]+/ << mystringvalue ~= match; >>
 string: /"/ <<
   $mode(default);
   return $token(str);
@@ -530,6 +569,28 @@ It also returns the `str` token now that the token is complete.
 Note that the token name `str` above could have been `string` instead - the
 namespace for token names is distinct from the namespace for lexer modes.
 
+Multiple modes can be specified for a token or pattern or drop statement.
+For example, if the grammar wanted to only recognize an identifier following
+a `.` token and not other keywords, it could switch to an `identonly` mode
+when matching a `.`
+The `ident` token pattern will be matched in either the `default` or
+`identonly` mode.
+
+```
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\s+/;
+```
+
 ##> Specifying parser value types - the `ptype` statement
 
 The `ptype` statement is used to define parser value type(s).
@@ -601,6 +662,10 @@ This can be changed with the `start` statement.
 The grammar file must define a rule with the name of the start rule name which
 will be used as the top-level starting rule that the parser attempts to reduce.
 
+Rule statements are composed of the name of the rule, a `->` token, the fields
+defining the rule pattern that must be matched, and a terminating semicolon or
+user code block.
+
 Example:
 
 ```
@@ -629,11 +694,16 @@ E4 -> lparen E1 rparen << $$ = $2; >>
 
 This example uses the default start rule name of `Start`.
 
-A parser rule has zero or more terms on the right side of its definition.
-Each of these terms is either a token name or a rule name.
-A term can be immediately followed by a `?` character to signify that it is
+A parser rule has zero or more fields on the right side of its definition.
+Each of these fields is either a token name or a rule name.
+A field can be immediately followed by a `?` character to signify that it is
 optional.
-Another example:
+A field can optionally be followed by a `:` and then a field alias name.
+If present, the field alias name is used to refer to the field value in user
+code blocks, or if AST mode is active, the field alias name is used as the
+field name in the generated AST node structure.
+An optional and named field must use the format `field?:name`.
+Example:
 
 ```
 token public;
@@ -641,14 +711,16 @@ token private;
 token int;
 token ident /[a-zA-Z_][a-zA-Z_0-9]*/;
 token semicolon /;/;
-IntegerDeclaration -> Visibility? int ident semicolon;
+IntegerDeclaration -> Visibility?:visibility int ident:name semicolon;
 Visibility -> public;
 Visibility -> private;
 ```
 
-In a parser rule code block, parser values for the right side terms are
-accessible as `$1` for the first term's parser value, `$2` for the second
-term's parser value, etc...
+In a parser rule code block, parser values for the right side fields are
+accessible as `$1` for the first field's parser value, `$2` for the second
+field's parser value, etc...
+For the `IntegerDeclaration` rule, the first field value can also be referred to as `${visibility}` and the third field value can also be referred
+to as `${name}`.
 The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
@@ -709,7 +781,7 @@ Some example uses of this functionality could be to:
 
   * Detect integer overflow when lexing an integer literal constant.
   * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
-  * Determine whether parsing should stop and instead be performed using a different parser version.
+  * Determine whether parsing should stop and instead be retried using a different parser version.
 
 To terminate parsing from a lexer or parser user code block, use the
 `$terminate(code)` function, passing an integer expression argument.
@@ -745,10 +817,16 @@ Propane generates the following result code constants:
 * `P_EOF`: The lexer reached the end of the input string.
 * `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
 
-Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
+Result codes are returned by the API functions `p_decode_code_point()`, `p_lex()`, and `p_parse()`.
 
 ##> Types
 
+### `p_code_point_t`
+
+The `p_code_point_t` type is aliased to a 32-bit unsigned integer.
+It is used to store decoded code points from the input text and perform
+lexing based on the grammar's lexer patterns.
+
 ### `p_context_t`
 
 Propane defines a `p_context_t` structure type.
@@ -759,8 +837,25 @@ A pointer to this instance is passed to the generated functions.
 
 ### `p_position_t`
 
-The `p_position_t` structure contains two fields `row` and `col`.
-These fields contain the 0-based row and column describing a parser position.
+The `p_position_t` structure contains two fields: `row` and `col`.
+These fields contain the 1-based row and column describing a parser position.
+
+For D targets, the `p_position_t` structure can be checked for validity by
+querying the `valid` property.
+
+For C targets, the `p_position_t` structure can be checked for validity by
+calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
+instance.
+
+### `p_token_info_t`
+
+The `p_token_info_t` structure contains the following fields:
+
+* `position` (`p_position_t`) holds the text position of the first code point in the token.
+* `end_position` (`p_position_t`) holds the text position of the last code point in the token.
+* `length` (`size_t`) holds the number of input bytes used by the token.
+* `token` (`p_token_t`) holds the token ID of the lexed token
+* `pvalue` (`p_value_t`) holds the parser value associated with the token.
 
 ### AST Node Types
 
@@ -772,13 +867,26 @@ AST node which refers to a raw parser token rather than a composite rule.
 
 #### AST Node Fields
 
-A `Token` node has two fields:
+All AST nodes have a `position` field specifying the text position of the
+beginning of the matched token or rule, and an `end_position` field specifying
+the text position of the end of the matched token or rule.
+Each of these fields are instances of the `p_position_t` structure.
+
+A `Token` node will always have a valid `position` and `end_position`.
+A rule node may not have valid positions if the rule allows for an empty match.
+In this case the `position` structure should be checked for validity before
+using it.
+For C targets this can be accomplished with
+`if (p_position_valid(node->position))` and for D targets this can be
+accomplished with `if (node.position.valid)`.
+
+A `Token` node has the following additional fields:
 
   * `token` which specifies which token was parsed (one of `TOKEN_*`)
   * `pvalue` which specifies the parser value for the token. If a lexer user
   code block assigned to `$$`, the assigned value will be stored here.
 
-The other generated AST node structures have fields generated based on the
+AST node structures for rules contain generated fields based on the
 right hand side components specified for all rules of a given name.
 
 In this example:
@@ -802,7 +910,7 @@ The `Items` structure will have fields:
 
 If a rule can be empty (for example in the second `Items` rule above), then
 an instance of a pointer to that rule's generated AST node will be null if the
-parser matches the empty rule definition.
+parser matches the empty rule pattern.
 
 The non-positional AST node field pointer will not be generated if there are
 multiple positions in which an instance of the node it points to could be
@@ -823,6 +931,19 @@ If the first rule is matched, then `pOne1` and `pTwo2` will be non-null while
 `pTwo1` and `pOne2` will be null.
 If the second rule is matched instead, then the opposite would be the case.
 
+If a field alias is present in a rule definition, an additional field will be
+generated in the AST node with the field alias name.
+For example:
+
+```
+Exp -> Exp:left plus ExpB:right;
+```
+
+In the generated `Exp` structure, the fields `pExp`, `pExp1`, and `left` will
+all point to the same child node (an instance of the `Exp` structure), and the
+fields `pExpB`, `pExpB3`, and `right` will all point to the same child node
+(an instance of the `ExpB` structure).
+
 ##> Functions
 
 ### `p_context_init`
@@ -846,6 +967,44 @@ p_context_t context;
 p_context_init(&context, input);
 ```
 
+### `p_lex`
+
+The `p_lex()` function is the main entry point to the lexer.
+It is normally called automatically by the generated parser to retrieve the
+next input token for the parser and does not need to be called by the user.
+However, the user may initialize a context and call `p_lex()` to use the
+generated lexer in a standalone mode.
+
+Example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+p_token_info_t token_info;
+size_t result = p_lex(&context, &token_info);
+switch (result)
+{
+case P_DECODE_ERROR:
+    /* UTF-8 decode error */
+    break;
+case P_UNEXPECTED_INPUT:
+    /* Input text does not match any lexer pattern. */
+    break;
+case P_USER_TERMINATED:
+    /* Lexer user code block requested to terminate the lexer. */
+    break;
+case P_SUCCESS:
+    /*
+     * token_info.position holds the text position of the first code point in the token.
+     * token_info.end_position holds the text position of the last code point in the token.
+     * token_info.length holds the number of input bytes used by the token.
+     * token_info.token holds the token ID of the lexed token
+     * token_info.pvalue holds the parser value associated with the token.
+     */
+    break;
+}
+```
+
 ### `p_parse`
 
 The `p_parse()` function is the main entry point to the parser.
@@ -859,6 +1018,24 @@ p_context_init(&context, input, input_length);
 size_t result = p_parse(&context);
 ```
 
+### `p_position_valid`
+
+The `p_position_valid()` function is only generated for C targets.
+it is used to determine whether or not a `p_position_t` structure is valid.
+
+Example:
+
+```
+if (p_position_valid(node->position))
+{
+    ....
+}
+```
+
+For D targets, rather than using `p_position_valid()`, the `valid` property
+function of the `p_position_t` structure can be queried
+(e.g. `if (node.position.valid)`).
+
 ### `p_result`
 
 The `p_result()` function can be used to retrieve the final parse value after
@@ -931,6 +1108,40 @@ if (p_parse(&context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_decode_code_point`
+
+The `p_decode_code_point()` function can be used to decode code points from a
+UTF-8 string.
+It does not require a lexer/parser context structure and can be used as a
+standalone UTF-8 decoder or from within a lexer or parser user code block.
+
+D Example:
+
+```
+size_t result;
+p_code_point_t code_point;
+ubyte code_point_length;
+
+result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+assert(result == P_SUCCESS);
+assert(code_point == 0x1F9E1u);
+assert(code_point_length == 4u);
+```
+
+### `p_free_ast`
+
+The `p_free_ast()` function can be used to free the memory used by the AST.
+It should be passed the same value that is returned by `p_result()`.
+
+The `p_free_ast()` function is only available for C/C++ output targets.
+
+Note that if any lexer user code block allocates memory to store in a token's
+`pvalue`, in order to properly free this memory a `free_token_node` function
+should be specified in the grammar file.
+If specified, the `free_token_node` function will be called during the
+`p_free_ast()` process to allow user code to free any memory associated with
+a token node's `pvalue`.
+
 ##> Data
 
 ### `p_token_names`

extra/vim/ftdetect/propane.vim

@@ -0,0 +1 @@
+au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -0,0 +1,33 @@
+" Vim syntax file for Propane
+" Language: propane
+" Maintainer: Josh Holtrop
+" URL: https://github.com/holtrop/propane
+
+if exists("b:current_syntax")
+  finish
+endif
+
+if !exists("b:propane_subtype")
+  let b:propane_subtype = "d"
+endif
+
+exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
+
+syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
+
+syn match propaneComment "#.*"
+syn match propaneOperator "->"
+syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
+syn match propaneFieldOperator ":" contained
+syn match propaneOperator "?"
+syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
+
+syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
+
+hi def link propaneComment Comment
+hi def link propaneKeyword Keyword
+hi def link propaneRegex String
+hi def link propaneOperator Operator
+hi def link propaneFieldOperator Operator
+hi def link propaneDelimiter Delimiter
+hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -31,10 +31,10 @@ class Propane
 
   class << self
 
-    def run(input_file, output_file, log_file)
+    def run(input_file, output_file, log_file, options)
       begin
         grammar = Grammar.new(File.read(input_file))
-        generator = Generator.new(grammar, output_file, log_file)
+        generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e
         $stderr.puts e.message

lib/propane/cli.rb

@@ -4,15 +4,21 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  --log LOG   Write log file
-  --version   Show program version and exit
-  -h, --help  Show this usage and exit
+  -h, --help  Show this usage and exit.
+  --log LOG   Write log file. This will show all parser states and their
+              associated shifts and reduces. It can be helpful when
+              debugging a grammar.
+  --version   Show program version and exit.
+  -w          Treat warnings as errors. This option will treat shift/reduce
+              conflicts as fatal errors and will print them to stderr in
+              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
+        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -29,6 +35,8 @@ EOF
           when "-h", "--help"
             puts USAGE
             return 0
+          when "-w"
+            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -45,7 +53,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file)
+        Propane.run(*params, log_file, options)
       end
 
     end

lib/propane/generator.rb

@@ -2,7 +2,7 @@ class Propane
 
   class Generator
 
-    def initialize(grammar, output_file, log_file)
+    def initialize(grammar, output_file, log_file, options)
       @grammar = grammar
       @output_file = output_file
       if log_file
@@ -11,23 +11,28 @@ class Propane
         @log = StringIO.new
       end
       @language =
-        if output_file =~ /\.([a-z]+)$/
-          $1
-        else
+        if output_file.end_with?(".d")
           "d"
+        else
+          "c"
         end
+      @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [@language]
+      extensions = [nil]
       if @language == "c"
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension}.erb")
+        template = Assets.get("parser.#{extension || @language}.erb")
+        if extension
+          output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        else
+          output_file = @output_file
+        end
         erb = ERB.new(template, trim_mode: "<>")
-        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         result = erb.result(binding.clone)
         File.open(output_file, "wb") do |fh|
           fh.write(result)
@@ -42,8 +47,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.mode.nil?
-          pattern.mode = "default"
+        if pattern.modes.empty?
+          pattern.modes << "default"
           found_default = true
         end
         pattern.ptypename ||= "default"
@@ -66,12 +71,14 @@ class Propane
         end
         tokens_by_name[token.name] = token
       end
-      # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
-        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
+      # Check for user start rules.
+      @grammar.start_rules.each_with_index do |start_rule, i|
+        unless @grammar.rules.find {|rule| rule.name == start_rule}
+          raise Error.new("Start rule `#{start_rule}` not found")
+        end
+        # Add "real" start rule.
+        @grammar.rules.unshift(Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil))
       end
-      # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
       # Generate and add rules for optional components.
       generate_optional_component_rules!(tokens_by_name)
       # Build rule sets.
@@ -129,7 +136,7 @@ class Propane
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, @log)
+      @parser = Parser.new(@grammar, rule_sets, @log, @options)
     end
 
     # Check that any referenced ptypes have been defined.
@@ -275,6 +282,19 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
+        code = code.gsub(/\$\{(\w+)\}/) do |match|
+          aliasname = $1
+          if index = rule.aliases[aliasname]
+            case @language
+            when "c"
+              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
+            when "d"
+              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
+            end
+          else
+            raise Error.new("Field alias '#{aliasname}' not found")
+          end
+        end
       else
         code = code.gsub(/\$\$/) do |match|
           if @grammar.ast
@@ -314,9 +334,9 @@ class Propane
     #
     # @return [Array<String>]
     #   Start rule parser value type name and type string.
-    def start_rule_type
+    def start_rule_type(start_rule_index = 0)
       start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rule
+        rule.name == @grammar.start_rules[start_rule_index]
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -8,10 +8,11 @@ class Propane
     attr_reader :ast
     attr_reader :ast_prefix
     attr_reader :ast_suffix
+    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
-    attr_reader :start_rule
+    attr_reader :start_rules
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
@@ -19,20 +20,22 @@ class Propane
 
     def initialize(input)
       @patterns = []
-      @start_rule = "Start"
+      @start_rules = []
       @tokens = []
       @rules = []
       @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
-      @mode = nil
+      @modeline = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
       @ast = false
       @ast_prefix = ""
       @ast_suffix = ""
+      @free_token_node = nil
       parse_grammar!
+      @start_rules << "Start" if @start_rules.empty?
     end
 
     def ptype
@@ -58,10 +61,11 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @mode.nil? && parse_mode_label!
+      elsif @modeline.nil? && parse_mode_label!
       elsif parse_ast_statement!
       elsif parse_ast_prefix_statement!
       elsif parse_ast_suffix_statement!
+      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
@@ -81,8 +85,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
-        @mode = md[1]
+      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
+        @modeline = md[1]
       end
     end
 
@@ -112,12 +116,18 @@ class Propane
       end
     end
 
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\s+(\w+)\s*;/)
+        @free_token_node = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -153,9 +163,9 @@ class Propane
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
         @patterns << pattern
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -173,7 +183,7 @@ class Propane
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -186,8 +196,8 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
-        @mode = nil
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
+        @modeline = nil
         true
       end
     end
@@ -198,7 +208,7 @@ class Propane
         if @ast && ptypename
           raise Error.new("Multiple ptypes are unsupported in AST mode")
         end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\??\s*)*)\s*/, "expected rule component list")
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         if @ast
           consume!(/;/, "expected `;'")
@@ -208,7 +218,7 @@ class Propane
           end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -225,15 +235,15 @@ class Propane
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
-        @mode = nil
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
+        @modeline = nil
         true
       end
     end
 
     def parse_start_statement!
       if md = consume!(/start\s+(\w+)\s*;/)
-        @start_rule = md[1]
+        @start_rules << md[1] unless @start_rules.include?(md[1])
       end
     end
 
@@ -247,7 +257,7 @@ class Propane
         else
           @code_blocks[name] = code
         end
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -315,6 +325,14 @@ class Propane
       end
     end
 
+    def get_modes_from_modeline
+      if @modeline
+        Set[*@modeline.split(",").map(&:strip)]
+      else
+        Set.new
+      end
+    end
+
   end
 
 end

lib/propane/lexer.rb

@@ -26,8 +26,14 @@ class Propane
     private
 
     def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
-        pattern.mode
+      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
+        result + pattern.modes
+      end
+      @modes = modenames.reduce({}) do |result, modename|
+        result[modename] = @grammar.patterns.select do |pattern|
+          pattern.modes.include?(modename)
+        end
+        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -7,12 +7,15 @@ class Propane
     attr_reader :reduce_table
     attr_reader :rule_sets
 
-    def initialize(grammar, rule_sets, log)
+    def initialize(grammar, rule_sets, log, options)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
       @item_sets = []
       @item_sets_set = {}
+      @warnings = Set.new
+      @errors = Set.new
+      @options = options
       start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
 
@@ -23,10 +26,10 @@ class Propane
           item_set.id = @item_sets.size
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
-          item_set.following_symbols.each do |following_symbol|
-            unless following_symbol.name == "$EOF"
-              following_set = item_set.build_following_item_set(following_symbol)
-              eval_item_sets << following_set
+          item_set.next_symbols.each do |next_symbol|
+            unless next_symbol.name == "$EOF"
+              next_item_set = item_set.build_next_item_set(next_symbol)
+              eval_item_sets << next_item_set
             end
           end
         end
@@ -37,8 +40,21 @@ class Propane
       end
 
       build_reduce_actions!
-      write_log!
       build_tables!
+      write_log!
+      errormessage = ""
+      if @errors.size > 0
+        errormessage += @errors.join("\n")
+      end
+      if @warnings.size > 0 && @options[:warnings_as_errors]
+        if errormessage != ""
+          errormessage += "\n"
+        end
+        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
+      end
+      if errormessage != ""
+        raise Error.new(errormessage)
+      end
     end
 
     private
@@ -48,27 +64,24 @@ class Propane
       @shift_table = []
       @reduce_table = []
       @item_sets.each do |item_set|
-        shift_entries = item_set.following_symbols.map do |following_symbol|
-          state_id =
-            if following_symbol.name == "$EOF"
-              0
-            else
-              item_set.following_item_set[following_symbol].id
+        unless item_set.reduce_rules.empty?
+          item_set.shift_entries.each do |shift_entry|
+            token = shift_entry[:symbol]
+            if item_set.reduce_actions
+              if rule = item_set.reduce_actions[token]
+                @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+              end
             end
-          {
-            symbol_id: following_symbol.id,
-            state_id: state_id,
-          }
+          end
         end
         reduce_entries =
-          case ra = item_set.reduce_actions
-          when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size,
-              propagate_optional_target: ra.optional? && ra.components.size == 1}]
-          when Hash
-            ra.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id, rule: rule,
+          if rule = item_set.reduce_rule
+            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
+              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+              propagate_optional_target: rule.optional? && rule.components.size == 1}]
+          elsif reduce_actions = item_set.reduce_actions
+            reduce_actions.map do |token, rule|
+              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
                rule_set_id: rule.rule_set.id, n_states: rule.components.size,
                propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
@@ -77,21 +90,21 @@ class Propane
           end
         @state_table << {
           shift_index: @shift_table.size,
-          n_shifts: shift_entries.size,
+          n_shifts: item_set.shift_entries.size,
           reduce_index: @reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += shift_entries
+        @shift_table += item_set.shift_entries
         @reduce_table += reduce_entries
       end
     end
 
     def process_item_set(item_set)
-      item_set.following_symbols.each do |following_symbol|
-        unless following_symbol.name == "$EOF"
-          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
-          item_set.following_item_set[following_symbol] = following_set
-          following_set.in_sets << item_set
+      item_set.next_symbols.each do |next_symbol|
+        unless next_symbol.name == "$EOF"
+          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
+          item_set.next_item_set[next_symbol] = next_item_set
+          next_item_set.in_sets << item_set
         end
       end
     end
@@ -101,7 +114,109 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
-        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
+        build_shift_entries(item_set)
+        build_reduce_actions_for_item_set(item_set)
+      end
+      item_sets_to_process = @item_sets.select do |item_set|
+        # We need lookahead reduce actions if:
+        # 1) There is more than one possible rule to reduce. In this case the
+        #    lookahead token can help choose which rule to reduce.
+        # 2) There is at least one shift action and one reduce action for
+        #    this item set. In this case the lookahead reduce actions are
+        #    needed to test for a Shift/Reduce conflict.
+        item_set.reduce_rules.size > 1 ||
+          (item_set.reduce_rules.size > 0 && item_set.shift_entries.size > 0)
+      end
+      if RbConfig::CONFIG["host_os"] =~ /linux/
+        item_sets_by_id = {}
+        item_sets_to_process.each do |item_set|
+          item_sets_by_id[item_set.object_id] = item_set
+        end
+        tokens_by_id = {}
+        @grammar.tokens.each do |token|
+          tokens_by_id[token.object_id] = token
+        end
+        rules_by_id = {}
+        @grammar.rules.each do |rule|
+          rules_by_id[rule.object_id] = rule
+        end
+        n_threads = Util.determine_n_threads
+        semaphore = Mutex.new
+        queue = Queue.new
+        threads = {}
+        n_threads.times do
+          piper, pipew = IO.pipe
+          thread = Thread.new do
+            loop do
+              item_set = nil
+              semaphore.synchronize do
+                item_set = item_sets_to_process.slice!(0)
+              end
+              break if item_set.nil?
+              fork do
+                piper.close
+                build_lookahead_reduce_actions_for_item_set(item_set, pipew)
+              end
+            end
+            queue.push(Thread.current)
+          end
+          threads[thread] = [piper, pipew]
+        end
+        until threads.empty?
+          thread = queue.pop
+          piper, pipew = threads[thread]
+          pipew.close
+          thread_txt = piper.read
+          thread_txt.each_line do |line|
+            if line.start_with?("RA,")
+              parts = line.split(",")
+              item_set_id, token_id, rule_id = parts[1..3].map(&:to_i)
+              item_set = item_sets_by_id[item_set_id]
+              unless item_set
+                raise "Internal error: could not find item set from thread"
+              end
+              token = tokens_by_id[token_id]
+              unless item_set
+                raise "Internal error: could not find token from thread"
+              end
+              rule = rules_by_id[rule_id]
+              unless item_set
+                raise "Internal error: could not find rule from thread"
+              end
+              item_set.reduce_actions ||= {}
+              item_set.reduce_actions[token] = rule
+            elsif line.start_with?("Error: ")
+              @errors << line.chomp
+            else
+              raise "Internal error: unhandled thread line #{line}"
+            end
+          end
+          thread.join
+          threads.delete(thread)
+        end
+      else
+        # Fall back to single threaded algorithm.
+        item_sets_to_process.each do |item_set|
+          item_set.reduce_actions = build_lookahead_reduce_actions_for_item_set(item_set)
+        end
+      end
+    end
+
+    # Build the shift entries for a single item set.
+    #
+    # @return [void]
+    def build_shift_entries(item_set)
+      item_set.shift_entries = item_set.next_symbols.map do |next_symbol|
+        state_id =
+          if next_symbol.name == "$EOF"
+            0
+          else
+            item_set.next_item_set[next_symbol].id
+          end
+        {
+          symbol: next_symbol,
+          state_id: state_id,
+        }
       end
     end
 
@@ -110,40 +225,45 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [nil, Rule, Hash]
-    #   If no reduce actions are possible for the given item set, nil.
-    #   If only one reduce action is possible for the given item set, the Rule
-    #   to reduce.
-    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
+    # @return [void]
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
       # "complete" items, i.e., items where the parse position is at the
       # end of a rule. These are the only rules that are candidates for
       # reduction in the current ItemSet.
-      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      # If there are no rules to reduce for this ItemSet, we're done here.
-      return nil if reduce_rules.size == 0
-
-      # If there is exactly one rule to reduce for this ItemSet, then do not
-      # figure out the lookaheads; just reduce it.
-      return reduce_rules.first if reduce_rules.size == 1
-
-      # Otherwise, we have more than one possible rule to reduce.
+      if item_set.reduce_rules.size == 1
+        item_set.reduce_rule = item_set.reduce_rules.first
+      end
+    end
 
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    # @param fh [File]
+    #   Output file handle for multiprocessing mode.
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def build_lookahead_reduce_actions_for_item_set(item_set, fh = nil)
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
       item_sets = Set[item_set] + item_set.leading_item_sets
-      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            raise Error.new("Error: reduce/reduce conflict between rule #{existing_reduce_rule.id} (#{existing_reduce_rule.name}) and rule #{reduce_rule.id} (#{reduce_rule.name})")
+            error = "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
+            @errors << error
+            fh.puts(error) if fh
           end
           reduce_actions[lookahead_token] = reduce_rule
+          fh.puts "RA,#{item_set.object_id},#{lookahead_token.object_id},#{reduce_rule.object_id}" if fh
         end
         reduce_actions
       end
@@ -183,13 +303,14 @@ class Propane
         # tokens to form the lookahead token set.
         item_sets.each do |item_set|
           item_set.items.each do |item|
-            if item.following_symbol == rule_set
+            if item.next_symbol == rule_set
               (1..).each do |offset|
-                case symbol = item.following_symbol(offset)
+                case symbol = item.next_symbol(offset)
                 when nil
                   rule_set = item.rule.rule_set
                   unless checked_rule_sets.include?(rule_set)
                     rule_sets_to_check_after << rule_set
+                    checked_rule_sets << rule_set
                   end
                   break
                 when Token
@@ -242,20 +363,26 @@ class Propane
         @log.puts
         @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
         @log.puts "  Outgoing states:"
-        item_set.following_item_set.each do |following_symbol, following_item_set|
-          @log.puts "    #{following_symbol.name} => #{following_item_set.id}"
+        item_set.next_item_set.each do |next_symbol, next_item_set|
+          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
         end
         @log.puts
         @log.puts "  Reduce actions:"
-        case item_set.reduce_actions
-        when Rule
-          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
-        when Hash
+        if item_set.reduce_rule
+          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
+        elsif item_set.reduce_actions
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
           end
         end
       end
+      if @warnings.size > 0
+        @log.puts
+        @log.puts "Warnings:"
+        @warnings.each do |warning|
+          @log.puts "  #{warning}"
+        end
+      end
     end
 
   end

lib/propane/parser/item.rb

@@ -22,6 +22,7 @@ class Propane
       def initialize(rule, position)
         @rule = rule
         @position = position
+        @_hash = [@rule, @position].hash
       end
 
       # Hash function.
@@ -29,7 +30,7 @@ class Propane
       # @return [Integer]
       #   Hash code.
       def hash
-        [@rule, @position].hash
+        @_hash
       end
 
       # Compare Item objects.
@@ -56,7 +57,7 @@ class Propane
 
       # Return the set of Items obtained by "closing" the current item.
       #
-      # If the following symbol for the current item is another Rule name, then
+      # If the next symbol for the current item is another Rule name, then
       # this method will return all Items for that Rule with a position of 0.
       # Otherwise, an empty Array is returned.
       #
@@ -81,17 +82,17 @@ class Propane
         @position == @rule.components.size
       end
 
-      # Get the following symbol for the Item.
+      # Get the next symbol for the Item.
       #
-      # That is, the symbol which follows the parse position marker in the
+      # That is, the symbol which is after the parse position marker in the
       # current Item.
       #
       # @param offset [Integer]
       #   Offset from current parse position to examine.
       #
       # @return [Token, RuleSet, nil]
-      #   Following symbol for the Item.
-      def following_symbol(offset = 0)
+      #   Next symbol for the Item.
+      def next_symbol(offset = 0)
         @rule.components[@position + offset]
       end
 
@@ -108,25 +109,25 @@ class Propane
         end
       end
 
-      # Get whether this Item is followed by the provided symbol.
+      # Get whether this Item's next symbol is the given symbol.
       #
       # @param symbol [Token, RuleSet]
       #   Symbol to query.
       #
       # @return [Boolean]
-      #   Whether this Item is followed by the provided symbol.
-      def followed_by?(symbol)
-        following_symbol == symbol
+      #   Whether this Item's next symbol is the given symbol.
+      def next_symbol?(symbol)
+        next_symbol == symbol
       end
 
-      # Get the following item for this Item.
+      # Get the next item for this Item.
       #
       # That is, the Item formed by moving the parse position marker one place
       # forward from its position in this Item.
       #
       # @return [Item]
-      #   The following item for this Item.
-      def following_item
+      #   The next item for this Item.
+      def next_item
         Item.new(@rule, @position + 1)
       end
 

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing.
+    # parser could currently be parsing. This is equivalent to a parser state.
     class ItemSet
 
       # @return [Set<Item>]
@@ -14,45 +14,58 @@ class Propane
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a following symbol to its ItemSet.
-      attr_reader :following_item_set
+      #   Maps a next symbol to its ItemSet.
+      attr_reader :next_item_set
 
       # @return [Set<ItemSet>]
       #   ItemSets leading to this item set.
       attr_reader :in_sets
 
-      # @return [nil, Rule, Hash]
-      #   Reduce actions, mapping lookahead tokens to rules.
+      # @return [nil, Rule]
+      #   Rule to reduce if there is only one possibility.
+      attr_accessor :reduce_rule
+
+      # @return [Set<Rule>]
+      #   Set of rules that could be reduced in this parser state.
+      attr_accessor :reduce_rules
+
+      # @return [nil, Hash]
+      #   Reduce actions, mapping lookahead tokens to rules, if there is
+      #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
+      # @return [Array<Hash>]
+      #   Shift table entries.
+      attr_accessor :shift_entries
+
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
       #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @following_item_set = {}
+        @next_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      # Get the set of following symbols for all Items in this ItemSet.
+      # Get the set of next symbols for all Items in this ItemSet.
       #
       # @return [Set<Token, RuleSet>]
-      #   Set of following symbols for all Items in this ItemSet.
-      def following_symbols
-        Set.new(@items.map(&:following_symbol).compact)
+      #   Set of next symbols for all Items in this ItemSet.
+      def next_symbols
+        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
       end
 
-      # Build a following ItemSet for the given following symbol.
+      # Build a next ItemSet for the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol to build the following ItemSet for.
+      #   Next symbol to build the next ItemSet for.
       #
       # @return [ItemSet]
-      #   Following ItemSet for the given following symbol.
-      def build_following_item_set(symbol)
-        ItemSet.new(items_followed_by(symbol).map(&:following_item))
+      #   Next ItemSet for the given next symbol.
+      def build_next_item_set(symbol)
+        ItemSet.new(items_with_next(symbol).map(&:next_item))
       end
 
       # Hash function.
@@ -90,21 +103,24 @@ class Propane
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        result = Set.new
-        eval_sets = Set[self]
-        evaled = Set.new
-        while eval_sets.size > 0
-          eval_set = eval_sets.first
-          eval_sets.delete(eval_set)
-          evaled << eval_set
-          eval_set.in_sets.each do |in_set|
-            result << in_set
-            unless evaled.include?(in_set)
-              eval_sets << in_set
+        @_leading_item_sets ||=
+          begin
+            result = Set.new
+            eval_sets = Set[self]
+            evaled = Set.new
+            while eval_sets.size > 0
+              eval_set = eval_sets.first
+              eval_sets.delete(eval_set)
+              evaled << eval_set
+              eval_set.in_sets.each do |in_set|
+                result << in_set
+                unless evaled.include?(in_set)
+                  eval_sets << in_set
+                end
+              end
             end
+            result
           end
-        end
-        result
       end
 
       # Represent the ItemSet as a String.
@@ -137,16 +153,16 @@ class Propane
         end
       end
 
-      # Get the Items followed by the given following symbol.
+      # Get the Items with the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol.
+      #   Next symbol.
       #
       # @return [Array<Item>]
-      #   Items followed by the given following symbol.
-      def items_followed_by(symbol)
+      #   Items with the given next symbol.
+      def items_with_next(symbol)
         @items.select do |item|
-          item.followed_by?(symbol)
+          item.next_symbol?(symbol)
         end
       end
 

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [String, nil]
-    #   Lexer mode for this pattern.
-    attr_accessor :mode
+    # @return [Set]
+    #   Lexer modes for this pattern.
+    attr_accessor :modes
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,16 +46,16 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :mode
-    #   Lexer mode for this pattern.
+    # @option options [String, nil] :modes
+    #   Lexer modes for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @mode = options[:mode]
+      @modes = options[:modes]
       @ptypename = options[:ptypename]
-      regex = Regex.new(@pattern)
+      regex = Regex.new(@pattern, @line_number)
       regex.nfa.end_state.accepts = self
       @nfa = regex.nfa
     end

lib/propane/regex.rb

@@ -4,12 +4,13 @@ class Propane
     attr_reader :unit
     attr_reader :nfa
 
-    def initialize(pattern)
+    def initialize(pattern, line_number)
       @pattern = pattern.dup
+      @line_number = line_number
       @unit = parse_alternates
       @nfa = @unit.to_nfa
       if @pattern != ""
-        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
+        raise Error.new(%[Line #{@line_number}: unexpected "#{@pattern}" in pattern])
       end
     end
 
@@ -41,7 +42,7 @@ class Propane
             mu = MultiplicityUnit.new(last_unit, min_count, max_count)
             au.replace_last!(mu)
           else
-            raise Error.new("#{c} follows nothing")
+            raise Error.new("Line #{@line_number}: #{c} follows nothing")
           end
         when "|"
           au.new_alternate!
@@ -59,7 +60,7 @@ class Propane
     def parse_group
       au = parse_alternates
       if @pattern[0] != ")"
-        raise Error.new("Unterminated group in pattern")
+        raise Error.new("Line #{@line_number}: unterminated group in pattern")
       end
       @pattern.slice!(0)
       au
@@ -70,7 +71,7 @@ class Propane
       index = 0
       loop do
         if @pattern == ""
-          raise Error.new("Unterminated character class")
+          raise Error.new("Line #{@line_number}: unterminated character class")
         end
         c = @pattern.slice!(0)
         if c == "]"
@@ -84,13 +85,13 @@ class Propane
         elsif c == "-" && @pattern[0] != "]"
           begin_cu = ccu.last_unit
           unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
-            raise Error.new("Character range must be between single characters")
+            raise Error.new("Line #{@line_number}: character range must be between single characters")
           end
           if @pattern[0] == "\\"
             @pattern.slice!(0)
             end_cu = parse_backslash
             unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
-              raise Error.new("Character range must be between single characters")
+              raise Error.new("Line #{@line_number}: character range must be between single characters")
             end
             max_code_point = end_cu.code_point
           else
@@ -116,7 +117,7 @@ class Propane
         elsif max_count.to_s != ""
           max_count = max_count.to_i
           if max_count < min_count
-            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
+            raise Error.new("Line #{@line_number}: maximum repetition count cannot be less than minimum repetition count")
           end
         else
           max_count = nil
@@ -124,28 +125,33 @@ class Propane
         @pattern = pattern
         [min_count, max_count]
       else
-        raise Error.new("Unexpected match count at #{@pattern}")
+        raise Error.new("Line #{@line_number}: unexpected match count following {")
       end
     end
 
     def parse_backslash
       if @pattern == ""
-        raise Error.new("Error: unfollowed \\")
+        raise Error.new("Line #{@line_number}: error: unfollowed \\")
       else
         c = @pattern.slice!(0)
         case c
         when "a"
-          CharacterRangeUnit.new("\a", "\a")
+          CharacterRangeUnit.new("\a")
         when "b"
-          CharacterRangeUnit.new("\b", "\b")
+          CharacterRangeUnit.new("\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "D"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu.negate = true
+          ccu
         when "f"
-          CharacterRangeUnit.new("\f", "\f")
+          CharacterRangeUnit.new("\f")
         when "n"
-          CharacterRangeUnit.new("\n", "\n")
+          CharacterRangeUnit.new("\n")
         when "r"
-          CharacterRangeUnit.new("\r", "\r")
+          CharacterRangeUnit.new("\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -155,10 +161,35 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "S"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new(" ")
+          ccu << CharacterRangeUnit.new("\t")
+          ccu << CharacterRangeUnit.new("\r")
+          ccu << CharacterRangeUnit.new("\n")
+          ccu << CharacterRangeUnit.new("\f")
+          ccu << CharacterRangeUnit.new("\v")
+          ccu.negate = true
+          ccu
         when "t"
-          CharacterRangeUnit.new("\t", "\t")
+          CharacterRangeUnit.new("\t")
         when "v"
-          CharacterRangeUnit.new("\v", "\v")
+          CharacterRangeUnit.new("\v")
+        when "w"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu
+        when "W"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu.negate = true
+          ccu
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/regex/unit.rb

@@ -92,16 +92,19 @@ class Propane
         @units = []
         @negate = false
       end
-      def initialize
-        @units = []
-      end
-      def method_missing(*args)
-        @units.__send__(*args)
+      def method_missing(*args, &block)
+        @units.__send__(*args, &block)
       end
       def <<(thing)
         if thing.is_a?(CharacterClassUnit)
-          thing.each do |ccu_unit|
-            @units << ccu_unit
+          if thing.negate
+            CodePointRange.invert_ranges(thing.map(&:code_point_range)).each do |cpr|
+              CharacterRangeUnit.new(cpr.first, cpr.last)
+            end
+          else
+            thing.each do |ccu_unit|
+              @units << ccu_unit
+            end
           end
         else
           @units << thing

lib/propane/rule.rb

@@ -6,6 +6,10 @@ class Propane
     #   Rule components.
     attr_reader :components
 
+    # @return [Hash]
+    #   Field aliases.
+    attr_reader :aliases
+
     # @return [String]
     #   User code associated with the rule.
     attr_reader :code
@@ -49,7 +53,19 @@ class Propane
     #   Line number where the rule was defined in the input grammar.
     def initialize(name, components, code, ptypename, line_number)
       @name = name
-      @components = components
+      @aliases = {}
+      @components = components.each_with_index.map do |component, i|
+        if component =~ /(\S+):(\S+)/
+          c, aliasname = $1, $2
+          if @aliases[aliasname]
+            raise Error.new("Error: duplicate field alias `#{aliasname}` for rule #{name} defined on line #{line_number}")
+          end
+          @aliases[aliasname] = i
+          c
+        else
+          component
+        end
+      end
       @rule_set_node_field_index_map = components.map {0}
       @code = code
       @ptypename = ptypename

lib/propane/rule_set.rb

@@ -100,7 +100,9 @@ class Propane
 
     # Finalize a RuleSet after adding all Rules to it.
     def finalize(grammar)
-      build_ast_fields(grammar)
+      if grammar.ast
+        build_ast_fields(grammar)
+      end
     end
 
     private
@@ -117,6 +119,8 @@ class Propane
     def build_ast_fields(grammar)
       field_ast_node_indexes = {}
       field_indexes_across_all_rules = {}
+      # Stores the index into @ast_fields by field alias name.
+      field_aliases = {}
       @ast_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
@@ -134,6 +138,16 @@ class Propane
             field_ast_node_indexes[field_name] = @ast_fields.size
             @ast_fields << {field_name => struct_name}
           end
+          rule.aliases.each do |alias_name, index|
+            if index == i
+              alias_ast_fields_index = field_ast_node_indexes[field_name]
+              if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index
+                raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
+              end
+              field_aliases[alias_name] = alias_ast_fields_index
+              @ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1]
+            end
+          end
           field_indexes_across_all_rules[node_name] ||= Set.new
           field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
           rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]

lib/propane/util.rb

@@ -10,6 +10,32 @@ class Propane
         "#{s}\n* #{message} *\n#{s}\n"
       end
 
+      # Determine the number of threads to use.
+      #
+      # @return [Integer]
+      #   The number of threads to use.
+      def determine_n_threads
+        # Try to figure out how many threads are available on the host hardware.
+        begin
+          case RbConfig::CONFIG["host_os"]
+          when /linux/
+            return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
+          when /mswin|mingw|msys/
+            if `wmic cpu get NumberOfLogicalProcessors -value` =~ /NumberOfLogicalProcessors=(\d+)/
+              return $1.to_i
+            end
+          when /darwin/
+            if `sysctl -n hw.ncpu` =~ /(\d+)/
+              return $1.to_i
+            end
+          end
+        rescue
+        end
+
+        # If we can't figure it out, default to 4.
+        4
+      end
+
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.4.0"
+  VERSION = "2.3.0"
 end

spec/ast_node_memory_remains.c.propane

@@ -0,0 +1,151 @@
+ast;
+ast_prefix P;
+
+<<header
+#include <stdio.h>
+#include <stdint.h>
+
+typedef union
+{
+    uint64_t i64;
+    const uint8_t * s;
+    double dou;
+} TokenVal;
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.i64 = 64u;
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  $$.s = match;
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/ast_node_memory_remains.d.propane

@@ -0,0 +1,177 @@
+ast;
+ast_prefix P;
+
+<<
+import std.bigint;
+
+private string stringvalue;
+
+union TokenVal
+{
+    BigInt bi;
+    string s;
+    double dou;
+}
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.bi = BigInt(match[0..3]);
+  foreach (c; match[3..$])
+  {
+    if (('0' <= c) && (c <= '9'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - '0');
+    }
+    if (('a' <= c) && (c <= 'f'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'a' + 10);
+    }
+    if (('A' <= c) && (c <= 'F'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'A' + 10);
+    }
+  }
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  if (match[0] == '$')
+  {
+    $$.s = match[1..$];
+  }
+  else
+  {
+    $$.s = match;
+  }
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/json_parser.c.propane

@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {match[0], 0};
+  char s[] = {(char)match[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_types.c

@@ -5,7 +5,7 @@
 
 JSONValue * JSONValue_new(size_t id)
 {
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
     jv->id = id;
     return jv;
 }
@@ -29,7 +29,7 @@ void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
         }
     }
     size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
     if (size > 0)
     {
         memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
@@ -52,7 +52,7 @@ void JSONArray_append(JSONValue * array, JSONValue * value)
 {
     size_t const size = array->array.size;
     size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
     if (array->array.size > 0)
     {
         memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);

spec/json_types.h

@@ -11,6 +11,12 @@
 #define JSON_FALSE 5u
 #define JSON_NULL 6u
 
+typedef struct JSONObjectEntry_s
+{
+    char const * name;
+    struct JSONValue_s * value;
+} JSONObjectEntry;
+
 typedef struct JSONValue_s
 {
     size_t id;
@@ -19,11 +25,7 @@ typedef struct JSONValue_s
         struct
         {
             size_t size;
-            struct
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
+            JSONObjectEntry * entries;
         } object;
         struct
         {

spec/propane/grammar_spec.rb

@@ -151,30 +151,30 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m1"
+      expect(o.modes).to eq Set["m1"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m2"
+      expect(o.modes).to eq Set["m2"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m3"
+      expect(o.modes).to eq Set["m3"]
     end
 
     it "allows assigning ptypes to tokens and rules" do

spec/propane/lexer/dfa_spec.rb

@@ -126,6 +126,74 @@ EOF
     ]
     expect(run(<<EOF, ";")).to eq expected
 token semicolon /;/;
+EOF
+  end
+
+  it "matches a negated character class" do
+    expected = [
+      ["pattern", "/abc/"],
+    ]
+    expect(run(<<EOF, "/abc/")).to eq expected
+token pattern /\\/[^\\s]*\\//;
+EOF
+  end
+
+  it "matches special character classes " do
+    expected = [
+      ["a", "abc123_FOO"],
+    ]
+    expect(run(<<EOF, "abc123_FOO")).to eq expected
+token a /\\w+/;
+EOF
+    expected = [
+      ["b", "FROG*%$#"],
+    ]
+    expect(run(<<EOF, "FROG*%$#")).to eq expected
+token b /FROG\\D{1,4}/;
+EOF
+    expected = [
+      ["c", "$883366"],
+    ]
+    expect(run(<<EOF, "$883366")).to eq expected
+token c /$\\d+/;
+EOF
+    expected = [
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "^&$@")).to eq expected
+token d /^\\W+/;
+EOF
+    expected = [
+      ["a", "abc123_FOO"],
+      [nil, " "],
+      ["b", "FROG*%$#"],
+      [nil, " "],
+      ["c", "$883366"],
+      [nil, " "],
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "abc123_FOO FROG*%$# $883366 ^&$@")).to eq expected
+token a /\\w+/;
+token b /FROG\\D{1,4}/;
+token c /$\\d+/;
+token d /^\\W+/;
+drop /\\s+/;
+EOF
+  end
+
+  it "matches a negated character class with a nested inner negated character class" do
+    expected = [
+      ["t", "$&*"],
+    ]
+    expect(run(<<EOF, "$&*")).to eq expected
+token t /[^%\\W]+/;
+EOF
+  end
+
+  it "\\s matches a newline" do
+    expected = [["s", "\n"]]
+    expect(run(<<EOF, "\n")).to eq expected
+token s /\\s/;
 EOF
   end
 end

spec/propane/regex_spec.rb

@@ -2,14 +2,14 @@ class Propane
   RSpec.describe Regex do
 
     it "parses an empty expression" do
-      regex = Regex.new("")
+      regex = Regex.new("", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0].size).to eq 0
     end
 
     it "parses a single character unit expression" do
-      regex = Regex.new("a")
+      regex = Regex.new("a", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -19,7 +19,7 @@ class Propane
     end
 
     it "parses a group with a single character unit expression" do
-      regex = Regex.new("(a)")
+      regex = Regex.new("(a)", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -33,7 +33,7 @@ class Propane
     end
 
     it "parses a *" do
-      regex = Regex.new("a*")
+      regex = Regex.new("a*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -47,7 +47,7 @@ class Propane
     end
 
     it "parses a +" do
-      regex = Regex.new("a+")
+      regex = Regex.new("a+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -61,7 +61,7 @@ class Propane
     end
 
     it "parses a ?" do
-      regex = Regex.new("a?")
+      regex = Regex.new("a?", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -75,7 +75,7 @@ class Propane
     end
 
     it "parses a multiplicity count" do
-      regex = Regex.new("a{5}")
+      regex = Regex.new("a{5}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -89,7 +89,7 @@ class Propane
     end
 
     it "parses a minimum-only multiplicity count" do
-      regex = Regex.new("a{5,}")
+      regex = Regex.new("a{5,}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -103,7 +103,7 @@ class Propane
     end
 
     it "parses a minimum and maximum multiplicity count" do
-      regex = Regex.new("a{5,8}")
+      regex = Regex.new("a{5,8}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -118,7 +118,7 @@ class Propane
     end
 
     it "parses an escaped *" do
-      regex = Regex.new("a\\*")
+      regex = Regex.new("a\\*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -131,7 +131,7 @@ class Propane
     end
 
     it "parses an escaped +" do
-      regex = Regex.new("a\\+")
+      regex = Regex.new("a\\+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -144,7 +144,7 @@ class Propane
     end
 
     it "parses an escaped \\" do
-      regex = Regex.new("\\\\d")
+      regex = Regex.new("\\\\d", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -157,7 +157,7 @@ class Propane
     end
 
     it "parses a character class" do
-      regex = Regex.new("[a-z_]")
+      regex = Regex.new("[a-z_]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -175,7 +175,7 @@ class Propane
     end
 
     it "parses a negated character class" do
-      regex = Regex.new("[^xyz]")
+      regex = Regex.new("[^xyz]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -189,8 +189,25 @@ class Propane
       expect(ccu[0].first).to eq "x".ord
     end
 
+    it "parses a negated character class with inner character classes" do
+      regex = Regex.new("[^x\\sz]", 1)
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_truthy
+      expect(ccu.size).to eq 8
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "x".ord
+      expect(ccu[1].first).to eq " ".ord
+      expect(ccu[7].first).to eq "z".ord
+    end
+
     it "parses - as a plain character at beginning of a character class" do
-      regex = Regex.new("[-9]")
+      regex = Regex.new("[-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -204,7 +221,7 @@ class Propane
     end
 
     it "parses - as a plain character at end of a character class" do
-      regex = Regex.new("[0-]")
+      regex = Regex.new("[0-]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -220,7 +237,7 @@ class Propane
     end
 
     it "parses - as a plain character at beginning of a negated character class" do
-      regex = Regex.new("[^-9]")
+      regex = Regex.new("[^-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -235,7 +252,7 @@ class Propane
     end
 
     it "parses . as a plain character in a character class" do
-      regex = Regex.new("[.]")
+      regex = Regex.new("[.]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -250,7 +267,7 @@ class Propane
     end
 
     it "parses - as a plain character when escaped in middle of character class" do
-      regex = Regex.new("[0\\-9]")
+      regex = Regex.new("[0\\-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -269,7 +286,7 @@ class Propane
     end
 
     it "parses alternates" do
-      regex = Regex.new("ab|c")
+      regex = Regex.new("ab|c", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 2
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -279,7 +296,7 @@ class Propane
     end
 
     it "parses a ." do
-      regex = Regex.new("a.b")
+      regex = Regex.new("a.b", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -290,7 +307,7 @@ class Propane
     end
 
     it "parses something complex" do
-      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 4
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit

spec/propane_spec.rb

@@ -54,6 +54,7 @@ EOF
     else
       command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     end
+    command += (options[:extra_args] || [])
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -64,26 +65,47 @@ EOF
   end
 
   def compile(test_files, options = {})
-    test_files = Array(test_files)
+    test_files = Array(test_files).map do |test_file|
+      if !File.exist?(test_file) && test_file.end_with?(".cpp")
+        test_file.sub(%r{\.cpp$}, ".c")
+      else
+        test_file
+      end
+    end
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
       "spec/run/testparser#{name}.#{options[:language]}"
     end
     case options[:language]
     when "c"
-      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
+      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+    when "cpp"
+      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
     end
+    result = system(*command)
     expect(result).to be_truthy
   end
 
-  def run_test
+  def run_test(options)
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
-    Results.new(stdout, stderr, status)
+    results = Results.new(stdout, stderr, status)
+    if %w[c cpp].include?(options[:language])
+      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+      vgout = stdout + stderr
+      File.binwrite("spec/run/.vgout", vgout)
+      vgout.scan(/lost: (\d+) bytes/) do |match|
+        bytes = $1.to_i
+        if bytes > 0
+          raise "Valgrind detected memory leak"
+        end
+      end
+    end
+    results
   end
 
   def lines(str)
@@ -184,7 +206,83 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c].each do |language|
+  it "warns on shift/reduce conflicts" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> As? b?;
+As -> a As2?;
+As2 -> b a As2?;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+    expect(File.binread("spec/run/testparser.log")).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}
+  end
+
+  it "errors on shift/reduce conflicts with -w" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> As? b?;
+As -> a As2?;
+As2 -> b a As2?;
+EOF
+    results = run_propane(extra_args: %w[-w], capture: true)
+    expect(results.stderr).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}m
+    expect(results.status).to_not eq 0
+    expect(File.binread("spec/run/testparser.log")).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}
+  end
+
+  it "errors on duplicate field aliases in a rule" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> a:foo b:foo;
+EOF
+    results = run_propane(extra_args: %w[-w], capture: true)
+    expect(results.stderr).to match %r{Error: duplicate field alias `foo` for rule Start defined on line 3}
+    expect(results.status).to_not eq 0
+  end
+
+  it "errors when an alias is in different positions for different rules in a rule set when AST mode is enabled" do
+    write_grammar <<EOF
+ast;
+token a;
+token b;
+Start -> a:foo b;
+Start -> b b:foo;
+EOF
+    results = run_propane(extra_args: %w[-w], capture: true)
+    expect(results.stderr).to match %r{Error: conflicting AST node field positions for alias `foo`}
+    expect(results.status).to_not eq 0
+  end
+
+  it "does not error when an alias is in different positions for different rules in a rule set when AST mode is not enabled" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> a:foo b;
+Start -> b b:foo;
+EOF
+    results = run_propane(extra_args: %w[-w], capture: true)
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "shows error line number for unmatched left curly brace" do
+    write_grammar <<EOF
+# Line 1
+# Line 2
+token a /a{/;
+Start -> a;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Line 3: unexpected match count following \{/
+    expect(results.status).to_not eq 0
+  end
+
+  %w[d c cpp].each do |language|
 
     context "#{language.upcase} language" do
 
@@ -200,14 +298,14 @@ Foo -> plus <<>>
 EOF
         run_propane(language: language)
         compile("spec/test_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "detects a lexer error when an unknown character is seen" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype int;
 token int /\\d+/ <<
@@ -238,7 +336,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -261,7 +359,7 @@ EOF
 
       it "generates a parser that does basic math - user guide example" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <math.h>
@@ -332,7 +430,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -358,7 +456,7 @@ R2 -> a b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
@@ -373,13 +471,13 @@ R1 -> b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
       it "executes user code when matching lexer token" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -408,7 +506,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "abc!",
@@ -421,7 +519,7 @@ EOF
 
       it "supports a pattern statement" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -444,7 +542,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -457,7 +555,7 @@ EOF
 
       it "supports returning tokens from pattern code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -486,7 +584,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -497,7 +595,7 @@ EOF
 
       it "supports lexer modes" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -544,7 +642,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "begin string mode",
@@ -556,9 +654,65 @@ EOF
         ])
       end
 
+      it "multiple lexer modes may apply to a pattern" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  printf("ident: %c\\n", $3);
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  writeln("ident: ", $3);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_multiple_modes.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "ident: d",
+          "pass1",
+          "ident: a",
+          "pass2",
+        ])
+      end
+
       it "executes user code associated with a parser rule" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -583,7 +737,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "A!",
@@ -594,7 +748,7 @@ EOF
 
       it "parses lists" do
         write_grammar <<EOF
-ptype #{language == "c" ? "uint32_t" : "uint"};
+ptype #{language == "d" ? "uint" : "uint32_t"};
 token a;
 Start -> As << $$ = $1; >>
 As -> << $$ = 0u; >>
@@ -602,7 +756,7 @@ As -> As a << $$ = $1 + 1u; >>
 EOF
         run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         expect(results.stderr).to eq ""
       end
@@ -623,19 +777,20 @@ F -> e;
 EOF
         results = run_propane(capture: true, language: language)
         expect(results.status).to_not eq 0
-        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+        expect(results.stderr).to match %r{Error: reduce/reduce conflict \(state \d+\) between rule E#\d+ \(defined on line 10\) and rule F#\d+ \(defined on line 11\)}
+        expect(File.binread("spec/run/testparser.log")).to match %r{Reduce.actions:}
       end
 
       it "provides matched text to user code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
 #include <stdlib.h>
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * t = malloc(match_length + 1);
+  char * t = (char *)malloc(match_length + 1);
   strncpy(t, (char *)match, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
@@ -655,7 +810,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "Matched token is identifier_123",
@@ -665,7 +820,7 @@ EOF
 
       it "allows storing a result value for the lexer" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype uint64_t;
 token word /[a-z]+/ <<
@@ -688,7 +843,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -703,13 +858,14 @@ Start -> a num;
 EOF
         run_propane(language: language)
         compile("spec/test_error_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/json_parser.#{ext}.propane"))
         run_propane(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end
@@ -731,7 +887,7 @@ Start -> b c b;
 EOF
         run_propane(name: "myp2", language: language)
         compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -750,7 +906,7 @@ Any -> c;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -768,14 +924,14 @@ Any -> ;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "matches backslash escape sequences" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
   #include <stdio.h>
@@ -812,7 +968,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -877,7 +1033,7 @@ Two -> two;
 EOF
         run_propane(language: language)
         compile("spec/test_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -921,7 +1077,7 @@ Two -> two;
 EOF
         run_propane(language: language)
         compile("spec/test_ast_ps.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -989,13 +1145,13 @@ Start -> a? b R? <<
   printf("b: %d\\n", $2);
   printf("R: %s\\n", $3 == NULL ? "" : $3);
 >>
-R -> c d << $$ = "cd"; >>
-R (string) -> d c << $$ = "dc"; >>
+R -> c d << $$ = (char *)"cd"; >>
+R (string) -> d c << $$ = (char *)"dc"; >>
 EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -1047,10 +1203,272 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "allows naming an optional rule component in AST generation mode" do
+        if language == "d"
+          write_grammar <<EOF
+ast;
+
+<<
+import std.stdio;
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        else
+          write_grammar <<EOF
+ast;
+
+<<
+#include <stdio.h>
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "stores token and rule positions in AST nodes" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token bb;
+token c /c(.|\\n)*c/;
+drop /\\s+/;
+Start -> T T T;
+T -> a;
+T -> bb;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_token_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "stores invalid positions for empty rule matches" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token bb;
+token c /c(.|\\n)*c/;
+drop /\\s+/;
+Start -> T Start;
+Start -> ;
+T -> a A;
+A -> bb? c?;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_invalid_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows specifying field aliases in AST mode" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> T:first T:second T:third;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "aliases the correct field when multiple rules are in a rule set in AST mode" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> a;
+Start -> Foo;
+Start -> T:first T:second T:third;
+Foo -> b;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows specifying field aliases when AST mode is not enabled" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype string;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = match;
+>>
+drop /\\s+/;
+Start -> id:first id:second <<
+  writeln("first is ", ${first});
+  writeln("second is ", ${second});
+>>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+ptype char *;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * s = (char *)malloc(match_length + 1);
+  strncpy(s, (char const *)match, match_length);
+  s[match_length] = 0;
+  $$ = s;
+>>
+drop /\\s+/;
+Start -> id:first id:second <<
+  printf("first is %s\\n", ${first});
+  printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to match /first is foo1.*second is bar2/m
+      end
+
+      it "aliases the correct field when multiple rules are in a rule set when AST mode is not enabled" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype string;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = match;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  writeln("first is ", ${first});
+  writeln("second is ", ${second});
+>>
+Foo -> ;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+ptype char *;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * s = (char *)malloc(match_length + 1);
+  strncpy(s, (char const *)match, match_length);
+  s[match_length] = 0;
+  $$ = s;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  printf("first is %s\\n", ${first});
+  printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
+>>
+Foo -> ;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to match /first is foo1.*second is bar2/m
+      end
+
+      it "does not free memory allocated for AST nodes" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_ast_node_memory_remains.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      if %w[c cpp].include?(language)
+        it "allows a user function to free token node memory in AST mode" do
+    write_grammar <<EOF
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+EOF
+          run_propane(language: language)
+          compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
     end
   end
 end

spec/spec_helper.rb

@@ -2,6 +2,10 @@ unless ENV["dist_specs"]
   require "bundler/setup"
   require "simplecov"
 
+  class MyFormatter
+    def format(*args)
+    end
+  end
   SimpleCov.start do
     add_filter "/spec/"
     add_filter "/.bundle/"
@@ -12,6 +16,7 @@ unless ENV["dist_specs"]
     end
     project_name "Propane"
     merge_timeout 3600
+    formatter(MyFormatter)
   end
 
   RSpec.configure do |config|

spec/test_ast.c

@@ -33,12 +33,16 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_free_ast(start);
+
     input = "";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
     start = p_result(&context);
     assert(start->pItems == NULL);
 
+    p_free_ast(start);
+
     input = "2 1";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
@@ -51,5 +55,7 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_field_aliases.c

@@ -0,0 +1,21 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "\na\nb\nc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(TOKEN_a, start->first->pToken->token);
+    assert_eq(TOKEN_b, start->second->pToken->token);
+    assert_eq(TOKEN_c, start->third->pToken->token);
+
+    p_free_ast(start);
+
+    return 0;
+}

spec/test_ast_field_aliases.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\na\nb\nc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(TOKEN_a, start.first.pToken.token);
+    assert_eq(TOKEN_b, start.second.pToken.token);
+    assert_eq(TOKEN_c, start.third.pToken.token);
+}

spec/test_ast_invalid_positions.c

@@ -0,0 +1,110 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "\na\n  bb ccc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(2, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(3, start->pT1->pA->position.row);
+    assert_eq(3, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(8, start->pT1->pA->end_position.col);
+    assert_eq(2, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(8, start->pT1->end_position.col);
+
+    assert_eq(2, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(8, start->end_position.col);
+
+    p_free_ast(start);
+
+    input = "a\nbb";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(2, start->pT1->pA->end_position.row);
+    assert_eq(2, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(2, start->pT1->end_position.col);
+
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(2, start->end_position.row);
+    assert_eq(2, start->end_position.col);
+
+    p_free_ast(start);
+
+    input = "a\nc\nc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert(p_position_valid(start->pT1->pA->position));
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(1, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
+
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(1, start->end_position.col);
+
+    p_free_ast(start);
+
+    input = "a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert(!p_position_valid(start->pT1->pA->position));
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
+
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(1, start->end_position.col);
+
+    p_free_ast(start);
+
+    return 0;
+}

spec/test_ast_invalid_positions.d

@@ -0,0 +1,104 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\na\n  bb ccc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(2, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(3, start.pT1.pA.position.row);
+    assert_eq(3, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(8, start.pT1.pA.end_position.col);
+    assert_eq(2, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(8, start.pT1.end_position.col);
+
+    assert_eq(2, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(8, start.end_position.col);
+
+    input = "a\nbb";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(2, start.pT1.pA.end_position.row);
+    assert_eq(2, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(2, start.pT1.end_position.col);
+
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(2, start.end_position.row);
+    assert_eq(2, start.end_position.col);
+
+    input = "a\nc\nc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert(start.pT1.pA.position.valid);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(1, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
+
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(1, start.end_position.col);
+
+    input = "a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert(!start.pT1.pA.position.valid);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
+
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(1, start.end_position.col);
+}

spec/test_ast_node_memory_remains.c

@@ -0,0 +1,418 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+int main(int argc, char * argv[])
+{
+    const char * input =
+        "# 0\n"
+        "def byte_val() -> byte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 1\n"
+        "def short_val() -> short\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 2\n"
+        "def int_val() -> int\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 3\n"
+        "def long_val() -> long\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 4\n"
+        "def ssize_t_val() -> ssize_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 5\n"
+        "def byte_to_short() -> short\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 6\n"
+        "def byte_to_int() -> int\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 7\n"
+        "def byte_to_long() -> long\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 8\n"
+        "def byte_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 9\n"
+        "def short_to_byte() -> byte\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 10\n"
+        "def short_to_int() -> int\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 11\n"
+        "def short_to_long() -> long\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 12\n"
+        "def short_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 13\n"
+        "def int_to_byte() -> byte\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 14\n"
+        "def int_to_short() -> short\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 15\n"
+        "def int_to_long() -> long\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 16\n"
+        "def int_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 17\n"
+        "def long_to_byte() -> byte\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 18\n"
+        "def long_to_short() -> short\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 19\n"
+        "def long_to_int() -> int\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 20\n"
+        "def long_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 21\n"
+        "def ssize_t_to_byte() -> byte\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 22\n"
+        "def ssize_t_to_short() -> short\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 23\n"
+        "def ssize_t_to_int() -> int\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 24\n"
+        "def ssize_t_to_long() -> long\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 25\n"
+        "def ubyte_val() -> ubyte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 26\n"
+        "def ushort_val() -> ushort\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 27\n"
+        "def uint_val() -> uint\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 28\n"
+        "def ulong_val() -> ulong\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 29\n"
+        "def size_t_val() -> size_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 30\n"
+        "def ubyte_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 31\n"
+        "def ubyte_to_uint() -> uint\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 32\n"
+        "def ubyte_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 33\n"
+        "def ubyte_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 34\n"
+        "def ushort_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 35\n"
+        "def ushort_to_uint() -> uint\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 36\n"
+        "def ushort_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 37\n"
+        "def ushort_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 38\n"
+        "def uint_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 39\n"
+        "def uint_to_ushort() -> ushort\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 40\n"
+        "def uint_to_ulong() -> ulong\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 41\n"
+        "def uint_to_size_t() -> size_t\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 42\n"
+        "def ulong_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 43\n"
+        "def ulong_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 44\n"
+        "def ulong_to_uint() -> uint\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 45\n"
+        "def ulong_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 46\n"
+        "def size_t_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 47\n"
+        "def size_t_to_ushort() -> ushort\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 48\n"
+        "def size_t_to_int() -> int\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 49\n"
+        "def size_t_to_ulong() -> ulong\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 50\n"
+        "def main() -> int\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n";
+    struct
+    {
+        const char * name;
+        p_token_t token;
+    } expected[] = {
+        {"byte_val", TOKEN_byte},
+        {"short_val", TOKEN_short},
+        {"int_val", TOKEN_int},
+        {"long_val", TOKEN_long},
+        {"ssize_t_val", TOKEN_ssize_t},
+        {"byte_to_short", TOKEN_short},
+        {"byte_to_int", TOKEN_int},
+        {"byte_to_long", TOKEN_long},
+        {"byte_to_ssize_t", TOKEN_ssize_t},
+        {"short_to_byte", TOKEN_byte},
+        {"short_to_int", TOKEN_int},
+        {"short_to_long", TOKEN_long},
+        {"short_to_ssize_t", TOKEN_ssize_t},
+        {"int_to_byte", TOKEN_byte},
+        {"int_to_short", TOKEN_short},
+        {"int_to_long", TOKEN_long},
+        {"int_to_ssize_t", TOKEN_ssize_t},
+        {"long_to_byte", TOKEN_byte},
+        {"long_to_short", TOKEN_short},
+        {"long_to_int", TOKEN_int},
+        {"long_to_ssize_t", TOKEN_ssize_t},
+        {"ssize_t_to_byte", TOKEN_byte},
+        {"ssize_t_to_short", TOKEN_short},
+        {"ssize_t_to_int", TOKEN_int},
+        {"ssize_t_to_long", TOKEN_long},
+        {"ubyte_val", TOKEN_ubyte},
+        {"ushort_val", TOKEN_ushort},
+        {"uint_val", TOKEN_uint},
+        {"ulong_val", TOKEN_ulong},
+        {"size_t_val", TOKEN_size_t},
+        {"ubyte_to_ushort", TOKEN_ushort},
+        {"ubyte_to_uint", TOKEN_uint},
+        {"ubyte_to_ulong", TOKEN_ulong},
+        {"ubyte_to_size_t", TOKEN_size_t},
+        {"ushort_to_ubyte", TOKEN_ubyte},
+        {"ushort_to_uint", TOKEN_uint},
+        {"ushort_to_ulong", TOKEN_ulong},
+        {"ushort_to_size_t", TOKEN_size_t},
+        {"uint_to_ubyte", TOKEN_ubyte},
+        {"uint_to_ushort", TOKEN_ushort},
+        {"uint_to_ulong", TOKEN_ulong},
+        {"uint_to_size_t", TOKEN_size_t},
+        {"ulong_to_ubyte", TOKEN_ubyte},
+        {"ulong_to_ushort", TOKEN_ushort},
+        {"ulong_to_uint", TOKEN_uint},
+        {"ulong_to_size_t", TOKEN_size_t},
+        {"size_t_to_ubyte", TOKEN_ubyte},
+        {"size_t_to_ushort", TOKEN_ushort},
+        {"size_t_to_int", TOKEN_int},
+        {"size_t_to_ulong", TOKEN_ulong},
+        {"main", TOKEN_int},
+    };
+    p_context_t context;
+    p_context_init(&context, (const uint8_t *)input, strlen(input));
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod->pModuleItems;
+    PFunctionDefinition ** pfds;
+    size_t n_pfds = 0u;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        if (pmi->pFunctionDefinition != NULL)
+        {
+            n_pfds++;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
+    pmis = pmod->pModuleItems;
+    size_t pfd_i = n_pfds;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        PFunctionDefinition * pfd = pmi->pFunctionDefinition;
+        if (pfd != NULL)
+        {
+            pfd_i--;
+            assert(pfd_i < n_pfds);
+            pfds[pfd_i] = pfd;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    assert_eq(51, n_pfds);
+    for (size_t i = 0; i < n_pfds; i++)
+    {
+        if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
+            (expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
+        {
+            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
+        }
+    }
+
+    free(pfds);
+    p_free_ast(pmod);
+
+    return 0;
+}

spec/test_ast_node_memory_remains.d

@@ -0,0 +1,408 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "
+# 0
+def byte_val() -> byte
+{
+    return 0x42;
+}
+
+# 1
+def short_val() -> short
+{
+    return 0x4242;
+}
+
+# 2
+def int_val() -> int
+{
+    return 0x42424242;
+}
+
+# 3
+def long_val() -> long
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 4
+def ssize_t_val() -> ssize_t
+{
+    return 0x42424242;
+}
+
+# 5
+def byte_to_short() -> short
+{
+    return byte_val();
+}
+
+# 6
+def byte_to_int() -> int
+{
+    return byte_val();
+}
+
+# 7
+def byte_to_long() -> long
+{
+    return byte_val();
+}
+
+# 8
+def byte_to_ssize_t() -> ssize_t
+{
+    return byte_val();
+}
+
+# 9
+def short_to_byte() -> byte
+{
+    return short_val();
+}
+
+# 10
+def short_to_int() -> int
+{
+    return short_val();
+}
+
+# 11
+def short_to_long() -> long
+{
+    return short_val();
+}
+
+# 12
+def short_to_ssize_t() -> ssize_t
+{
+    return short_val();
+}
+
+# 13
+def int_to_byte() -> byte
+{
+    return int_val();
+}
+
+# 14
+def int_to_short() -> short
+{
+    return int_val();
+}
+
+# 15
+def int_to_long() -> long
+{
+    return int_val();
+}
+
+# 16
+def int_to_ssize_t() -> ssize_t
+{
+    return int_val();
+}
+
+# 17
+def long_to_byte() -> byte
+{
+    return long_val();
+}
+
+# 18
+def long_to_short() -> short
+{
+    return long_val();
+}
+
+# 19
+def long_to_int() -> int
+{
+    return long_val();
+}
+
+# 20
+def long_to_ssize_t() -> ssize_t
+{
+    return long_val();
+}
+
+# 21
+def ssize_t_to_byte() -> byte
+{
+    return ssize_t_val();
+}
+
+# 22
+def ssize_t_to_short() -> short
+{
+    return ssize_t_val();
+}
+
+# 23
+def ssize_t_to_int() -> int
+{
+    return ssize_t_val();
+}
+
+# 24
+def ssize_t_to_long() -> long
+{
+    return ssize_t_val();
+}
+
+# 25
+def ubyte_val() -> ubyte
+{
+    return 0x42;
+}
+
+# 26
+def ushort_val() -> ushort
+{
+    return 0x4242;
+}
+
+# 27
+def uint_val() -> uint
+{
+    return 0x42424242;
+}
+
+# 28
+def ulong_val() -> ulong
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 29
+def size_t_val() -> size_t
+{
+    return 0x42424242;
+}
+
+# 30
+def ubyte_to_ushort() -> ushort
+{
+    return ubyte_val();
+}
+
+# 31
+def ubyte_to_uint() -> uint
+{
+    return ubyte_val();
+}
+
+# 32
+def ubyte_to_ulong() -> ulong
+{
+    return ubyte_val();
+}
+
+# 33
+def ubyte_to_size_t() -> size_t
+{
+    return ubyte_val();
+}
+
+# 34
+def ushort_to_ubyte() -> ubyte
+{
+    return ushort_val();
+}
+
+# 35
+def ushort_to_uint() -> uint
+{
+    return ushort_val();
+}
+
+# 36
+def ushort_to_ulong() -> ulong
+{
+    return ushort_val();
+}
+
+# 37
+def ushort_to_size_t() -> size_t
+{
+    return ushort_val();
+}
+
+# 38
+def uint_to_ubyte() -> ubyte
+{
+    return uint_val();
+}
+
+# 39
+def uint_to_ushort() -> ushort
+{
+    return uint_val();
+}
+
+# 40
+def uint_to_ulong() -> ulong
+{
+    return uint_val();
+}
+
+# 41
+def uint_to_size_t() -> size_t
+{
+    return uint_val();
+}
+
+# 42
+def ulong_to_ubyte() -> ubyte
+{
+    return ulong_val();
+}
+
+# 43
+def ulong_to_ushort() -> ushort
+{
+    return ulong_val();
+}
+
+# 44
+def ulong_to_uint() -> uint
+{
+    return ulong_val();
+}
+
+# 45
+def ulong_to_size_t() -> size_t
+{
+    return ulong_val();
+}
+
+# 46
+def size_t_to_ubyte() -> ubyte
+{
+    return size_t_val();
+}
+
+# 47
+def size_t_to_ushort() -> ushort
+{
+    return size_t_val();
+}
+
+# 48
+def size_t_to_int() -> int
+{
+    return size_t_val();
+}
+
+# 49
+def size_t_to_ulong() -> ulong
+{
+    return size_t_val();
+}
+
+# 50
+def main() -> int
+{
+    return int_val();
+}
+";
+    struct Expected
+    {
+        string name;
+        p_token_t token;
+    }
+    Expected[] expected = [
+        Expected("byte_val", TOKEN_byte),
+        Expected("short_val", TOKEN_short),
+        Expected("int_val", TOKEN_int),
+        Expected("long_val", TOKEN_long),
+        Expected("ssize_t_val", TOKEN_ssize_t),
+        Expected("byte_to_short", TOKEN_short),
+        Expected("byte_to_int", TOKEN_int),
+        Expected("byte_to_long", TOKEN_long),
+        Expected("byte_to_ssize_t", TOKEN_ssize_t),
+        Expected("short_to_byte", TOKEN_byte),
+        Expected("short_to_int", TOKEN_int),
+        Expected("short_to_long", TOKEN_long),
+        Expected("short_to_ssize_t", TOKEN_ssize_t),
+        Expected("int_to_byte", TOKEN_byte),
+        Expected("int_to_short", TOKEN_short),
+        Expected("int_to_long", TOKEN_long),
+        Expected("int_to_ssize_t", TOKEN_ssize_t),
+        Expected("long_to_byte", TOKEN_byte),
+        Expected("long_to_short", TOKEN_short),
+        Expected("long_to_int", TOKEN_int),
+        Expected("long_to_ssize_t", TOKEN_ssize_t),
+        Expected("ssize_t_to_byte", TOKEN_byte),
+        Expected("ssize_t_to_short", TOKEN_short),
+        Expected("ssize_t_to_int", TOKEN_int),
+        Expected("ssize_t_to_long", TOKEN_long),
+        Expected("ubyte_val", TOKEN_ubyte),
+        Expected("ushort_val", TOKEN_ushort),
+        Expected("uint_val", TOKEN_uint),
+        Expected("ulong_val", TOKEN_ulong),
+        Expected("size_t_val", TOKEN_size_t),
+        Expected("ubyte_to_ushort", TOKEN_ushort),
+        Expected("ubyte_to_uint", TOKEN_uint),
+        Expected("ubyte_to_ulong", TOKEN_ulong),
+        Expected("ubyte_to_size_t", TOKEN_size_t),
+        Expected("ushort_to_ubyte", TOKEN_ubyte),
+        Expected("ushort_to_uint", TOKEN_uint),
+        Expected("ushort_to_ulong", TOKEN_ulong),
+        Expected("ushort_to_size_t", TOKEN_size_t),
+        Expected("uint_to_ubyte", TOKEN_ubyte),
+        Expected("uint_to_ushort", TOKEN_ushort),
+        Expected("uint_to_ulong", TOKEN_ulong),
+        Expected("uint_to_size_t", TOKEN_size_t),
+        Expected("ulong_to_ubyte", TOKEN_ubyte),
+        Expected("ulong_to_ushort", TOKEN_ushort),
+        Expected("ulong_to_uint", TOKEN_uint),
+        Expected("ulong_to_size_t", TOKEN_size_t),
+        Expected("size_t_to_ubyte", TOKEN_ubyte),
+        Expected("size_t_to_ushort", TOKEN_ushort),
+        Expected("size_t_to_int", TOKEN_int),
+        Expected("size_t_to_ulong", TOKEN_ulong),
+        Expected("main", TOKEN_int),
+    ];
+    p_context_t context;
+    p_context_init(&context, input);
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod.pModuleItems;
+    PFunctionDefinition *[] pfds;
+    while (pmis !is null)
+    {
+        PModuleItem * pmi = pmis.pModuleItem;
+        if (pmi is null)
+        {
+            stderr.writeln("pmi is null!!!?");
+            assert(0);
+        }
+        PFunctionDefinition * pfd = pmi.pFunctionDefinition;
+        if (pfd !is null)
+        {
+            pfds = [pfd] ~ pfds;
+        }
+        pmis = pmis.pModuleItems;
+    }
+    assert_eq(51, pfds.length);
+    for (size_t i = 0; i < pfds.length; i++)
+    {
+        if ((expected[i].name != pfds[i].name.pvalue.s) ||
+            (expected[i].token != pfds[i].returntype.pType.pTypeBase.pToken1.token))
+        {
+            stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
+        }
+    }
+}

spec/test_ast_ps.c

@@ -33,12 +33,16 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_free_ast(start);
+
     input = "";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
     start = p_result(&context);
     assert(start->pItems == NULL);
 
+    p_free_ast(start);
+
     input = "2 1";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
@@ -51,5 +55,7 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_token_positions.c

@@ -0,0 +1,88 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "abbccc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
+
+    assert_eq(1, start->pT2->pToken->position.row);
+    assert_eq(2, start->pT2->pToken->position.col);
+    assert_eq(1, start->pT2->pToken->end_position.row);
+    assert_eq(3, start->pT2->pToken->end_position.col);
+    assert_eq(1, start->pT2->position.row);
+    assert_eq(2, start->pT2->position.col);
+    assert_eq(1, start->pT2->end_position.row);
+    assert_eq(3, start->pT2->end_position.col);
+
+    assert_eq(1, start->pT3->pToken->position.row);
+    assert_eq(4, start->pT3->pToken->position.col);
+    assert_eq(1, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(1, start->pT3->position.row);
+    assert_eq(4, start->pT3->position.col);
+    assert_eq(1, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
+
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(6, start->end_position.col);
+
+    p_free_ast(start);
+
+    input = "\n\n  bb\nc\ncc\n\n     a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(3, start->pT1->pToken->position.row);
+    assert_eq(3, start->pT1->pToken->position.col);
+    assert_eq(3, start->pT1->pToken->end_position.row);
+    assert_eq(4, start->pT1->pToken->end_position.col);
+    assert_eq(3, start->pT1->position.row);
+    assert_eq(3, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(4, start->pT1->end_position.col);
+
+    assert_eq(4, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(5, start->pT2->pToken->end_position.row);
+    assert_eq(2, start->pT2->pToken->end_position.col);
+    assert_eq(4, start->pT2->position.row);
+    assert_eq(1, start->pT2->position.col);
+    assert_eq(5, start->pT2->end_position.row);
+    assert_eq(2, start->pT2->end_position.col);
+
+    assert_eq(7, start->pT3->pToken->position.row);
+    assert_eq(6, start->pT3->pToken->position.col);
+    assert_eq(7, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(7, start->pT3->position.row);
+    assert_eq(6, start->pT3->position.col);
+    assert_eq(7, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
+
+    assert_eq(3, start->position.row);
+    assert_eq(3, start->position.col);
+    assert_eq(7, start->end_position.row);
+    assert_eq(6, start->end_position.col);
+
+    p_free_ast(start);
+
+    return 0;
+}

spec/test_ast_token_positions.d

@@ -0,0 +1,86 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abbccc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
+
+    assert_eq(1, start.pT2.pToken.position.row);
+    assert_eq(2, start.pT2.pToken.position.col);
+    assert_eq(1, start.pT2.pToken.end_position.row);
+    assert_eq(3, start.pT2.pToken.end_position.col);
+    assert_eq(1, start.pT2.position.row);
+    assert_eq(2, start.pT2.position.col);
+    assert_eq(1, start.pT2.end_position.row);
+    assert_eq(3, start.pT2.end_position.col);
+
+    assert_eq(1, start.pT3.pToken.position.row);
+    assert_eq(4, start.pT3.pToken.position.col);
+    assert_eq(1, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(1, start.pT3.position.row);
+    assert_eq(4, start.pT3.position.col);
+    assert_eq(1, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
+
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(6, start.end_position.col);
+
+    input = "\n\n  bb\nc\ncc\n\n     a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+
+    assert_eq(3, start.pT1.pToken.position.row);
+    assert_eq(3, start.pT1.pToken.position.col);
+    assert_eq(3, start.pT1.pToken.end_position.row);
+    assert_eq(4, start.pT1.pToken.end_position.col);
+    assert_eq(3, start.pT1.position.row);
+    assert_eq(3, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(4, start.pT1.end_position.col);
+
+    assert_eq(4, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(5, start.pT2.pToken.end_position.row);
+    assert_eq(2, start.pT2.pToken.end_position.col);
+    assert_eq(4, start.pT2.position.row);
+    assert_eq(1, start.pT2.position.col);
+    assert_eq(5, start.pT2.end_position.row);
+    assert_eq(2, start.pT2.end_position.col);
+
+    assert_eq(7, start.pT3.pToken.position.row);
+    assert_eq(6, start.pT3.pToken.position.col);
+    assert_eq(7, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(7, start.pT3.position.row);
+    assert_eq(6, start.pT3.position.col);
+    assert_eq(7, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
+
+    assert_eq(3, start.position.row);
+    assert_eq(3, start.position.col);
+    assert_eq(7, start.end_position.row);
+    assert_eq(6, start.end_position.col);
+}

spec/test_error_positions.c

@@ -12,28 +12,28 @@ int main()
     input = "a\n123\na  a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 3);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 4);
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 0);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 1);
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 2);
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 5);
-    assert(p_position(&context).col == 4);
+    assert(p_position(&context).row == 6);
+    assert(p_position(&context).col == 5);
 
     assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
     assert(strcmp(p_token_names[TOKEN_num], "num") == 0);

spec/test_error_positions.d

@@ -16,24 +16,24 @@ unittest
     input = "a\n123\na  a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
+    assert(p_position(&context) == p_position_t(3, 4));
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
+    assert(p_position(&context) == p_position_t(1, 1));
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
+    assert(p_position(&context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(5, 4));
+    assert(p_position(&context) == p_position_t(6, 5));
 
     assert(p_token_names[TOKEN_a] == "a");
     assert(p_token_names[TOKEN_num] == "num");

spec/test_field_aliases.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "foo1\nbar2";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    return 0;
+}

spec/test_field_aliases.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "foo1\nbar2";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_free_ast_token_node_memory.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->a != NULL);
+    assert(*start->a->pvalue == 1);
+    assert(start->b != NULL);
+    assert(*start->b->pvalue == 2);
+
+    p_free_ast(start);
+}

spec/test_lexer.c

@@ -41,50 +41,68 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 2u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 3u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 4u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 6u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 0u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 4u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 6u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 9u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 10u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 
     p_context_init(&context, (uint8_t const *)"", 0u);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 

spec/test_lexer.d

@@ -47,23 +47,23 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
+    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
     p_context_init(&context, "");
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_lexer_multiple_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc.def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc .  abc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_multiple_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc.def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc .  abc`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_named_optional_rule_component_ast.c

@@ -0,0 +1,51 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+    assert(start->r == NULL);
+
+    p_free_ast(start);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->r != NULL);
+    assert(start->pR == start->pR3);
+    assert(start->pR == start->r);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    p_free_ast(start);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->r != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    p_free_ast(start);
+
+    return 0;
+}
+

spec/test_named_optional_rule_component_ast.d

@@ -0,0 +1,46 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+    assert(start.r is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.r != null);
+    assert(start.pR == start.pR3);
+    assert(start.pR == start.r);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_optional_rule_component_ast.c

@@ -16,6 +16,8 @@ int main()
     assert(start->pR3 == NULL);
     assert(start->pR == NULL);
 
+    p_free_ast(start);
+
     input = "abcd";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -28,6 +30,8 @@ int main()
     assert(start->pR == start->pR3);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     input = "bdc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -37,6 +41,8 @@ int main()
     assert(start->pR != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     return 0;
 }
 

spec/test_parser_rule_from_multiple_states.c

@@ -8,8 +8,8 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 2);
     assert(context.token == TOKEN___EOF);
 
     input = "a b";

spec/test_parser_rule_from_multiple_states.d

@@ -12,7 +12,7 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
+    assert(p_position(&context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";

spec/test_start_rule_ast.c

@@ -13,5 +13,7 @@ int main()
     assert(top->pToken != NULL);
     assert_eq(TOKEN_hi, top->pToken->token);
 
+    p_free_ast(top);
+
     return 0;
 }

spec/testutils.c

@@ -17,7 +17,7 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 void str_init(str_t * str, char const * cs)
 {
     size_t length = strlen(cs);
-    str->cs = malloc(length + 1u);
+    str->cs = (char *)malloc(length + 1u);
     strcpy(str->cs, cs);
 }
 
@@ -25,7 +25,7 @@ void str_append(str_t * str, char const * cs)
 {
     size_t length = strlen(str->cs);
     size_t length2 = strlen(cs);
-    char * new_cs = malloc(length + length2 + 1u);
+    char * new_cs = (char *)malloc(length + length2 + 1u);
     memcpy(new_cs, str->cs, length);
     strcpy(&new_cs[length], cs);
     free(str->cs);