wip

This issue manifested when multiple ptypes are present with different sizes.

.github/workflows/run-tests.yml

@@ -0,0 +1,38 @@
+name: Run Propane Tests
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ruby-version: ['3.4']
+
+    steps:
+    - name: Install dependencies (Linux)
+      if: runner.os == 'Linux'
+      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
+
+    - name: Install dependencies (macOS)
+      if: runner.os == 'macOS'
+      run: brew install gcc ldc
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests
+      run: rake all

.rspec

@@ -1,3 +1,2 @@
---format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,3 +1,47 @@
+## v2.3.0
+
+### New Features
+
+- Add \D, \S, \w, \W special character classes
+
+### Improvements
+
+- Include line numbers for pattern errors
+- Improve performance in a few places
+- Parallelize parser table generation on Linux hosts
+- Add github workflow to run unit tests
+
+### Fixes
+
+- Fix a couple clang warnings for C backend
+- Fix C backend not fully initializing pvalues when multiple ptypes are used with different sizes.
+- Fix some user guide examples
+
+## v2.2.1
+
+### Fixes
+
+- Fix GC issue for D backend when AST is enabled (#36)
+
+## v2.2.0
+
+### Improvements
+
+- Allow multiple lexer modes to be specified for a lexer pattern (#35)
+- Document p_decode_code_point() API function (#34)
+
+## v2.1.1
+
+### Fixes
+
+- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
+
+## v2.1.0
+
+### Improvements
+
+- Report rule name and line number for conflicting AST node field positions errors (#32)
+
 ## v2.0.0
 
 ### Improvements

Gemfile

@@ -1,5 +1,6 @@
 source "https://rubygems.org"
 
+gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"

Gemfile.lock

@@ -1,40 +1,48 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.5.0)
+    base64 (0.3.0)
-    docile (1.4.0)
+    date (3.5.1)
-    psych (5.1.0)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    erb (6.0.1)
+    psych (5.3.1)
+      date
       stringio
-    rake (13.0.6)
+    rake (13.3.1)
-    rdoc (6.5.0)
+    rdoc (7.1.0)
+      erb
       psych (>= 4.0.0)
-    redcarpet (3.6.0)
+      tsort
-    rspec (3.12.0)
+    redcarpet (3.6.1)
-      rspec-core (~> 3.12.0)
+    rspec (3.13.2)
-      rspec-expectations (~> 3.12.0)
+      rspec-core (~> 3.13.0)
-      rspec-mocks (~> 3.12.0)
+      rspec-expectations (~> 3.13.0)
-    rspec-core (3.12.2)
+      rspec-mocks (~> 3.13.0)
-      rspec-support (~> 3.12.0)
+    rspec-core (3.13.6)
-    rspec-expectations (3.12.3)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
+      rspec-support (~> 3.13.0)
-    rspec-mocks (3.12.6)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
+      rspec-support (~> 3.13.0)
-    rspec-support (3.12.1)
+    rspec-support (3.13.6)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    stringio (3.0.7)
+    stringio (3.2.0)
     syntax (1.2.2)
+    tsort (0.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  base64
   rake
   rdoc
   redcarpet

README.md

@@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C or D language outputs
+  * targets C, C++, or D language outputs
   * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script

Rakefile.rb

@@ -1,5 +1,7 @@
 require "rake/clean"
 require "rspec/core/rake_task"
+require "simplecov"
+require "stringio"
 
 CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
 
@@ -12,6 +14,18 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
   end
 end
+task :spec do |task, args|
+  unless ENV["dist_specs"]
+    original_stdout = $stdout
+    sio = StringIO.new
+    $stdout = sio
+    SimpleCov.collate Dir["coverage/.resultset.json"]
+    $stdout = original_stdout
+    sio.string.lines.each do |line|
+      $stdout.write(line) unless line =~ /Coverage report generated for/
+    end
+  end
+end
 
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.

assets/parser.c.erb

@@ -55,7 +55,8 @@ const char * <%= @grammar.prefix %>token_names[] = {
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
 {
     /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t newcontext = {0};
+    <%= @grammar.prefix %>context_t newcontext;
+    memset(&newcontext, 0, sizeof(newcontext));
 
     /* Lexer initialization. */
     newcontext.input = input;
@@ -344,8 +345,10 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
     lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
 {
-    lexer_match_info_t longest_match = {0};
+    lexer_match_info_t longest_match;
-    lexer_match_info_t attempt_match = {0};
+    memset(&longest_match, 0, sizeof(longest_match));
+    lexer_match_info_t attempt_match;
+    memset(&attempt_match, 0, sizeof(attempt_match));
     *out_match_info = longest_match;
     uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
     for (;;)
@@ -359,6 +362,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
         switch (result)
         {
         case P_SUCCESS:
+            {
             lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
@@ -390,6 +394,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 *out_unexpected_input_length = attempt_match.length + code_point_length;
                 return P_UNEXPECTED_INPUT;
             }
+            }
             break;
 
         case P_EOF:
@@ -447,7 +452,8 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
-    <%= @grammar.prefix %>token_info_t token_info = {0};
+    <%= @grammar.prefix %>token_info_t token_info;
+    memset(&token_info, 0, sizeof(token_info));
     token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
     lexer_match_info_t match_info;
@@ -456,6 +462,7 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     switch (result)
     {
     case P_SUCCESS:
+        {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
@@ -507,6 +514,7 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
         }
         *out_token_info = token_info;
+        }
         return P_SUCCESS;
 
     case P_EOF:
@@ -687,11 +695,13 @@ typedef struct
 } state_value_t;
 
 /** Common AST node structure. */
-typedef struct
+typedef struct ASTNode_s
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
-    void * fields[];
+    uint16_t n_fields;
+    uint8_t is_token;
+    struct ASTNode_s * fields[];
 } ASTNode;
 
 /** Parser shift table. */
@@ -712,17 +722,22 @@ const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    {
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , NULL
+        NULL, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u},
 <%     end %>
-    },
 <%   end %>
 };
 
@@ -792,7 +807,7 @@ static void state_values_stack_push(state_values_stack_t * stack)
     if (current_length >= current_capacity)
     {
         size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
+        state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
         memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
@@ -959,7 +974,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
+                context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -976,9 +991,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
                 token_ast_node->position = token_info.position;
                 token_ast_node->end_position = token_info.end_position;
+                token_ast_node->n_fields = 0u;
+                token_ast_node->is_token = 1u;
                 token_ast_node->token = token;
                 token_ast_node->pvalue = token_info.pvalue;
                 state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@@ -994,7 +1011,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                 state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
-                <%= @grammar.prefix %>value_t new_parse_result = {0};
+                <%= @grammar.prefix %>value_t new_parse_result;
+                memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
 <% end %>
                 reduced_rule_set = INVALID_ID;
@@ -1014,31 +1032,30 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                ASTNode * node = (ASTNode *)malloc(sizeof(ASTNode) + n_fields * sizeof(void *));
+                size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *);
+                ASTNode * node = (ASTNode *)malloc(bytes);
+                memset(node, 0, bytes);
                 node->position = INVALID_POSITION;
                 node->end_position = INVALID_POSITION;
-                for (size_t i = 0; i < n_fields; i++)
+                node->n_fields = n_fields;
-                {
-                    node->fields[i] = NULL;
-                }
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                     }
                 }
                 else
                 {
                     for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
                     {
-                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                        node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
                     }
                 }
                 bool position_found = false;
                 for (size_t i = 0; i < n_fields; i++)
                 {
-                    ASTNode * child = (ASTNode *)node->fields[i];
+                    ASTNode * child = node->fields[i];
                     if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
                     {
                         if (!position_found)
@@ -1056,9 +1073,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                 reduced_parser_node = NULL;
             }
 <% else %>
-            <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
+            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
             {
+                state_values_stack_free(&statevalues);
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
@@ -1091,13 +1110,13 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context->parse_result;
+    return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result;
 <% else %>
     return context->parse_result.v_<%= start_rule_type[0] %>;
 <% end %>
@@ -1138,3 +1157,35 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->token;
 }
+<% if @grammar.ast %>
+
+static void free_ast_node(ASTNode * node)
+{
+    if (node->is_token)
+    {
+<%   if @grammar.free_token_node %>
+        <%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
+<%   end %>
+        /* TODO: free value_t */
+    }
+    else if (node->n_fields > 0u)
+    {
+        for (size_t i = 0u; i < node->n_fields; i++)
+        {
+            if (node->fields[i] != NULL)
+            {
+                free_ast_node(node->fields[i]);
+            }
+        }
+    }
+    free(node);
+}
+
+/**
+ * Free all AST node memory.
+ */
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast)
+{
+    free_ast_node((ASTNode *)ast);
+}
+<% end %>

assets/parser.d.erb

@@ -8,6 +8,7 @@
 module <%= @grammar.modulename %>;
 <% end %>
 
+import core.memory;
 import core.stdc.stdlib : malloc;
 
 /**************************************************************************
@@ -172,7 +173,7 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -875,17 +876,22 @@ immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    reduce_t(
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , null
+        null, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u), /* Number of states */
 <%     end %>
-            ),
 <%   end %>
 ];
 
@@ -1025,7 +1031,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
+                context.parse_result = statevalues[$-1].ast_node;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -1074,7 +1080,9 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                ASTNode * node = cast(ASTNode *)malloc(ASTNode.sizeof + n_fields * (void *).sizeof);
+                size_t node_size = ASTNode.sizeof + n_fields * (void *).sizeof;
+                ASTNode * node = cast(ASTNode *)malloc(node_size);
+                GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
                 foreach (i; 0..n_fields)
@@ -1148,13 +1156,13 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>
 {
 <% if @grammar.ast %>
-    return context.parse_result;
+    return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result;
 <% else %>
     return context.parse_result.v_<%= start_rule_type[0] %>;
 <% end %>

assets/parser.h.erb

@@ -75,9 +75,11 @@ typedef union
 /** AST node types. @{ */
 typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
-    /* ASTNode fields must be present in the same order here. */
+    <% # ASTNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
@@ -93,8 +95,11 @@ struct <%= name %>;
 <%     next if rule_set.optional? %>
 typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
+    <% # ASTNode fields must be present in the same order here. # %>
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    uint16_t n_fields;
+    uint8_t is_token;
 <%     rule_set.ast_fields.each do |fields| %>
     union
     {
@@ -157,7 +162,7 @@ typedef struct
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+    void * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -186,7 +191,9 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 
 <% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+
+void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <% end %>

doc/user_guide.md

@@ -13,7 +13,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * targets C or D language outputs
+  * targets C, C++, or D language outputs
   * optionally supports automatic full AST generation
   * tracks input text start and end positions for all matched tokens/rules
   * is MIT-licensed
@@ -67,10 +67,10 @@ import std.math;
 ptype ulong;
 
 # A few basic arithmetic operators.
-token plus /\\+/;
+token plus /\+/;
-token times /\\*/;
+token times /\*/;
-token power /\\*\\*/;
+token power /\*\*/;
-token integer /\\d+/ <<
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -79,10 +79,10 @@ token integer /\\d+/ <<
   }
   $$ = v;
 >>
-token lparen /\\(/;
+token lparen /\(/;
-token rparen /\\)/;
+token rparen /\)/;
 # Drop whitespace.
-drop /\\s+/;
+drop /\s+/;
 
 Start -> E1 << $$ = $1; >>
 E1 -> E2 << $$ = $1; >>
@@ -155,7 +155,7 @@ Example:
 ```
 ptype ulong;
 
-token integer /\\d+/ <<
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -228,9 +228,9 @@ token two /2/;
 token comma /,/ <<
   $$ = 42;
 >>
-token lparen /\\(/;
+token lparen /\(/;
-token rparen /\\)/;
+token rparen /\)/;
-drop /\\s+/;
+drop /\s+/;
 
 Start -> Items;
 
@@ -326,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null);
 assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
 ```
 
+## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
+
+If user lexer code block allocates memory to store in a token node's `pvalue`,
+the `free_token_node` grammar statement can be used to specify the name of a
+function which will be called during the `p_free_ast()` call to free the memory
+associated with a token node.
+
+Example:
+
+```
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+```
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -432,10 +462,10 @@ A regular expression begins and ends with a `/` character.
 Example:
 
 ```
-/#.*$/
+/#.*/
 ```
 
-Regular expressions can include many special characters:
+Regular expressions can include many special characters/sequences:
 
   * The `.` character matches any input character other than a newline.
   * The `*` character matches any number of the previous regex element.
@@ -447,14 +477,17 @@ Regular expressions can include many special characters:
   * The `\` character escapes the following character and changes its meaning:
     * The `\a` sequence matches an ASCII bell character (0x07).
     * The `\b` sequence matches an ASCII backspace character (0x08).
-    * The `\d` sequence matches any character `0` through `9`.
+    * The `\d` sequence is shorthand for the `[0-9]` character class.
+    * The `\D` sequence matches every code point not matched by `\d`.
     * The `\f` sequence matches an ASCII form feed character (0x0C).
     * The `\n` sequence matches an ASCII new line character (0x0A).
     * The `\r` sequence matches an ASCII carriage return character (0x0D).
-    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
+    * The `\s` sequence is shorthand for the `[ \t\r\n\f\v]` character class.
-    `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * The `\S` sequence matches every code point not matched by `\s`.
     * The `\t` sequence matches an ASCII tab character (0x09).
     * The `\v` sequence matches an ASCII vertical tab character (0x0B).
+    * The `\w` sequence is shorthand for the `[a-zA-Z0-9_]` character class.
+    * The `\W` sequence matches every code point not matched by `\w`.
     * Any other character matches itself.
   * The `|` character creates an alternate match.
 
@@ -536,6 +569,28 @@ It also returns the `str` token now that the token is complete.
 Note that the token name `str` above could have been `string` instead - the
 namespace for token names is distinct from the namespace for lexer modes.
 
+Multiple modes can be specified for a token or pattern or drop statement.
+For example, if the grammar wanted to only recognize an identifier following
+a `.` token and not other keywords, it could switch to an `identonly` mode
+when matching a `.`
+The `ident` token pattern will be matched in either the `default` or
+`identonly` mode.
+
+```
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\s+/;
+```
+
 ##> Specifying parser value types - the `ptype` statement
 
 The `ptype` statement is used to define parser value type(s).
@@ -726,7 +781,7 @@ Some example uses of this functionality could be to:
 
   * Detect integer overflow when lexing an integer literal constant.
   * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
-  * Determine whether parsing should stop and instead be performed using a different parser version.
+  * Determine whether parsing should stop and instead be retried using a different parser version.
 
 To terminate parsing from a lexer or parser user code block, use the
 `$terminate(code)` function, passing an integer expression argument.
@@ -762,10 +817,16 @@ Propane generates the following result code constants:
 * `P_EOF`: The lexer reached the end of the input string.
 * `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
 
-Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
+Result codes are returned by the API functions `p_decode_code_point()`, `p_lex()`, and `p_parse()`.
 
 ##> Types
 
+### `p_code_point_t`
+
+The `p_code_point_t` type is aliased to a 32-bit unsigned integer.
+It is used to store decoded code points from the input text and perform
+lexing based on the grammar's lexer patterns.
+
 ### `p_context_t`
 
 Propane defines a `p_context_t` structure type.
@@ -776,7 +837,7 @@ A pointer to this instance is passed to the generated functions.
 
 ### `p_position_t`
 
-The `p_position_t` structure contains two fields `row` and `col`.
+The `p_position_t` structure contains two fields: `row` and `col`.
 These fields contain the 1-based row and column describing a parser position.
 
 For D targets, the `p_position_t` structure can be checked for validity by
@@ -786,6 +847,16 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_token_info_t`
+
+The `p_token_info_t` structure contains the following fields:
+
+* `position` (`p_position_t`) holds the text position of the first code point in the token.
+* `end_position` (`p_position_t`) holds the text position of the last code point in the token.
+* `length` (`size_t`) holds the number of input bytes used by the token.
+* `token` (`p_token_t`) holds the token ID of the lexed token
+* `pvalue` (`p_value_t`) holds the parser value associated with the token.
+
 ### AST Node Types
 
 If AST generation mode is enabled, a structure type for each rule will be
@@ -896,6 +967,44 @@ p_context_t context;
 p_context_init(&context, input);
 ```
 
+### `p_lex`
+
+The `p_lex()` function is the main entry point to the lexer.
+It is normally called automatically by the generated parser to retrieve the
+next input token for the parser and does not need to be called by the user.
+However, the user may initialize a context and call `p_lex()` to use the
+generated lexer in a standalone mode.
+
+Example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+p_token_info_t token_info;
+size_t result = p_lex(&context, &token_info);
+switch (result)
+{
+case P_DECODE_ERROR:
+    /* UTF-8 decode error */
+    break;
+case P_UNEXPECTED_INPUT:
+    /* Input text does not match any lexer pattern. */
+    break;
+case P_USER_TERMINATED:
+    /* Lexer user code block requested to terminate the lexer. */
+    break;
+case P_SUCCESS:
+    /*
+     * token_info.position holds the text position of the first code point in the token.
+     * token_info.end_position holds the text position of the last code point in the token.
+     * token_info.length holds the number of input bytes used by the token.
+     * token_info.token holds the token ID of the lexed token
+     * token_info.pvalue holds the parser value associated with the token.
+     */
+    break;
+}
+```
+
 ### `p_parse`
 
 The `p_parse()` function is the main entry point to the parser.
@@ -999,6 +1108,40 @@ if (p_parse(&context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_decode_code_point`
+
+The `p_decode_code_point()` function can be used to decode code points from a
+UTF-8 string.
+It does not require a lexer/parser context structure and can be used as a
+standalone UTF-8 decoder or from within a lexer or parser user code block.
+
+D Example:
+
+```
+size_t result;
+p_code_point_t code_point;
+ubyte code_point_length;
+
+result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+assert(result == P_SUCCESS);
+assert(code_point == 0x1F9E1u);
+assert(code_point_length == 4u);
+```
+
+### `p_free_ast`
+
+The `p_free_ast()` function can be used to free the memory used by the AST.
+It should be passed the same value that is returned by `p_result()`.
+
+The `p_free_ast()` function is only available for C/C++ output targets.
+
+Note that if any lexer user code block allocates memory to store in a token's
+`pvalue`, in order to properly free this memory a `free_token_node` function
+should be specified in the grammar file.
+If specified, the `free_token_node` function will be called during the
+`p_free_ast()` process to allow user code to free any memory associated with
+a token node's `pvalue`.
+
 ##> Data
 
 ### `p_token_names`

extra/vim/syntax/propane.vim

@@ -22,7 +22,7 @@ syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
 syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
 
-syn region propaneRegex start="/" end="/" skip="\\/"
+syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword

lib/propane/generator.rb

@@ -11,24 +11,28 @@ class Propane
         @log = StringIO.new
       end
       @language =
-        if output_file =~ /\.([a-z]+)$/
+        if output_file.end_with?(".d")
-          $1
-        else
           "d"
+        else
+          "c"
         end
       @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [@language]
+      extensions = [nil]
       if @language == "c"
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template = Assets.get("parser.#{extension}.erb")
+        template = Assets.get("parser.#{extension || @language}.erb")
+        if extension
+          output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
+        else
+          output_file = @output_file
+        end
         erb = ERB.new(template, trim_mode: "<>")
-        output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
         result = erb.result(binding.clone)
         File.open(output_file, "wb") do |fh|
           fh.write(result)
@@ -43,8 +47,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.mode.nil?
+        if pattern.modes.empty?
-          pattern.mode = "default"
+          pattern.modes << "default"
           found_default = true
         end
         pattern.ptypename ||= "default"
@@ -67,12 +71,14 @@ class Propane
         end
         tokens_by_name[token.name] = token
       end
-      # Check for user start rule.
+      # Check for user start rules.
-      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
+      @grammar.start_rules.each_with_index do |start_rule, i|
-        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
+        unless @grammar.rules.find {|rule| rule.name == start_rule}
+          raise Error.new("Start rule `#{start_rule}` not found")
+        end
+        # Add "real" start rule.
+        @grammar.rules.unshift(Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil))
       end
-      # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
       # Generate and add rules for optional components.
       generate_optional_component_rules!(tokens_by_name)
       # Build rule sets.
@@ -328,9 +334,9 @@ class Propane
     #
     # @return [Array<String>]
     #   Start rule parser value type name and type string.
-    def start_rule_type
+    def start_rule_type(start_rule_index = 0)
       start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rule
+        rule.name == @grammar.start_rules[start_rule_index]
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -8,10 +8,11 @@ class Propane
     attr_reader :ast
     attr_reader :ast_prefix
     attr_reader :ast_suffix
+    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
-    attr_reader :start_rule
+    attr_reader :start_rules
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
@@ -19,20 +20,22 @@ class Propane
 
     def initialize(input)
       @patterns = []
-      @start_rule = "Start"
+      @start_rules = []
       @tokens = []
       @rules = []
       @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
-      @mode = nil
+      @modeline = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
       @ast = false
       @ast_prefix = ""
       @ast_suffix = ""
+      @free_token_node = nil
       parse_grammar!
+      @start_rules << "Start" if @start_rules.empty?
     end
 
     def ptype
@@ -58,10 +61,11 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @mode.nil? && parse_mode_label!
+      elsif @modeline.nil? && parse_mode_label!
       elsif parse_ast_statement!
       elsif parse_ast_prefix_statement!
       elsif parse_ast_suffix_statement!
+      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
@@ -81,8 +85,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
+      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
-        @mode = md[1]
+        @modeline = md[1]
       end
     end
 
@@ -112,12 +116,18 @@ class Propane
       end
     end
 
+    def parse_free_token_node_statement!
+      if md = consume!(/free_token_node\s+(\w+)\s*;/)
+        @free_token_node = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -153,9 +163,9 @@ class Propane
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
         @patterns << pattern
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -173,7 +183,7 @@ class Propane
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -186,8 +196,8 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -208,7 +218,7 @@ class Propane
           end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -225,15 +235,15 @@ class Propane
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
 
     def parse_start_statement!
       if md = consume!(/start\s+(\w+)\s*;/)
-        @start_rule = md[1]
+        @start_rules << md[1] unless @start_rules.include?(md[1])
       end
     end
 
@@ -247,7 +257,7 @@ class Propane
         else
           @code_blocks[name] = code
         end
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -315,6 +325,14 @@ class Propane
       end
     end
 
+    def get_modes_from_modeline
+      if @modeline
+        Set[*@modeline.split(",").map(&:strip)]
+      else
+        Set.new
+      end
+    end
+
   end
 
 end

lib/propane/lexer.rb

@@ -26,8 +26,14 @@ class Propane
     private
 
     def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
+      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
-        pattern.mode
+        result + pattern.modes
+      end
+      @modes = modenames.reduce({}) do |result, modename|
+        result[modename] = @grammar.patterns.select do |pattern|
+          pattern.modes.include?(modename)
+        end
+        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -64,24 +64,13 @@ class Propane
       @shift_table = []
       @reduce_table = []
       @item_sets.each do |item_set|
-        shift_entries = item_set.next_symbols.map do |next_symbol|
-          state_id =
-            if next_symbol.name == "$EOF"
-              0
-            else
-              item_set.next_item_set[next_symbol].id
-            end
-          {
-            symbol: next_symbol,
-            state_id: state_id,
-          }
-        end
         unless item_set.reduce_rules.empty?
-          shift_entries.each do |shift_entry|
+          item_set.shift_entries.each do |shift_entry|
             token = shift_entry[:symbol]
-            if get_lookahead_reduce_actions_for_item_set(item_set).include?(token)
+            if item_set.reduce_actions
-              rule = item_set.reduce_actions[token]
+              if rule = item_set.reduce_actions[token]
-              @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+                @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+              end
             end
           end
         end
@@ -92,7 +81,7 @@ class Propane
               propagate_optional_target: rule.optional? && rule.components.size == 1}]
           elsif reduce_actions = item_set.reduce_actions
             reduce_actions.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id, rule: rule,
+              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
                rule_set_id: rule.rule_set.id, n_states: rule.components.size,
                propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
@@ -101,11 +90,11 @@ class Propane
           end
         @state_table << {
           shift_index: @shift_table.size,
-          n_shifts: shift_entries.size,
+          n_shifts: item_set.shift_entries.size,
           reduce_index: @reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += shift_entries
+        @shift_table += item_set.shift_entries
         @reduce_table += reduce_entries
       end
     end
@@ -125,8 +114,110 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
+        build_shift_entries(item_set)
         build_reduce_actions_for_item_set(item_set)
       end
+      item_sets_to_process = @item_sets.select do |item_set|
+        # We need lookahead reduce actions if:
+        # 1) There is more than one possible rule to reduce. In this case the
+        #    lookahead token can help choose which rule to reduce.
+        # 2) There is at least one shift action and one reduce action for
+        #    this item set. In this case the lookahead reduce actions are
+        #    needed to test for a Shift/Reduce conflict.
+        item_set.reduce_rules.size > 1 ||
+          (item_set.reduce_rules.size > 0 && item_set.shift_entries.size > 0)
+      end
+      if RbConfig::CONFIG["host_os"] =~ /linux/
+        item_sets_by_id = {}
+        item_sets_to_process.each do |item_set|
+          item_sets_by_id[item_set.object_id] = item_set
+        end
+        tokens_by_id = {}
+        @grammar.tokens.each do |token|
+          tokens_by_id[token.object_id] = token
+        end
+        rules_by_id = {}
+        @grammar.rules.each do |rule|
+          rules_by_id[rule.object_id] = rule
+        end
+        n_threads = Util.determine_n_threads
+        semaphore = Mutex.new
+        queue = Queue.new
+        threads = {}
+        n_threads.times do
+          piper, pipew = IO.pipe
+          thread = Thread.new do
+            loop do
+              item_set = nil
+              semaphore.synchronize do
+                item_set = item_sets_to_process.slice!(0)
+              end
+              break if item_set.nil?
+              fork do
+                piper.close
+                build_lookahead_reduce_actions_for_item_set(item_set, pipew)
+              end
+            end
+            queue.push(Thread.current)
+          end
+          threads[thread] = [piper, pipew]
+        end
+        until threads.empty?
+          thread = queue.pop
+          piper, pipew = threads[thread]
+          pipew.close
+          thread_txt = piper.read
+          thread_txt.each_line do |line|
+            if line.start_with?("RA,")
+              parts = line.split(",")
+              item_set_id, token_id, rule_id = parts[1..3].map(&:to_i)
+              item_set = item_sets_by_id[item_set_id]
+              unless item_set
+                raise "Internal error: could not find item set from thread"
+              end
+              token = tokens_by_id[token_id]
+              unless item_set
+                raise "Internal error: could not find token from thread"
+              end
+              rule = rules_by_id[rule_id]
+              unless item_set
+                raise "Internal error: could not find rule from thread"
+              end
+              item_set.reduce_actions ||= {}
+              item_set.reduce_actions[token] = rule
+            elsif line.start_with?("Error: ")
+              @errors << line.chomp
+            else
+              raise "Internal error: unhandled thread line #{line}"
+            end
+          end
+          thread.join
+          threads.delete(thread)
+        end
+      else
+        # Fall back to single threaded algorithm.
+        item_sets_to_process.each do |item_set|
+          item_set.reduce_actions = build_lookahead_reduce_actions_for_item_set(item_set)
+        end
+      end
+    end
+
+    # Build the shift entries for a single item set.
+    #
+    # @return [void]
+    def build_shift_entries(item_set)
+      item_set.shift_entries = item_set.next_symbols.map do |next_symbol|
+        state_id =
+          if next_symbol.name == "$EOF"
+            0
+          else
+            item_set.next_item_set[next_symbol].id
+          end
+        {
+          symbol: next_symbol,
+          state_id: state_id,
+        }
+      end
     end
 
     # Build the reduce actions for a single item set (parser state).
@@ -145,33 +236,18 @@ class Propane
       if item_set.reduce_rules.size == 1
         item_set.reduce_rule = item_set.reduce_rules.first
       end
-
-      if item_set.reduce_rules.size > 1
-        # Force item_set.reduce_actions to be built to store the lookahead
-        # tokens for the possible reduce rules if there is more than one.
-        get_lookahead_reduce_actions_for_item_set(item_set)
-      end
-    end
-
-    # Get the reduce actions for a single item set (parser state).
-    #
-    # @param item_set [ItemSet]
-    #   ItemSet (parser state)
-    #
-    # @return [Hash]
-    #   Mapping of lookahead Tokens to the Rules to reduce.
-    def get_lookahead_reduce_actions_for_item_set(item_set)
-      item_set.reduce_actions ||= build_lookahead_reduce_actions_for_item_set(item_set)
     end
 
     # Build the reduce actions for a single item set (parser state).
     #
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
+    # @param fh [File]
+    #   Output file handle for multiprocessing mode.
     #
     # @return [Hash]
     #   Mapping of lookahead Tokens to the Rules to reduce.
-    def build_lookahead_reduce_actions_for_item_set(item_set)
+    def build_lookahead_reduce_actions_for_item_set(item_set, fh = nil)
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
@@ -182,9 +258,12 @@ class Propane
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            @errors << "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
+            error = "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
+            @errors << error
+            fh.puts(error) if fh
           end
           reduce_actions[lookahead_token] = reduce_rule
+          fh.puts "RA,#{item_set.object_id},#{lookahead_token.object_id},#{reduce_rule.object_id}" if fh
         end
         reduce_actions
       end
@@ -231,6 +310,7 @@ class Propane
                   rule_set = item.rule.rule_set
                   unless checked_rule_sets.include?(rule_set)
                     rule_sets_to_check_after << rule_set
+                    checked_rule_sets << rule_set
                   end
                   break
                 when Token

lib/propane/parser/item.rb

@@ -22,6 +22,7 @@ class Propane
       def initialize(rule, position)
         @rule = rule
         @position = position
+        @_hash = [@rule, @position].hash
       end
 
       # Hash function.
@@ -29,7 +30,7 @@ class Propane
       # @return [Integer]
       #   Hash code.
       def hash
-        [@rule, @position].hash
+        @_hash
       end
 
       # Compare Item objects.

lib/propane/parser/item_set.rb

@@ -34,6 +34,10 @@ class Propane
       #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
+      # @return [Array<Hash>]
+      #   Shift table entries.
+      attr_accessor :shift_entries
+
       # Build an ItemSet.
       #
       # @param items [Array<Item>]

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [String, nil]
+    # @return [Set]
-    #   Lexer mode for this pattern.
+    #   Lexer modes for this pattern.
-    attr_accessor :mode
+    attr_accessor :modes
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,16 +46,16 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :mode
+    # @option options [String, nil] :modes
-    #   Lexer mode for this pattern.
+    #   Lexer modes for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @mode = options[:mode]
+      @modes = options[:modes]
       @ptypename = options[:ptypename]
-      regex = Regex.new(@pattern)
+      regex = Regex.new(@pattern, @line_number)
       regex.nfa.end_state.accepts = self
       @nfa = regex.nfa
     end

lib/propane/regex.rb

@@ -4,12 +4,13 @@ class Propane
     attr_reader :unit
     attr_reader :nfa
 
-    def initialize(pattern)
+    def initialize(pattern, line_number)
       @pattern = pattern.dup
+      @line_number = line_number
       @unit = parse_alternates
       @nfa = @unit.to_nfa
       if @pattern != ""
-        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
+        raise Error.new(%[Line #{@line_number}: unexpected "#{@pattern}" in pattern])
       end
     end
 
@@ -41,7 +42,7 @@ class Propane
             mu = MultiplicityUnit.new(last_unit, min_count, max_count)
             au.replace_last!(mu)
           else
-            raise Error.new("#{c} follows nothing")
+            raise Error.new("Line #{@line_number}: #{c} follows nothing")
           end
         when "|"
           au.new_alternate!
@@ -59,7 +60,7 @@ class Propane
     def parse_group
       au = parse_alternates
       if @pattern[0] != ")"
-        raise Error.new("Unterminated group in pattern")
+        raise Error.new("Line #{@line_number}: unterminated group in pattern")
       end
       @pattern.slice!(0)
       au
@@ -70,7 +71,7 @@ class Propane
       index = 0
       loop do
         if @pattern == ""
-          raise Error.new("Unterminated character class")
+          raise Error.new("Line #{@line_number}: unterminated character class")
         end
         c = @pattern.slice!(0)
         if c == "]"
@@ -84,13 +85,13 @@ class Propane
         elsif c == "-" && @pattern[0] != "]"
           begin_cu = ccu.last_unit
           unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
-            raise Error.new("Character range must be between single characters")
+            raise Error.new("Line #{@line_number}: character range must be between single characters")
           end
           if @pattern[0] == "\\"
             @pattern.slice!(0)
             end_cu = parse_backslash
             unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
-              raise Error.new("Character range must be between single characters")
+              raise Error.new("Line #{@line_number}: character range must be between single characters")
             end
             max_code_point = end_cu.code_point
           else
@@ -116,7 +117,7 @@ class Propane
         elsif max_count.to_s != ""
           max_count = max_count.to_i
           if max_count < min_count
-            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
+            raise Error.new("Line #{@line_number}: maximum repetition count cannot be less than minimum repetition count")
           end
         else
           max_count = nil
@@ -124,28 +125,33 @@ class Propane
         @pattern = pattern
         [min_count, max_count]
       else
-        raise Error.new("Unexpected match count at #{@pattern}")
+        raise Error.new("Line #{@line_number}: unexpected match count following {")
       end
     end
 
     def parse_backslash
       if @pattern == ""
-        raise Error.new("Error: unfollowed \\")
+        raise Error.new("Line #{@line_number}: error: unfollowed \\")
       else
         c = @pattern.slice!(0)
         case c
         when "a"
-          CharacterRangeUnit.new("\a", "\a")
+          CharacterRangeUnit.new("\a")
         when "b"
-          CharacterRangeUnit.new("\b", "\b")
+          CharacterRangeUnit.new("\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "D"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu.negate = true
+          ccu
         when "f"
-          CharacterRangeUnit.new("\f", "\f")
+          CharacterRangeUnit.new("\f")
         when "n"
-          CharacterRangeUnit.new("\n", "\n")
+          CharacterRangeUnit.new("\n")
         when "r"
-          CharacterRangeUnit.new("\r", "\r")
+          CharacterRangeUnit.new("\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -155,10 +161,35 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "S"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new(" ")
+          ccu << CharacterRangeUnit.new("\t")
+          ccu << CharacterRangeUnit.new("\r")
+          ccu << CharacterRangeUnit.new("\n")
+          ccu << CharacterRangeUnit.new("\f")
+          ccu << CharacterRangeUnit.new("\v")
+          ccu.negate = true
+          ccu
         when "t"
-          CharacterRangeUnit.new("\t", "\t")
+          CharacterRangeUnit.new("\t")
         when "v"
-          CharacterRangeUnit.new("\v", "\v")
+          CharacterRangeUnit.new("\v")
+        when "w"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu
+        when "W"
+          ccu = CharacterClassUnit.new
+          ccu << CharacterRangeUnit.new("_")
+          ccu << CharacterRangeUnit.new("0", "9")
+          ccu << CharacterRangeUnit.new("a", "z")
+          ccu << CharacterRangeUnit.new("A", "Z")
+          ccu.negate = true
+          ccu
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/regex/unit.rb

@@ -92,16 +92,19 @@ class Propane
         @units = []
         @negate = false
       end
-      def initialize
+      def method_missing(*args, &block)
-        @units = []
+        @units.__send__(*args, &block)
-      end
-      def method_missing(*args)
-        @units.__send__(*args)
       end
       def <<(thing)
         if thing.is_a?(CharacterClassUnit)
-          thing.each do |ccu_unit|
+          if thing.negate
-            @units << ccu_unit
+            CodePointRange.invert_ranges(thing.map(&:code_point_range)).each do |cpr|
+              CharacterRangeUnit.new(cpr.first, cpr.last)
+            end
+          else
+            thing.each do |ccu_unit|
+              @units << ccu_unit
+            end
           end
         else
           @units << thing

lib/propane/rule_set.rb

@@ -119,6 +119,8 @@ class Propane
     def build_ast_fields(grammar)
       field_ast_node_indexes = {}
       field_indexes_across_all_rules = {}
+      # Stores the index into @ast_fields by field alias name.
+      field_aliases = {}
       @ast_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
@@ -136,6 +138,16 @@ class Propane
             field_ast_node_indexes[field_name] = @ast_fields.size
             @ast_fields << {field_name => struct_name}
           end
+          rule.aliases.each do |alias_name, index|
+            if index == i
+              alias_ast_fields_index = field_ast_node_indexes[field_name]
+              if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index
+                raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
+              end
+              field_aliases[alias_name] = alias_ast_fields_index
+              @ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1]
+            end
+          end
           field_indexes_across_all_rules[node_name] ||= Set.new
           field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
           rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
@@ -150,18 +162,6 @@ class Propane
             "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
         end
       end
-      # Now merge in the field aliases as given by the user in the
-      # grammar.
-      field_aliases = {}
-      @rules.each do |rule|
-        rule.aliases.each do |alias_name, index|
-          if field_aliases[alias_name] && field_aliases[alias_name] != index
-            raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}`")
-          end
-          field_aliases[alias_name] = index
-          @ast_fields[index][alias_name] = @ast_fields[index].first[1]
-        end
-      end
     end
 
   end

lib/propane/util.rb

@@ -10,6 +10,32 @@ class Propane
         "#{s}\n* #{message} *\n#{s}\n"
       end
 
+      # Determine the number of threads to use.
+      #
+      # @return [Integer]
+      #   The number of threads to use.
+      def determine_n_threads
+        # Try to figure out how many threads are available on the host hardware.
+        begin
+          case RbConfig::CONFIG["host_os"]
+          when /linux/
+            return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
+          when /mswin|mingw|msys/
+            if `wmic cpu get NumberOfLogicalProcessors -value` =~ /NumberOfLogicalProcessors=(\d+)/
+              return $1.to_i
+            end
+          when /darwin/
+            if `sysctl -n hw.ncpu` =~ /(\d+)/
+              return $1.to_i
+            end
+          end
+        rescue
+        end
+
+        # If we can't figure it out, default to 4.
+        4
+      end
+
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "2.0.0"
+  VERSION = "2.3.0"
 end

spec/ast_node_memory_remains.c.propane

@@ -0,0 +1,151 @@
+ast;
+ast_prefix P;
+
+<<header
+#include <stdio.h>
+#include <stdint.h>
+
+typedef union
+{
+    uint64_t i64;
+    const uint8_t * s;
+    double dou;
+} TokenVal;
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.i64 = 64u;
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  $$.s = match;
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/ast_node_memory_remains.d.propane

@@ -0,0 +1,177 @@
+ast;
+ast_prefix P;
+
+<<
+import std.bigint;
+
+private string stringvalue;
+
+union TokenVal
+{
+    BigInt bi;
+    string s;
+    double dou;
+}
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.bi = BigInt(match[0..3]);
+  foreach (c; match[3..$])
+  {
+    if (('0' <= c) && (c <= '9'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - '0');
+    }
+    if (('a' <= c) && (c <= 'f'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'a' + 10);
+    }
+    if (('A' <= c) && (c <= 'F'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'A' + 10);
+    }
+  }
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  if (match[0] == '$')
+  {
+    $$.s = match[1..$];
+  }
+  else
+  {
+    $$.s = match;
+  }
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/json_parser.c.propane

@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {match[0], 0};
+  char s[] = {(char)match[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_types.c

@@ -5,7 +5,7 @@
 
 JSONValue * JSONValue_new(size_t id)
 {
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
     jv->id = id;
     return jv;
 }
@@ -29,7 +29,7 @@ void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
         }
     }
     size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
     if (size > 0)
     {
         memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
@@ -52,7 +52,7 @@ void JSONArray_append(JSONValue * array, JSONValue * value)
 {
     size_t const size = array->array.size;
     size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
     if (array->array.size > 0)
     {
         memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);

spec/json_types.h

@@ -11,6 +11,12 @@
 #define JSON_FALSE 5u
 #define JSON_NULL 6u
 
+typedef struct JSONObjectEntry_s
+{
+    char const * name;
+    struct JSONValue_s * value;
+} JSONObjectEntry;
+
 typedef struct JSONValue_s
 {
     size_t id;
@@ -19,11 +25,7 @@ typedef struct JSONValue_s
         struct
         {
             size_t size;
-            struct
+            JSONObjectEntry * entries;
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
         } object;
         struct
         {

spec/propane/grammar_spec.rb

@@ -151,30 +151,30 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m1"
+      expect(o.modes).to eq Set["m1"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m2"
+      expect(o.modes).to eq Set["m2"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m3"
+      expect(o.modes).to eq Set["m3"]
     end
 
     it "allows assigning ptypes to tokens and rules" do

spec/propane/lexer/dfa_spec.rb

@@ -126,6 +126,74 @@ EOF
     ]
     expect(run(<<EOF, ";")).to eq expected
 token semicolon /;/;
+EOF
+  end
+
+  it "matches a negated character class" do
+    expected = [
+      ["pattern", "/abc/"],
+    ]
+    expect(run(<<EOF, "/abc/")).to eq expected
+token pattern /\\/[^\\s]*\\//;
+EOF
+  end
+
+  it "matches special character classes " do
+    expected = [
+      ["a", "abc123_FOO"],
+    ]
+    expect(run(<<EOF, "abc123_FOO")).to eq expected
+token a /\\w+/;
+EOF
+    expected = [
+      ["b", "FROG*%$#"],
+    ]
+    expect(run(<<EOF, "FROG*%$#")).to eq expected
+token b /FROG\\D{1,4}/;
+EOF
+    expected = [
+      ["c", "$883366"],
+    ]
+    expect(run(<<EOF, "$883366")).to eq expected
+token c /$\\d+/;
+EOF
+    expected = [
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "^&$@")).to eq expected
+token d /^\\W+/;
+EOF
+    expected = [
+      ["a", "abc123_FOO"],
+      [nil, " "],
+      ["b", "FROG*%$#"],
+      [nil, " "],
+      ["c", "$883366"],
+      [nil, " "],
+      ["d", "^&$@"],
+    ]
+    expect(run(<<EOF, "abc123_FOO FROG*%$# $883366 ^&$@")).to eq expected
+token a /\\w+/;
+token b /FROG\\D{1,4}/;
+token c /$\\d+/;
+token d /^\\W+/;
+drop /\\s+/;
+EOF
+  end
+
+  it "matches a negated character class with a nested inner negated character class" do
+    expected = [
+      ["t", "$&*"],
+    ]
+    expect(run(<<EOF, "$&*")).to eq expected
+token t /[^%\\W]+/;
+EOF
+  end
+
+  it "\\s matches a newline" do
+    expected = [["s", "\n"]]
+    expect(run(<<EOF, "\n")).to eq expected
+token s /\\s/;
 EOF
   end
 end

spec/propane/regex_spec.rb

@@ -2,14 +2,14 @@ class Propane
   RSpec.describe Regex do
 
     it "parses an empty expression" do
-      regex = Regex.new("")
+      regex = Regex.new("", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0].size).to eq 0
     end
 
     it "parses a single character unit expression" do
-      regex = Regex.new("a")
+      regex = Regex.new("a", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -19,7 +19,7 @@ class Propane
     end
 
     it "parses a group with a single character unit expression" do
-      regex = Regex.new("(a)")
+      regex = Regex.new("(a)", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -33,7 +33,7 @@ class Propane
     end
 
     it "parses a *" do
-      regex = Regex.new("a*")
+      regex = Regex.new("a*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -47,7 +47,7 @@ class Propane
     end
 
     it "parses a +" do
-      regex = Regex.new("a+")
+      regex = Regex.new("a+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -61,7 +61,7 @@ class Propane
     end
 
     it "parses a ?" do
-      regex = Regex.new("a?")
+      regex = Regex.new("a?", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -75,7 +75,7 @@ class Propane
     end
 
     it "parses a multiplicity count" do
-      regex = Regex.new("a{5}")
+      regex = Regex.new("a{5}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -89,7 +89,7 @@ class Propane
     end
 
     it "parses a minimum-only multiplicity count" do
-      regex = Regex.new("a{5,}")
+      regex = Regex.new("a{5,}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -103,7 +103,7 @@ class Propane
     end
 
     it "parses a minimum and maximum multiplicity count" do
-      regex = Regex.new("a{5,8}")
+      regex = Regex.new("a{5,8}", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -118,7 +118,7 @@ class Propane
     end
 
     it "parses an escaped *" do
-      regex = Regex.new("a\\*")
+      regex = Regex.new("a\\*", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -131,7 +131,7 @@ class Propane
     end
 
     it "parses an escaped +" do
-      regex = Regex.new("a\\+")
+      regex = Regex.new("a\\+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -144,7 +144,7 @@ class Propane
     end
 
     it "parses an escaped \\" do
-      regex = Regex.new("\\\\d")
+      regex = Regex.new("\\\\d", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -157,7 +157,7 @@ class Propane
     end
 
     it "parses a character class" do
-      regex = Regex.new("[a-z_]")
+      regex = Regex.new("[a-z_]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -175,7 +175,7 @@ class Propane
     end
 
     it "parses a negated character class" do
-      regex = Regex.new("[^xyz]")
+      regex = Regex.new("[^xyz]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -189,8 +189,25 @@ class Propane
       expect(ccu[0].first).to eq "x".ord
     end
 
+    it "parses a negated character class with inner character classes" do
+      regex = Regex.new("[^x\\sz]", 1)
+      expect(regex.unit).to be_a Regex::AlternatesUnit
+      expect(regex.unit.alternates.size).to eq 1
+      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
+      seq_unit = regex.unit.alternates[0]
+      expect(seq_unit.size).to eq 1
+      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
+      ccu = seq_unit[0]
+      expect(ccu.negate).to be_truthy
+      expect(ccu.size).to eq 8
+      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
+      expect(ccu[0].first).to eq "x".ord
+      expect(ccu[1].first).to eq " ".ord
+      expect(ccu[7].first).to eq "z".ord
+    end
+
     it "parses - as a plain character at beginning of a character class" do
-      regex = Regex.new("[-9]")
+      regex = Regex.new("[-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -204,7 +221,7 @@ class Propane
     end
 
     it "parses - as a plain character at end of a character class" do
-      regex = Regex.new("[0-]")
+      regex = Regex.new("[0-]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -220,7 +237,7 @@ class Propane
     end
 
     it "parses - as a plain character at beginning of a negated character class" do
-      regex = Regex.new("[^-9]")
+      regex = Regex.new("[^-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -235,7 +252,7 @@ class Propane
     end
 
     it "parses . as a plain character in a character class" do
-      regex = Regex.new("[.]")
+      regex = Regex.new("[.]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -250,7 +267,7 @@ class Propane
     end
 
     it "parses - as a plain character when escaped in middle of character class" do
-      regex = Regex.new("[0\\-9]")
+      regex = Regex.new("[0\\-9]", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -269,7 +286,7 @@ class Propane
     end
 
     it "parses alternates" do
-      regex = Regex.new("ab|c")
+      regex = Regex.new("ab|c", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 2
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -279,7 +296,7 @@ class Propane
     end
 
     it "parses a ." do
-      regex = Regex.new("a.b")
+      regex = Regex.new("a.b", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -290,7 +307,7 @@ class Propane
     end
 
     it "parses something complex" do
-      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+", 1)
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 4
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit

spec/propane_spec.rb

@@ -65,26 +65,47 @@ EOF
   end
 
   def compile(test_files, options = {})
-    test_files = Array(test_files)
+    test_files = Array(test_files).map do |test_file|
+      if !File.exist?(test_file) && test_file.end_with?(".cpp")
+        test_file.sub(%r{\.cpp$}, ".c")
+      else
+        test_file
+      end
+    end
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
       "spec/run/testparser#{name}.#{options[:language]}"
     end
     case options[:language]
     when "c"
-      result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
+      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+    when "cpp"
+      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
     end
+    result = system(*command)
     expect(result).to be_truthy
   end
 
-  def run_test
+  def run_test(options)
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
-    Results.new(stdout, stderr, status)
+    results = Results.new(stdout, stderr, status)
+    if %w[c cpp].include?(options[:language])
+      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+      vgout = stdout + stderr
+      File.binwrite("spec/run/.vgout", vgout)
+      vgout.scan(/lost: (\d+) bytes/) do |match|
+        bytes = $1.to_i
+        if bytes > 0
+          raise "Valgrind detected memory leak"
+        end
+      end
+    end
+    results
   end
 
   def lines(str)
@@ -249,7 +270,19 @@ EOF
     expect(results.status).to eq 0
   end
 
-  %w[d c].each do |language|
+  it "shows error line number for unmatched left curly brace" do
+    write_grammar <<EOF
+# Line 1
+# Line 2
+token a /a{/;
+Start -> a;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Line 3: unexpected match count following \{/
+    expect(results.status).to_not eq 0
+  end
+
+  %w[d c cpp].each do |language|
 
     context "#{language.upcase} language" do
 
@@ -265,14 +298,14 @@ Foo -> plus <<>>
 EOF
         run_propane(language: language)
         compile("spec/test_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "detects a lexer error when an unknown character is seen" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype int;
 token int /\\d+/ <<
@@ -303,7 +336,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -326,7 +359,7 @@ EOF
 
       it "generates a parser that does basic math - user guide example" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <math.h>
@@ -397,7 +430,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -423,7 +456,7 @@ R2 -> a b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
@@ -438,13 +471,13 @@ R1 -> b;
 EOF
         run_propane(language: language)
         compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
       end
 
       it "executes user code when matching lexer token" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -473,7 +506,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "abc!",
@@ -486,7 +519,7 @@ EOF
 
       it "supports a pattern statement" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -509,7 +542,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -522,7 +555,7 @@ EOF
 
       it "supports returning tokens from pattern code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -551,7 +584,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -562,7 +595,7 @@ EOF
 
       it "supports lexer modes" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -609,7 +642,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "begin string mode",
@@ -621,9 +654,65 @@ EOF
         ])
       end
 
+      it "multiple lexer modes may apply to a pattern" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  printf("ident: %c\\n", $3);
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  writeln("ident: ", $3);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_multiple_modes.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "ident: d",
+          "pass1",
+          "ident: a",
+          "pass2",
+        ])
+      end
+
       it "executes user code associated with a parser rule" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
@@ -648,7 +737,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "A!",
@@ -659,7 +748,7 @@ EOF
 
       it "parses lists" do
         write_grammar <<EOF
-ptype #{language == "c" ? "uint32_t" : "uint"};
+ptype #{language == "d" ? "uint" : "uint32_t"};
 token a;
 Start -> As << $$ = $1; >>
 As -> << $$ = 0u; >>
@@ -667,7 +756,7 @@ As -> As a << $$ = $1 + 1u; >>
 EOF
         run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         expect(results.stderr).to eq ""
       end
@@ -694,14 +783,14 @@ EOF
 
       it "provides matched text to user code blocks" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
 #include <stdio.h>
 #include <stdlib.h>
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * t = malloc(match_length + 1);
+  char * t = (char *)malloc(match_length + 1);
   strncpy(t, (char *)match, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
@@ -721,7 +810,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "Matched token is identifier_123",
@@ -731,7 +820,7 @@ EOF
 
       it "allows storing a result value for the lexer" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 ptype uint64_t;
 token word /[a-z]+/ <<
@@ -754,7 +843,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -769,13 +858,14 @@ Start -> a num;
 EOF
         run_propane(language: language)
         compile("spec/test_error_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/json_parser.#{ext}.propane"))
         run_propane(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end
@@ -797,7 +887,7 @@ Start -> b c b;
 EOF
         run_propane(name: "myp2", language: language)
         compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -816,7 +906,7 @@ Any -> c;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -834,14 +924,14 @@ Any -> ;
 EOF
         run_propane(language: language)
         compile("spec/test_user_terminate.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "matches backslash escape sequences" do
         case language
-        when "c"
+        when "c", "cpp"
           write_grammar <<EOF
 <<
   #include <stdio.h>
@@ -878,7 +968,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -943,7 +1033,7 @@ Two -> two;
 EOF
         run_propane(language: language)
         compile("spec/test_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -987,7 +1077,7 @@ Two -> two;
 EOF
         run_propane(language: language)
         compile("spec/test_ast_ps.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1055,13 +1145,13 @@ Start -> a? b R? <<
   printf("b: %d\\n", $2);
   printf("R: %s\\n", $3 == NULL ? "" : $3);
 >>
-R -> c d << $$ = "cd"; >>
+R -> c d << $$ = (char *)"cd"; >>
-R (string) -> d c << $$ = "dc"; >>
+R (string) -> d c << $$ = (char *)"dc"; >>
 EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -1113,7 +1203,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1154,7 +1244,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1174,7 +1264,7 @@ T -> c;
 EOF
         run_propane(language: language)
         compile("spec/test_ast_token_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1194,7 +1284,7 @@ A -> bb? c?;
 EOF
         run_propane(language: language)
         compile("spec/test_ast_invalid_positions.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1214,7 +1304,30 @@ T -> c;
 EOF
         run_propane(language: language)
         compile("spec/test_ast_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "aliases the correct field when multiple rules are in a rule set in AST mode" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> a;
+Start -> Foo;
+Start -> T:first T:second T:third;
+Foo -> b;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -1241,9 +1354,9 @@ EOF
 #include <stdio.h>
 #include <string.h>
 >>
-ptype char const *;
+ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  char * s = malloc(match_length + 1);
+  char * s = (char *)malloc(match_length + 1);
   strncpy(s, (char const *)match, match_length);
   s[match_length] = 0;
   $$ = s;
@@ -1252,16 +1365,110 @@ drop /\\s+/;
 Start -> id:first id:second <<
   printf("first is %s\\n", ${first});
   printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
 >>
 EOF
         end
         run_propane(language: language)
         compile("spec/test_field_aliases.#{language}", language: language)
-        results = run_test
+        results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
+
+      it "aliases the correct field when multiple rules are in a rule set when AST mode is not enabled" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype string;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = match;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  writeln("first is ", ${first});
+  writeln("second is ", ${second});
+>>
+Foo -> ;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+ptype char *;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * s = (char *)malloc(match_length + 1);
+  strncpy(s, (char const *)match, match_length);
+  s[match_length] = 0;
+  $$ = s;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  printf("first is %s\\n", ${first});
+  printf("second is %s\\n", ${second});
+  free(${first});
+  free(${second});
+>>
+Foo -> ;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_field_aliases.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to match /first is foo1.*second is bar2/m
+      end
+
+      it "does not free memory allocated for AST nodes" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_ast_node_memory_remains.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      if %w[c cpp].include?(language)
+        it "allows a user function to free token node memory in AST mode" do
+    write_grammar <<EOF
+<<
+static void free_token(Token * token)
+{
+    free(token->pvalue);
+}
+>>
+ast;
+free_token_node free_token;
+ptype int *;
+token a <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 1;
+>>
+token b <<
+  $$ = (int *)malloc(sizeof(int));
+  *$$ = 2;
+>>
+Start -> a:a b:b;
+EOF
+          run_propane(language: language)
+          compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
     end
   end
 end

spec/spec_helper.rb

@@ -2,6 +2,10 @@ unless ENV["dist_specs"]
   require "bundler/setup"
   require "simplecov"
 
+  class MyFormatter
+    def format(*args)
+    end
+  end
   SimpleCov.start do
     add_filter "/spec/"
     add_filter "/.bundle/"
@@ -12,6 +16,7 @@ unless ENV["dist_specs"]
     end
     project_name "Propane"
     merge_timeout 3600
+    formatter(MyFormatter)
   end
 
   RSpec.configure do |config|

spec/test_ast.c

@@ -33,12 +33,16 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_free_ast(start);
+
     input = "";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
     start = p_result(&context);
     assert(start->pItems == NULL);
 
+    p_free_ast(start);
+
     input = "2 1";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
@@ -51,5 +55,7 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_field_aliases.c

@@ -15,5 +15,7 @@ int main()
     assert_eq(TOKEN_b, start->second->pToken->token);
     assert_eq(TOKEN_c, start->third->pToken->token);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_invalid_positions.c

@@ -30,6 +30,8 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(8, start->end_position.col);
 
+    p_free_ast(start);
+
     input = "a\nbb";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -54,6 +56,8 @@ int main()
     assert_eq(2, start->end_position.row);
     assert_eq(2, start->end_position.col);
 
+    p_free_ast(start);
+
     input = "a\nc\nc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -78,6 +82,8 @@ int main()
     assert_eq(3, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
+    p_free_ast(start);
+
     input = "a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -98,5 +104,7 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(1, start->end_position.col);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_node_memory_remains.c

@@ -0,0 +1,418 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+int main(int argc, char * argv[])
+{
+    const char * input =
+        "# 0\n"
+        "def byte_val() -> byte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 1\n"
+        "def short_val() -> short\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 2\n"
+        "def int_val() -> int\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 3\n"
+        "def long_val() -> long\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 4\n"
+        "def ssize_t_val() -> ssize_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 5\n"
+        "def byte_to_short() -> short\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 6\n"
+        "def byte_to_int() -> int\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 7\n"
+        "def byte_to_long() -> long\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 8\n"
+        "def byte_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 9\n"
+        "def short_to_byte() -> byte\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 10\n"
+        "def short_to_int() -> int\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 11\n"
+        "def short_to_long() -> long\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 12\n"
+        "def short_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 13\n"
+        "def int_to_byte() -> byte\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 14\n"
+        "def int_to_short() -> short\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 15\n"
+        "def int_to_long() -> long\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 16\n"
+        "def int_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 17\n"
+        "def long_to_byte() -> byte\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 18\n"
+        "def long_to_short() -> short\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 19\n"
+        "def long_to_int() -> int\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 20\n"
+        "def long_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 21\n"
+        "def ssize_t_to_byte() -> byte\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 22\n"
+        "def ssize_t_to_short() -> short\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 23\n"
+        "def ssize_t_to_int() -> int\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 24\n"
+        "def ssize_t_to_long() -> long\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 25\n"
+        "def ubyte_val() -> ubyte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 26\n"
+        "def ushort_val() -> ushort\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 27\n"
+        "def uint_val() -> uint\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 28\n"
+        "def ulong_val() -> ulong\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 29\n"
+        "def size_t_val() -> size_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 30\n"
+        "def ubyte_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 31\n"
+        "def ubyte_to_uint() -> uint\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 32\n"
+        "def ubyte_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 33\n"
+        "def ubyte_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 34\n"
+        "def ushort_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 35\n"
+        "def ushort_to_uint() -> uint\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 36\n"
+        "def ushort_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 37\n"
+        "def ushort_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 38\n"
+        "def uint_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 39\n"
+        "def uint_to_ushort() -> ushort\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 40\n"
+        "def uint_to_ulong() -> ulong\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 41\n"
+        "def uint_to_size_t() -> size_t\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 42\n"
+        "def ulong_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 43\n"
+        "def ulong_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 44\n"
+        "def ulong_to_uint() -> uint\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 45\n"
+        "def ulong_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 46\n"
+        "def size_t_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 47\n"
+        "def size_t_to_ushort() -> ushort\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 48\n"
+        "def size_t_to_int() -> int\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 49\n"
+        "def size_t_to_ulong() -> ulong\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 50\n"
+        "def main() -> int\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n";
+    struct
+    {
+        const char * name;
+        p_token_t token;
+    } expected[] = {
+        {"byte_val", TOKEN_byte},
+        {"short_val", TOKEN_short},
+        {"int_val", TOKEN_int},
+        {"long_val", TOKEN_long},
+        {"ssize_t_val", TOKEN_ssize_t},
+        {"byte_to_short", TOKEN_short},
+        {"byte_to_int", TOKEN_int},
+        {"byte_to_long", TOKEN_long},
+        {"byte_to_ssize_t", TOKEN_ssize_t},
+        {"short_to_byte", TOKEN_byte},
+        {"short_to_int", TOKEN_int},
+        {"short_to_long", TOKEN_long},
+        {"short_to_ssize_t", TOKEN_ssize_t},
+        {"int_to_byte", TOKEN_byte},
+        {"int_to_short", TOKEN_short},
+        {"int_to_long", TOKEN_long},
+        {"int_to_ssize_t", TOKEN_ssize_t},
+        {"long_to_byte", TOKEN_byte},
+        {"long_to_short", TOKEN_short},
+        {"long_to_int", TOKEN_int},
+        {"long_to_ssize_t", TOKEN_ssize_t},
+        {"ssize_t_to_byte", TOKEN_byte},
+        {"ssize_t_to_short", TOKEN_short},
+        {"ssize_t_to_int", TOKEN_int},
+        {"ssize_t_to_long", TOKEN_long},
+        {"ubyte_val", TOKEN_ubyte},
+        {"ushort_val", TOKEN_ushort},
+        {"uint_val", TOKEN_uint},
+        {"ulong_val", TOKEN_ulong},
+        {"size_t_val", TOKEN_size_t},
+        {"ubyte_to_ushort", TOKEN_ushort},
+        {"ubyte_to_uint", TOKEN_uint},
+        {"ubyte_to_ulong", TOKEN_ulong},
+        {"ubyte_to_size_t", TOKEN_size_t},
+        {"ushort_to_ubyte", TOKEN_ubyte},
+        {"ushort_to_uint", TOKEN_uint},
+        {"ushort_to_ulong", TOKEN_ulong},
+        {"ushort_to_size_t", TOKEN_size_t},
+        {"uint_to_ubyte", TOKEN_ubyte},
+        {"uint_to_ushort", TOKEN_ushort},
+        {"uint_to_ulong", TOKEN_ulong},
+        {"uint_to_size_t", TOKEN_size_t},
+        {"ulong_to_ubyte", TOKEN_ubyte},
+        {"ulong_to_ushort", TOKEN_ushort},
+        {"ulong_to_uint", TOKEN_uint},
+        {"ulong_to_size_t", TOKEN_size_t},
+        {"size_t_to_ubyte", TOKEN_ubyte},
+        {"size_t_to_ushort", TOKEN_ushort},
+        {"size_t_to_int", TOKEN_int},
+        {"size_t_to_ulong", TOKEN_ulong},
+        {"main", TOKEN_int},
+    };
+    p_context_t context;
+    p_context_init(&context, (const uint8_t *)input, strlen(input));
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod->pModuleItems;
+    PFunctionDefinition ** pfds;
+    size_t n_pfds = 0u;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        if (pmi->pFunctionDefinition != NULL)
+        {
+            n_pfds++;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
+    pmis = pmod->pModuleItems;
+    size_t pfd_i = n_pfds;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        PFunctionDefinition * pfd = pmi->pFunctionDefinition;
+        if (pfd != NULL)
+        {
+            pfd_i--;
+            assert(pfd_i < n_pfds);
+            pfds[pfd_i] = pfd;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    assert_eq(51, n_pfds);
+    for (size_t i = 0; i < n_pfds; i++)
+    {
+        if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
+            (expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
+        {
+            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
+        }
+    }
+
+    free(pfds);
+    p_free_ast(pmod);
+
+    return 0;
+}

spec/test_ast_node_memory_remains.d

@@ -0,0 +1,408 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "
+# 0
+def byte_val() -> byte
+{
+    return 0x42;
+}
+
+# 1
+def short_val() -> short
+{
+    return 0x4242;
+}
+
+# 2
+def int_val() -> int
+{
+    return 0x42424242;
+}
+
+# 3
+def long_val() -> long
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 4
+def ssize_t_val() -> ssize_t
+{
+    return 0x42424242;
+}
+
+# 5
+def byte_to_short() -> short
+{
+    return byte_val();
+}
+
+# 6
+def byte_to_int() -> int
+{
+    return byte_val();
+}
+
+# 7
+def byte_to_long() -> long
+{
+    return byte_val();
+}
+
+# 8
+def byte_to_ssize_t() -> ssize_t
+{
+    return byte_val();
+}
+
+# 9
+def short_to_byte() -> byte
+{
+    return short_val();
+}
+
+# 10
+def short_to_int() -> int
+{
+    return short_val();
+}
+
+# 11
+def short_to_long() -> long
+{
+    return short_val();
+}
+
+# 12
+def short_to_ssize_t() -> ssize_t
+{
+    return short_val();
+}
+
+# 13
+def int_to_byte() -> byte
+{
+    return int_val();
+}
+
+# 14
+def int_to_short() -> short
+{
+    return int_val();
+}
+
+# 15
+def int_to_long() -> long
+{
+    return int_val();
+}
+
+# 16
+def int_to_ssize_t() -> ssize_t
+{
+    return int_val();
+}
+
+# 17
+def long_to_byte() -> byte
+{
+    return long_val();
+}
+
+# 18
+def long_to_short() -> short
+{
+    return long_val();
+}
+
+# 19
+def long_to_int() -> int
+{
+    return long_val();
+}
+
+# 20
+def long_to_ssize_t() -> ssize_t
+{
+    return long_val();
+}
+
+# 21
+def ssize_t_to_byte() -> byte
+{
+    return ssize_t_val();
+}
+
+# 22
+def ssize_t_to_short() -> short
+{
+    return ssize_t_val();
+}
+
+# 23
+def ssize_t_to_int() -> int
+{
+    return ssize_t_val();
+}
+
+# 24
+def ssize_t_to_long() -> long
+{
+    return ssize_t_val();
+}
+
+# 25
+def ubyte_val() -> ubyte
+{
+    return 0x42;
+}
+
+# 26
+def ushort_val() -> ushort
+{
+    return 0x4242;
+}
+
+# 27
+def uint_val() -> uint
+{
+    return 0x42424242;
+}
+
+# 28
+def ulong_val() -> ulong
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 29
+def size_t_val() -> size_t
+{
+    return 0x42424242;
+}
+
+# 30
+def ubyte_to_ushort() -> ushort
+{
+    return ubyte_val();
+}
+
+# 31
+def ubyte_to_uint() -> uint
+{
+    return ubyte_val();
+}
+
+# 32
+def ubyte_to_ulong() -> ulong
+{
+    return ubyte_val();
+}
+
+# 33
+def ubyte_to_size_t() -> size_t
+{
+    return ubyte_val();
+}
+
+# 34
+def ushort_to_ubyte() -> ubyte
+{
+    return ushort_val();
+}
+
+# 35
+def ushort_to_uint() -> uint
+{
+    return ushort_val();
+}
+
+# 36
+def ushort_to_ulong() -> ulong
+{
+    return ushort_val();
+}
+
+# 37
+def ushort_to_size_t() -> size_t
+{
+    return ushort_val();
+}
+
+# 38
+def uint_to_ubyte() -> ubyte
+{
+    return uint_val();
+}
+
+# 39
+def uint_to_ushort() -> ushort
+{
+    return uint_val();
+}
+
+# 40
+def uint_to_ulong() -> ulong
+{
+    return uint_val();
+}
+
+# 41
+def uint_to_size_t() -> size_t
+{
+    return uint_val();
+}
+
+# 42
+def ulong_to_ubyte() -> ubyte
+{
+    return ulong_val();
+}
+
+# 43
+def ulong_to_ushort() -> ushort
+{
+    return ulong_val();
+}
+
+# 44
+def ulong_to_uint() -> uint
+{
+    return ulong_val();
+}
+
+# 45
+def ulong_to_size_t() -> size_t
+{
+    return ulong_val();
+}
+
+# 46
+def size_t_to_ubyte() -> ubyte
+{
+    return size_t_val();
+}
+
+# 47
+def size_t_to_ushort() -> ushort
+{
+    return size_t_val();
+}
+
+# 48
+def size_t_to_int() -> int
+{
+    return size_t_val();
+}
+
+# 49
+def size_t_to_ulong() -> ulong
+{
+    return size_t_val();
+}
+
+# 50
+def main() -> int
+{
+    return int_val();
+}
+";
+    struct Expected
+    {
+        string name;
+        p_token_t token;
+    }
+    Expected[] expected = [
+        Expected("byte_val", TOKEN_byte),
+        Expected("short_val", TOKEN_short),
+        Expected("int_val", TOKEN_int),
+        Expected("long_val", TOKEN_long),
+        Expected("ssize_t_val", TOKEN_ssize_t),
+        Expected("byte_to_short", TOKEN_short),
+        Expected("byte_to_int", TOKEN_int),
+        Expected("byte_to_long", TOKEN_long),
+        Expected("byte_to_ssize_t", TOKEN_ssize_t),
+        Expected("short_to_byte", TOKEN_byte),
+        Expected("short_to_int", TOKEN_int),
+        Expected("short_to_long", TOKEN_long),
+        Expected("short_to_ssize_t", TOKEN_ssize_t),
+        Expected("int_to_byte", TOKEN_byte),
+        Expected("int_to_short", TOKEN_short),
+        Expected("int_to_long", TOKEN_long),
+        Expected("int_to_ssize_t", TOKEN_ssize_t),
+        Expected("long_to_byte", TOKEN_byte),
+        Expected("long_to_short", TOKEN_short),
+        Expected("long_to_int", TOKEN_int),
+        Expected("long_to_ssize_t", TOKEN_ssize_t),
+        Expected("ssize_t_to_byte", TOKEN_byte),
+        Expected("ssize_t_to_short", TOKEN_short),
+        Expected("ssize_t_to_int", TOKEN_int),
+        Expected("ssize_t_to_long", TOKEN_long),
+        Expected("ubyte_val", TOKEN_ubyte),
+        Expected("ushort_val", TOKEN_ushort),
+        Expected("uint_val", TOKEN_uint),
+        Expected("ulong_val", TOKEN_ulong),
+        Expected("size_t_val", TOKEN_size_t),
+        Expected("ubyte_to_ushort", TOKEN_ushort),
+        Expected("ubyte_to_uint", TOKEN_uint),
+        Expected("ubyte_to_ulong", TOKEN_ulong),
+        Expected("ubyte_to_size_t", TOKEN_size_t),
+        Expected("ushort_to_ubyte", TOKEN_ubyte),
+        Expected("ushort_to_uint", TOKEN_uint),
+        Expected("ushort_to_ulong", TOKEN_ulong),
+        Expected("ushort_to_size_t", TOKEN_size_t),
+        Expected("uint_to_ubyte", TOKEN_ubyte),
+        Expected("uint_to_ushort", TOKEN_ushort),
+        Expected("uint_to_ulong", TOKEN_ulong),
+        Expected("uint_to_size_t", TOKEN_size_t),
+        Expected("ulong_to_ubyte", TOKEN_ubyte),
+        Expected("ulong_to_ushort", TOKEN_ushort),
+        Expected("ulong_to_uint", TOKEN_uint),
+        Expected("ulong_to_size_t", TOKEN_size_t),
+        Expected("size_t_to_ubyte", TOKEN_ubyte),
+        Expected("size_t_to_ushort", TOKEN_ushort),
+        Expected("size_t_to_int", TOKEN_int),
+        Expected("size_t_to_ulong", TOKEN_ulong),
+        Expected("main", TOKEN_int),
+    ];
+    p_context_t context;
+    p_context_init(&context, input);
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod.pModuleItems;
+    PFunctionDefinition *[] pfds;
+    while (pmis !is null)
+    {
+        PModuleItem * pmi = pmis.pModuleItem;
+        if (pmi is null)
+        {
+            stderr.writeln("pmi is null!!!?");
+            assert(0);
+        }
+        PFunctionDefinition * pfd = pmi.pFunctionDefinition;
+        if (pfd !is null)
+        {
+            pfds = [pfd] ~ pfds;
+        }
+        pmis = pmis.pModuleItems;
+    }
+    assert_eq(51, pfds.length);
+    for (size_t i = 0; i < pfds.length; i++)
+    {
+        if ((expected[i].name != pfds[i].name.pvalue.s) ||
+            (expected[i].token != pfds[i].returntype.pType.pTypeBase.pToken1.token))
+        {
+            stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
+        }
+    }
+}

spec/test_ast_ps.c

@@ -33,12 +33,16 @@ int main()
     assert_eq(22, itemsmore->pItem->pToken1->pvalue);
     assert(itemsmore->pItemsMore == NULL);
 
+    p_free_ast(start);
+
     input = "";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
     start = p_result(&context);
     assert(start->pItems == NULL);
 
+    p_free_ast(start);
+
     input = "2 1";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert_eq(P_SUCCESS, p_parse(&context));
@@ -51,5 +55,7 @@ int main()
     assert(start->pItems->pItem->pDual->pTwo2 == NULL);
     assert(start->pItems->pItem->pDual->pOne1 == NULL);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_ast_token_positions.c

@@ -43,6 +43,8 @@ int main()
     assert_eq(1, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
+    p_free_ast(start);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -80,5 +82,7 @@ int main()
     assert_eq(7, start->end_position.row);
     assert_eq(6, start->end_position.col);
 
+    p_free_ast(start);
+
     return 0;
 }

spec/test_free_ast_token_node_memory.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "ab";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->a != NULL);
+    assert(*start->a->pvalue == 1);
+    assert(start->b != NULL);
+    assert(*start->b->pvalue == 2);
+
+    p_free_ast(start);
+}

spec/test_lexer_multiple_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc.def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc .  abc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_multiple_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc.def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc .  abc`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_named_optional_rule_component_ast.c

@@ -17,6 +17,8 @@ int main()
     assert(start->pR == NULL);
     assert(start->r == NULL);
 
+    p_free_ast(start);
+
     input = "abcd";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -31,6 +33,8 @@ int main()
     assert(start->pR == start->r);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     input = "bdc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -40,6 +44,8 @@ int main()
     assert(start->r != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     return 0;
 }
 

spec/test_optional_rule_component_ast.c

@@ -16,6 +16,8 @@ int main()
     assert(start->pR3 == NULL);
     assert(start->pR == NULL);
 
+    p_free_ast(start);
+
     input = "abcd";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -28,6 +30,8 @@ int main()
     assert(start->pR == start->pR3);
     assert_eq(TOKEN_c, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     input = "bdc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
@@ -37,6 +41,8 @@ int main()
     assert(start->pR != NULL);
     assert_eq(TOKEN_d, start->pR->pToken1->token);
 
+    p_free_ast(start);
+
     return 0;
 }
 

spec/test_start_rule_ast.c

@@ -13,5 +13,7 @@ int main()
     assert(top->pToken != NULL);
     assert_eq(TOKEN_hi, top->pToken->token);
 
+    p_free_ast(top);
+
     return 0;
 }

spec/testutils.c

@@ -17,7 +17,7 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 void str_init(str_t * str, char const * cs)
 {
     size_t length = strlen(cs);
-    str->cs = malloc(length + 1u);
+    str->cs = (char *)malloc(length + 1u);
     strcpy(str->cs, cs);
 }
 
@@ -25,7 +25,7 @@ void str_append(str_t * str, char const * cs)
 {
     size_t length = strlen(str->cs);
     size_t length2 = strlen(cs);
-    char * new_cs = malloc(length + length2 + 1u);
+    char * new_cs = (char *)malloc(length + length2 + 1u);
     memcpy(new_cs, str->cs, length);
     strcpy(&new_cs[length], cs);
     free(str->cs);