Remove some Rust module allow directives

- clean coverage directory before running specs to avoid simplecov
  slowdowns
- add valgrind task to run valgrind; normal spec does not run it
- delay combining and reporting simplecov results until end of task to
  speed up task execution

CHANGELOG.md

@@ -6,6 +6,11 @@
 
 ### API Changes
 
+- The matched text argument passed to lexer user code blocks is now named
+  `match_text` for every target language. It was previously named `match` for
+  C, C++, and D, and `match_` for Rust. Any lexer user code block which
+  references the matched text must be updated to use the new name. The
+  `match_length` argument (C and C++) is unchanged.
 - Tree generation mode now stores all tree nodes in a compact arena owned by
   the parser context (a flat node array plus a shared child-link array).
   This replaces the previous design of one heap allocation per node with

README.md

@@ -69,7 +69,7 @@ token times /\*/;
 token power /\*\*/;
 token integer /\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');

Rakefile.rb

@@ -1,3 +1,4 @@
+require "fileutils"
 require "rake/clean"
 require "rspec/core/rake_task"
 require "simplecov"
@@ -11,7 +12,10 @@ end
 
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
+    ENV["partial_specs"] = "1"
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  else
+    FileUtils.rm_rf("coverage")
   end
 end
 task :spec do |task, args|
@@ -19,7 +23,7 @@ task :spec do |task, args|
     original_stdout = $stdout
     sio = StringIO.new
     $stdout = sio
-    SimpleCov.collate Dir["coverage/.resultset.json"]
+    SimpleCov.collate Dir["coverage/parts/*/.resultset.json"]
     $stdout = original_stdout
     sio.string.lines.each do |line|
       $stdout.write(line) unless line =~ /Coverage report generated for/
@@ -27,6 +31,15 @@ task :spec do |task, args|
   end
 end
 
+task :valgrind do
+  begin
+    ENV["spec-valgrind"] = "1"
+    Rake::Task[:spec].execute
+  ensure
+    ENV.delete("spec-valgrind")
+  end
+end
+
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.
 desc "Dist Specs"
@@ -43,4 +56,4 @@ task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
 
-task :all => [:spec, :dspec, :user_guide]
+task :all => [:valgrind, :dspec, :user_guide]

UPGRADING.md

@@ -2,7 +2,20 @@
 
 The generated API for tree generation mode (`tree;`) has been changed
 significantly for this version.
-The lexer/parser value APIs for non-tree grammars are unchanged.
+Aside from the lexer user code block matched text rename described below, the
+lexer/parser value APIs for non-tree grammars are unchanged.
+
+### Lexer user code block matched text
+
+The matched text argument passed to lexer user code blocks has been renamed
+from `match` to `match_text` for all target languages.
+
+- C, C++, and D: rename references to `match` in lexer user code blocks to
+  `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
+
+The `match_length` argument (C, C++) is unchanged.
+This rename only affects lexer user code blocks; parser rule user code blocks
+never had a matched text argument.
 
 ### Tree memory management
 

assets/parser.c.erb

@@ -319,7 +319,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param match_length
  *   Matched text length.
@@ -330,7 +330,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   not explicitly return a token.
  */
 static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match,
+    lexer_user_code_id_t code_id, uint8_t const * match_text,
     size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -534,9 +534,9 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match = &context->input[context->input_index];
+            uint8_t const * match_text = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -1075,7 +1075,7 @@ static size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {

assets/parser.d.erb

@@ -322,8 +322,6 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *
  * @param input
  *   Text input.
- * @param input_length
- *   Text input length.
  *
  * @return Context structure for lexer/parser.
  */
@@ -562,7 +560,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param out_token_info
  *   Lexer token info in progress.
@@ -571,7 +569,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   not explicitly return a token.
  */
 private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, string match,
+    lexer_user_code_id_t code_id, string match_text,
     <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -767,9 +765,9 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
         }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
-            string match = context.input[context.input_index..(context.input_index + match_info.length)];
+            string match_text = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state.code_id, match, &token_info);
+                match_info.accepting_state.code_id, match_text, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -1141,7 +1139,7 @@ private size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1476,7 +1474,6 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
-
 /**
  * Get the current text input position.
  *

assets/parser.rs.erb

@@ -7,18 +7,12 @@
 #![allow(non_upper_case_globals)]
 #![allow(dead_code)]
 #![allow(unused_variables)]
-#![allow(unused_mut)]
 #![allow(unused_parens)]
-#![allow(unused_assignments)]
-#![allow(unreachable_patterns)]
 
 /**************************************************************************
  * User code blocks
  *************************************************************************/
 
-<% unless @grammar.code_blocks.fetch("header", "").empty? %>
-<%= @grammar.code_blocks.fetch("header", "") %>
-<% end %>
 <%= @grammar.code_blocks.fetch("", "") %>
 
 /**************************************************************************
@@ -33,17 +27,6 @@ pub const <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN: usize = 3;
 pub const <%= @grammar.prefix.upcase %>DROP: usize = 4;
 pub const <%= @grammar.prefix.upcase %>EOF: usize = 5;
 pub const <%= @grammar.prefix.upcase %>USER_TERMINATED: usize = 6;
-<% if @grammar.prefix.upcase != "P_" %>
-
-/* Internal result code aliases used by the generated implementation below. */
-const P_SUCCESS: usize = 0;
-const P_DECODE_ERROR: usize = 1;
-const P_UNEXPECTED_INPUT: usize = 2;
-const P_UNEXPECTED_TOKEN: usize = 3;
-const P_DROP: usize = 4;
-const P_EOF: usize = 5;
-const P_USER_TERMINATED: usize = 6;
-<% end %>
 
 /** Token type. */
 pub type <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
@@ -227,8 +210,6 @@ pub struct <%= @grammar.prefix %>context_t {
 
     /** Input text. */
     input: Vec<u8>,
-    /** Input text length. */
-    input_length: usize,
     /** Input text index (byte offset). */
     input_index: usize,
     /** Input text position (row/column). */
@@ -268,6 +249,24 @@ pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>]
 <% end %>
 ];
 
+/**************************************************************************
+ * Private types
+ *************************************************************************/
+
+<% if @grammar.prefix.upcase != "P_" %>
+/* Result codes. */
+const P_SUCCESS: usize = 0;
+const P_DECODE_ERROR: usize = 1;
+const P_UNEXPECTED_INPUT: usize = 2;
+const P_UNEXPECTED_TOKEN: usize = 3;
+const P_DROP: usize = 4;
+const P_EOF: usize = 5;
+const P_USER_TERMINATED: usize = 6;
+<% end %>
+
+/* An invalid ID value. */
+const INVALID_ID: usize = usize::MAX;
+
 /**************************************************************************
  * State initialization
  *************************************************************************/
@@ -275,6 +274,8 @@ pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>]
 /**
  * Allocate and initialize lexer/parser context structure.
  *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
+ *
  * @param input
  *   Text input.
  *
@@ -285,7 +286,6 @@ pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>
 
     /* Lexer initialization. */
     context.input = input.to_vec();
-    context.input_length = input.len();
     context.text_position.row = 1;
     context.text_position.col = 1;
     context.mode = <%= @lexer.mode_id("default") %>;
@@ -304,16 +304,22 @@ pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>
  * @param context
  *   Lexer/parser context structure.
  */
-pub fn <%= @grammar.prefix %>context_delete(mut context: <%= @grammar.prefix %>context_t) {
-<% if @grammar.tree && @grammar.free_token_node != "" %>
+<% free_token_node_used = @grammar.tree && @grammar.free_token_node != "" %>
+<% if free_token_node_used %>
+/* The context is taken as mutable for the benefit of the free_token_node user
+ * code block below, which is permitted but not required to modify the node it
+ * is freeing. */
+#[allow(unused_mut)]
+<% end %>
+pub fn <%= @grammar.prefix %>context_delete(<%= free_token_node_used ? "mut " : "" %>context: <%= @grammar.prefix %>context_t) {
+<% if free_token_node_used %>
     for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() {
         if context.<%= @grammar.prefix %>tree_nodes[i].is_token {
             let token_node_id = i;
-<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
+<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
         }
     }
 <% end %>
-    drop(context);
 }
 
 /**************************************************************************
@@ -387,14 +393,21 @@ pub fn <%= @grammar.prefix %>decode_code_point(input: &[u8],
  * Lexer
  *************************************************************************/
 
+type lexer_state_id_t = <%= get_type_for(@lexer.state_table.size) %>;
+
 /** Invalid lexer state ID. */
-const INVALID_LEXER_STATE_ID: <%= get_type_for(@lexer.state_table.size) %> = <%= @lexer.state_table.size %>;
+const INVALID_LEXER_STATE_ID: lexer_state_id_t = <%= @lexer.state_table.size %>;
 
 /** Invalid lexer user code ID. */
 <% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
 const INVALID_USER_CODE_ID: <%= get_type_for(user_code_id_count) %> = <%= user_code_id_count %>;
 
-/** Lexer transition table entry. */
+/**
+ * Lexer transition table entry.
+ *
+ * An incoming code point matching the range for a transition entry will cause
+ * the lexer to progress to the destination state.
+ */
 #[derive(Clone, Copy)]
 struct lexer_transition_t {
     /** First code point in the range for this transition. */
@@ -402,7 +415,7 @@ struct lexer_transition_t {
     /** Last code point in the range for this transition. */
     last: <%= @grammar.prefix %>code_point_t,
     /** Destination lexer state ID for this transition. */
-    destination_state: <%= get_type_for(@lexer.state_table.size) %>,
+    destination_state: lexer_state_id_t,
 }
 
 /** Lexer state table entry. */
@@ -442,7 +455,7 @@ struct lexer_match_info_t {
     /** Input text position delta to next code point after token end. */
     delta_position: <%= @grammar.prefix %>position_t,
     /** Accepting lexer state from the match (state ID, or INVALID). */
-    accepting_state: <%= get_type_for(@lexer.state_table.size) %>,
+    accepting_state: lexer_state_id_t,
 }
 
 /** Lexer transition table. */
@@ -469,12 +482,21 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
 /**
  * Execute user code associated with a lexer pattern.
  *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param code_id
+ *   The ID of the user code block to execute.
+ * @param match_text
+ *   Matched text for this pattern.
+ * @param out_token_info
+ *   Lexer token info in progress.
+ *
  * @return Token to accept, or invalid token if the user code does
  *   not explicitly return a token.
  */
 fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
-    code_id: <%= get_type_for(user_code_id_count) %>, match_: &[u8],
-    match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
+    code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8],
+    out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
     match code_id {
 <% @grammar.patterns.each do |pattern| %>
 <%   if pattern.code_id %>
@@ -496,9 +518,14 @@ fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
  * Check if there is a transition from the current lexer state to another
  * based on the given input code point.
  *
+ * @param current_state
+ *   Current lexer state.
+ * @param code_point
+ *   Input code point.
+ *
  * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
  */
-fn check_lexer_transition(current_state: u32, code_point: u32) -> <%= get_type_for(@lexer.state_table.size) %> {
+fn check_lexer_transition(current_state: u32, code_point: u32) -> lexer_state_id_t {
     let transition_table_index = lexer_state_table[current_state as usize].transition_table_index as u32;
     for i in 0..(lexer_state_table[current_state as usize].n_transitions as u32) {
         let t = &lexer_transition_table[(transition_table_index + i) as usize];
@@ -511,6 +538,24 @@ fn check_lexer_transition(current_state: u32, code_point: u32) -> <%= get_type_f
 
 /**
  * Find the longest lexer pattern match at the current position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_EOF
+ *   The end of the text input was reached.
  */
 fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
     out_match_info: &mut lexer_match_info_t, out_unexpected_input_length: &mut usize) -> usize {
@@ -540,7 +585,7 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                     }
                     current_state = transition_state as u32;
                     if lexer_state_table[current_state as usize].accepts {
-                        attempt_match.accepting_state = current_state as <%= get_type_for(@lexer.state_table.size) %>;
+                        attempt_match.accepting_state = current_state as lexer_state_id_t;
                         longest_match = attempt_match;
                     }
                 } else if longest_match.length > 0 {
@@ -557,6 +602,7 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                     *out_match_info = longest_match;
                     return P_SUCCESS;
                 } else if attempt_match.length != 0 {
+                    /* There is a partial match - error! */
                     *out_unexpected_input_length = attempt_match.length;
                     return P_UNEXPECTED_INPUT;
                 } else {
@@ -564,6 +610,9 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                 }
             }
             P_DECODE_ERROR => {
+                /* If we see a decode error, we may be partially in the middle of
+                 * matching a pattern, so return the attempted match info so that
+                 * the input text position can be updated. */
                 *out_match_info = attempt_match;
                 return result;
             }
@@ -576,6 +625,23 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
 
 /**
  * Attempt to lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_DROP
+ *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
     let mut token_info = <%= @grammar.prefix %>token_info_t::default();
@@ -587,6 +653,11 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
     match result {
         P_SUCCESS => {
             let mut token_to_accept = lexer_state_table[match_info.accepting_state as usize].token;
+            /* Calculate the token length and start/end positions before invoking
+             * the lexer user code so that the user code can access them. The
+             * context input text position tracking is not updated until after the
+             * user code has run so that it is left unchanged if the user code
+             * requests to terminate the lexer. */
             token_info.length = match_info.length;
             if match_info.end_delta_position.row != 0 {
                 token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
@@ -599,10 +670,16 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
                 let match_start = context.input_index;
                 let match_slice = context.input[match_start..(match_start + match_info.length)].to_vec();
                 let user_code_token = lexer_user_code(context,
-                    lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, match_info.length, &mut token_info);
+                    lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, &mut token_info);
+                /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+                 * that the user code is requesting to terminate the lexer. */
                 if user_code_token == TERMINATE_TOKEN_ID {
                     return P_USER_TERMINATED;
                 }
+                /* An invalid token returned from lexer_user_code() means that the
+                 * user code did not explicitly return a token. So only override
+                 * the token to return if the user code does explicitly return a
+                 * token. */
                 if user_code_token != INVALID_TOKEN_ID {
                     token_to_accept = user_code_token;
                 }
@@ -649,6 +726,21 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
 
 /**
  * Lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
     loop {
@@ -663,16 +755,33 @@ pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t,
  * Parser
  *************************************************************************/
 
-/* An invalid ID value. */
-const INVALID_ID: usize = usize::MAX;
+/** Reduce ID type. */
+type reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
+
+/**
+ * A symbol ID can hold either a token ID or a rule set ID.
+ *
+ * Token IDs and rule set IDs share the same namespace, with rule set IDs
+ * beginning after token IDs end.
+ */
+type symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
+
+/** Parser state ID type. */
+type parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
+
+/** Parser rule ID type. */
+type rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
+
+/** Parser shift ID type. */
+type shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
 
 /** Shift table entry. */
 #[derive(Clone, Copy)]
 struct shift_t {
     /** Token or rule set ID. */
-    symbol_id: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>,
+    symbol_id: symbol_id_t,
     /** Parser state to shift to. */
-    state_id: <%= get_type_for(@parser.state_table.size) %>,
+    state_id: parser_state_id_t,
 }
 
 /** Reduce table entry. */
@@ -680,18 +789,47 @@ struct shift_t {
 struct reduce_t {
     /** Lookahead token. */
     token: <%= @grammar.prefix %>token_t,
-    /** Rule ID. */
-    rule: <%= get_type_for(@grammar.rules.size) %>,
-    /** Rule set ID. */
-    rule_set: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>,
-    /** Number of states leading to this reduce action. */
-    n_states: <%= get_type_for(@parser.state_table.size) %>,
+
+    /**
+     * Rule ID.
+     *
+     * This is used to execute the parser user code block associated with a
+     * grammar rule.
+     */
+    rule: rule_id_t,
+
+    /**
+     * Rule set ID.
+     *
+     * This is used as the new top symbol ID of the parse stack after this
+     * reduce action.
+     */
+    rule_set: symbol_id_t,
+
+    /**
+     * Number of states leading to this reduce action.
+     *
+     * This is the number of entries popped from the parse stack after this
+     * reduce action.
+     */
+    n_states: parser_state_id_t,
 <% if @grammar.tree %>
-    /** Map of rule components to rule set child fields (None for a flat map). */
+
+    /**
+     * Map of rule components to rule set child fields (None for a flat map).
+     */
     rule_set_node_field_index_map: Option<&'static [u16]>,
-    /** Number of rule set tree node fields. */
+
+    /**
+     * Number of rule set tree node fields.
+     */
     rule_set_node_field_array_size: u16,
-    /** Whether this rule propagates a matched optional target node. */
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
     propagate_optional_target: bool,
 <% end %>
 }
@@ -700,13 +838,16 @@ struct reduce_t {
 #[derive(Clone, Copy)]
 struct parser_state_t {
     /** First shift table entry for this parser state. */
-    shift_table_index: <%= get_type_for(@parser.shift_table.size) %>,
+    shift_table_index: shift_id_t,
+
     /** Number of shift table entries for this parser state. */
-    n_shift_entries: <%= get_type_for(@parser.shift_table.size) %>,
+    n_shift_entries: shift_id_t,
+
     /** First reduce table entry for this parser state. */
-    reduce_table_index: <%= get_type_for(@parser.reduce_table.size) %>,
+    reduce_table_index: reduce_id_t,
+
     /** Number of reduce table entries for this parser state. */
-    n_reduce_entries: <%= get_type_for(@parser.reduce_table.size) %>,
+    n_reduce_entries: reduce_id_t,
 }
 
 /**
@@ -718,6 +859,7 @@ struct parser_state_t {
 struct state_value_t {
     /** Parser state ID. */
     state_id: usize,
+
 <% if @grammar.tree %>
     /** Tree node ID. */
     node_id: <%= @grammar.prefix %>node_id_t,
@@ -829,6 +971,13 @@ fn get_rule_position(statevalues: &[state_value_t], i: usize, n_states: usize, g
 <% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
+ *
+ * @param rule The ID of the rule.
+ *
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
 fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.tree ? "_node_id: #{@grammar.prefix}node_id_t" : "_pvalue: &mut #{@grammar.prefix}value_t" %>, rule: u32, statevalues: &[state_value_t], n_states: usize) -> usize {
     match rule {
@@ -852,6 +1001,11 @@ fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.
 /**
  * Check if the parser should shift to a new state.
  *
+ * @param state_id
+ *   Parser state ID.
+ * @param symbol_id
+ *   Incoming token/rule set ID.
+ *
  * @return State to shift to, or INVALID_ID if none.
  */
 fn check_shift(state_id: usize, symbol_id: usize) -> usize {
@@ -868,6 +1022,11 @@ fn check_shift(state_id: usize, symbol_id: usize) -> usize {
 /**
  * Check if the parser should reduce to a new state.
  *
+ * @param state_id
+ *   Parser state ID.
+ * @param token
+ *   Incoming token.
+ *
  * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize {
@@ -883,6 +1042,30 @@ fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize
 
 /**
  * Run the parser.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param start_state_id
+ *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
+ *
+ * @retval P_SUCCESS
+ *   The parser successfully matched the input text. The parse result value
+ *   can be accessed with <%= @grammar.prefix %>result().
+ * @retval P_UNEXPECTED_TOKEN
+ *   An unexpected token was encountered that does not match any grammar rule.
+ *   The function p_token(&context) can be used to get the unexpected token.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
  */
 fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usize,
     start_rule_set_id: usize, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize {
@@ -901,16 +1084,17 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
     statevalues.push(state_value_t::default());
     let sv_len = statevalues.len();
     statevalues[sv_len - 1].state_id = start_state_id;
-    let result;
     loop {
         if token == INVALID_TOKEN_ID {
             let lexer_result = <%= lex_fn %>(context, &mut token_info);
             if lexer_result != P_SUCCESS {
-                result = lexer_result;
-                break;
+                return lexer_result;
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
         let mut token_is_follow = false;
         for &ft in follow_tokens {
             if token == ft {
@@ -931,12 +1115,23 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 <% else %>
                 context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
 <% end %>
-                result = P_SUCCESS;
-                break;
+                return P_SUCCESS;
             }
             if (shift_state == INVALID_ID) && token_is_follow {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
                 let retry_shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF as usize);
-                if (retry_shift_state != INVALID_ID) && (statevalues.len() == 2) && (last_shifted_rule_set_id == start_rule_set_id) {
+                if (retry_shift_state != INVALID_ID) &&
+                    (statevalues.len() == 2) &&
+                    (last_shifted_rule_set_id == start_rule_set_id) {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
                     context.input_index -= token_info.length;
                     context.text_position = token_info.position;
 <% if @grammar.tree %>
@@ -944,12 +1139,15 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 <% else %>
                     context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
 <% end %>
-                    result = P_SUCCESS;
-                    break;
+                    return P_SUCCESS;
                 }
             }
         }
         if shift_state != INVALID_ID {
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
             last_shifted_rule_set_id = reduced_rule_set;
             statevalues.push(state_value_t::default());
             let new_index = statevalues.len() - 1;
@@ -967,7 +1165,7 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
                     token_tree_node.token = token;
                     token_tree_node.pvalue = token_info.pvalue.clone();
                 }
-<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %>
+<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %>
                 statevalues[new_index].node_id = token_node_id;
 <% else %>
                 statevalues[new_index].position = token_info.position;
@@ -992,6 +1190,11 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 
         let mut reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, token);
         if (reduce_index == INVALID_ID) && token_is_follow {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
             reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF);
         }
         if reduce_index != INVALID_ID {
@@ -1063,13 +1266,15 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
             continue;
         }
 
-        /* Unexpected token. Reset the context text position to the token. */
+        /* A token was successfully lexed, so the input text position was
+         * advanced. However, this is an unexpected token, so we want to reset
+         * the context text position to point to the token rather than the text
+         * after it, so that if the caller wants to report the error position,
+         * it will point to the correct position of the unexpected token. */
         context.text_position = token_info.position;
         context.token = token;
-        result = P_UNEXPECTED_TOKEN;
-        break;
+        return P_UNEXPECTED_TOKEN;
     }
-    result
 }
 
 pub fn <%= @grammar.prefix %>parse(context: &mut <%= @grammar.prefix %>context_t) -> usize {
@@ -1088,6 +1293,11 @@ pub fn <%= @grammar.prefix %>parse_inner_<%= start_rule %>(context: &mut <%= @gr
 
 /**
  * Get the parse result value.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Parse result value.
  */
 <% if @grammar.tree %>
 pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(@grammar.start_rules[0]) %><'_> {
@@ -1109,32 +1319,84 @@ pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.pre
 <%   end %>
 <% end %>
 
-/** Get the current text input position. */
+/**
+ * Get the current text input position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current text position.
+ */
 pub fn <%= @grammar.prefix %>position(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>position_t {
     context.text_position
 }
 
-/** Set the current text input position. */
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
 pub fn <%= @grammar.prefix %>set_position(context: &mut <%= @grammar.prefix %>context_t, position: <%= @grammar.prefix %>position_t) {
     context.text_position = position;
 }
 
-/** Get the current input text byte offset. */
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
 pub fn <%= @grammar.prefix %>input_index(context: &<%= @grammar.prefix %>context_t) -> usize {
     context.input_index
 }
 
-/** Set the current input text byte offset. */
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
 pub fn <%= @grammar.prefix %>set_input_index(context: &mut <%= @grammar.prefix %>context_t, input_index: usize) {
     context.input_index = input_index;
 }
 
-/** Get the user terminate code. */
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
 pub fn <%= @grammar.prefix %>user_terminate_code(context: &<%= @grammar.prefix %>context_t) -> usize {
     context.user_terminate_code
 }
 
-/** Get the parse token. */
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
 pub fn <%= @grammar.prefix %>token(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>token_t {
     context.token
 }

spec/json_parser.c.propane

@@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n = 0.0;
     bool negative = false;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative = false;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
+  char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {(char)match[0], 0};
+  char s[] = {(char)match_text[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_parser.d.propane

@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n;
     bool negative;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
+  string_value ~= "{" ~ match_text[2..6] ~ "}";
 >>
 string: /[^\\]/ <<
-  string_value ~= match;
+  string_value ~= match_text;
 >>
 Start -> Value <<
   $$ = $1;

spec/json_parser.rust.propane

@@ -1,4 +1,4 @@
-<<header
+<<
 pub const JSON_OBJECT: usize = 0;
 pub const JSON_ARRAY: usize = 1;
 pub const JSON_NUMBER: usize = 2;
@@ -60,7 +60,7 @@ token rbracket /\]/;
 token comma /,/;
 token colon /:/;
 token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
+    let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
     $$ = JSONValue::Number(n);
 >>
 token true <<
@@ -106,11 +106,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
+  let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
   ${context.string_value}.push_str(&s);
 >>
 string: /[^\\]/ <<
-  ${context.string_value}.push(match_[0] as char);
+  ${context.string_value}.push(match_text[0] as char);
 >>
 Start -> Value <<
   $$ = $1;

spec/macros.c.propane

@@ -14,7 +14,7 @@ token rbrace /\}/;
 token plus /\+/;
 token macro;
 token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/macros.d.propane

@@ -14,7 +14,7 @@ token macro;
 token macroname /@[a-zA-Z_]\w*/;
 token num /\d+/ <<
     int n = 0;
-    foreach (c; match)
+    foreach (c; match_text)
     {
         n *= 10;
         n += (c - '0');

spec/macros.rust.propane

@@ -67,7 +67,7 @@ token macro;
 token macroname /@[a-zA-Z_]\w*/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/parse_inner_nested.c.propane

@@ -12,7 +12,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.d.propane

@@ -10,7 +10,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.rust.propane

@@ -32,7 +32,7 @@ token rparen /\)/;
 token plus /\+/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/propane_spec.rb

@@ -39,6 +39,11 @@ class MyFormatter
 end
 SimpleCov.start do
   command_name(#{command_name.inspect})
+  # Write this process's results to its own directory. Sharing one resultset
+  # file means every propane invocation must read, merge, and rewrite the
+  # results of all previous invocations, which grows quadratically over the
+  # suite. The parts are merged once at the end by the spec Rake task.
+  coverage_dir(#{"coverage/parts/#{command_name}".inspect})
   filters.clear
   add_filter do |src|
     !(src.filename[SimpleCov.root])
@@ -86,26 +91,29 @@ EOF
     end
     case options[:language]
     when "c"
-      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[gcc -g -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "cpp"
-      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[g++ -g -x c++ -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
+      command = [*%w[ldc2 -g -w -de --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
     when "rust"
       # Compile each generated parser to an rlib, then compile the test crate
       # against them. Safe Rust needs no valgrind, but it is run anyway.
+      # Both compiles deny warnings so that a warning introduced into the
+      # generated parser (or into a spec test file) fails the spec rather than
+      # scrolling past unnoticed.
       externs = []
       options[:parsers].each do |name|
         crate = "testparser#{name}"
         rlib = "spec/run/lib#{crate}.rlib"
-        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -A warnings],
+        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -D warnings],
                  "--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"]
         expect(system(*rustc)).to be_truthy
         externs += ["--extern", "#{crate}=#{rlib}"]
       end
       # rustc takes a single crate root; any additional Rust test files are
       # expected to be pulled in as modules or provided by the parser crate.
-      command = [*%w[rustc --edition 2021 -A warnings -o spec/run/testparser], *externs, test_files.first]
+      command = [*%w[rustc --edition 2021 -D warnings -o spec/run/testparser], *externs, test_files.first]
     end
     result = system(*command)
     expect(result).to be_truthy
@@ -119,7 +127,7 @@ EOF
     results = Results.new(stdout, stderr, status)
     # Valgrind is only reliably available on Linux, so limit the leak check to
     # Linux platforms.
-    if RUBY_PLATFORM =~ /linux/
+    if RUBY_PLATFORM =~ /linux/ && ENV["spec-valgrind"]
       stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
       vgout = stdout + stderr
       File.binwrite("spec/run/.vgout", vgout)
@@ -307,9 +315,9 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c cpp rust].each do |language|
+  {"d" => "D", "c" => "C", "cpp" => "C++", "rust" => "Rust"}.each do |language, language_name|
 
-    context "#{language.upcase} language" do
+    context "#{language_name} language" do
 
       it "generates a lexer" do
         write_grammar <<EOF
@@ -338,7 +346,7 @@ token int /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -349,7 +357,7 @@ EOF
 ptype int;
 token int /\\d+/ <<
   int v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -363,7 +371,7 @@ EOF
 ptype i64;
 token int /\\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 Start -> int << $$ = $1; >>
@@ -410,7 +418,7 @@ token integer /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -441,7 +449,7 @@ token times /\\*/;
 token power /\\*\\*/;
 token integer /\\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -470,7 +478,7 @@ token times /\\*/;
 token power /\\*\\*/;
 token integer /\\d+/ <<
   let mut v: u64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as u64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as u64; }
   $$ = v;
 >>
 token lparen /\\(/;
@@ -770,7 +778,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
   return $token(ident);
 >>
@@ -791,7 +799,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
 >>
 token dot /\\./ <<
@@ -808,7 +816,7 @@ ptype u8;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match_[0];
+  $$ = match_text[0];
   $mode(default);
   return $token(ident);
 >>
@@ -1059,7 +1067,7 @@ EOF
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * t = (char *)malloc(match_length + 1);
-  strncpy(t, (char *)match, match_length);
+  strncpy(t, (char *)match_text, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
 >>
@@ -1071,14 +1079,14 @@ EOF
 import std.stdio;
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  writeln("Matched token is ", match);
+  writeln("Matched token is ", match_text);
 >>
 Start -> id;
 EOF
         when "rust"
           write_grammar <<EOF
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
+  println!("Matched token is {}", std::str::from_utf8(match_text).unwrap());
 >>
 Start -> id;
 EOF
@@ -1109,7 +1117,7 @@ EOF
           write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
-  $$ = match.length;
+  $$ = match_text.length;
 >>
 Start -> word <<
   $$ = $1;
@@ -1119,7 +1127,7 @@ EOF
           write_grammar <<EOF
 ptype u64;
 token word /[a-z]+/ <<
-  $$ = match_length as u64;
+  $$ = match_text.len() as u64;
 >>
 Start -> word <<
   $$ = $1;
@@ -1788,7 +1796,7 @@ EOF
           write_grammar <<EOF
 ptype String;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = std::str::from_utf8(match_).unwrap().to_string();
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
 >>
 drop /\\s+/;
 Start -> id:first id:second <<
@@ -1803,7 +1811,7 @@ import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id:first id:second <<
@@ -1820,7 +1828,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -1846,7 +1854,7 @@ EOF
           write_grammar <<EOF
 ptype String;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = std::str::from_utf8(match_).unwrap().to_string();
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
 >>
 drop /\\s+/;
 Start -> id;
@@ -1864,7 +1872,7 @@ import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id;
@@ -1884,7 +1892,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -2082,7 +2090,7 @@ EOF
           write_grammar <<EOF
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
+  eprint!("comment: {}", std::str::from_utf8(match_text).unwrap());
 >>
 token a;
 Start -> a;
@@ -2094,7 +2102,7 @@ import std.stdio;
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  stderr.write("comment: ", match);
+  stderr.write("comment: ", match_text);
 >>
 token a;
 Start -> a;
@@ -2107,7 +2115,7 @@ EOF
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+  fprintf(stderr, "comment: %.*s", (int)match_length, match_text);
 >>
 token a;
 Start -> a;
@@ -2129,7 +2137,7 @@ context_user_fields <<
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::str::from_utf8(match_).unwrap();
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
 >>
 token a <<
     ${context.acount} += 1;
@@ -2146,7 +2154,7 @@ context_user_fields <<
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token a <<
     ${context.acount}++;
@@ -2173,7 +2181,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -2212,7 +2220,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::str::from_utf8(match_).unwrap();
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -2234,7 +2242,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -2270,7 +2278,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -2301,7 +2309,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::string((const char *)match, match_length);
+    ${context.comments} += std::string((const char *)match_text, match_length);
 >>
 token id /\\w+/;
 Start -> IDs;

spec/rewind.c.propane

@@ -14,7 +14,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.d.propane

@@ -11,7 +11,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.rust.propane

@@ -56,7 +56,7 @@ token rbrace /\}/;
 token plus /\+/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/spec_helper.rb

@@ -15,6 +15,10 @@ unless ENV["dist_specs"]
       command_name "RSpec"
     end
     project_name "Propane"
+    # Keep this process's results separate from the propane subprocess results
+    # so that nothing has to merge on the fly; the spec Rake task collates all
+    # of the parts once the suite is done.
+    coverage_dir "coverage/parts/rspec"
     merge_timeout 3600
     formatter(MyFormatter)
   end

spec/test_input_index.rs

@@ -1,7 +1,7 @@
 use testparser::*;
 
 fn main() {
-    let mut c = p_context_new(b"ab");
+    let c = p_context_new(b"ab");
     assert_eq!(0, p_input_index(&c));
     p_context_delete(c);
 

spec/tree_node_memory_remains.c.propane

@@ -46,7 +46,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  $$.s = match;
+  $$.s = match_text;
   $mode(default);
   return $token(ident);
 >>

spec/tree_node_memory_remains.d.propane

@@ -42,8 +42,8 @@ token semicolon /;/;
 
 # Integer literals.
 token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
-  $$.bi = BigInt(match[0..3]);
-  foreach (c; match[3..$])
+  $$.bi = BigInt(match_text[0..3]);
+  foreach (c; match_text[3..$])
   {
     if (('0' <= c) && (c <= '9'))
     {
@@ -65,13 +65,13 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  if (match[0] == '$')
+  if (match_text[0] == '$')
   {
-    $$.s = match[1..$];
+    $$.s = match_text[1..$];
   }
   else
   {
-    $$.s = match;
+    $$.s = match_text;
   }
   $mode(default);
   return $token(ident);

spec/tree_node_memory_remains.rust.propane

@@ -1,7 +1,7 @@
 tree;
 tree_prefix P;
 
-<<header
+<<
 #[derive(Clone, Default)]
 pub struct TokenVal {
     pub i64_: u64,
@@ -43,7 +43,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  $$.s = std::str::from_utf8(match_).unwrap().to_string();
+  $$.s = std::str::from_utf8(match_text).unwrap().to_string();
   $mode(default);
   return $token(ident);
 >>