v5.1.0

This takes the place of #line directives

.github/workflows/run-tests.yml

@@ -31,6 +31,9 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby-version }}
 
+    - name: Set up Rust
+      uses: dtolnay/rust-toolchain@stable
+
     - name: Install dependencies
       run: bundle install
 

CHANGELOG.md

@@ -1,17 +1,30 @@
+## v5.1.0
+
+### New Features
+
+- Add a `node_id()` accessor to the C++ and D tree node handle types, for node
+  identity comparison. This matches the existing `p_node_id()` macro (C) and
+  `node_id()` method (Rust).
+
 ## v5.0.0
 
 ### New Features
 
 - Add Rust target language output.
+- Add Rust language detection in propane.vim.
 
 ### API Changes
 
+- The matched text argument passed to lexer user code blocks is now named
+  `match_text` instead of `match`, since `match` is a keyword in Rust. The
+  `match_length` argument (C and C++) is unchanged.
 - Tree generation mode now stores all tree nodes in a compact arena owned by
   the parser context (a flat node array plus a shared child-link array).
   This replaces the previous design of one heap allocation per node with
   layout-punned typed structs.
 - Tree nodes are now referenced by lightweight handles rather than pointers.
-  `p_result()` and the field accessors return handle values.
+  `p_result()` and the field accessors now return handle values in tree
+  generation mode.
 - The whole tree is freed together with the context by `p_context_delete()`.
   The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
   tree node handles are only valid while the context is alive.
@@ -29,6 +42,11 @@
   Reference child fields through the target-language accessors described above
   rather than through struct pointer members.
 
+### Improvements
+
+- Improve D language detection in propane.vim
+- Speed up specs
+
 ## v4.8.1
 
 ### Fixes

README.md

@@ -69,7 +69,7 @@ token times /\*/;
 token power /\*\*/;
 token integer /\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');

Rakefile.rb

@@ -1,3 +1,4 @@
+require "fileutils"
 require "rake/clean"
 require "rspec/core/rake_task"
 require "simplecov"
@@ -11,7 +12,10 @@ end
 
 RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
   if args.example_pattern
+    ENV["partial_specs"] = "1"
     task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  else
+    FileUtils.rm_rf("coverage")
   end
 end
 task :spec do |task, args|
@@ -19,7 +23,7 @@ task :spec do |task, args|
     original_stdout = $stdout
     sio = StringIO.new
     $stdout = sio
-    SimpleCov.collate Dir["coverage/.resultset.json"]
+    SimpleCov.collate Dir["coverage/parts/*/.resultset.json"]
     $stdout = original_stdout
     sio.string.lines.each do |line|
       $stdout.write(line) unless line =~ /Coverage report generated for/
@@ -27,6 +31,15 @@ task :spec do |task, args|
   end
 end
 
+task :valgrind do
+  begin
+    ENV["spec-valgrind"] = "1"
+    Rake::Task[:spec].execute
+  ensure
+    ENV.delete("spec-valgrind")
+  end
+end
+
 # dspec task is useful to test the distributable release script, but is not
 # useful for coverage information.
 desc "Dist Specs"
@@ -43,4 +56,4 @@ task :user_guide do
   system("ruby", "-Ilib", "rb/gen_user_guide.rb")
 end
 
-task :all => [:spec, :dspec, :user_guide]
+task :all => [:valgrind, :dspec, :user_guide]

UPGRADING.md

@@ -2,7 +2,18 @@
 
 The generated API for tree generation mode (`tree;`) has been changed
 significantly for this version.
-The lexer/parser value APIs for non-tree grammars are unchanged.
+Aside from the lexer user code block matched text rename described below, the
+lexer/parser value APIs for non-tree grammars are unchanged.
+
+### Lexer user code block matched text
+
+The matched text argument passed to lexer user code blocks has been renamed
+from `match` to `match_text` for all target languages.
+
+- C, C++, and D: rename references to `match` in lexer user code blocks to
+  `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
+
+The `match_length` argument (C, C++) is unchanged.
 
 ### Tree memory management
 
@@ -38,6 +49,44 @@ Reference child fields through the target-language accessors above (for example
 `$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
 in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
 
+### Pointers into tree node storage
+
+Tree nodes previously each had their own allocation, so a pointer to a node
+stayed valid for the life of the tree. They are now held in a single array
+which is reallocated as it grows, so a pointer or reference into that array may
+be invalidated whenever a new node is created.
+
+New nodes are created while parsing, so this matters for a pointer taken in a
+tree-mode parser rule user code block, which runs before the parse has
+finished. Keep the node handle instead, which stores a node ID rather than an
+address and stays valid, and obtain the pointer from it when it is needed.
+
+For example, replace a saved pointer:
+
+```
+context_user_fields <<
+    p_node_data_t * saved;
+>>
+Items -> Items a << ${context.saved} = p_node_data($$); >>
+```
+
+with a saved handle:
+
+```
+context_user_fields <<
+    Items saved_node;
+>>
+Items -> Items a << ${context.saved_node} = $$; >>
+```
+
+```
+p_node_data_t * data = p_node_data(context->saved_node);
+```
+
+Once parsing has finished, no further nodes are created, so a pointer obtained
+after `p_parse()` returns stays valid until the context is deleted, as long as
+no further parsing is performed with the same context.
+
 ## v4.0.0
 
 ### API Changes

assets/parser.c.erb

@@ -319,7 +319,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param match_length
  *   Matched text length.
@@ -330,7 +330,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   not explicitly return a token.
  */
 static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match,
+    lexer_user_code_id_t code_id, uint8_t const * match_text,
     size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -534,9 +534,9 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match = &context->input[context->input_index];
+            uint8_t const * match_text = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -1075,7 +1075,7 @@ static size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {

assets/parser.d.erb

@@ -149,6 +149,12 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
         return __id != 0u;
     }
 
+    /** Return the node ID (for identity comparison). */
+    @property <%= @grammar.prefix %>node_id_t node_id()
+    {
+        return __id;
+    }
+
     /** Access the underlying node record (token, pvalue, and user fields). */
     @property ref <%= @grammar.prefix %>node_data_t __node()
     {
@@ -177,6 +183,12 @@ public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_s
         return __id != 0u;
     }
 
+    /** Return the node ID (for identity comparison). */
+    @property <%= @grammar.prefix %>node_id_t node_id()
+    {
+        return __id;
+    }
+
     /** Text position of the first code point spanned by this node. */
     @property <%= @grammar.prefix %>position_t position()
     {
@@ -322,8 +334,6 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *
  * @param input
  *   Text input.
- * @param input_length
- *   Text input length.
  *
  * @return Context structure for lexer/parser.
  */
@@ -562,7 +572,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match
+ * @param match_text
  *   Matched text for this pattern.
  * @param out_token_info
  *   Lexer token info in progress.
@@ -571,7 +581,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
  *   not explicitly return a token.
  */
 private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, string match,
+    lexer_user_code_id_t code_id, string match_text,
     <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -767,9 +777,9 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
         }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
-            string match = context.input[context.input_index..(context.input_index + match_info.length)];
+            string match_text = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state.code_id, match, &token_info);
+                match_info.accepting_state.code_id, match_text, &token_info);
             /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
              * that the user code is requesting to terminate the lexer. */
             if (user_code_token == TERMINATE_TOKEN_ID)
@@ -1141,7 +1151,7 @@ private size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return State to reduce to, or INVALID_ID if none.
+ * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1476,7 +1486,6 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
-
 /**
  * Get the current text input position.
  *

assets/parser.h.erb

@@ -135,13 +135,19 @@ typedef struct
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.prefix %>token_info_t;
 
+typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t;
+
+<% if @grammar.tree %>
+<%= c_tree_handle_types_header %>
+
+<% end %>
 /**
  * Lexer and parser context.
  *
  * The user must allocate an instance of this structure and pass it to any
  * public API function.
  */
-typedef struct
+struct <%= @grammar.prefix %>context_s
 {
     /* Lexer context data. */
 
@@ -194,7 +200,7 @@ typedef struct
     size_t user_terminate_code;
 
 <%= @grammar.context_user_fields %>
-} <%= @grammar.prefix %>context_t;
+};
 
 <% if @grammar.tree %>
 <%= c_tree_types_header %>

assets/parser.rs.erb

@@ -5,20 +5,11 @@
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 #![allow(non_upper_case_globals)]
-#![allow(dead_code)]
-#![allow(unused_variables)]
-#![allow(unused_mut)]
-#![allow(unused_parens)]
-#![allow(unused_assignments)]
-#![allow(unreachable_patterns)]
 
 /**************************************************************************
  * User code blocks
  *************************************************************************/
 
-<% unless @grammar.code_blocks.fetch("header", "").empty? %>
-<%= @grammar.code_blocks.fetch("header", "") %>
-<% end %>
 <%= @grammar.code_blocks.fetch("", "") %>
 
 /**************************************************************************
@@ -33,17 +24,6 @@ pub const <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN: usize = 3;
 pub const <%= @grammar.prefix.upcase %>DROP: usize = 4;
 pub const <%= @grammar.prefix.upcase %>EOF: usize = 5;
 pub const <%= @grammar.prefix.upcase %>USER_TERMINATED: usize = 6;
-<% if @grammar.prefix.upcase != "P_" %>
-
-/* Internal result code aliases used by the generated implementation below. */
-const P_SUCCESS: usize = 0;
-const P_DECODE_ERROR: usize = 1;
-const P_UNEXPECTED_INPUT: usize = 2;
-const P_UNEXPECTED_TOKEN: usize = 3;
-const P_DROP: usize = 4;
-const P_EOF: usize = 5;
-const P_USER_TERMINATED: usize = 6;
-<% end %>
 
 /** Token type. */
 pub type <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
@@ -66,7 +46,7 @@ pub type <%= @grammar.prefix %>code_point_t = u32;
  *
  * This is useful for reporting errors, etc...
  */
-#[derive(Clone, Copy, Default, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, PartialEq)]
 pub struct <%= @grammar.prefix %>position_t {
     /** Input text row (1-based). */
     pub row: u32,
@@ -100,6 +80,8 @@ pub enum <%= @grammar.prefix %>value_t {
 
 impl <%= @grammar.prefix %>value_t {
 <%   @grammar.ptypes.each do |name, typestring| %>
+    /* A grammar need not assign to $$ for every declared ptype. */
+    #[allow(dead_code)]
     fn v_<%= name %>_mut(&mut self) -> &mut <%= rust_ptype(typestring) %> {
         match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v, _ => unreachable!() }
     }
@@ -227,8 +209,6 @@ pub struct <%= @grammar.prefix %>context_t {
 
     /** Input text. */
     input: Vec<u8>,
-    /** Input text length. */
-    input_length: usize,
     /** Input text index (byte offset). */
     input_index: usize,
     /** Input text position (row/column). */
@@ -268,6 +248,24 @@ pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>]
 <% end %>
 ];
 
+/**************************************************************************
+ * Private types
+ *************************************************************************/
+
+<% if @grammar.prefix.upcase != "P_" %>
+/* Result codes. */
+const P_SUCCESS: usize = 0;
+const P_DECODE_ERROR: usize = 1;
+const P_UNEXPECTED_INPUT: usize = 2;
+const P_UNEXPECTED_TOKEN: usize = 3;
+const P_DROP: usize = 4;
+const P_EOF: usize = 5;
+const P_USER_TERMINATED: usize = 6;
+<% end %>
+
+/* An invalid ID value. */
+const INVALID_ID: usize = usize::MAX;
+
 /**************************************************************************
  * State initialization
  *************************************************************************/
@@ -275,6 +273,8 @@ pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>]
 /**
  * Allocate and initialize lexer/parser context structure.
  *
+ * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
+ *
  * @param input
  *   Text input.
  *
@@ -285,7 +285,6 @@ pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>
 
     /* Lexer initialization. */
     context.input = input.to_vec();
-    context.input_length = input.len();
     context.text_position.row = 1;
     context.text_position.col = 1;
     context.mode = <%= @lexer.mode_id("default") %>;
@@ -298,22 +297,36 @@ pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>
     context
 }
 
+<% free_token_node_used = @grammar.tree && @grammar.free_token_node != "" %>
+<% if free_token_node_used %>
+impl Drop for <%= @grammar.prefix %>context_t {
+    /* Run the free_token_node user code block for every token node in the tree. */
+    fn drop(&mut self) {
+        /* Named so that ${context.<field>} expansions resolve here. */
+        let context = self;
+        for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() {
+            if context.<%= @grammar.prefix %>tree_nodes[i].is_token {
+                let token_node_id = i;
+<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
+            }
+        }
+    }
+}
+
+<% end %>
 /**
  * Deinitialize and deallocate lexer/parser context structure.
  *
+ * The memory owned by the context is released when the context is dropped, so
+ * this function only has to consume it. It is provided for symmetry with
+ * <%= @grammar.prefix %>context_new() and with the other target languages;
+ * letting the context go out of scope has the same effect.
+ *
  * @param context
  *   Lexer/parser context structure.
  */
-pub fn <%= @grammar.prefix %>context_delete(mut context: <%= @grammar.prefix %>context_t) {
-<% if @grammar.tree && @grammar.free_token_node != "" %>
-    for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() {
-        if context.<%= @grammar.prefix %>tree_nodes[i].is_token {
-            let token_node_id = i;
-<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
-        }
-    }
-<% end %>
-    drop(context);
+#[allow(unused_variables)]
+pub fn <%= @grammar.prefix %>context_delete(context: <%= @grammar.prefix %>context_t) {
 }
 
 /**************************************************************************
@@ -387,14 +400,21 @@ pub fn <%= @grammar.prefix %>decode_code_point(input: &[u8],
  * Lexer
  *************************************************************************/
 
+type lexer_state_id_t = <%= get_type_for(@lexer.state_table.size) %>;
+
 /** Invalid lexer state ID. */
-const INVALID_LEXER_STATE_ID: <%= get_type_for(@lexer.state_table.size) %> = <%= @lexer.state_table.size %>;
+const INVALID_LEXER_STATE_ID: lexer_state_id_t = <%= @lexer.state_table.size %>;
 
 /** Invalid lexer user code ID. */
 <% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
 const INVALID_USER_CODE_ID: <%= get_type_for(user_code_id_count) %> = <%= user_code_id_count %>;
 
-/** Lexer transition table entry. */
+/**
+ * Lexer transition table entry.
+ *
+ * An incoming code point matching the range for a transition entry will cause
+ * the lexer to progress to the destination state.
+ */
 #[derive(Clone, Copy)]
 struct lexer_transition_t {
     /** First code point in the range for this transition. */
@@ -402,7 +422,7 @@ struct lexer_transition_t {
     /** Last code point in the range for this transition. */
     last: <%= @grammar.prefix %>code_point_t,
     /** Destination lexer state ID for this transition. */
-    destination_state: <%= get_type_for(@lexer.state_table.size) %>,
+    destination_state: lexer_state_id_t,
 }
 
 /** Lexer state table entry. */
@@ -442,7 +462,7 @@ struct lexer_match_info_t {
     /** Input text position delta to next code point after token end. */
     delta_position: <%= @grammar.prefix %>position_t,
     /** Accepting lexer state from the match (state ID, or INVALID). */
-    accepting_state: <%= get_type_for(@lexer.state_table.size) %>,
+    accepting_state: lexer_state_id_t,
 }
 
 /** Lexer transition table. */
@@ -469,12 +489,27 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
 /**
  * Execute user code associated with a lexer pattern.
  *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param code_id
+ *   The ID of the user code block to execute.
+ * @param match_text
+ *   Matched text for this pattern.
+ * @param out_token_info
+ *   Lexer token info in progress.
+ *
  * @return Token to accept, or invalid token if the user code does
  *   not explicitly return a token.
  */
+/* The parameters of this function are provided for the user code blocks
+ * inlined into it, which are not obliged to reference any of them. A $$ or $N
+ * reference in a user code block also expands to a parenthesized dereference,
+ * since it may be followed there by a field or method access; those
+ * parentheses are redundant where the reference is a complete argument. */
+#[allow(unused_parens, unused_variables)]
 fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
-    code_id: <%= get_type_for(user_code_id_count) %>, match_: &[u8],
-    match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
+    code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8],
+    out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
     match code_id {
 <% @grammar.patterns.each do |pattern| %>
 <%   if pattern.code_id %>
@@ -496,9 +531,14 @@ fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
  * Check if there is a transition from the current lexer state to another
  * based on the given input code point.
  *
+ * @param current_state
+ *   Current lexer state.
+ * @param code_point
+ *   Input code point.
+ *
  * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
  */
-fn check_lexer_transition(current_state: u32, code_point: u32) -> <%= get_type_for(@lexer.state_table.size) %> {
+fn check_lexer_transition(current_state: u32, code_point: u32) -> lexer_state_id_t {
     let transition_table_index = lexer_state_table[current_state as usize].transition_table_index as u32;
     for i in 0..(lexer_state_table[current_state as usize].n_transitions as u32) {
         let t = &lexer_transition_table[(transition_table_index + i) as usize];
@@ -511,6 +551,24 @@ fn check_lexer_transition(current_state: u32, code_point: u32) -> <%= get_type_f
 
 /**
  * Find the longest lexer pattern match at the current position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_EOF
+ *   The end of the text input was reached.
  */
 fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
     out_match_info: &mut lexer_match_info_t, out_unexpected_input_length: &mut usize) -> usize {
@@ -540,7 +598,7 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                     }
                     current_state = transition_state as u32;
                     if lexer_state_table[current_state as usize].accepts {
-                        attempt_match.accepting_state = current_state as <%= get_type_for(@lexer.state_table.size) %>;
+                        attempt_match.accepting_state = current_state as lexer_state_id_t;
                         longest_match = attempt_match;
                     }
                 } else if longest_match.length > 0 {
@@ -557,6 +615,7 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                     *out_match_info = longest_match;
                     return P_SUCCESS;
                 } else if attempt_match.length != 0 {
+                    /* There is a partial match - error! */
                     *out_unexpected_input_length = attempt_match.length;
                     return P_UNEXPECTED_INPUT;
                 } else {
@@ -564,6 +623,9 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
                 }
             }
             P_DECODE_ERROR => {
+                /* If we see a decode error, we may be partially in the middle of
+                 * matching a pattern, so return the attempted match info so that
+                 * the input text position can be updated. */
                 *out_match_info = attempt_match;
                 return result;
             }
@@ -576,6 +638,23 @@ fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
 
 /**
  * Attempt to lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_DROP
+ *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
     let mut token_info = <%= @grammar.prefix %>token_info_t::default();
@@ -587,6 +666,11 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
     match result {
         P_SUCCESS => {
             let mut token_to_accept = lexer_state_table[match_info.accepting_state as usize].token;
+            /* Calculate the token length and start/end positions before invoking
+             * the lexer user code so that the user code can access them. The
+             * context input text position tracking is not updated until after the
+             * user code has run so that it is left unchanged if the user code
+             * requests to terminate the lexer. */
             token_info.length = match_info.length;
             if match_info.end_delta_position.row != 0 {
                 token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
@@ -599,10 +683,16 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
                 let match_start = context.input_index;
                 let match_slice = context.input[match_start..(match_start + match_info.length)].to_vec();
                 let user_code_token = lexer_user_code(context,
-                    lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, match_info.length, &mut token_info);
+                    lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, &mut token_info);
+                /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+                 * that the user code is requesting to terminate the lexer. */
                 if user_code_token == TERMINATE_TOKEN_ID {
                     return P_USER_TERMINATED;
                 }
+                /* An invalid token returned from lexer_user_code() means that the
+                 * user code did not explicitly return a token. So only override
+                 * the token to return if the user code does explicitly return a
+                 * token. */
                 if user_code_token != INVALID_TOKEN_ID {
                     token_to_accept = user_code_token;
                 }
@@ -649,6 +739,21 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in
 
 /**
  * Lex the next token in the input stream.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param[out] out_token_info
+ *   The lexed token information is stored here if the return value is
+ *   P_SUCCESS.
+ *
+ * @reval P_SUCCESS
+ *   A token was successfully lexed.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
     loop {
@@ -663,16 +768,36 @@ pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t,
  * Parser
  *************************************************************************/
 
-/* An invalid ID value. */
-const INVALID_ID: usize = usize::MAX;
+/** Reduce ID type. */
+type reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
+
+/**
+ * A symbol ID can hold either a token ID or a rule set ID.
+ *
+ * Token IDs and rule set IDs share the same namespace, with rule set IDs
+ * beginning after token IDs end.
+ */
+type symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
+
+/** Parser state ID type. */
+type parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
+
+<% parser_user_code_called = !@grammar.tree || @grammar.parser_user_code_used? %>
+<% if parser_user_code_called %>
+/** Parser rule ID type. */
+type rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
+<% end %>
+
+/** Parser shift ID type. */
+type shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
 
 /** Shift table entry. */
 #[derive(Clone, Copy)]
 struct shift_t {
     /** Token or rule set ID. */
-    symbol_id: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>,
+    symbol_id: symbol_id_t,
     /** Parser state to shift to. */
-    state_id: <%= get_type_for(@parser.state_table.size) %>,
+    state_id: parser_state_id_t,
 }
 
 /** Reduce table entry. */
@@ -680,18 +805,49 @@ struct shift_t {
 struct reduce_t {
     /** Lookahead token. */
     token: <%= @grammar.prefix %>token_t,
-    /** Rule ID. */
-    rule: <%= get_type_for(@grammar.rules.size) %>,
-    /** Rule set ID. */
-    rule_set: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>,
-    /** Number of states leading to this reduce action. */
-    n_states: <%= get_type_for(@parser.state_table.size) %>,
+<% if parser_user_code_called %>
+
+    /**
+     * Rule ID.
+     *
+     * This is used to execute the parser user code block associated with a
+     * grammar rule.
+     */
+    rule: rule_id_t,
+<% end %>
+
+    /**
+     * Rule set ID.
+     *
+     * This is used as the new top symbol ID of the parse stack after this
+     * reduce action.
+     */
+    rule_set: symbol_id_t,
+
+    /**
+     * Number of states leading to this reduce action.
+     *
+     * This is the number of entries popped from the parse stack after this
+     * reduce action.
+     */
+    n_states: parser_state_id_t,
 <% if @grammar.tree %>
-    /** Map of rule components to rule set child fields (None for a flat map). */
+
+    /**
+     * Map of rule components to rule set child fields (None for a flat map).
+     */
     rule_set_node_field_index_map: Option<&'static [u16]>,
-    /** Number of rule set tree node fields. */
+
+    /**
+     * Number of rule set tree node fields.
+     */
     rule_set_node_field_array_size: u16,
-    /** Whether this rule propagates a matched optional target node. */
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
     propagate_optional_target: bool,
 <% end %>
 }
@@ -700,13 +856,16 @@ struct reduce_t {
 #[derive(Clone, Copy)]
 struct parser_state_t {
     /** First shift table entry for this parser state. */
-    shift_table_index: <%= get_type_for(@parser.shift_table.size) %>,
+    shift_table_index: shift_id_t,
+
     /** Number of shift table entries for this parser state. */
-    n_shift_entries: <%= get_type_for(@parser.shift_table.size) %>,
+    n_shift_entries: shift_id_t,
+
     /** First reduce table entry for this parser state. */
-    reduce_table_index: <%= get_type_for(@parser.reduce_table.size) %>,
+    reduce_table_index: reduce_id_t,
+
     /** Number of reduce table entries for this parser state. */
-    n_reduce_entries: <%= get_type_for(@parser.reduce_table.size) %>,
+    n_reduce_entries: reduce_id_t,
 }
 
 /**
@@ -718,6 +877,7 @@ struct parser_state_t {
 struct state_value_t {
     /** Parser state ID. */
     state_id: usize,
+
 <% if @grammar.tree %>
     /** Tree node ID. */
     node_id: <%= @grammar.prefix %>node_id_t,
@@ -749,7 +909,9 @@ static parser_reduce_table: [reduce_t; <%= @parser.reduce_table.size %>] = [
 <%   @parser.reduce_table.each do |reduce| %>
     reduce_t {
         token: <%= reduce[:token_id] %>, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+<%     if parser_user_code_called %>
         rule: <%= reduce[:rule_id] %>, /* Rule ID */
+<%     end %>
         rule_set: <%= reduce[:rule_set_id] %>, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
         n_states: <%= reduce[:n_states] %>, /* Number of states */
 <%     if @grammar.tree %>
@@ -829,7 +991,20 @@ fn get_rule_position(statevalues: &[state_value_t], i: usize, n_states: usize, g
 <% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
+ *
+ * @param rule The ID of the rule.
+ *
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
+/* The parameters of this function are provided for the user code blocks
+ * inlined into it, which are not obliged to reference any of them. A $$ or $N
+ * reference in a user code block also expands to a parenthesized dereference,
+ * since it may be followed there by a field or method access; those
+ * parentheses are redundant where the reference is a complete argument. */
+#[allow(unused_parens, unused_variables)]
 fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.tree ? "_node_id: #{@grammar.prefix}node_id_t" : "_pvalue: &mut #{@grammar.prefix}value_t" %>, rule: u32, statevalues: &[state_value_t], n_states: usize) -> usize {
     match rule {
 <%   @grammar.rules.each do |rule| %>
@@ -852,6 +1027,11 @@ fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.
 /**
  * Check if the parser should shift to a new state.
  *
+ * @param state_id
+ *   Parser state ID.
+ * @param symbol_id
+ *   Incoming token/rule set ID.
+ *
  * @return State to shift to, or INVALID_ID if none.
  */
 fn check_shift(state_id: usize, symbol_id: usize) -> usize {
@@ -868,6 +1048,11 @@ fn check_shift(state_id: usize, symbol_id: usize) -> usize {
 /**
  * Check if the parser should reduce to a new state.
  *
+ * @param state_id
+ *   Parser state ID.
+ * @param token
+ *   Incoming token.
+ *
  * @return Reduce table index to reduce with, or INVALID_ID if none.
  */
 fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize {
@@ -883,6 +1068,30 @@ fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize
 
 /**
  * Run the parser.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param start_state_id
+ *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
+ *
+ * @retval P_SUCCESS
+ *   The parser successfully matched the input text. The parse result value
+ *   can be accessed with <%= @grammar.prefix %>result().
+ * @retval P_UNEXPECTED_TOKEN
+ *   An unexpected token was encountered that does not match any grammar rule.
+ *   The function p_token(&context) can be used to get the unexpected token.
+ * @reval P_DECODE_ERROR
+ *   The decoder encountered invalid text encoding.
+ * @reval P_UNEXPECTED_INPUT
+ *   Input text does not match any lexer pattern.
  */
 fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usize,
     start_rule_set_id: usize, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize {
@@ -901,16 +1110,17 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
     statevalues.push(state_value_t::default());
     let sv_len = statevalues.len();
     statevalues[sv_len - 1].state_id = start_state_id;
-    let result;
     loop {
         if token == INVALID_TOKEN_ID {
             let lexer_result = <%= lex_fn %>(context, &mut token_info);
             if lexer_result != P_SUCCESS {
-                result = lexer_result;
-                break;
+                return lexer_result;
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
         let mut token_is_follow = false;
         for &ft in follow_tokens {
             if token == ft {
@@ -931,12 +1141,23 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 <% else %>
                 context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
 <% end %>
-                result = P_SUCCESS;
-                break;
+                return P_SUCCESS;
             }
             if (shift_state == INVALID_ID) && token_is_follow {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
                 let retry_shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF as usize);
-                if (retry_shift_state != INVALID_ID) && (statevalues.len() == 2) && (last_shifted_rule_set_id == start_rule_set_id) {
+                if (retry_shift_state != INVALID_ID) &&
+                    (statevalues.len() == 2) &&
+                    (last_shifted_rule_set_id == start_rule_set_id) {
+                    /* Successful parse via follow token. Rewind the input
+                     * position so that the follow token is not consumed from
+                     * the input stream and remains available for a subsequent
+                     * call to <%= @grammar.prefix %>lex() or a
+                     * <%= @grammar.prefix %>parse*() function. */
                     context.input_index -= token_info.length;
                     context.text_position = token_info.position;
 <% if @grammar.tree %>
@@ -944,12 +1165,15 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 <% else %>
                     context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
 <% end %>
-                    result = P_SUCCESS;
-                    break;
+                    return P_SUCCESS;
                 }
             }
         }
         if shift_state != INVALID_ID {
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
             last_shifted_rule_set_id = reduced_rule_set;
             statevalues.push(state_value_t::default());
             let new_index = statevalues.len() - 1;
@@ -967,7 +1191,7 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
                     token_tree_node.token = token;
                     token_tree_node.pvalue = token_info.pvalue.clone();
                 }
-<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %>
+<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %>
                 statevalues[new_index].node_id = token_node_id;
 <% else %>
                 statevalues[new_index].position = token_info.position;
@@ -992,6 +1216,11 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
 
         let mut reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, token);
         if (reduce_index == INVALID_ID) && token_is_follow {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
             reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF);
         }
         if reduce_index != INVALID_ID {
@@ -1063,13 +1292,15 @@ fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usi
             continue;
         }
 
-        /* Unexpected token. Reset the context text position to the token. */
+        /* A token was successfully lexed, so the input text position was
+         * advanced. However, this is an unexpected token, so we want to reset
+         * the context text position to point to the token rather than the text
+         * after it, so that if the caller wants to report the error position,
+         * it will point to the correct position of the unexpected token. */
         context.text_position = token_info.position;
         context.token = token;
-        result = P_UNEXPECTED_TOKEN;
-        break;
+        return P_UNEXPECTED_TOKEN;
     }
-    result
 }
 
 pub fn <%= @grammar.prefix %>parse(context: &mut <%= @grammar.prefix %>context_t) -> usize {
@@ -1088,14 +1319,19 @@ pub fn <%= @grammar.prefix %>parse_inner_<%= start_rule %>(context: &mut <%= @gr
 
 /**
  * Get the parse result value.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Parse result value.
  */
 <% if @grammar.tree %>
 pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(@grammar.start_rules[0]) %><'_> {
-    <%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result") %>
+    <%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result", false) %>
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(start_rule) %><'_> {
-    <%= tree_handle(h_type(start_rule), "context.parse_result") %>
+    <%= tree_handle(h_type(start_rule), "context.parse_result", false) %>
 }
 <%   end %>
 <% else %>
@@ -1109,32 +1345,84 @@ pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.pre
 <%   end %>
 <% end %>
 
-/** Get the current text input position. */
+/**
+ * Get the current text input position.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current text position.
+ */
 pub fn <%= @grammar.prefix %>position(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>position_t {
     context.text_position
 }
 
-/** Set the current text input position. */
+/**
+ * Set the current text input position.
+ *
+ * This can be used to set the initial text position to something other than
+ * (1, 1) for a nested parse operation so that error positions reported by
+ * subsequent lexer/parser calls are relative to a larger enclosing document.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param position
+ *   Text position to set.
+ */
 pub fn <%= @grammar.prefix %>set_position(context: &mut <%= @grammar.prefix %>context_t, position: <%= @grammar.prefix %>position_t) {
     context.text_position = position;
 }
 
-/** Get the current input text byte offset. */
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
 pub fn <%= @grammar.prefix %>input_index(context: &<%= @grammar.prefix %>context_t) -> usize {
     context.input_index
 }
 
-/** Set the current input text byte offset. */
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
 pub fn <%= @grammar.prefix %>set_input_index(context: &mut <%= @grammar.prefix %>context_t, input_index: usize) {
     context.input_index = input_index;
 }
 
-/** Get the user terminate code. */
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
 pub fn <%= @grammar.prefix %>user_terminate_code(context: &<%= @grammar.prefix %>context_t) -> usize {
     context.user_terminate_code
 }
 
-/** Get the parse token. */
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
 pub fn <%= @grammar.prefix %>token(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>token_t {
     context.token
 }

extra/vim/syntax/propane.vim

@@ -7,12 +7,46 @@ if exists("b:current_syntax")
   finish
 endif
 
+" Guess the language of the user code blocks from their contents so that the
+" matching syntax file can be included below. b:propane_subtype may also be set
+" before this file is sourced to select the language explicitly.
 if !exists("b:propane_subtype")
-  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+  " Rust markers. Each keyword requires the syntax that follows it in Rust so
+  " that a plain identifier of the same name in another language does not match
+  " (`int fn = 3;' in C, for example). Type names are only accepted within a
+  " `ptype' statement for the same reason.
+  let s:rust = '\<let\s\+\%(mut\s\+\)\?\w'
+  let s:rust .= '\|\<fn\s\+\w\+\s*('
+  let s:rust .= '\|&mut\>\|\<pub\s\+\w\|\<impl\s\+\w'
+  let s:rust .= '\|#\[\|\<use\s\+\%(std\|core\)::'
+  let s:rust .= '\|\<ptype\>[^;]*\<\%(isize\|usize\|i8\|i16\|i32\|i64\|i128'
+  let s:rust .= '\|u8\|u16\|u32\|u64\|u128\|f32\|f64\|String\)\>'
+  " D markers. These are spellings that have no valid C, C++, or Rust
+  " equivalent, so `import' is deliberately not among them: it is a D keyword
+  " but is also a C++20 module declaration.
+  let s:d = '\<foreach\%(_reverse\)\?\s*([^)]*;'
+  let s:d .= '\|\~=\|\<static\s\+if\s*(\|\<version\s*(\s*\w\+\s*)'
+  let s:d .= '\|\<scope\s*(\s*\%(exit\|failure\|success\)\s*)'
+  let s:d .= '\|\<\%(unittest\|mixin\|immutable\|__gshared\|invariant\)\>'
+  let s:d .= '\|\<alias\s\+\w\+\s*=\|\<enum\s\+\w\+\s*='
+  let s:d .= '\|@\%(property\|safe\|trusted\|system\|nogc\|disable\)\>'
+  let s:d .= '\|\<is\s\+null\>\|\<cast\s*(\s*\w\+\s*)'
+  let s:d .= '\|\<write\%(ln\|fln\|f\)\s*('
+  let s:d .= '\|\<\%(dchar\|dstring\|wstring\|cent\|ucent\)\>'
+  " A module import on its own is ambiguous between D and C++20, so only take
+  " it as D when nothing else in the file looks like C++.
+  let s:import = '\<import\s\+[A-Za-z_][A-Za-z0-9_.]*\s*;'
+  let s:cpp = '::\|\<template\s*<\|\<namespace\>\|\<nullptr\>\|#include\s*[<"]'
+  if search(s:rust, 'nw') > 0
+    let b:propane_subtype = "rust"
+  elseif search(s:d, 'nw') > 0
+    let b:propane_subtype = "d"
+  elseif search(s:import, 'nw') > 0 && search(s:cpp, 'nw') == 0
     let b:propane_subtype = "d"
   else
     let b:propane_subtype = "cpp"
   endif
+  unlet s:rust s:d s:import s:cpp
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"

lib/propane/generator.rb

@@ -41,12 +41,23 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
+        # Rust has no #line directive support. For a Rust target the directives
+        # that the grammar embeds around user code blocks are replaced with
+        # comments naming the grammar file and line number the code came from,
+        # so that the origin of a section of user code can still be found by
+        # reading up from a compiler diagnostic pointing into the generated
+        # module.
+        user_code_origin = nil
         result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
           if @language == "rust"
-            # Rust has no #line directive support, so strip the directives that
-            # the grammar embeds in user code blocks.
-            line = line.sub(/^#line \d+ "[^"]*"/, "")
-            line == "#linereset\n" ? "" : line
+            if md = line.match(/^#line (\d+) "([^"]*)"/)
+              user_code_origin = "#{md[2]} line #{md[1]}"
+              line.sub(/^#line \d+ "[^"]*"/, %[/* Begin user code from #{user_code_origin}. */])
+            elsif line == "#linereset\n"
+              %[/* End user code from #{user_code_origin}. */\n]
+            else
+              line
+            end
           elsif line == "#linereset\n"
             %[#line #{i + 2} "#{output_file}"\n]
           else
@@ -470,16 +481,24 @@ class Propane
     #   Handle type name.
     # @param id_expr [String]
     #   Expression yielding the node ID.
+    # @param parenthesize [Boolean]
+    #   Whether to parenthesize the expression. Parentheses are required where
+    #   the expression is substituted into a user code block, since the
+    #   expression could be followed there by a field access or appear in a
+    #   position where a bare Rust struct literal is not accepted. They are
+    #   unnecessary where the expression stands alone, and Rust warns about
+    #   them there, so this can be disabled for those uses.
     #
     # @return [String]
     #   Handle constructor expression.
-    def tree_handle(typename, id_expr)
+    def tree_handle(typename, id_expr, parenthesize = true)
       if @cpp
         "(#{typename}{context, #{id_expr}})"
       elsif @language == "c"
         "((#{typename}){context, #{id_expr}})"
       elsif @language == "rust"
-        "(#{typename} { context, id: #{id_expr} })"
+        expr = "#{typename} { context, id: #{id_expr} }"
+        parenthesize ? "(#{expr})" : expr
       else
         "#{typename}(context, #{id_expr})"
       end
@@ -533,16 +552,30 @@ class Propane
       end
     end
 
-    # Generate the C/C++ tree node handle type section for the header.
+    # Generate the tree node handle type declarations for the header.
+    #
+    # These are emitted before the context structure definition so that a
+    # context_user_fields block can declare a field of a handle type.
     #
     # @return [String]
-    #   Header handle section.
+    #   Handle type declarations.
+    def c_tree_handle_types_header
+      @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header
+    end
+
+    # Generate the remainder of the tree node section for the header.
+    #
+    # This is emitted after the context structure definition since it
+    # dereferences the context and so requires the complete type.
+    #
+    # @return [String]
+    #   Accessors, macros, and out-of-line handle method definitions.
     def c_tree_types_header
       @cpp ? cpp_tree_types_header : c_only_tree_types_header
     end
 
     # Generate the C (non-C++) tree node handle type section for the header.
-    def c_only_tree_types_header
+    def c_only_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
@@ -550,6 +583,11 @@ class Propane
         out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
       end
       out << ""
+      out.join("\n")
+    end
+
+    def c_only_tree_types_header
+      out = []
       out << c_common_accessors_header
       out << "/** @} */"
       out.join("\n")
@@ -657,29 +695,32 @@ class Propane
       out.join("\n")
     end
 
-    # Generate the C++ tree node handle type section for the header.
-    def cpp_tree_types_header
+    # Generate the C++ tree node handle class declarations for the header.
+    # Only valid() and node_id() are defined inline; every other method
+    # dereferences the context, which is still an incomplete type here, so
+    # those are declared and defined out of line once the context is
+    # complete.
+    def cpp_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
       tree_handle_types.each {|t| out << "struct #{t};"}
       out << ""
-      # Token handle (all methods inline; no handle-typed returns).
       tt = h_type("Token")
       out << "struct #{tt}"
       out << "{"
       out << "    #{p}context_t * __context;"
       out << "    #{p}node_id_t __id;"
       out << "    bool valid() const { return __id != 0u; }"
-      out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
-      out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
-      out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-      out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
-      out << "    #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
-      out << "    #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << "    #{p}node_id_t node_id() const { return __id; }"
+      out << "    #{p}node_data_t * data() const;"
+      out << "    #{p}position_t position() const;"
+      out << "    #{p}position_t end_position() const;"
+      out << "    uint16_t n_fields() const;"
+      out << "    #{p}token_t token() const;"
+      out << "    #{p}value_t pvalue() const;"
       out << "};"
       out << ""
-      # Rule set handles: navigation methods declared, defined out-of-line below.
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)
         out << "struct #{rtype}"
@@ -687,16 +728,36 @@ class Propane
         out << "    #{p}context_t * __context;"
         out << "    #{p}node_id_t __id;"
         out << "    bool valid() const { return __id != 0u; }"
-        out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
-        out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
-        out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-        out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+        out << "    #{p}node_id_t node_id() const { return __id; }"
+        out << "    #{p}node_data_t * data() const;"
+        out << "    #{p}position_t position() const;"
+        out << "    #{p}position_t end_position() const;"
+        out << "    uint16_t n_fields() const;"
         each_tree_field(rule_set) do |rt, field_name, child_type, slot|
           out << "    #{child_type} #{field_name}() const;"
         end
         out << "};"
         out << ""
       end
+      out.join("\n")
+    end
+
+    # Generate the out-of-line C++ handle method definitions plus the C-style
+    # accessors. Emitted after the context structure definition.
+    def cpp_tree_types_header
+      p = @grammar.prefix
+      out = []
+      # Common node methods, now that the context type is complete.
+      ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht|
+        out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      end
+      tt = h_type("Token")
+      out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << ""
       # Out-of-line navigation method bodies (all handle types now complete).
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.8.1"
+  VERSION = "5.1.0"
 end

spec/json_parser.c.propane

@@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n = 0.0;
     bool negative = false;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative = false;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -120,11 +120,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
+  char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
   str_append(&string_value, s);
 >>
 string: /[^\\]/ <<
-  char s[] = {(char)match[0], 0};
+  char s[] = {(char)match_text[0], 0};
   str_append(&string_value, s);
 >>
 Start -> Value <<

spec/json_parser.d.propane

@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n;
     bool negative;
     size_t i = 0u;
-    if (match[i] == '-')
+    if (match_text[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match[i] && match[i] <= '9')
+    while ('0' <= match_text[i] && match_text[i] <= '9')
     {
         n *= 10.0;
-        n += (match[i] - '0');
+        n += (match_text[i] - '0');
         i++;
     }
-    if (match[i] == '.')
+    if (match_text[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
-            n += mult * (match[i] - '0');
+            n += mult * (match_text[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match[i] == 'e' || match[i] == 'E')
+    if (match_text[i] == 'e' || match_text[i] == 'E')
     {
         bool exp_negative;
         i++;
-        if (match[i] == '-')
+        if (match_text[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match[i] == '+')
+        else if (match_text[i] == '+')
         {
             i++;
         }
         long exp;
-        while ('0' <= match[i] && match[i] <= '9')
+        while ('0' <= match_text[i] && match_text[i] <= '9')
         {
             exp *= 10;
-            exp += (match[i] - '0');
+            exp += (match_text[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
+  string_value ~= "{" ~ match_text[2..6] ~ "}";
 >>
 string: /[^\\]/ <<
-  string_value ~= match;
+  string_value ~= match_text;
 >>
 Start -> Value <<
   $$ = $1;

spec/json_parser.rust.propane

@@ -1,4 +1,4 @@
-<<header
+<<
 pub const JSON_OBJECT: usize = 0;
 pub const JSON_ARRAY: usize = 1;
 pub const JSON_NUMBER: usize = 2;
@@ -60,7 +60,7 @@ token rbracket /\]/;
 token comma /,/;
 token colon /:/;
 token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
+    let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
     $$ = JSONValue::Number(n);
 >>
 token true <<
@@ -106,11 +106,11 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
+  let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
   ${context.string_value}.push_str(&s);
 >>
 string: /[^\\]/ <<
-  ${context.string_value}.push(match_[0] as char);
+  ${context.string_value}.push(match_text[0] as char);
 >>
 Start -> Value <<
   $$ = $1;

spec/macros.c.propane

@@ -14,7 +14,7 @@ token rbrace /\}/;
 token plus /\+/;
 token macro;
 token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/macros.d.propane

@@ -14,7 +14,7 @@ token macro;
 token macroname /@[a-zA-Z_]\w*/;
 token num /\d+/ <<
     int n = 0;
-    foreach (c; match)
+    foreach (c; match_text)
     {
         n *= 10;
         n += (c - '0');

spec/macros.rust.propane

@@ -67,7 +67,7 @@ token macro;
 token macroname /@[a-zA-Z_]\w*/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/parse_inner_nested.c.propane

@@ -12,7 +12,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.d.propane

@@ -10,7 +10,7 @@ drop /\s+/;
 token lparen /\(/;
 token rparen /\)/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Expr << $$ = $1; >>
 Expr -> num << $$ = $1; >>

spec/parse_inner_nested.rust.propane

@@ -32,7 +32,7 @@ token rparen /\)/;
 token plus /\+/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/propane_spec.rb

@@ -39,6 +39,11 @@ class MyFormatter
 end
 SimpleCov.start do
   command_name(#{command_name.inspect})
+  # Write this process's results to its own directory. Sharing one resultset
+  # file means every propane invocation must read, merge, and rewrite the
+  # results of all previous invocations, which grows quadratically over the
+  # suite. The parts are merged once at the end by the spec Rake task.
+  coverage_dir(#{"coverage/parts/#{command_name}".inspect})
   filters.clear
   add_filter do |src|
     !(src.filename[SimpleCov.root])
@@ -86,26 +91,29 @@ EOF
     end
     case options[:language]
     when "c"
-      command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[gcc -g -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "cpp"
-      command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
+      command = [*%w[g++ -g -x c++ -Wall -Werror -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
-      command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
+      command = [*%w[ldc2 -g -w -de --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
     when "rust"
       # Compile each generated parser to an rlib, then compile the test crate
       # against them. Safe Rust needs no valgrind, but it is run anyway.
+      # Both compiles deny warnings so that a warning introduced into the
+      # generated parser (or into a spec test file) fails the spec rather than
+      # scrolling past unnoticed.
       externs = []
       options[:parsers].each do |name|
         crate = "testparser#{name}"
         rlib = "spec/run/lib#{crate}.rlib"
-        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -A warnings],
+        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -D warnings],
                  "--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"]
         expect(system(*rustc)).to be_truthy
         externs += ["--extern", "#{crate}=#{rlib}"]
       end
       # rustc takes a single crate root; any additional Rust test files are
       # expected to be pulled in as modules or provided by the parser crate.
-      command = [*%w[rustc --edition 2021 -A warnings -o spec/run/testparser], *externs, test_files.first]
+      command = [*%w[rustc --edition 2021 -D warnings -o spec/run/testparser], *externs, test_files.first]
     end
     result = system(*command)
     expect(result).to be_truthy
@@ -119,7 +127,7 @@ EOF
     results = Results.new(stdout, stderr, status)
     # Valgrind is only reliably available on Linux, so limit the leak check to
     # Linux platforms.
-    if RUBY_PLATFORM =~ /linux/
+    if RUBY_PLATFORM =~ /linux/ && ENV["spec-valgrind"]
       stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
       vgout = stdout + stderr
       File.binwrite("spec/run/.vgout", vgout)
@@ -307,9 +315,9 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c cpp rust].each do |language|
+  {"d" => "D", "c" => "C", "cpp" => "C++", "rust" => "Rust"}.each do |language, language_name|
 
-    context "#{language.upcase} language" do
+    context "#{language_name} language" do
 
       it "generates a lexer" do
         write_grammar <<EOF
@@ -338,7 +346,7 @@ token int /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -349,7 +357,7 @@ EOF
 ptype int;
 token int /\\d+/ <<
   int v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -363,7 +371,7 @@ EOF
 ptype i64;
 token int /\\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 Start -> int << $$ = $1; >>
@@ -410,7 +418,7 @@ token integer /\\d+/ <<
   for (size_t i = 0u; i < match_length; i++)
   {
     v *= 10;
-    v += (match[i] - '0');
+    v += (match_text[i] - '0');
   }
   $$ = v;
 >>
@@ -441,7 +449,7 @@ token times /\\*/;
 token power /\\*\\*/;
 token integer /\\d+/ <<
   ulong v;
-  foreach (c; match)
+  foreach (c; match_text)
   {
     v *= 10;
     v += (c - '0');
@@ -470,7 +478,7 @@ token times /\\*/;
 token power /\\*\\*/;
 token integer /\\d+/ <<
   let mut v: u64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as u64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as u64; }
   $$ = v;
 >>
 token lparen /\\(/;
@@ -770,7 +778,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
   return $token(ident);
 >>
@@ -791,7 +799,7 @@ ptype char;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match[0];
+  $$ = match_text[0];
   $mode(default);
 >>
 token dot /\\./ <<
@@ -808,7 +816,7 @@ ptype u8;
 token abc;
 token def;
 default, identonly: token ident /[a-z]+/ <<
-  $$ = match_[0];
+  $$ = match_text[0];
   $mode(default);
   return $token(ident);
 >>
@@ -1059,7 +1067,7 @@ EOF
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * t = (char *)malloc(match_length + 1);
-  strncpy(t, (char *)match, match_length);
+  strncpy(t, (char *)match_text, match_length);
   printf("Matched token is %s\\n", t);
   free(t);
 >>
@@ -1071,14 +1079,14 @@ EOF
 import std.stdio;
 >>
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  writeln("Matched token is ", match);
+  writeln("Matched token is ", match_text);
 >>
 Start -> id;
 EOF
         when "rust"
           write_grammar <<EOF
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
+  println!("Matched token is {}", std::str::from_utf8(match_text).unwrap());
 >>
 Start -> id;
 EOF
@@ -1109,7 +1117,7 @@ EOF
           write_grammar <<EOF
 ptype ulong;
 token word /[a-z]+/ <<
-  $$ = match.length;
+  $$ = match_text.length;
 >>
 Start -> word <<
   $$ = $1;
@@ -1119,7 +1127,7 @@ EOF
           write_grammar <<EOF
 ptype u64;
 token word /[a-z]+/ <<
-  $$ = match_length as u64;
+  $$ = match_text.len() as u64;
 >>
 Start -> word <<
   $$ = $1;
@@ -1788,7 +1796,7 @@ EOF
           write_grammar <<EOF
 ptype String;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = std::str::from_utf8(match_).unwrap().to_string();
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
 >>
 drop /\\s+/;
 Start -> id:first id:second <<
@@ -1803,7 +1811,7 @@ import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id:first id:second <<
@@ -1820,7 +1828,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -1846,7 +1854,7 @@ EOF
           write_grammar <<EOF
 ptype String;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = std::str::from_utf8(match_).unwrap().to_string();
+  $$ = std::str::from_utf8(match_text).unwrap().to_string();
 >>
 drop /\\s+/;
 Start -> id;
@@ -1864,7 +1872,7 @@ import std.stdio;
 >>
 ptype string;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
-  $$ = match;
+  $$ = match_text;
 >>
 drop /\\s+/;
 Start -> id;
@@ -1884,7 +1892,7 @@ EOF
 ptype char *;
 token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   char * s = (char *)malloc(match_length + 1);
-  strncpy(s, (char const *)match, match_length);
+  strncpy(s, (char const *)match_text, match_length);
   s[match_length] = 0;
   $$ = s;
 >>
@@ -2051,9 +2059,36 @@ EOF
         expect(results.status).to eq 0
       end
 
-      if %w[c cpp].include?(language)
+      # D is excluded since it is garbage collected, so Propane does not emit a
+      # free_token_node code block for it.
+      if %w[c cpp rust].include?(language)
         it "allows a user function to free token node memory in tree mode" do
-          write_grammar <<EOF
+          if language == "rust"
+            write_grammar <<EOF
+<<
+use std::sync::atomic::{AtomicU32, Ordering};
+
+/** Number of token nodes freed by the free_token_node code block. */
+pub static FREED: AtomicU32 = AtomicU32::new(0);
+>>
+tree;
+free_token_node <<
+    if !${token.pvalue}.is_null() {
+        unsafe { drop(Box::from_raw(${token.pvalue})); }
+        FREED.fetch_add(1, Ordering::SeqCst);
+    }
+>>
+ptype *mut i32;
+token a <<
+  $$ = Box::into_raw(Box::new(1));
+>>
+token b <<
+  $$ = Box::into_raw(Box::new(2));
+>>
+Start -> a:a b:b;
+EOF
+          else
+            write_grammar <<EOF
 tree;
 free_token_node <<
     free(${token.pvalue});
@@ -2069,6 +2104,7 @@ token b <<
 >>
 Start -> a:a b:b;
 EOF
+          end
           run_propane(language: language)
           compile("spec/test_tree_delete_token_node_memory.#{language}", language: language)
           results = run_test(language: language)
@@ -2077,12 +2113,82 @@ EOF
         end
       end
 
+      # Rust is excluded since a Rust tree node handle borrows the context, so
+      # storing one in a context field would make the context self-referential.
+      if %w[c cpp d].include?(language)
+        it "allows a tree node handle type in a context user field" do
+          write_grammar <<EOF
+tree;
+ptype int;
+context_user_fields <<
+    Item first_item;
+    int have_first;
+>>
+drop /\\s+/;
+token a /a/ << $$ = 7; >>
+Item -> a;
+Items -> ;
+Items -> Items Item <<
+    if (${context.have_first} == 0)
+    {
+        ${context.first_item} = $2;
+        ${context.have_first} = 1;
+    }
+>>
+Start -> Items;
+EOF
+          run_propane(language: language)
+          compile("spec/test_context_field_handle.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
+
+      if language == "rust"
+        it "marks user code sections with their grammar file and line number" do
+          write_grammar <<EOF
+ptype i64;
+drop /\\s+/;
+token num /\\d+/ <<
+  $$ = 42;
+>>
+Start -> num << $$ = $1; >>
+EOF
+          run_propane(language: language)
+          parser = File.binread("spec/run/testparser.rs")
+          # The lexer code block body begins on grammar line 4 and the parser
+          # rule code block is on grammar line 6.
+          expect(parser).to include %[/* Begin user code from spec/run/testparser.propane line 4. */]
+          expect(parser).to include %[/* End user code from spec/run/testparser.propane line 4. */]
+          expect(parser).to include %[/* Begin user code from spec/run/testparser.propane line 6. */]
+          expect(parser).to include %[/* End user code from spec/run/testparser.propane line 6. */]
+          expect(parser).to_not include "#line"
+        end
+
+        it "omits user code section markers when noline is specified" do
+          write_grammar <<EOF
+noline;
+ptype i64;
+drop /\\s+/;
+token num /\\d+/ <<
+  $$ = 42;
+>>
+Start -> num << $$ = $1; >>
+EOF
+          run_propane(language: language)
+          parser = File.binread("spec/run/testparser.rs")
+          expect(parser).to_not include "user code from"
+          expect(parser).to_not include "#line"
+        end
+      end
+
       it "executes code blocks associated with drop statements" do
         if language == "rust"
           write_grammar <<EOF
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
+  eprint!("comment: {}", std::str::from_utf8(match_text).unwrap());
 >>
 token a;
 Start -> a;
@@ -2094,7 +2200,7 @@ import std.stdio;
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  stderr.write("comment: ", match);
+  stderr.write("comment: ", match_text);
 >>
 token a;
 Start -> a;
@@ -2107,7 +2213,7 @@ EOF
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+  fprintf(stderr, "comment: %.*s", (int)match_length, match_text);
 >>
 token a;
 Start -> a;
@@ -2129,7 +2235,7 @@ context_user_fields <<
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::str::from_utf8(match_).unwrap();
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
 >>
 token a <<
     ${context.acount} += 1;
@@ -2146,7 +2252,7 @@ context_user_fields <<
 >>
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token a <<
     ${context.acount}++;
@@ -2173,7 +2279,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -2212,7 +2318,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::str::from_utf8(match_).unwrap();
+    ${context.comments} += std::str::from_utf8(match_text).unwrap();
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -2234,7 +2340,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} ~= match;
+    ${context.comments} ~= match_text;
 >>
 token id /\\w+/;
 Start -> IDs;
@@ -2270,7 +2376,7 @@ drop /#(.*)\\n/ <<
     char * commentsnew = (char *)malloc(cur_len + match_length + 1);
     if (${context.comments} != NULL)
         memcpy(commentsnew, ${context.comments}, cur_len);
-    memcpy(&commentsnew[cur_len], match, match_length);
+    memcpy(&commentsnew[cur_len], match_text, match_length);
     commentsnew[cur_len + match_length] = '\\0';
     if (${context.comments} != NULL)
     {
@@ -2301,7 +2407,7 @@ on_token_node <<
 tree;
 drop /\\s+/;
 drop /#(.*)\\n/ <<
-    ${context.comments} += std::string((const char *)match, match_length);
+    ${context.comments} += std::string((const char *)match_text, match_length);
 >>
 token id /\\w+/;
 Start -> IDs;

spec/rewind.c.propane

@@ -14,7 +14,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.d.propane

@@ -11,7 +11,7 @@ token repeat /repeat/;
 token lbrace /\{/;
 token rbrace /\}/;
 token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
 
 Start -> Statements;
 Statements -> ;

spec/rewind.rust.propane

@@ -56,7 +56,7 @@ token rbrace /\}/;
 token plus /\+/;
 token num /\d+/ <<
   let mut v: i64 = 0;
-  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
   $$ = v;
 >>
 

spec/spec_helper.rb

@@ -15,6 +15,10 @@ unless ENV["dist_specs"]
       command_name "RSpec"
     end
     project_name "Propane"
+    # Keep this process's results separate from the propane subprocess results
+    # so that nothing has to merge on the fly; the spec Rake task collates all
+    # of the parts once the suite is done.
+    coverage_dir "coverage/parts/rspec"
     merge_timeout 3600
     formatter(MyFormatter)
   end

spec/test_context_field_handle.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+    size_t i;
+    p_context_t * context;
+    Token token;
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(p_node_valid(context->first_item));
+    token = p_Item_pToken1(context->first_item);
+    assert(p_node_valid(token));
+    assert_eq(TOKEN_a, p_Token_token(token));
+    assert_eq(7u, p_Token_pvalue(token));
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, p_node_position(context->first_item).row);
+    assert_eq(1u, p_node_position(context->first_item).col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.cpp

@@ -0,0 +1,36 @@
+#include "testparser.h"
+#include <cassert>
+#include <cstring>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (size_t i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(context->first_item.valid());
+    Token token = context->first_item.pToken1();
+    assert(token.valid());
+    assert_eq(TOKEN_a, token.token());
+    assert_eq(7u, token.pvalue());
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context->first_item.position().row);
+    assert_eq(1u, context->first_item.position().col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.d

@@ -0,0 +1,35 @@
+import testparser;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* Enough tokens that the tree node array is reallocated during the parse. */
+    string input;
+    foreach (i; 0 .. 40)
+    {
+        input ~= "a";
+    }
+
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1, context.have_first);
+    assert(context.first_item.valid);
+    Token token = context.first_item.pToken1;
+    assert(token.valid);
+    assert_eq(TOKEN_a, token.token);
+    assert_eq(7, token.pvalue);
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context.first_item.position.row);
+    assert_eq(1u, context.first_item.position.col);
+
+    p_context_delete(context);
+}

spec/test_input_index.rs

@@ -1,7 +1,7 @@
 use testparser::*;
 
 fn main() {
-    let mut c = p_context_new(b"ab");
+    let c = p_context_new(b"ab");
     assert_eq!(0, p_input_index(&c));
     p_context_delete(c);
 

spec/test_optional_rule_component_tree.d

@@ -31,6 +31,7 @@ unittest
     assert(start.pR3.valid);
     assert(start.pR.valid);
     assert(start.pR == start.pR3);
+    assert_eq(start.pR.node_id, start.pR3.node_id);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
     p_context_delete(context);

spec/test_set_position.rs

@@ -17,9 +17,7 @@ fn main() {
     /* p_set_position overrides the initial position. */
     let mut c = p_context_new(b"ab");
     p_set_position(&mut c, p_position_t { row: 5, col: 20 });
-    let pos = p_position(&c);
-    assert_eq!(5, pos.row);
-    assert_eq!(20, pos.col);
+    assert_eq!(p_position_t { row: 5, col: 20 }, p_position(&c));
     assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
     assert_eq!(TOKEN_a, ti.token);
     assert_eq!(5, ti.position.row);

spec/test_tree.cpp

@@ -13,6 +13,8 @@ int main()
     assert(start.pItems1().valid());
     assert(start.pItems().valid());
     Items items = start.pItems();
+    assert_ne(0u, items.node_id());
+    assert_eq(start.pItems().node_id(), items.node_id());
     assert(items.pItem().valid());
     assert(items.pItem().pToken1().valid());
     assert_eq(TOKEN_a, items.pItem().pToken1().token());
@@ -40,6 +42,7 @@ int main()
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(!start.pItems().valid());
+    assert_eq(0u, start.pItems().node_id());
 
     p_context_delete(context);
 

spec/test_tree_delete_token_node_memory.rs

@@ -0,0 +1,27 @@
+use testparser::*;
+use std::sync::atomic::Ordering;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    let start = p_result(&context);
+    assert!(start.a().valid());
+    assert_eq!(1, unsafe { *start.a().pvalue() });
+    assert!(start.b().valid());
+    assert_eq!(2, unsafe { *start.b().pvalue() });
+
+    /* The free_token_node code block runs when the context is disposed of, not
+     * before, and frees each of the two token nodes exactly once. */
+    assert_eq!(0, FREED.load(Ordering::SeqCst));
+    p_context_delete(context);
+    assert_eq!(2, FREED.load(Ordering::SeqCst));
+
+    /* Letting the context go out of scope runs the code block too, so a caller
+     * which never calls p_context_delete() does not leak. */
+    {
+        let mut context = p_context_new(b"ab");
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(2, FREED.load(Ordering::SeqCst));
+    }
+    assert_eq!(4, FREED.load(Ordering::SeqCst));
+}

spec/tree_node_memory_remains.c.propane

@@ -46,7 +46,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  $$.s = match;
+  $$.s = match_text;
   $mode(default);
   return $token(ident);
 >>

spec/tree_node_memory_remains.d.propane

@@ -42,8 +42,8 @@ token semicolon /;/;
 
 # Integer literals.
 token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
-  $$.bi = BigInt(match[0..3]);
-  foreach (c; match[3..$])
+  $$.bi = BigInt(match_text[0..3]);
+  foreach (c; match_text[3..$])
   {
     if (('0' <= c) && (c <= '9'))
     {
@@ -65,13 +65,13 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  if (match[0] == '$')
+  if (match_text[0] == '$')
   {
-    $$.s = match[1..$];
+    $$.s = match_text[1..$];
   }
   else
   {
-    $$.s = match;
+    $$.s = match_text;
   }
   $mode(default);
   return $token(ident);

spec/tree_node_memory_remains.rust.propane

@@ -1,7 +1,7 @@
 tree;
 tree_prefix P;
 
-<<header
+<<
 #[derive(Clone, Default)]
 pub struct TokenVal {
     pub i64_: u64,
@@ -43,7 +43,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
 
 # Identifier.
 token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  $$.s = std::str::from_utf8(match_).unwrap().to_string();
+  $$.s = std::str::from_utf8(match_text).unwrap().to_string();
   $mode(default);
   return $token(ident);
 >>