v5.1.0

This takes the place of #line directives

.github/workflows/run-tests.yml

@@ -31,6 +31,9 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby-version }}
 
+    - name: Set up Rust
+      uses: dtolnay/rust-toolchain@stable
+
     - name: Install dependencies
       run: bundle install
 

CHANGELOG.md

@@ -1,22 +1,30 @@
+## v5.1.0
+
+### New Features
+
+- Add a `node_id()` accessor to the C++ and D tree node handle types, for node
+  identity comparison. This matches the existing `p_node_id()` macro (C) and
+  `node_id()` method (Rust).
+
 ## v5.0.0
 
 ### New Features
 
 - Add Rust target language output.
+- Add Rust language detection in propane.vim.
 
 ### API Changes
 
 - The matched text argument passed to lexer user code blocks is now named
-  `match_text` for every target language. It was previously named `match` for
+  `match_text` instead of `match`, since `match` is a keyword in Rust. The
-  C, C++, and D, and `match_` for Rust. Any lexer user code block which
-  references the matched text must be updated to use the new name. The
   `match_length` argument (C and C++) is unchanged.
 - Tree generation mode now stores all tree nodes in a compact arena owned by
   the parser context (a flat node array plus a shared child-link array).
   This replaces the previous design of one heap allocation per node with
   layout-punned typed structs.
 - Tree nodes are now referenced by lightweight handles rather than pointers.
-  `p_result()` and the field accessors return handle values.
+  `p_result()` and the field accessors now return handle values in tree
+  generation mode.
 - The whole tree is freed together with the context by `p_context_delete()`.
   The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
   tree node handles are only valid while the context is alive.
@@ -34,6 +42,11 @@
   Reference child fields through the target-language accessors described above
   rather than through struct pointer members.
 
+### Improvements
+
+- Improve D language detection in propane.vim
+- Speed up specs
+
 ## v4.8.1
 
 ### Fixes

UPGRADING.md

@@ -14,8 +14,6 @@ from `match` to `match_text` for all target languages.
   `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
 
 The `match_length` argument (C, C++) is unchanged.
-This rename only affects lexer user code blocks; parser rule user code blocks
-never had a matched text argument.
 
 ### Tree memory management
 
@@ -51,6 +49,44 @@ Reference child fields through the target-language accessors above (for example
 `$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
 in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
 
+### Pointers into tree node storage
+
+Tree nodes previously each had their own allocation, so a pointer to a node
+stayed valid for the life of the tree. They are now held in a single array
+which is reallocated as it grows, so a pointer or reference into that array may
+be invalidated whenever a new node is created.
+
+New nodes are created while parsing, so this matters for a pointer taken in a
+tree-mode parser rule user code block, which runs before the parse has
+finished. Keep the node handle instead, which stores a node ID rather than an
+address and stays valid, and obtain the pointer from it when it is needed.
+
+For example, replace a saved pointer:
+
+```
+context_user_fields <<
+    p_node_data_t * saved;
+>>
+Items -> Items a << ${context.saved} = p_node_data($$); >>
+```
+
+with a saved handle:
+
+```
+context_user_fields <<
+    Items saved_node;
+>>
+Items -> Items a << ${context.saved_node} = $$; >>
+```
+
+```
+p_node_data_t * data = p_node_data(context->saved_node);
+```
+
+Once parsing has finished, no further nodes are created, so a pointer obtained
+after `p_parse()` returns stays valid until the context is deleted, as long as
+no further parsing is performed with the same context.
+
 ## v4.0.0
 
 ### API Changes

assets/parser.d.erb

@@ -149,6 +149,12 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
         return __id != 0u;
     }
 
+    /** Return the node ID (for identity comparison). */
+    @property <%= @grammar.prefix %>node_id_t node_id()
+    {
+        return __id;
+    }
+
     /** Access the underlying node record (token, pvalue, and user fields). */
     @property ref <%= @grammar.prefix %>node_data_t __node()
     {
@@ -177,6 +183,12 @@ public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_s
         return __id != 0u;
     }
 
+    /** Return the node ID (for identity comparison). */
+    @property <%= @grammar.prefix %>node_id_t node_id()
+    {
+        return __id;
+    }
+
     /** Text position of the first code point spanned by this node. */
     @property <%= @grammar.prefix %>position_t position()
     {

assets/parser.h.erb

@@ -135,13 +135,19 @@ typedef struct
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.prefix %>token_info_t;
 
+typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t;
+
+<% if @grammar.tree %>
+<%= c_tree_handle_types_header %>
+
+<% end %>
 /**
  * Lexer and parser context.
  *
  * The user must allocate an instance of this structure and pass it to any
  * public API function.
  */
-typedef struct
+struct <%= @grammar.prefix %>context_s
 {
     /* Lexer context data. */
 
@@ -194,7 +200,7 @@ typedef struct
     size_t user_terminate_code;
 
 <%= @grammar.context_user_fields %>
-} <%= @grammar.prefix %>context_t;
+};
 
 <% if @grammar.tree %>
 <%= c_tree_types_header %>

assets/parser.rs.erb

@@ -5,9 +5,6 @@
 #![allow(non_camel_case_types)]
 #![allow(non_snake_case)]
 #![allow(non_upper_case_globals)]
-#![allow(dead_code)]
-#![allow(unused_variables)]
-#![allow(unused_parens)]
 
 /**************************************************************************
  * User code blocks
@@ -49,7 +46,7 @@ pub type <%= @grammar.prefix %>code_point_t = u32;
  *
  * This is useful for reporting errors, etc...
  */
-#[derive(Clone, Copy, Default, PartialEq)]
+#[derive(Clone, Copy, Debug, Default, PartialEq)]
 pub struct <%= @grammar.prefix %>position_t {
     /** Input text row (1-based). */
     pub row: u32,
@@ -83,6 +80,8 @@ pub enum <%= @grammar.prefix %>value_t {
 
 impl <%= @grammar.prefix %>value_t {
 <%   @grammar.ptypes.each do |name, typestring| %>
+    /* A grammar need not assign to $$ for every declared ptype. */
+    #[allow(dead_code)]
     fn v_<%= name %>_mut(&mut self) -> &mut <%= rust_ptype(typestring) %> {
         match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v, _ => unreachable!() }
     }
@@ -298,28 +297,36 @@ pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>
     context
 }
 
-/**
- * Deinitialize and deallocate lexer/parser context structure.
- *
- * @param context
- *   Lexer/parser context structure.
- */
 <% free_token_node_used = @grammar.tree && @grammar.free_token_node != "" %>
 <% if free_token_node_used %>
-/* The context is taken as mutable for the benefit of the free_token_node user
+impl Drop for <%= @grammar.prefix %>context_t {
- * code block below, which is permitted but not required to modify the node it
+    /* Run the free_token_node user code block for every token node in the tree. */
- * is freeing. */
+    fn drop(&mut self) {
-#[allow(unused_mut)]
+        /* Named so that ${context.<field>} expansions resolve here. */
-<% end %>
+        let context = self;
-pub fn <%= @grammar.prefix %>context_delete(<%= free_token_node_used ? "mut " : "" %>context: <%= @grammar.prefix %>context_t) {
-<% if free_token_node_used %>
         for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() {
             if context.<%= @grammar.prefix %>tree_nodes[i].is_token {
                 let token_node_id = i;
 <%= expand_code(@grammar.free_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
             }
         }
+    }
+}
+
 <% end %>
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * The memory owned by the context is released when the context is dropped, so
+ * this function only has to consume it. It is provided for symmetry with
+ * <%= @grammar.prefix %>context_new() and with the other target languages;
+ * letting the context go out of scope has the same effect.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ */
+#[allow(unused_variables)]
+pub fn <%= @grammar.prefix %>context_delete(context: <%= @grammar.prefix %>context_t) {
 }
 
 /**************************************************************************
@@ -494,6 +501,12 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
  * @return Token to accept, or invalid token if the user code does
  *   not explicitly return a token.
  */
+/* The parameters of this function are provided for the user code blocks
+ * inlined into it, which are not obliged to reference any of them. A $$ or $N
+ * reference in a user code block also expands to a parenthesized dereference,
+ * since it may be followed there by a field or method access; those
+ * parentheses are redundant where the reference is a complete argument. */
+#[allow(unused_parens, unused_variables)]
 fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
     code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8],
     out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
@@ -769,8 +782,11 @@ type symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max)
 /** Parser state ID type. */
 type parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
 
+<% parser_user_code_called = !@grammar.tree || @grammar.parser_user_code_used? %>
+<% if parser_user_code_called %>
 /** Parser rule ID type. */
 type rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
+<% end %>
 
 /** Parser shift ID type. */
 type shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
@@ -789,6 +805,7 @@ struct shift_t {
 struct reduce_t {
     /** Lookahead token. */
     token: <%= @grammar.prefix %>token_t,
+<% if parser_user_code_called %>
 
     /**
      * Rule ID.
@@ -797,6 +814,7 @@ struct reduce_t {
      * grammar rule.
      */
     rule: rule_id_t,
+<% end %>
 
     /**
      * Rule set ID.
@@ -891,7 +909,9 @@ static parser_reduce_table: [reduce_t; <%= @parser.reduce_table.size %>] = [
 <%   @parser.reduce_table.each do |reduce| %>
     reduce_t {
         token: <%= reduce[:token_id] %>, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+<%     if parser_user_code_called %>
         rule: <%= reduce[:rule_id] %>, /* Rule ID */
+<%     end %>
         rule_set: <%= reduce[:rule_set_id] %>, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
         n_states: <%= reduce[:n_states] %>, /* Number of states */
 <%     if @grammar.tree %>
@@ -979,6 +999,12 @@ fn get_rule_position(statevalues: &[state_value_t], i: usize, n_states: usize, g
  * @retval P_USER_TERMINATED
  *   User requested to terminate parsing.
  */
+/* The parameters of this function are provided for the user code blocks
+ * inlined into it, which are not obliged to reference any of them. A $$ or $N
+ * reference in a user code block also expands to a parenthesized dereference,
+ * since it may be followed there by a field or method access; those
+ * parentheses are redundant where the reference is a complete argument. */
+#[allow(unused_parens, unused_variables)]
 fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.tree ? "_node_id: #{@grammar.prefix}node_id_t" : "_pvalue: &mut #{@grammar.prefix}value_t" %>, rule: u32, statevalues: &[state_value_t], n_states: usize) -> usize {
     match rule {
 <%   @grammar.rules.each do |rule| %>
@@ -1301,11 +1327,11 @@ pub fn <%= @grammar.prefix %>parse_inner_<%= start_rule %>(context: &mut <%= @gr
  */
 <% if @grammar.tree %>
 pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(@grammar.start_rules[0]) %><'_> {
-    <%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result") %>
+    <%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result", false) %>
 }
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(start_rule) %><'_> {
-    <%= tree_handle(h_type(start_rule), "context.parse_result") %>
+    <%= tree_handle(h_type(start_rule), "context.parse_result", false) %>
 }
 <%   end %>
 <% else %>

doc/user_guide.md

@@ -349,10 +349,11 @@ accessors on a node handle:
     for token payload and user fields), and `p_node_id(node)` (for identity
     comparison).
   * C++: handle methods called with `()`, e.g. `node.field()`, `node.valid()`,
-    `node.position()`, `node.token()`, `node.pvalue()`, and `node.data()`. The
+    `node.position()`, `node.token()`, `node.pvalue()`, `node.data()`, and
-    C-style functions and macros above are also available.
+    `node.node_id()` (for identity comparison). The C-style functions and
+    macros above are also available.
   * D: `@property` accessors, e.g. `node.field`, `node.valid`, `node.position`,
-    `node.token`, `node.pvalue`.
+    `node.token`, `node.pvalue`, and `node.node_id` (for identity comparison).
   * Rust: handle methods called with `()`, e.g. `node.field()`, `node.valid()`,
     `node.position()`, `node.end_position()`, `node.n_fields()`,
     `node.token()`, `node.pvalue()`, `node.data()` (a reference to the node
@@ -530,15 +531,16 @@ Start -> a:a b:b;
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
-The code block is emitted for the Rust target, where it runs from
+The code block is emitted for the Rust target, where it is run from a `Drop`
-`p_context_delete()`.
+implementation generated for `p_context_t`.
+It therefore runs exactly once however the context is disposed of, whether that
+is by calling `p_context_delete()` or by simply letting the context go out of
+scope.
 A `ptype` or token user field which owns its memory (a `String`, a `Vec`, a
 `Box`, and so on) is released when the context is dropped and does not need a
 `free_token_node` code block.
 The statement is only needed for memory which Rust does not track, such as a
 raw pointer obtained from `Box::into_raw()`.
-Note that the generated `p_context_t` does not implement `Drop`, so a
-`free_token_node` code block only runs if `p_context_delete()` is called.
 
 ##> `lex_fn` statement - specifying a custom lexer function
 
@@ -695,10 +697,21 @@ generated output without any surrounding `#line` directives.
 This can be useful when debugging the generated parser itself, or when the
 `#line` directives interfere with other tooling.
 
-The `noline` statement only affects the C, C++, and D targets.
+Rust has no `#line` directive equivalent.
-Rust has no `#line` directive equivalent, so `#line` directives are never
+For a Rust target, Propane instead emits a comment before and after each
-emitted into Rust output and the `noline` statement has no effect for the Rust
+section of user code naming the grammar file and the line number the code was
-target.
+taken from:
+
+```
+/* Begin user code from myparser.propane line 42. */
+  let mut v: i64 = 0;
+/* End user code from myparser.propane line 42. */
+```
+
+A compiler diagnostic that points into the generated Rust module can be traced
+back to the grammar by reading up to the nearest such comment.
+The `noline` statement suppresses these comments for a Rust target, in the same
+way that it suppresses `#line` directives for the other targets.
 
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
@@ -1680,9 +1693,9 @@ provide a code block which frees that memory; if specified, the
 
 For Rust targets, `p_context_delete()` takes the context by value and consumes
 it.
-The memory owned by the context is released when the context is dropped, so the
+Everything the context owns is released when it is dropped, including running
-call is only strictly required when the grammar supplies a `free_token_node`
+any `free_token_node` code block, so calling this function is optional for a
-code block, which runs from `p_context_delete()`.
+Rust target; letting the context go out of scope has the same effect.
 
 Rust example:
 

extra/vim/syntax/propane.vim

@@ -7,12 +7,46 @@ if exists("b:current_syntax")
   finish
 endif
 
+" Guess the language of the user code blocks from their contents so that the
+" matching syntax file can be included below. b:propane_subtype may also be set
+" before this file is sourced to select the language explicitly.
 if !exists("b:propane_subtype")
-  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+  " Rust markers. Each keyword requires the syntax that follows it in Rust so
+  " that a plain identifier of the same name in another language does not match
+  " (`int fn = 3;' in C, for example). Type names are only accepted within a
+  " `ptype' statement for the same reason.
+  let s:rust = '\<let\s\+\%(mut\s\+\)\?\w'
+  let s:rust .= '\|\<fn\s\+\w\+\s*('
+  let s:rust .= '\|&mut\>\|\<pub\s\+\w\|\<impl\s\+\w'
+  let s:rust .= '\|#\[\|\<use\s\+\%(std\|core\)::'
+  let s:rust .= '\|\<ptype\>[^;]*\<\%(isize\|usize\|i8\|i16\|i32\|i64\|i128'
+  let s:rust .= '\|u8\|u16\|u32\|u64\|u128\|f32\|f64\|String\)\>'
+  " D markers. These are spellings that have no valid C, C++, or Rust
+  " equivalent, so `import' is deliberately not among them: it is a D keyword
+  " but is also a C++20 module declaration.
+  let s:d = '\<foreach\%(_reverse\)\?\s*([^)]*;'
+  let s:d .= '\|\~=\|\<static\s\+if\s*(\|\<version\s*(\s*\w\+\s*)'
+  let s:d .= '\|\<scope\s*(\s*\%(exit\|failure\|success\)\s*)'
+  let s:d .= '\|\<\%(unittest\|mixin\|immutable\|__gshared\|invariant\)\>'
+  let s:d .= '\|\<alias\s\+\w\+\s*=\|\<enum\s\+\w\+\s*='
+  let s:d .= '\|@\%(property\|safe\|trusted\|system\|nogc\|disable\)\>'
+  let s:d .= '\|\<is\s\+null\>\|\<cast\s*(\s*\w\+\s*)'
+  let s:d .= '\|\<write\%(ln\|fln\|f\)\s*('
+  let s:d .= '\|\<\%(dchar\|dstring\|wstring\|cent\|ucent\)\>'
+  " A module import on its own is ambiguous between D and C++20, so only take
+  " it as D when nothing else in the file looks like C++.
+  let s:import = '\<import\s\+[A-Za-z_][A-Za-z0-9_.]*\s*;'
+  let s:cpp = '::\|\<template\s*<\|\<namespace\>\|\<nullptr\>\|#include\s*[<"]'
+  if search(s:rust, 'nw') > 0
+    let b:propane_subtype = "rust"
+  elseif search(s:d, 'nw') > 0
+    let b:propane_subtype = "d"
+  elseif search(s:import, 'nw') > 0 && search(s:cpp, 'nw') == 0
     let b:propane_subtype = "d"
   else
     let b:propane_subtype = "cpp"
   endif
+  unlet s:rust s:d s:import s:cpp
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"

lib/propane/generator.rb

@@ -41,12 +41,23 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
+        # Rust has no #line directive support. For a Rust target the directives
+        # that the grammar embeds around user code blocks are replaced with
+        # comments naming the grammar file and line number the code came from,
+        # so that the origin of a section of user code can still be found by
+        # reading up from a compiler diagnostic pointing into the generated
+        # module.
+        user_code_origin = nil
         result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
           if @language == "rust"
-            # Rust has no #line directive support, so strip the directives that
+            if md = line.match(/^#line (\d+) "([^"]*)"/)
-            # the grammar embeds in user code blocks.
+              user_code_origin = "#{md[2]} line #{md[1]}"
-            line = line.sub(/^#line \d+ "[^"]*"/, "")
+              line.sub(/^#line \d+ "[^"]*"/, %[/* Begin user code from #{user_code_origin}. */])
-            line == "#linereset\n" ? "" : line
+            elsif line == "#linereset\n"
+              %[/* End user code from #{user_code_origin}. */\n]
+            else
+              line
+            end
           elsif line == "#linereset\n"
             %[#line #{i + 2} "#{output_file}"\n]
           else
@@ -470,16 +481,24 @@ class Propane
     #   Handle type name.
     # @param id_expr [String]
     #   Expression yielding the node ID.
+    # @param parenthesize [Boolean]
+    #   Whether to parenthesize the expression. Parentheses are required where
+    #   the expression is substituted into a user code block, since the
+    #   expression could be followed there by a field access or appear in a
+    #   position where a bare Rust struct literal is not accepted. They are
+    #   unnecessary where the expression stands alone, and Rust warns about
+    #   them there, so this can be disabled for those uses.
     #
     # @return [String]
     #   Handle constructor expression.
-    def tree_handle(typename, id_expr)
+    def tree_handle(typename, id_expr, parenthesize = true)
       if @cpp
         "(#{typename}{context, #{id_expr}})"
       elsif @language == "c"
         "((#{typename}){context, #{id_expr}})"
       elsif @language == "rust"
-        "(#{typename} { context, id: #{id_expr} })"
+        expr = "#{typename} { context, id: #{id_expr} }"
+        parenthesize ? "(#{expr})" : expr
       else
         "#{typename}(context, #{id_expr})"
       end
@@ -533,16 +552,30 @@ class Propane
       end
     end
 
-    # Generate the C/C++ tree node handle type section for the header.
+    # Generate the tree node handle type declarations for the header.
+    #
+    # These are emitted before the context structure definition so that a
+    # context_user_fields block can declare a field of a handle type.
     #
     # @return [String]
-    #   Header handle section.
+    #   Handle type declarations.
+    def c_tree_handle_types_header
+      @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header
+    end
+
+    # Generate the remainder of the tree node section for the header.
+    #
+    # This is emitted after the context structure definition since it
+    # dereferences the context and so requires the complete type.
+    #
+    # @return [String]
+    #   Accessors, macros, and out-of-line handle method definitions.
     def c_tree_types_header
       @cpp ? cpp_tree_types_header : c_only_tree_types_header
     end
 
     # Generate the C (non-C++) tree node handle type section for the header.
-    def c_only_tree_types_header
+    def c_only_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
@@ -550,6 +583,11 @@ class Propane
         out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
       end
       out << ""
+      out.join("\n")
+    end
+
+    def c_only_tree_types_header
+      out = []
       out << c_common_accessors_header
       out << "/** @} */"
       out.join("\n")
@@ -657,29 +695,32 @@ class Propane
       out.join("\n")
     end
 
-    # Generate the C++ tree node handle type section for the header.
+    # Generate the C++ tree node handle class declarations for the header.
-    def cpp_tree_types_header
+    # Only valid() and node_id() are defined inline; every other method
+    # dereferences the context, which is still an incomplete type here, so
+    # those are declared and defined out of line once the context is
+    # complete.
+    def cpp_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
       tree_handle_types.each {|t| out << "struct #{t};"}
       out << ""
-      # Token handle (all methods inline; no handle-typed returns).
       tt = h_type("Token")
       out << "struct #{tt}"
       out << "{"
       out << "    #{p}context_t * __context;"
       out << "    #{p}node_id_t __id;"
       out << "    bool valid() const { return __id != 0u; }"
-      out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+      out << "    #{p}node_id_t node_id() const { return __id; }"
-      out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+      out << "    #{p}node_data_t * data() const;"
-      out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+      out << "    #{p}position_t position() const;"
-      out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      out << "    #{p}position_t end_position() const;"
-      out << "    #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "    uint16_t n_fields() const;"
-      out << "    #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << "    #{p}token_t token() const;"
+      out << "    #{p}value_t pvalue() const;"
       out << "};"
       out << ""
-      # Rule set handles: navigation methods declared, defined out-of-line below.
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)
         out << "struct #{rtype}"
@@ -687,16 +728,36 @@ class Propane
         out << "    #{p}context_t * __context;"
         out << "    #{p}node_id_t __id;"
         out << "    bool valid() const { return __id != 0u; }"
-        out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "    #{p}node_id_t node_id() const { return __id; }"
-        out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "    #{p}node_data_t * data() const;"
-        out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "    #{p}position_t position() const;"
-        out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+        out << "    #{p}position_t end_position() const;"
+        out << "    uint16_t n_fields() const;"
         each_tree_field(rule_set) do |rt, field_name, child_type, slot|
           out << "    #{child_type} #{field_name}() const;"
         end
         out << "};"
         out << ""
       end
+      out.join("\n")
+    end
+
+    # Generate the out-of-line C++ handle method definitions plus the C-style
+    # accessors. Emitted after the context structure definition.
+    def cpp_tree_types_header
+      p = @grammar.prefix
+      out = []
+      # Common node methods, now that the context type is complete.
+      ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht|
+        out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      end
+      tt = h_type("Token")
+      out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << ""
       # Out-of-line navigation method bodies (all handle types now complete).
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.8.1"
+  VERSION = "5.1.0"
 end

spec/propane_spec.rb

@@ -2059,8 +2059,35 @@ EOF
         expect(results.status).to eq 0
       end
 
-      if %w[c cpp].include?(language)
+      # D is excluded since it is garbage collected, so Propane does not emit a
+      # free_token_node code block for it.
+      if %w[c cpp rust].include?(language)
         it "allows a user function to free token node memory in tree mode" do
+          if language == "rust"
+            write_grammar <<EOF
+<<
+use std::sync::atomic::{AtomicU32, Ordering};
+
+/** Number of token nodes freed by the free_token_node code block. */
+pub static FREED: AtomicU32 = AtomicU32::new(0);
+>>
+tree;
+free_token_node <<
+    if !${token.pvalue}.is_null() {
+        unsafe { drop(Box::from_raw(${token.pvalue})); }
+        FREED.fetch_add(1, Ordering::SeqCst);
+    }
+>>
+ptype *mut i32;
+token a <<
+  $$ = Box::into_raw(Box::new(1));
+>>
+token b <<
+  $$ = Box::into_raw(Box::new(2));
+>>
+Start -> a:a b:b;
+EOF
+          else
             write_grammar <<EOF
 tree;
 free_token_node <<
@@ -2077,6 +2104,7 @@ token b <<
 >>
 Start -> a:a b:b;
 EOF
+          end
           run_propane(language: language)
           compile("spec/test_tree_delete_token_node_memory.#{language}", language: language)
           results = run_test(language: language)
@@ -2085,6 +2113,76 @@ EOF
         end
       end
 
+      # Rust is excluded since a Rust tree node handle borrows the context, so
+      # storing one in a context field would make the context self-referential.
+      if %w[c cpp d].include?(language)
+        it "allows a tree node handle type in a context user field" do
+          write_grammar <<EOF
+tree;
+ptype int;
+context_user_fields <<
+    Item first_item;
+    int have_first;
+>>
+drop /\\s+/;
+token a /a/ << $$ = 7; >>
+Item -> a;
+Items -> ;
+Items -> Items Item <<
+    if (${context.have_first} == 0)
+    {
+        ${context.first_item} = $2;
+        ${context.have_first} = 1;
+    }
+>>
+Start -> Items;
+EOF
+          run_propane(language: language)
+          compile("spec/test_context_field_handle.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
+
+      if language == "rust"
+        it "marks user code sections with their grammar file and line number" do
+          write_grammar <<EOF
+ptype i64;
+drop /\\s+/;
+token num /\\d+/ <<
+  $$ = 42;
+>>
+Start -> num << $$ = $1; >>
+EOF
+          run_propane(language: language)
+          parser = File.binread("spec/run/testparser.rs")
+          # The lexer code block body begins on grammar line 4 and the parser
+          # rule code block is on grammar line 6.
+          expect(parser).to include %[/* Begin user code from spec/run/testparser.propane line 4. */]
+          expect(parser).to include %[/* End user code from spec/run/testparser.propane line 4. */]
+          expect(parser).to include %[/* Begin user code from spec/run/testparser.propane line 6. */]
+          expect(parser).to include %[/* End user code from spec/run/testparser.propane line 6. */]
+          expect(parser).to_not include "#line"
+        end
+
+        it "omits user code section markers when noline is specified" do
+          write_grammar <<EOF
+noline;
+ptype i64;
+drop /\\s+/;
+token num /\\d+/ <<
+  $$ = 42;
+>>
+Start -> num << $$ = $1; >>
+EOF
+          run_propane(language: language)
+          parser = File.binread("spec/run/testparser.rs")
+          expect(parser).to_not include "user code from"
+          expect(parser).to_not include "#line"
+        end
+      end
+
       it "executes code blocks associated with drop statements" do
         if language == "rust"
           write_grammar <<EOF

spec/test_context_field_handle.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+    size_t i;
+    p_context_t * context;
+    Token token;
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(p_node_valid(context->first_item));
+    token = p_Item_pToken1(context->first_item);
+    assert(p_node_valid(token));
+    assert_eq(TOKEN_a, p_Token_token(token));
+    assert_eq(7u, p_Token_pvalue(token));
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, p_node_position(context->first_item).row);
+    assert_eq(1u, p_node_position(context->first_item).col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.cpp

@@ -0,0 +1,36 @@
+#include "testparser.h"
+#include <cassert>
+#include <cstring>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (size_t i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(context->first_item.valid());
+    Token token = context->first_item.pToken1();
+    assert(token.valid());
+    assert_eq(TOKEN_a, token.token());
+    assert_eq(7u, token.pvalue());
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context->first_item.position().row);
+    assert_eq(1u, context->first_item.position().col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.d

@@ -0,0 +1,35 @@
+import testparser;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* Enough tokens that the tree node array is reallocated during the parse. */
+    string input;
+    foreach (i; 0 .. 40)
+    {
+        input ~= "a";
+    }
+
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1, context.have_first);
+    assert(context.first_item.valid);
+    Token token = context.first_item.pToken1;
+    assert(token.valid);
+    assert_eq(TOKEN_a, token.token);
+    assert_eq(7, token.pvalue);
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context.first_item.position.row);
+    assert_eq(1u, context.first_item.position.col);
+
+    p_context_delete(context);
+}

spec/test_optional_rule_component_tree.d

@@ -31,6 +31,7 @@ unittest
     assert(start.pR3.valid);
     assert(start.pR.valid);
     assert(start.pR == start.pR3);
+    assert_eq(start.pR.node_id, start.pR3.node_id);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
     p_context_delete(context);

spec/test_set_position.rs

@@ -17,9 +17,7 @@ fn main() {
     /* p_set_position overrides the initial position. */
     let mut c = p_context_new(b"ab");
     p_set_position(&mut c, p_position_t { row: 5, col: 20 });
-    let pos = p_position(&c);
+    assert_eq!(p_position_t { row: 5, col: 20 }, p_position(&c));
-    assert_eq!(5, pos.row);
-    assert_eq!(20, pos.col);
     assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
     assert_eq!(TOKEN_a, ti.token);
     assert_eq!(5, ti.position.row);

spec/test_tree.cpp

@@ -13,6 +13,8 @@ int main()
     assert(start.pItems1().valid());
     assert(start.pItems().valid());
     Items items = start.pItems();
+    assert_ne(0u, items.node_id());
+    assert_eq(start.pItems().node_id(), items.node_id());
     assert(items.pItem().valid());
     assert(items.pItem().pToken1().valid());
     assert_eq(TOKEN_a, items.pItem().pToken1().token());
@@ -40,6 +42,7 @@ int main()
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(!start.pItems().valid());
+    assert_eq(0u, start.pItems().node_id());
 
     p_context_delete(context);
 

spec/test_tree_delete_token_node_memory.rs

@@ -0,0 +1,27 @@
+use testparser::*;
+use std::sync::atomic::Ordering;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    let start = p_result(&context);
+    assert!(start.a().valid());
+    assert_eq!(1, unsafe { *start.a().pvalue() });
+    assert!(start.b().valid());
+    assert_eq!(2, unsafe { *start.b().pvalue() });
+
+    /* The free_token_node code block runs when the context is disposed of, not
+     * before, and frees each of the two token nodes exactly once. */
+    assert_eq!(0, FREED.load(Ordering::SeqCst));
+    p_context_delete(context);
+    assert_eq!(2, FREED.load(Ordering::SeqCst));
+
+    /* Letting the context go out of scope runs the code block too, so a caller
+     * which never calls p_context_delete() does not leak. */
+    {
+        let mut context = p_context_new(b"ab");
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(2, FREED.load(Ordering::SeqCst));
+    }
+    assert_eq!(4, FREED.load(Ordering::SeqCst));
+}