Make tree node handle names visible in context user fields

assets/parser.h.erb

@@ -135,13 +135,19 @@ typedef struct
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.prefix %>token_info_t;
 
+typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t;
+
+<% if @grammar.tree %>
+<%= c_tree_handle_types_header %>
+
+<% end %>
 /**
  * Lexer and parser context.
  *
  * The user must allocate an instance of this structure and pass it to any
  * public API function.
  */
-typedef struct
+struct <%= @grammar.prefix %>context_s
 {
     /* Lexer context data. */
 
@@ -194,7 +200,7 @@ typedef struct
     size_t user_terminate_code;
 
 <%= @grammar.context_user_fields %>
-} <%= @grammar.prefix %>context_t;
+};
 
 <% if @grammar.tree %>
 <%= c_tree_types_header %>

lib/propane/generator.rb

@@ -552,16 +552,30 @@ class Propane
       end
     end
 
-    # Generate the C/C++ tree node handle type section for the header.
+    # Generate the tree node handle type declarations for the header.
+    #
+    # These are emitted before the context structure definition so that a
+    # context_user_fields block can declare a field of a handle type.
     #
     # @return [String]
-    #   Header handle section.
+    #   Handle type declarations.
+    def c_tree_handle_types_header
+      @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header
+    end
+
+    # Generate the remainder of the tree node section for the header.
+    #
+    # This is emitted after the context structure definition since it
+    # dereferences the context and so requires the complete type.
+    #
+    # @return [String]
+    #   Accessors, macros, and out-of-line handle method definitions.
     def c_tree_types_header
       @cpp ? cpp_tree_types_header : c_only_tree_types_header
     end
 
     # Generate the C (non-C++) tree node handle type section for the header.
-    def c_only_tree_types_header
+    def c_only_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
@@ -569,6 +583,11 @@ class Propane
         out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
       end
       out << ""
+      out.join("\n")
+    end
+
+    def c_only_tree_types_header
+      out = []
       out << c_common_accessors_header
       out << "/** @} */"
       out.join("\n")
@@ -676,29 +695,30 @@ class Propane
       out.join("\n")
     end
 
-    # Generate the C++ tree node handle type section for the header.
-    def cpp_tree_types_header
+    # Generate the C++ tree node handle class declarations for the header.
+    # Only valid() is defined inline; every other method dereferences the
+    # context, which is still an incomplete type here, so those are declared
+    # and defined out of line once the context is complete.
+    def cpp_tree_handle_types_header
       p = @grammar.prefix
       out = []
       out << "/** Tree node handle types. @{ */"
       tree_handle_types.each {|t| out << "struct #{t};"}
       out << ""
-      # Token handle (all methods inline; no handle-typed returns).
       tt = h_type("Token")
       out << "struct #{tt}"
       out << "{"
       out << "    #{p}context_t * __context;"
       out << "    #{p}node_id_t __id;"
       out << "    bool valid() const { return __id != 0u; }"
-      out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
-      out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
-      out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-      out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
-      out << "    #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
-      out << "    #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << "    #{p}node_data_t * data() const;"
+      out << "    #{p}position_t position() const;"
+      out << "    #{p}position_t end_position() const;"
+      out << "    uint16_t n_fields() const;"
+      out << "    #{p}token_t token() const;"
+      out << "    #{p}value_t pvalue() const;"
       out << "};"
       out << ""
-      # Rule set handles: navigation methods declared, defined out-of-line below.
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)
         out << "struct #{rtype}"
@@ -706,16 +726,35 @@ class Propane
         out << "    #{p}context_t * __context;"
         out << "    #{p}node_id_t __id;"
         out << "    bool valid() const { return __id != 0u; }"
-        out << "    #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
-        out << "    #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
-        out << "    #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-        out << "    uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+        out << "    #{p}node_data_t * data() const;"
+        out << "    #{p}position_t position() const;"
+        out << "    #{p}position_t end_position() const;"
+        out << "    uint16_t n_fields() const;"
         each_tree_field(rule_set) do |rt, field_name, child_type, slot|
           out << "    #{child_type} #{field_name}() const;"
         end
         out << "};"
         out << ""
       end
+      out.join("\n")
+    end
+
+    # Generate the out-of-line C++ handle method definitions plus the C-style
+    # accessors. Emitted after the context structure definition.
+    def cpp_tree_types_header
+      p = @grammar.prefix
+      out = []
+      # Common node methods, now that the context type is complete.
+      ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht|
+        out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }"
+        out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }"
+        out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
+        out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
+      end
+      tt = h_type("Token")
+      out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }"
+      out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
+      out << ""
       # Out-of-line navigation method bodies (all handle types now complete).
       tree_node_rule_sets.each do |rule_set|
         rtype = h_type(rule_set.name)

spec/propane_spec.rb

@@ -2113,6 +2113,38 @@ EOF
         end
       end
 
+      # Rust is excluded since a Rust tree node handle borrows the context, so
+      # storing one in a context field would make the context self-referential.
+      if %w[c cpp d].include?(language)
+        it "allows a tree node handle type in a context user field" do
+          write_grammar <<EOF
+tree;
+ptype int;
+context_user_fields <<
+    Item first_item;
+    int have_first;
+>>
+drop /\\s+/;
+token a /a/ << $$ = 7; >>
+Item -> a;
+Items -> ;
+Items -> Items Item <<
+    if (${context.have_first} == 0)
+    {
+        ${context.first_item} = $2;
+        ${context.have_first} = 1;
+    }
+>>
+Start -> Items;
+EOF
+          run_propane(language: language)
+          compile("spec/test_context_field_handle.#{language}", language: language)
+          results = run_test(language: language)
+          expect(results.stderr).to eq ""
+          expect(results.status).to eq 0
+        end
+      end
+
       if language == "rust"
         it "marks user code sections with their grammar file and line number" do
           write_grammar <<EOF

spec/test_context_field_handle.c

@@ -0,0 +1,39 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+    size_t i;
+    p_context_t * context;
+    Token token;
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(p_node_valid(context->first_item));
+    token = p_Item_pToken1(context->first_item);
+    assert(p_node_valid(token));
+    assert_eq(TOKEN_a, p_Token_token(token));
+    assert_eq(7u, p_Token_pvalue(token));
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, p_node_position(context->first_item).row);
+    assert_eq(1u, p_node_position(context->first_item).col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.cpp

@@ -0,0 +1,36 @@
+#include "testparser.h"
+#include <cassert>
+#include <cstring>
+#include "testutils.h"
+
+int main()
+{
+    char input[128];
+
+    /* Enough tokens that the tree node arena is reallocated during the parse. */
+    memset(input, 0, sizeof(input));
+    for (size_t i = 0u; i < 40u; i++)
+    {
+        input[i] = 'a';
+    }
+
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1u, context->have_first);
+    assert(context->first_item.valid());
+    Token token = context->first_item.pToken1();
+    assert(token.valid());
+    assert_eq(TOKEN_a, token.token());
+    assert_eq(7u, token.pvalue());
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context->first_item.position().row);
+    assert_eq(1u, context->first_item.position().col);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_context_field_handle.d

@@ -0,0 +1,35 @@
+import testparser;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* Enough tokens that the tree node array is reallocated during the parse. */
+    string input;
+    foreach (i; 0 .. 40)
+    {
+        input ~= "a";
+    }
+
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+
+    /* The handle was stored in a context user field during the parse, before
+     * the remaining nodes were created. It still refers to the same node. */
+    assert_eq(1, context.have_first);
+    assert(context.first_item.valid);
+    Token token = context.first_item.pToken1;
+    assert(token.valid);
+    assert_eq(TOKEN_a, token.token);
+    assert_eq(7, token.pvalue);
+
+    /* The stored handle refers to the first Item, which starts at column 1. */
+    assert_eq(1u, context.first_item.position.row);
+    assert_eq(1u, context.first_item.position.col);
+
+    p_context_delete(context);
+}