diff --git a/assets/parser.h.erb b/assets/parser.h.erb index 07aa072..5855e81 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -135,13 +135,19 @@ typedef struct <%= @grammar.prefix %>value_t pvalue; } <%= @grammar.prefix %>token_info_t; +typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t; + +<% if @grammar.tree %> +<%= c_tree_handle_types_header %> + +<% end %> /** * Lexer and parser context. * * The user must allocate an instance of this structure and pass it to any * public API function. */ -typedef struct +struct <%= @grammar.prefix %>context_s { /* Lexer context data. */ @@ -194,7 +200,7 @@ typedef struct size_t user_terminate_code; <%= @grammar.context_user_fields %> -} <%= @grammar.prefix %>context_t; +}; <% if @grammar.tree %> <%= c_tree_types_header %> diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index e75a4a1..4999e68 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -552,16 +552,30 @@ class Propane end end - # Generate the C/C++ tree node handle type section for the header. + # Generate the tree node handle type declarations for the header. + # + # These are emitted before the context structure definition so that a + # context_user_fields block can declare a field of a handle type. # # @return [String] - # Header handle section. + # Handle type declarations. + def c_tree_handle_types_header + @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header + end + + # Generate the remainder of the tree node section for the header. + # + # This is emitted after the context structure definition since it + # dereferences the context and so requires the complete type. + # + # @return [String] + # Accessors, macros, and out-of-line handle method definitions. def c_tree_types_header @cpp ? cpp_tree_types_header : c_only_tree_types_header end # Generate the C (non-C++) tree node handle type section for the header. - def c_only_tree_types_header + def c_only_tree_handle_types_header p = @grammar.prefix out = [] out << "/** Tree node handle types. @{ */" @@ -569,6 +583,11 @@ class Propane out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};" end out << "" + out.join("\n") + end + + def c_only_tree_types_header + out = [] out << c_common_accessors_header out << "/** @} */" out.join("\n") @@ -676,29 +695,30 @@ class Propane out.join("\n") end - # Generate the C++ tree node handle type section for the header. - def cpp_tree_types_header + # Generate the C++ tree node handle class declarations for the header. + # Only valid() is defined inline; every other method dereferences the + # context, which is still an incomplete type here, so those are declared + # and defined out of line once the context is complete. + def cpp_tree_handle_types_header p = @grammar.prefix out = [] out << "/** Tree node handle types. @{ */" tree_handle_types.each {|t| out << "struct #{t};"} out << "" - # Token handle (all methods inline; no handle-typed returns). tt = h_type("Token") out << "struct #{tt}" out << "{" out << " #{p}context_t * __context;" out << " #{p}node_id_t __id;" out << " bool valid() const { return __id != 0u; }" - out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }" - out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }" - out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }" - out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }" - out << " #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }" - out << " #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }" + out << " #{p}node_data_t * data() const;" + out << " #{p}position_t position() const;" + out << " #{p}position_t end_position() const;" + out << " uint16_t n_fields() const;" + out << " #{p}token_t token() const;" + out << " #{p}value_t pvalue() const;" out << "};" out << "" - # Rule set handles: navigation methods declared, defined out-of-line below. tree_node_rule_sets.each do |rule_set| rtype = h_type(rule_set.name) out << "struct #{rtype}" @@ -706,16 +726,35 @@ class Propane out << " #{p}context_t * __context;" out << " #{p}node_id_t __id;" out << " bool valid() const { return __id != 0u; }" - out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }" - out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }" - out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }" - out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }" + out << " #{p}node_data_t * data() const;" + out << " #{p}position_t position() const;" + out << " #{p}position_t end_position() const;" + out << " uint16_t n_fields() const;" each_tree_field(rule_set) do |rt, field_name, child_type, slot| out << " #{child_type} #{field_name}() const;" end out << "};" out << "" end + out.join("\n") + end + + # Generate the out-of-line C++ handle method definitions plus the C-style + # accessors. Emitted after the context structure definition. + def cpp_tree_types_header + p = @grammar.prefix + out = [] + # Common node methods, now that the context type is complete. + ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht| + out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }" + out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }" + out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }" + out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }" + end + tt = h_type("Token") + out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }" + out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }" + out << "" # Out-of-line navigation method bodies (all handle types now complete). tree_node_rule_sets.each do |rule_set| rtype = h_type(rule_set.name) diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index c62ca26..471a22e 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -2113,6 +2113,38 @@ EOF end end + # Rust is excluded since a Rust tree node handle borrows the context, so + # storing one in a context field would make the context self-referential. + if %w[c cpp d].include?(language) + it "allows a tree node handle type in a context user field" do + write_grammar <> +drop /\\s+/; +token a /a/ << $$ = 7; >> +Item -> a; +Items -> ; +Items -> Items Item << + if (${context.have_first} == 0) + { + ${context.first_item} = $2; + ${context.have_first} = 1; + } +>> +Start -> Items; +EOF + run_propane(language: language) + compile("spec/test_context_field_handle.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + end + if language == "rust" it "marks user code sections with their grammar file and line number" do write_grammar < +#include +#include "testutils.h" + +int main() +{ + char input[128]; + size_t i; + p_context_t * context; + Token token; + + /* Enough tokens that the tree node arena is reallocated during the parse. */ + memset(input, 0, sizeof(input)); + for (i = 0u; i < 40u; i++) + { + input[i] = 'a'; + } + + context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(context)); + + /* The handle was stored in a context user field during the parse, before + * the remaining nodes were created. It still refers to the same node. */ + assert_eq(1u, context->have_first); + assert(p_node_valid(context->first_item)); + token = p_Item_pToken1(context->first_item); + assert(p_node_valid(token)); + assert_eq(TOKEN_a, p_Token_token(token)); + assert_eq(7u, p_Token_pvalue(token)); + + /* The stored handle refers to the first Item, which starts at column 1. */ + assert_eq(1u, p_node_position(context->first_item).row); + assert_eq(1u, p_node_position(context->first_item).col); + + p_context_delete(context); + + return 0; +} diff --git a/spec/test_context_field_handle.cpp b/spec/test_context_field_handle.cpp new file mode 100644 index 0000000..fff7328 --- /dev/null +++ b/spec/test_context_field_handle.cpp @@ -0,0 +1,36 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +int main() +{ + char input[128]; + + /* Enough tokens that the tree node arena is reallocated during the parse. */ + memset(input, 0, sizeof(input)); + for (size_t i = 0u; i < 40u; i++) + { + input[i] = 'a'; + } + + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(context)); + + /* The handle was stored in a context user field during the parse, before + * the remaining nodes were created. It still refers to the same node. */ + assert_eq(1u, context->have_first); + assert(context->first_item.valid()); + Token token = context->first_item.pToken1(); + assert(token.valid()); + assert_eq(TOKEN_a, token.token()); + assert_eq(7u, token.pvalue()); + + /* The stored handle refers to the first Item, which starts at column 1. */ + assert_eq(1u, context->first_item.position().row); + assert_eq(1u, context->first_item.position().col); + + p_context_delete(context); + + return 0; +} diff --git a/spec/test_context_field_handle.d b/spec/test_context_field_handle.d new file mode 100644 index 0000000..5463e56 --- /dev/null +++ b/spec/test_context_field_handle.d @@ -0,0 +1,35 @@ +import testparser; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + /* Enough tokens that the tree node array is reallocated during the parse. */ + string input; + foreach (i; 0 .. 40) + { + input ~= "a"; + } + + p_context_t * context = p_context_new(input); + assert_eq(P_SUCCESS, p_parse(context)); + + /* The handle was stored in a context user field during the parse, before + * the remaining nodes were created. It still refers to the same node. */ + assert_eq(1, context.have_first); + assert(context.first_item.valid); + Token token = context.first_item.pToken1; + assert(token.valid); + assert_eq(TOKEN_a, token.token); + assert_eq(7, token.pvalue); + + /* The stored handle refers to the first Item, which starts at column 1. */ + assert_eq(1u, context.first_item.position.row); + assert_eq(1u, context.first_item.position.col); + + p_context_delete(context); +}