Rework tree generation mode and API
Store tree nodes in congruent, compact arena array. Define handle types to refer to tree nodes rather than pointers to structure instances. Free tree with context.
This commit is contained in:
josh committed
1 parent
dddb1b5089
commit
5fc712c6ee
39 files changed
+1349
-811
No files matched your search
+266
-5
@@ -298,11 +298,12 @@ class Propane
|
||||
if parser
|
||||
code = code.gsub(/\$\$/) do |match|
|
||||
if @grammar.tree
|
||||
typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
|
||||
case @language
|
||||
when "c"
|
||||
"((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
|
||||
tree_handle(typename, "_node_id")
|
||||
when "d"
|
||||
"(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
|
||||
tree_handle(typename, "_node_id")
|
||||
end
|
||||
else
|
||||
case @language
|
||||
@@ -402,7 +403,7 @@ class Propane
|
||||
def parser_component_reference(rule, index)
|
||||
component = rule.components[index - 1]
|
||||
if @grammar.tree
|
||||
# In tree mode a component reference yields a pointer to that
|
||||
# In tree mode a component reference yields a handle to that
|
||||
# component's tree node. An optional component propagates its target
|
||||
# node (or null), so use the optional target's node type.
|
||||
if component.is_a?(RuleSet) && component.optional?
|
||||
@@ -412,9 +413,9 @@ class Propane
|
||||
typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
|
||||
case @language
|
||||
when "c"
|
||||
"((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
|
||||
tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
|
||||
when "d"
|
||||
"(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
|
||||
tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
|
||||
end
|
||||
else
|
||||
case @language
|
||||
@@ -426,6 +427,266 @@ class Propane
|
||||
end
|
||||
end
|
||||
|
||||
# Construct a tree node handle expression for the target language.
|
||||
#
|
||||
# A handle is a small value pairing the parser context with a node ID
|
||||
# (an index into the context's node arena). All handle types share this
|
||||
# layout; the distinct types exist for documentation and, in C, to drive
|
||||
# the tree walk macro's type threading.
|
||||
#
|
||||
# @param typename [String]
|
||||
# Handle type name.
|
||||
# @param id_expr [String]
|
||||
# Expression yielding the node ID.
|
||||
#
|
||||
# @return [String]
|
||||
# Handle constructor expression.
|
||||
def tree_handle(typename, id_expr)
|
||||
if @cpp
|
||||
"(#{typename}{context, #{id_expr}})"
|
||||
elsif @language == "c"
|
||||
"((#{typename}){context, #{id_expr}})"
|
||||
else
|
||||
"#{typename}(context, #{id_expr})"
|
||||
end
|
||||
end
|
||||
|
||||
# Get the list of non-optional, non-internal rule sets that get a tree node
|
||||
# handle type generated for them.
|
||||
#
|
||||
# @return [Array<Propane::RuleSet>]
|
||||
# Rule sets with generated tree node handle types.
|
||||
def tree_node_rule_sets
|
||||
@parser.rule_sets.reject do |name, rule_set|
|
||||
name.start_with?("$") || rule_set.optional?
|
||||
end.map {|name, rule_set| rule_set}
|
||||
end
|
||||
|
||||
# Maximum number of chained fields supported by a single C tree walk macro
|
||||
# invocation. Deeper navigation can be expressed by nesting walk calls.
|
||||
C_TREE_WALK_MAX = 16
|
||||
|
||||
# Get the tree node handle type name for a node name.
|
||||
#
|
||||
# @param name [String]
|
||||
# Rule set name, or "Token".
|
||||
#
|
||||
# @return [String]
|
||||
# Handle type name.
|
||||
def h_type(name)
|
||||
"#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
|
||||
end
|
||||
|
||||
# Get the list of all tree node handle type names (Token plus rule sets).
|
||||
#
|
||||
# @return [Array<String>]
|
||||
# Handle type names.
|
||||
def tree_handle_types
|
||||
[h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
|
||||
end
|
||||
|
||||
# Enumerate the navigation fields of a rule set's tree node.
|
||||
#
|
||||
# @yield [rtype, field_name, child_type, slot]
|
||||
# Handle type name, field accessor name, child handle type, and child
|
||||
# slot index.
|
||||
def each_tree_field(rule_set)
|
||||
rtype = h_type(rule_set.name)
|
||||
rule_set.tree_fields.each_with_index do |fields, slot|
|
||||
fields.each do |field_name, child_type|
|
||||
yield rtype, field_name, child_type, slot
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Generate the C/C++ tree node handle type section for the header.
|
||||
#
|
||||
# @return [String]
|
||||
# Header handle section.
|
||||
def c_tree_types_header
|
||||
@cpp ? cpp_tree_types_header : c_only_tree_types_header
|
||||
end
|
||||
|
||||
# Generate the C (non-C++) tree node handle type section for the header.
|
||||
def c_only_tree_types_header
|
||||
p = @grammar.prefix
|
||||
out = []
|
||||
out << "/** Tree node handle types. @{ */"
|
||||
tree_handle_types.each do |t|
|
||||
out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
|
||||
end
|
||||
out << ""
|
||||
out << c_common_accessors_header
|
||||
out << "/** @} */"
|
||||
out.join("\n")
|
||||
end
|
||||
|
||||
# Generate the C-style (function + macro) tree node accessors shared by the
|
||||
# C and C++ headers. In C++ these are provided in addition to the handle
|
||||
# methods so that C-style code (and the tree walk macros) also works.
|
||||
def c_common_accessors_header
|
||||
p = @grammar.prefix
|
||||
out = []
|
||||
out << "/** Generic tree node accessors (usable on any handle type). */"
|
||||
out << "#define #{p}node_valid(h) ((h).__id != 0u)"
|
||||
out << "#define #{p}node_id(h) ((h).__id)"
|
||||
out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
|
||||
out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
|
||||
out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
|
||||
out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
|
||||
out << ""
|
||||
out << "/** Tree node field accessor functions. */"
|
||||
out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
|
||||
out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
|
||||
tree_node_rule_sets.each do |rule_set|
|
||||
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
|
||||
out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
|
||||
end
|
||||
end
|
||||
out << ""
|
||||
out << c_tree_walk_macros
|
||||
out.join("\n")
|
||||
end
|
||||
|
||||
# Generate the C tree walk macro machinery.
|
||||
def c_tree_walk_macros
|
||||
p = @grammar.prefix
|
||||
max = C_TREE_WALK_MAX
|
||||
out = []
|
||||
out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
|
||||
out << "#define #{p}CAT_(a, b) a##b"
|
||||
out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
|
||||
out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
|
||||
out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
|
||||
argn = (1..max).map {|i| "_#{i}"}.join(", ")
|
||||
rseq = (0..max).to_a.reverse.join(", ")
|
||||
out << "#define #{p}ARG_N(#{argn}, N, ...) N"
|
||||
out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
|
||||
(1..max).each do |n|
|
||||
fparams = (1..n).map {|k| "f#{k}"}.join(", ")
|
||||
call = "h"
|
||||
(1..n).each do |k|
|
||||
texpr = "R"
|
||||
(1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
|
||||
call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
|
||||
end
|
||||
out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
|
||||
end
|
||||
out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
|
||||
# Type transition map (navigation fields only).
|
||||
tree_node_rule_sets.each do |rule_set|
|
||||
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
|
||||
out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
|
||||
end
|
||||
end
|
||||
# Per-handle-type walk entry points.
|
||||
tree_handle_types.each do |t|
|
||||
out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
|
||||
end
|
||||
out.join("\n")
|
||||
end
|
||||
|
||||
# Generate the C tree node accessor function definitions for the source.
|
||||
#
|
||||
# @return [String]
|
||||
# Accessor function definitions.
|
||||
def c_tree_accessor_defs
|
||||
p = @grammar.prefix
|
||||
tt = h_type("Token")
|
||||
out = []
|
||||
out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
|
||||
out << "{"
|
||||
out << " return node.__context->#{p}tree_nodes[node.__id].token;"
|
||||
out << "}"
|
||||
out << ""
|
||||
out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
|
||||
out << "{"
|
||||
out << " return node.__context->#{p}tree_nodes[node.__id].pvalue;"
|
||||
out << "}"
|
||||
tree_node_rule_sets.each do |rule_set|
|
||||
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
|
||||
out << ""
|
||||
out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
|
||||
out << "{"
|
||||
out << " #{child_type} result;"
|
||||
out << " result.__context = node.__context;"
|
||||
out << " if (node.__id == 0u)"
|
||||
out << " {"
|
||||
out << " result.__id = 0u;"
|
||||
out << " return result;"
|
||||
out << " }"
|
||||
out << " result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
|
||||
out << " return result;"
|
||||
out << "}"
|
||||
end
|
||||
end
|
||||
out.join("\n")
|
||||
end
|
||||
|
||||
# Generate the C++ tree node handle type section for the header.
|
||||
def cpp_tree_types_header
|
||||
p = @grammar.prefix
|
||||
out = []
|
||||
out << "/** Tree node handle types. @{ */"
|
||||
tree_handle_types.each {|t| out << "struct #{t};"}
|
||||
out << ""
|
||||
# Token handle (all methods inline; no handle-typed returns).
|
||||
tt = h_type("Token")
|
||||
out << "struct #{tt}"
|
||||
out << "{"
|
||||
out << " #{p}context_t * __context;"
|
||||
out << " #{p}node_id_t __id;"
|
||||
out << " bool valid() const { return __id != 0u; }"
|
||||
out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
|
||||
out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
|
||||
out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
|
||||
out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
|
||||
out << " #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
|
||||
out << " #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
|
||||
out << "};"
|
||||
out << ""
|
||||
# Rule set handles: navigation methods declared, defined out-of-line below.
|
||||
tree_node_rule_sets.each do |rule_set|
|
||||
rtype = h_type(rule_set.name)
|
||||
out << "struct #{rtype}"
|
||||
out << "{"
|
||||
out << " #{p}context_t * __context;"
|
||||
out << " #{p}node_id_t __id;"
|
||||
out << " bool valid() const { return __id != 0u; }"
|
||||
out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
|
||||
out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
|
||||
out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
|
||||
out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
|
||||
each_tree_field(rule_set) do |rt, field_name, child_type, slot|
|
||||
out << " #{child_type} #{field_name}() const;"
|
||||
end
|
||||
out << "};"
|
||||
out << ""
|
||||
end
|
||||
# Out-of-line navigation method bodies (all handle types now complete).
|
||||
tree_node_rule_sets.each do |rule_set|
|
||||
rtype = h_type(rule_set.name)
|
||||
each_tree_field(rule_set) do |rt, field_name, child_type, slot|
|
||||
out << "inline #{child_type} #{rtype}::#{field_name}() const"
|
||||
out << "{"
|
||||
out << " if (__id == 0u)"
|
||||
out << " {"
|
||||
out << " return #{child_type}{__context, 0u};"
|
||||
out << " }"
|
||||
out << " return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
|
||||
out << "}"
|
||||
end
|
||||
end
|
||||
out << ""
|
||||
out << "/*"
|
||||
out << " * C-style function and macro accessors, provided in addition to the handle"
|
||||
out << " * methods above so that C-style code and the tree walk macros also work."
|
||||
out << " */"
|
||||
out << c_common_accessors_header
|
||||
out << "/** @} */"
|
||||
out.join("\n")
|
||||
end
|
||||
|
||||
# Get the lex function to use.
|
||||
#
|
||||
# @return [String]
|
||||
|
||||
Reference in new issue
Block a user