josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Rework tree generation mode and API

Browse Source 
Store tree nodes in congruent, compact arena array.
Define handle types to refer to tree nodes rather than pointers to
structure instances.
Free tree with context.
This commit is contained in:
Josh Holtrop 2026-07-27 20:57:44 -04:00
parent dddb1b5089
commit 5fc712c6ee
39 changed files with 1349 additions and 811 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
CHANGELOG.md
View File

@ -1,3 +1,30 @@
## v5.0.0
### API Changes
- Tree generation mode now stores all tree nodes in a compact arena owned by
the parser context (a flat node array plus a shared child-link array).
This replaces the previous design of one heap allocation per node with
layout-punned typed structs.
- Tree nodes are now referenced by lightweight handles rather than pointers.
`p_result()` and the field accessors return handle values.
- The whole tree is freed together with the context by `p_context_delete()`.
The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
tree node handles are only valid while the context is alive.
- Tree node field access changed per target language:
- C: per-field accessor functions (e.g. `p_Start_pItems(node)`) plus tree
walk macros (e.g. `p_tree_walk_Start(node, pItems, pItem, pToken1, token)`),
and generic accessors `p_node_valid()`, `p_node_position()`,
`p_node_end_position()`, `p_node_n_fields()`, `p_node_data()`, `p_node_id()`.
- C++: handle methods called with `()` (e.g. `node.pItems().pToken1().token()`),
plus the same C-style functions/macros for convenience.
- D: `@property` accessors preserving the previous field-access syntax
(e.g. `node.pItems.pToken1.token`); null checks use `.valid` instead of
`is null`.
- Tree-mode parser rule user code: `$$` and `$1` etc. now yield node handles.
Reference child fields through the target-language accessors described above
rather than through struct pointer members.
## v4.8.1
### Fixes

40
UPGRADING.md
View File

@ -1,3 +1,43 @@
## v5.0.0
The generated API for tree generation mode (`tree;`) has been changed
significantly for this version.
The lexer/parser value APIs for non-tree grammars are unchanged.
### Tree memory management
- Remove all calls to `p_tree_delete()` / `p_tree_delete_XXX()`. Tree nodes now
live in the parser context and are freed by `p_context_delete()`.
- Tree node handles (returned by `p_result()` and the field accessors) are only
valid while the context is alive. Do not use them after `p_context_delete()`.
### Tree node field access
Tree nodes are now referenced by handle values instead of pointers, and field
access differs per target language:
- C: replace `node->field` with the accessor function `p_TYPE_field(node)`, or
use the tree walk macro `p_tree_walk_TYPE(node, field1, field2, ...)`. Replace
`x != NULL` / `x == NULL` node checks with `p_node_valid(x)` /
`!p_node_valid(x)`. Read positions with `p_node_position(node)` /
`p_node_end_position(node)`, token payload with `p_TYPE_token(node)` /
`p_TYPE_pvalue(node)` or `p_node_data(node)->field`, and compare node identity
with `p_node_id(a) == p_node_id(b)`.
- C++: replace `node->field` with the handle method `node.field()`. Use
`node.valid()`, `node.position()`, `node.token()`, `node.pvalue()`, and
`node.data()->field` for user token fields. (The C-style functions and macros
above are also available.)
- D: replace pointer declarations (`Start * s`) with value handles (`Start s`)
and replace `x !is null` / `x is null` with `x.valid` / `!x.valid`. Field
access syntax (`node.field.field`) is otherwise unchanged.
### Tree-mode parser rule user code
In tree generation mode `$$` and `$1`, `$2`, ... now expand to node handles.
Reference child fields through the target-language accessors above (for example
`$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
## v4.0.0
### API Changes

227
assets/parser.c.erb
View File

@ -68,6 +68,18 @@ const char * <%= @grammar.prefix %>token_names[] = {
context->text_position.row = 1u;
context->text_position.col = 1u;
context->mode = <%= @lexer.mode_id("default") %>;
<% if @grammar.tree %>
/* Reserve node ID 0 as the null tree node. */
<% if @cpp %>
context-><%= @grammar.prefix %>tree_nodes.resize(1);
<% else %>
context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
context-><%= @grammar.prefix %>tree_nodes_length = 1u;
<% end %>
<% end %>
return context;
}
@ -84,9 +96,27 @@ const char * <%= @grammar.prefix %>token_names[] = {
*/
void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
{
<% if @grammar.tree && @grammar.free_token_node != "" %>
<% if @cpp %>
for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
<% else %>
for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
<% end %>
{
if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
{
<%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
}
}
<% end %>
<% if @cpp %>
delete context;
<% else %>
<% if @grammar.tree %>
free(context-><%= @grammar.prefix %>tree_nodes);
free(context-><%= @grammar.prefix %>tree_children);
<% end %>
free(context);
<% end %>
}
@ -713,8 +743,8 @@ typedef struct
size_t state_id;
<% if @grammar.tree %>
/** tree node. */
void * tree_node;
/** Tree node ID. */
<%= @grammar.prefix %>node_id_t node_id;
<% else %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
@ -723,18 +753,6 @@ typedef struct
<% end %>
} state_value_t;
<% if @grammar.tree %>
/** Common tree node structure. */
typedef struct TreeNode_s
{
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
struct TreeNode_s * fields[];
} TreeNode;
<% end %>
/** Parser shift table. */
static const shift_t parser_shift_table[] = {
<% @parser.shift_table.each do |shift| %>
@ -872,6 +890,77 @@ static void state_values_stack_free(state_values_stack_t * stack)
free(stack->entries);
}
<% if @grammar.tree %>
/* Tree arena helpers. */
/**
* Allocate a new (zeroed) tree node in the context arena.
*
* @return The new node ID.
*/
static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
{
<% if @cpp %>
<%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
context-><%= @grammar.prefix %>tree_nodes.emplace_back();
return id;
<% else %>
if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
{
size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
<%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
free(context-><%= @grammar.prefix %>tree_nodes);
context-><%= @grammar.prefix %>tree_nodes = new_nodes;
context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
}
<%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
context-><%= @grammar.prefix %>tree_nodes_length += 1u;
return id;
<% end %>
}
/**
* Reserve n contiguous (zeroed) child slots in the shared children array.
*
* @return The offset of the first reserved slot.
*/
static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
{
<% if @cpp %>
<%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
return offset;
<% else %>
size_t offset = context-><%= @grammar.prefix %>tree_children_length;
size_t needed = offset + n;
if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
{
size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
while (new_capacity < needed)
{
new_capacity *= 2u;
}
<%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
if (context-><%= @grammar.prefix %>tree_children != NULL)
{
memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
free(context-><%= @grammar.prefix %>tree_children);
}
context-><%= @grammar.prefix %>tree_children = new_children;
context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
}
memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
context-><%= @grammar.prefix %>tree_children_length = needed;
return (<%= @grammar.prefix %>node_id_t)offset;
<% end %>
}
/* Tree node field accessor functions. */
<%= c_tree_accessor_defs %>
<% end %>
<% unless @grammar.tree %>
/**
* Get the rule position (start or end) for the currently matched rule.
@ -936,7 +1025,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
* @retval P_USER_TERMINATED
* User requested to terminate parsing.
*/
static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
{
switch (rule)
{
@ -1041,7 +1130,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
size_t reduced_rule_set = INVALID_ID;
size_t last_shifted_rule_set_id = INVALID_ID;
<% if @grammar.tree %>
void * reduced_parser_node;
<%= @grammar.prefix %>node_id_t reduced_parser_node;
<% else %>
<%= @grammar.prefix %>position_t reduced_position;
<%= @grammar.prefix %>position_t reduced_end_position;
@ -1087,7 +1176,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
/* Successful parse. */
<% if @grammar.tree %>
context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
<% else %>
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
<% end %>
@ -1114,7 +1203,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
context->input_index -= token_info.length;
context->text_position = token_info.position;
<% if @grammar.tree %>
context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
<% else %>
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
<% end %>
@ -1137,11 +1226,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
/* We shifted a token, mark it consumed. */
<% if @grammar.tree %>
<% if @cpp %>
<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>();
<% else %>
<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
<% end %>
<%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
<%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
token_tree_node->position = token_info.position;
token_tree_node->end_position = token_info.end_position;
token_tree_node->n_fields = 0u;
@ -1149,7 +1235,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
token_tree_node->token = token;
token_tree_node->pvalue = token_info.pvalue;
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
new_state_info->tree_node = token_tree_node;
new_state_info->node_id = token_node_id;
<% else %>
new_state_info->position = token_info.position;
new_state_info->end_position = token_info.end_position;
@ -1161,7 +1247,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
/* We shifted a RuleSet. */
<% if @grammar.tree %>
new_state_info->tree_node = reduced_parser_node;
new_state_info->node_id = reduced_parser_node;
<% else %>
new_state_info->pvalue = reduced_parser_value;
new_state_info->position = reduced_position;
@ -1191,50 +1277,54 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
<% if @grammar.tree %>
if (parser_reduce_table[reduce_index].propagate_optional_target)
{
reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
}
else if (parser_reduce_table[reduce_index].n_states > 0)
{
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
TreeNode * node = (TreeNode *)malloc(bytes);
memset(node, 0, bytes);
node->position = INVALID_POSITION;
node->end_position = INVALID_POSITION;
node->n_fields = n_fields;
uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
/* Reserve child slots. New slots are zero-initialized
* (null node ID) so absent optional children remain null. */
<%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
{
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{
node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
}
}
else
{
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{
node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
}
}
<%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
<%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
node->position = INVALID_POSITION;
node->end_position = INVALID_POSITION;
node->child_offset = child_offset;
node->n_fields = n_fields;
node->is_token = 0u;
bool position_found = false;
for (size_t i = 0; i < n_fields; i++)
for (uint16_t i = 0; i < n_fields; i++)
{
TreeNode * child = node->fields[i];
if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
<%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
{
if (!position_found)
{
node->position = child->position;
node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
position_found = true;
}
node->end_position = child->end_position;
node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
}
}
reduced_parser_node = node;
reduced_parser_node = node_id;
}
else
{
reduced_parser_node = NULL;
reduced_parser_node = 0u;
}
<% if @grammar.parser_user_code_used? %>
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@ -1310,14 +1400,14 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
* @return Parse result value.
*/
<% if @grammar.tree %>
<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
}
<% end %>
<% else %>
@ -1420,48 +1510,3 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
{
return context->token;
}
<% if @grammar.tree %>
static void tree_delete(TreeNode * node)
{
if (node->is_token)
{
<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)node;
<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
<% if @cpp %>
delete token_tree_node;
<% else %>
free(token_tree_node);
<% end %>
}
else if (node->n_fields > 0u)
{
for (size_t i = 0u; i < node->n_fields; i++)
{
if (node->fields[i] != NULL)
{
tree_delete(node->fields[i]);
}
}
free(node);
}
}
/**
* Free all tree node memory.
*/
void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
{
tree_delete((TreeNode *)tree);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
/**
* Free all tree node memory.
*/
void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
{
tree_delete((TreeNode *)tree);
}
<% end %>
<% end %>

241
assets/parser.d.erb
View File

@ -105,22 +105,23 @@ public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ?
<% end %>
<% if @grammar.tree %>
/** Common tree node structure. */
private struct TreeNode
{
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
ushort n_fields;
bool is_token;
void *[0] fields;
}
/** Tree node ID type (index into the context node arena). ID 0 is null. */
public alias <%= @grammar.prefix %>node_id_t = uint;
/** Tree node types. @{ */
public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
/**
* Tree node record.
*
* All tree nodes are stored contiguously in the context node arena. Child
* links are stored in a shared children array: a node's children
* occupy children[child_offset .. child_offset + n_fields]. Token payload
* fields (token, pvalue, and any user fields) are only meaningful when
* is_token is true.
*/
private struct <%= @grammar.prefix %>node_data_t
{
/* TreeNode fields must be present in the same order here. */
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
<%= @grammar.prefix %>node_id_t child_offset;
ushort n_fields;
bool is_token;
<%= @grammar.prefix %>token_t token;
@ -128,22 +129,84 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
<%= @grammar.token_user_fields %>
}
<% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %>
<% next if rule_set.optional? %>
public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
/** Tree node handle types. @{ */
/** Token tree node handle. */
public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
{
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
ushort n_fields;
bool is_token;
<% rule_set.tree_fields.each do |fields| %>
union
private <%= @grammar.prefix %>context_t * __context;
private <%= @grammar.prefix %>node_id_t __id;
this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
{
<% fields.each do |field_name, type| %>
<%= type %> * <%= field_name %>;
<% end %>
this.__context = context;
this.__id = id;
}
/** Return whether this handle refers to a valid (non-null) node. */
@property bool valid()
{
return __id != 0u;
}
/** Access the underlying node record (token, pvalue, and user fields). */
@property ref <%= @grammar.prefix %>node_data_t __node()
{
return __context.<%= @grammar.prefix %>tree_nodes[__id];
}
alias __node this;
}
<% tree_node_rule_sets.each do |rule_set| %>
/** <%= rule_set.name %> tree node handle. */
public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_suffix %>
{
private <%= @grammar.prefix %>context_t * __context;
private <%= @grammar.prefix %>node_id_t __id;
this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
{
this.__context = context;
this.__id = id;
}
/** Return whether this handle refers to a valid (non-null) node. */
@property bool valid()
{
return __id != 0u;
}
/** Text position of the first code point spanned by this node. */
@property <%= @grammar.prefix %>position_t position()
{
return __context.<%= @grammar.prefix %>tree_nodes[__id].position;
}
/** Text position of the last code point spanned by this node. */
@property <%= @grammar.prefix %>position_t end_position()
{
return __context.<%= @grammar.prefix %>tree_nodes[__id].end_position;
}
/** Number of child fields in this node. */
@property ushort n_fields()
{
return __id ? __context.<%= @grammar.prefix %>tree_nodes[__id].n_fields : cast(ushort)0u;
}
<% rule_set.tree_fields.each_with_index do |fields, i| %>
<% fields.each do |field_name, type| %>
/** Access the <%= field_name %> child node. */
@property <%= type %> <%= field_name %>()
{
if (__id == 0u)
{
return <%= type %>(__context, 0u);
}
return <%= type %>(__context, __context.<%= @grammar.prefix %>tree_children[__context.<%= @grammar.prefix %>tree_nodes[__id].child_offset + <%= i %>u]);
}
<% end %>
<% end %>
}
@ -196,7 +259,13 @@ public struct <%= @grammar.prefix %>context_t
/** Parse result value. */
<% if @grammar.tree %>
void * parse_result;
<%= @grammar.prefix %>node_id_t parse_result;
/** Tree node arena. Node ID 0 is reserved as the null node. */
<%= @grammar.prefix %>node_data_t[] <%= @grammar.prefix %>tree_nodes;
/** Shared tree child links. */
<%= @grammar.prefix %>node_id_t[] <%= @grammar.prefix %>tree_children;
<% else %>
<%= @grammar.prefix %>value_t parse_result;
<% end %>
@ -268,6 +337,11 @@ private enum size_t INVALID_ID = cast(size_t)-1;
context.text_position.row = 1u;
context.text_position.col = 1u;
context.mode = <%= @lexer.mode_id("default") %>;
<% if @grammar.tree %>
/* Reserve node ID 0 as the null tree node. */
context.<%= @grammar.prefix %>tree_nodes = new <%= @grammar.prefix %>node_data_t[](1);
<% end %>
return context;
}
@ -280,6 +354,16 @@ private enum size_t INVALID_ID = cast(size_t)-1;
*/
void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
{
<% if @grammar.tree && @grammar.free_token_node != "" %>
foreach (ref node; context.<%= @grammar.prefix %>tree_nodes)
{
if (node.is_token)
{
<%= @grammar.prefix %>node_data_t * token_tree_node = &node;
<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
}
}
<% end %>
}
/**************************************************************************
@ -888,8 +972,8 @@ private struct state_value_t
size_t state_id;
<% if @grammar.tree %>
/** Tree node. */
void * tree_node;
/** Tree node ID. */
<%= @grammar.prefix %>node_id_t node_id;
<% else %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
@ -1007,7 +1091,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
* @retval P_USER_TERMINATED
* User requested to terminate parsing.
*/
private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
private size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
{
switch (rule)
{
@ -1112,7 +1196,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
size_t reduced_rule_set = INVALID_ID;
size_t last_shifted_rule_set_id = INVALID_ID;
<% if @grammar.tree %>
void * reduced_parser_node;
<%= @grammar.prefix %>node_id_t reduced_parser_node;
<% else %>
<%= @grammar.prefix %>position_t reduced_position;
<%= @grammar.prefix %>position_t reduced_end_position;
@ -1153,7 +1237,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{
/* Successful parse. */
<% if @grammar.tree %>
context.parse_result = statevalues[$-1].tree_node;
context.parse_result = statevalues[$-1].node_id;
<% else %>
context.parse_result = statevalues[$-1].pvalue;
<% end %>
@ -1179,7 +1263,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
context.input_index -= token_info.length;
context.text_position = token_info.position;
<% if @grammar.tree %>
context.parse_result = statevalues[$-1].tree_node;
context.parse_result = statevalues[$-1].node_id;
<% else %>
context.parse_result = statevalues[$-1].pvalue;
<% end %>
@ -1199,9 +1283,17 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{
/* We shifted a token, mark it consumed. */
<% if @grammar.tree %>
<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
<%= @grammar.prefix %>node_id_t token_node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
<%= @grammar.prefix %>node_data_t * token_tree_node = &context.<%= @grammar.prefix %>tree_nodes[token_node_id];
token_tree_node.position = token_info.position;
token_tree_node.end_position = token_info.end_position;
token_tree_node.n_fields = 0u;
token_tree_node.is_token = true;
token_tree_node.token = token;
token_tree_node.pvalue = token_info.pvalue;
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
statevalues[$-1].tree_node = token_tree_node;
statevalues[$-1].node_id = token_node_id;
<% else %>
statevalues[$-1].position = token_info.position;
statevalues[$-1].end_position = token_info.end_position;
@ -1213,7 +1305,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{
/* We shifted a RuleSet. */
<% if @grammar.tree %>
statevalues[$-1].tree_node = reduced_parser_node;
statevalues[$-1].node_id = reduced_parser_node;
<% else %>
statevalues[$-1].pvalue = reduced_parser_value;
statevalues[$-1].position = reduced_position;
@ -1242,55 +1334,56 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
<% if @grammar.tree %>
if (parser_reduce_table[reduce_index].propagate_optional_target)
{
reduced_parser_node = statevalues[$ - 1].tree_node;
reduced_parser_node = statevalues[$ - 1].node_id;
}
else if (parser_reduce_table[reduce_index].n_states > 0)
{
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
TreeNode * node = cast(TreeNode *)malloc(node_size);
GC.addRange(node, node_size);
node.position = <%= @grammar.prefix %>position_t.INVALID;
node.end_position = <%= @grammar.prefix %>position_t.INVALID;
node.n_fields = cast(ushort)n_fields;
node.is_token = false;
foreach (i; 0..n_fields)
{
node.fields[i] = null;
}
ushort n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
/* Reserve child slots. New slots are zero-initialized
* (null node ID) so absent optional children remain null. */
<%= @grammar.prefix %>node_id_t child_offset = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_children.length;
context.<%= @grammar.prefix %>tree_children.length += n_fields;
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
{
foreach (i; 0..parser_reduce_table[reduce_index].n_states)
{
node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
context.<%= @grammar.prefix %>tree_children[child_offset + i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
}
}
else
{
foreach (i; 0..parser_reduce_table[reduce_index].n_states)
{
node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
context.<%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
}
}
<%= @grammar.prefix %>node_id_t node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
<%= @grammar.prefix %>node_data_t * node = &context.<%= @grammar.prefix %>tree_nodes[node_id];
node.position = <%= @grammar.prefix %>position_t.INVALID;
node.end_position = <%= @grammar.prefix %>position_t.INVALID;
node.child_offset = child_offset;
node.n_fields = n_fields;
node.is_token = false;
bool position_found = false;
foreach (i; 0..n_fields)
{
TreeNode * child = cast(TreeNode *)node.fields[i];
if (child && child.position.valid)
<%= @grammar.prefix %>node_id_t child_id = context.<%= @grammar.prefix %>tree_children[child_offset + i];
if (child_id != 0u && context.<%= @grammar.prefix %>tree_nodes[child_id].position.valid)
{
if (!position_found)
{
node.position = child.position;
node.position = context.<%= @grammar.prefix %>tree_nodes[child_id].position;
position_found = true;
}
node.end_position = child.end_position;
node.end_position = context.<%= @grammar.prefix %>tree_nodes[child_id].end_position;
}
}
reduced_parser_node = node;
reduced_parser_node = node_id;
}
else
{
reduced_parser_node = null;
reduced_parser_node = 0u;
}
<% if @grammar.parser_user_code_used? %>
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
@ -1360,14 +1453,14 @@ public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.p
* @return Parse result value.
*/
<% if @grammar.tree %>
public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
return <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %>(context, context.parse_result);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
return <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %>(context, context.parse_result);
}
<% end %>
<% else %>
@ -1383,34 +1476,6 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
<% end %>
<% end %>
<% if @grammar.tree %>
private void tree_delete(TreeNode * node)
{
if (!node.is_token)
{
for (size_t i = 0u; i < node.n_fields; i++)
{
if (node.fields[i])
{
tree_delete(cast(TreeNode *)node.fields[i]);
}
}
GC.removeRange(node);
free(node);
}
}
void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
{
tree_delete(cast(TreeNode *)tree);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
{
tree_delete(cast(TreeNode *)tree);
}
<% end %>
<% end %>
/**
* Get the current text input position.

88
assets/parser.h.erb
View File

@ -8,6 +8,9 @@
#include <stdint.h>
#include <stddef.h>
<% if @cpp %>
#include <vector>
<% end %>
/**************************************************************************
* Public types
@ -88,47 +91,29 @@ static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "defa
<% end %>
<% if @grammar.tree %>
/** Tree node types. @{ */
typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
/** Tree node ID type (index into the context node arena). ID 0 is null. */
typedef uint32_t <%= @grammar.prefix %>node_id_t;
/**
* Tree node record.
*
* All tree nodes are stored contiguously in the context node arena. Child
* links are stored in a shared children array: a node's children
* occupy children[child_offset .. child_offset + n_fields]. Token payload
* fields (token, pvalue, and any user fields) are only meaningful when
* is_token is nonzero.
*/
typedef struct
{
<% # TreeNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
<%= @grammar.prefix %>node_id_t child_offset;
uint16_t n_fields;
uint8_t is_token;
<%= @grammar.token_user_fields %>
<%= @grammar.prefix %>token_t token;
<%= @grammar.prefix %>value_t pvalue;
} <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
<% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %>
<% next if rule_set.optional? %>
struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
<% end %>
<% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %>
<% next if rule_set.optional? %>
typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
{
<% # TreeNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
<% rule_set.tree_fields.each do |fields| %>
union
{
<% fields.each do |field_name, type| %>
struct <%= type %> * <%= field_name %>;
<% end %>
};
<% end %>
} <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
<% end %>
/** @} */
<%= @grammar.token_user_fields %>
} <%= @grammar.prefix %>node_data_t;
<% end %>
/** Lexed token information. */
@ -179,7 +164,25 @@ typedef struct
/** Parse result value. */
<% if @grammar.tree %>
void * parse_result;
<%= @grammar.prefix %>node_id_t parse_result;
<% if @cpp %>
/** Tree node arena. Node ID 0 is reserved as the null node. */
std::vector<<%= @grammar.prefix %>node_data_t> <%= @grammar.prefix %>tree_nodes;
/** Shared tree child links. */
std::vector<<%= @grammar.prefix %>node_id_t> <%= @grammar.prefix %>tree_children;
<% else %>
/** Tree node arena. Node ID 0 is reserved as the null node. */
<%= @grammar.prefix %>node_data_t * <%= @grammar.prefix %>tree_nodes;
size_t <%= @grammar.prefix %>tree_nodes_length;
size_t <%= @grammar.prefix %>tree_nodes_capacity;
/** Shared tree child links. */
<%= @grammar.prefix %>node_id_t * <%= @grammar.prefix %>tree_children;
size_t <%= @grammar.prefix %>tree_children_length;
size_t <%= @grammar.prefix %>tree_children_capacity;
<% end %>
<% else %>
<%= @grammar.prefix %>value_t parse_result;
<% end %>
@ -193,6 +196,10 @@ typedef struct
<%= @grammar.context_user_fields %>
} <%= @grammar.prefix %>context_t;
<% if @grammar.tree %>
<%= c_tree_types_header %>
<% end %>
/**************************************************************************
* Public data
*************************************************************************/
@ -217,9 +224,9 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
<% end %>
<% if @grammar.tree %>
<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
@ -228,13 +235,6 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
<% end %>
<% end %>
<% if @grammar.tree %>
void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
<% end %>
<% end %>
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);

45
doc/user_guide.md
View File

@ -276,19 +276,40 @@ Parser rule code blocks are still supported in tree generation mode, but they
behave differently than when tree generation mode is not active.
The code block for a rule is executed after the rule has been matched and its
tree node has been fully formed.
Within the code block, `$$` refers to the tree node for the reduced rule, typed
as a pointer to that rule's generated tree node structure.
The tree nodes for the rule components are accessed positionally with `$1`,
`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure
for that component (a rule node or a `Token` node).
Within the code block, `$$` refers to the tree node handle for the reduced
rule, and the rule components are accessed positionally with `$1`, `$2`, `$3`,
etc..., each a tree node handle for that component (a rule node or a `Token`
node).
Field aliases (see the "Specifying parser rules" section) may also be used to
reference a component tree node by name; a field alias behaves identically to
the positional reference for that component.
The positional position expansions (`${$.position}`, `${N.position}`, etc...)
are not available in tree generation mode; the `position` and `end_position`
fields of the tree nodes can be accessed directly instead.
Example:
Tree nodes are stored in a compact arena owned by the parser context and are
referenced by lightweight handles rather than pointers. The whole tree is freed
together with the context by `p_context_delete()`; there is no separate tree
delete function, and tree node handles are only valid while the context is
alive.
Child fields, positions, and token payloads are accessed through per-language
accessors on a node handle:
* C: field accessor functions `p_TYPE_field(node)` and tree walk macros
`p_tree_walk_TYPE(node, field1, field2, ...)`; generic accessors
`p_node_valid(node)`, `p_node_position(node)`, `p_node_end_position(node)`,
`p_node_n_fields(node)`, `p_node_data(node)` (a pointer to the node record,
for token payload and user fields), and `p_node_id(node)` (for identity
comparison).
* C++: handle methods called with `()`, e.g. `node.field()`, `node.valid()`,
`node.position()`, `node.token()`, `node.pvalue()`, and `node.data()`. The
C-style functions and macros above are also available.
* D: `@property` accessors, e.g. `node.field`, `node.valid`, `node.position`,
`node.token`, `node.pvalue`.
The positional position expansions (`${$.position}`, `${N.position}`, etc...)
are not available in tree generation mode; use the position accessors above
instead.
C example:
```
tree;
@ -297,9 +318,9 @@ Assignment -> ident equals Expr <<
/* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
* the Expr rule node. */
printf("assignment on row %d, col %d\n",
$$->position.row, $$->position.col);
p_node_position($$).row, p_node_position($$).col);
printf("target identifier ends on row %d, col %d\n",
$1->end_position.row, $1->end_position.col);
p_node_end_position($1).row, p_node_end_position($1).col);
>>
```
@ -1406,7 +1427,7 @@ In this case, the parser will start parsing with the `Statement` rule.
For each start rule, a `p_parse_inner_XXX()` function is also generated.
This variant of the parser entry point accepts a caller-provided array of
"follow tokens" — tokens the caller allows to appear immediately after the
"follow tokens" -- tokens the caller allows to appear immediately after the
start rule in some outer grammar context.
It is useful when embedding a Propane-generated sub-parser within an outer
parser and the outer parser knows which tokens naturally terminate the

271
lib/propane/generator.rb
View File

@ -298,11 +298,12 @@ class Propane
if parser
code = code.gsub(/\$\$/) do |match|
if @grammar.tree
typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
case @language
when "c"
"((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
tree_handle(typename, "_node_id")
when "d"
"(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
tree_handle(typename, "_node_id")
end
else
case @language
@ -402,7 +403,7 @@ class Propane
def parser_component_reference(rule, index)
component = rule.components[index - 1]
if @grammar.tree
# In tree mode a component reference yields a pointer to that
# In tree mode a component reference yields a handle to that
# component's tree node. An optional component propagates its target
# node (or null), so use the optional target's node type.
if component.is_a?(RuleSet) && component.optional?
@ -412,9 +413,9 @@ class Propane
typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
case @language
when "c"
"((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
when "d"
"(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
end
else
case @language
@ -426,6 +427,266 @@ class Propane
end
end
# Construct a tree node handle expression for the target language.
#
# A handle is a small value pairing the parser context with a node ID
# (an index into the context's node arena). All handle types share this
# layout; the distinct types exist for documentation and, in C, to drive
# the tree walk macro's type threading.
#
# @param typename [String]
# Handle type name.
# @param id_expr [String]
# Expression yielding the node ID.
#
# @return [String]
# Handle constructor expression.
def tree_handle(typename, id_expr)
if @cpp
"(#{typename}{context, #{id_expr}})"
elsif @language == "c"
"((#{typename}){context, #{id_expr}})"
else
"#{typename}(context, #{id_expr})"
end
end
# Get the list of non-optional, non-internal rule sets that get a tree node
# handle type generated for them.
#
# @return [Array<Propane::RuleSet>]
# Rule sets with generated tree node handle types.
def tree_node_rule_sets
@parser.rule_sets.reject do |name, rule_set|
name.start_with?("$") || rule_set.optional?
end.map {|name, rule_set| rule_set}
end
# Maximum number of chained fields supported by a single C tree walk macro
# invocation. Deeper navigation can be expressed by nesting walk calls.
C_TREE_WALK_MAX = 16
# Get the tree node handle type name for a node name.
#
# @param name [String]
# Rule set name, or "Token".
#
# @return [String]
# Handle type name.
def h_type(name)
"#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
end
# Get the list of all tree node handle type names (Token plus rule sets).
#
# @return [Array<String>]
# Handle type names.
def tree_handle_types
[h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
end
# Enumerate the navigation fields of a rule set's tree node.
#
# @yield [rtype, field_name, child_type, slot]
# Handle type name, field accessor name, child handle type, and child
# slot index.
def each_tree_field(rule_set)
rtype = h_type(rule_set.name)
rule_set.tree_fields.each_with_index do |fields, slot|
fields.each do |field_name, child_type|
yield rtype, field_name, child_type, slot
end
end
end
# Generate the C/C++ tree node handle type section for the header.
#
# @return [String]
# Header handle section.
def c_tree_types_header
@cpp ? cpp_tree_types_header : c_only_tree_types_header
end
# Generate the C (non-C++) tree node handle type section for the header.
def c_only_tree_types_header
p = @grammar.prefix
out = []
out << "/** Tree node handle types. @{ */"
tree_handle_types.each do |t|
out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
end
out << ""
out << c_common_accessors_header
out << "/** @} */"
out.join("\n")
end
# Generate the C-style (function + macro) tree node accessors shared by the
# C and C++ headers. In C++ these are provided in addition to the handle
# methods so that C-style code (and the tree walk macros) also works.
def c_common_accessors_header
p = @grammar.prefix
out = []
out << "/** Generic tree node accessors (usable on any handle type). */"
out << "#define #{p}node_valid(h) ((h).__id != 0u)"
out << "#define #{p}node_id(h) ((h).__id)"
out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
out << ""
out << "/** Tree node field accessor functions. */"
out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
tree_node_rule_sets.each do |rule_set|
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
end
end
out << ""
out << c_tree_walk_macros
out.join("\n")
end
# Generate the C tree walk macro machinery.
def c_tree_walk_macros
p = @grammar.prefix
max = C_TREE_WALK_MAX
out = []
out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
out << "#define #{p}CAT_(a, b) a##b"
out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
argn = (1..max).map {|i| "_#{i}"}.join(", ")
rseq = (0..max).to_a.reverse.join(", ")
out << "#define #{p}ARG_N(#{argn}, N, ...) N"
out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
(1..max).each do |n|
fparams = (1..n).map {|k| "f#{k}"}.join(", ")
call = "h"
(1..n).each do |k|
texpr = "R"
(1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
end
out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
end
out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
# Type transition map (navigation fields only).
tree_node_rule_sets.each do |rule_set|
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
end
end
# Per-handle-type walk entry points.
tree_handle_types.each do |t|
out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
end
out.join("\n")
end
# Generate the C tree node accessor function definitions for the source.
#
# @return [String]
# Accessor function definitions.
def c_tree_accessor_defs
p = @grammar.prefix
tt = h_type("Token")
out = []
out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
out << "{"
out << " return node.__context->#{p}tree_nodes[node.__id].token;"
out << "}"
out << ""
out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
out << "{"
out << " return node.__context->#{p}tree_nodes[node.__id].pvalue;"
out << "}"
tree_node_rule_sets.each do |rule_set|
each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
out << ""
out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
out << "{"
out << " #{child_type} result;"
out << " result.__context = node.__context;"
out << " if (node.__id == 0u)"
out << " {"
out << " result.__id = 0u;"
out << " return result;"
out << " }"
out << " result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
out << " return result;"
out << "}"
end
end
out.join("\n")
end
# Generate the C++ tree node handle type section for the header.
def cpp_tree_types_header
p = @grammar.prefix
out = []
out << "/** Tree node handle types. @{ */"
tree_handle_types.each {|t| out << "struct #{t};"}
out << ""
# Token handle (all methods inline; no handle-typed returns).
tt = h_type("Token")
out << "struct #{tt}"
out << "{"
out << " #{p}context_t * __context;"
out << " #{p}node_id_t __id;"
out << " bool valid() const { return __id != 0u; }"
out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
out << " #{p}token_t token() const { return __context->#{p}tree_nodes[__id].token; }"
out << " #{p}value_t pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
out << "};"
out << ""
# Rule set handles: navigation methods declared, defined out-of-line below.
tree_node_rule_sets.each do |rule_set|
rtype = h_type(rule_set.name)
out << "struct #{rtype}"
out << "{"
out << " #{p}context_t * __context;"
out << " #{p}node_id_t __id;"
out << " bool valid() const { return __id != 0u; }"
out << " #{p}node_data_t * data() const { return &__context->#{p}tree_nodes[__id]; }"
out << " #{p}position_t position() const { return __context->#{p}tree_nodes[__id].position; }"
out << " #{p}position_t end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
out << " uint16_t n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
each_tree_field(rule_set) do |rt, field_name, child_type, slot|
out << " #{child_type} #{field_name}() const;"
end
out << "};"
out << ""
end
# Out-of-line navigation method bodies (all handle types now complete).
tree_node_rule_sets.each do |rule_set|
rtype = h_type(rule_set.name)
each_tree_field(rule_set) do |rt, field_name, child_type, slot|
out << "inline #{child_type} #{rtype}::#{field_name}() const"
out << "{"
out << " if (__id == 0u)"
out << " {"
out << " return #{child_type}{__context, 0u};"
out << " }"
out << " return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
out << "}"
end
end
out << ""
out << "/*"
out << " * C-style function and macro accessors, provided in addition to the handle"
out << " * methods above so that C-style code and the tree walk macros also work."
out << " */"
out << c_common_accessors_header
out << "/** @} */"
out.join("\n")
end
# Get the lex function to use.
#
# @return [String]

55
spec/propane_spec.rb
View File

@ -751,7 +751,7 @@ EOF
it "executes user code associated with a parser rule in tree mode" do
case language
when "c", "cpp"
when "c"
write_grammar <<EOF
tree;
context_user_fields <<
@ -769,18 +769,49 @@ ptype int;
token a << $$ = 11; >>
token b << $$ = 22; >>
Start -> A:ay B:bee C <<
${context.start_n_fields} = $$->n_fields;
${context.start_a_value} = $$->pA->pToken1->pvalue;
${context.a_value} = $1->pToken1->pvalue;
${context.b_value} = $2->pToken1->pvalue;
${context.b_token} = $2->pToken1->token;
${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
${context.alias_a_value} = ${ay}->pToken1->pvalue;
${context.alias_b_value} = ${bee}->pToken1->pvalue;
${context.start_n_fields} = p_node_n_fields($$);
${context.start_a_value} = p_tree_walk_Start($$, pA, pToken1, pvalue);
${context.a_value} = p_tree_walk_A($1, pToken1, pvalue);
${context.b_value} = p_tree_walk_B($2, pToken1, pvalue);
${context.b_token} = p_tree_walk_B($2, pToken1, token);
${context.c_field_is_null} = p_node_valid(p_Start_pC($$)) ? 0 : 1;
${context.alias_a_value} = p_tree_walk_A(${ay}, pToken1, pvalue);
${context.alias_b_value} = p_tree_walk_B(${bee}, pToken1, pvalue);
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
C -> << ${context.c_is_null} = p_node_valid($$) ? 0 : 1; >>
EOF
when "cpp"
write_grammar <<EOF
tree;
context_user_fields <<
int start_n_fields;
int start_a_value;
int a_value;
int b_value;
p_token_t b_token;
int c_is_null;
int c_field_is_null;
int alias_a_value;
int alias_b_value;
>>
ptype int;
token a << $$ = 11; >>
token b << $$ = 22; >>
Start -> A:ay B:bee C <<
${context.start_n_fields} = $$.n_fields();
${context.start_a_value} = $$.pA().pToken1().pvalue();
${context.a_value} = $1.pToken1().pvalue();
${context.b_value} = $2.pToken1().pvalue();
${context.b_token} = $2.pToken1().token();
${context.c_field_is_null} = $$.pC().valid() ? 0 : 1;
${context.alias_a_value} = ${ay}.pToken1().pvalue();
${context.alias_b_value} = ${bee}.pToken1().pvalue();
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = $$.valid() ? 0 : 1; >>
EOF
when "d"
write_grammar <<EOF
@ -805,13 +836,13 @@ Start -> A:ay B:bee C <<
${context.a_value} = $1.pToken1.pvalue;
${context.b_value} = $2.pToken1.pvalue;
${context.b_token} = $2.pToken1.token;
${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
${context.c_field_is_null} = ($$.pC.valid) ? 0 : 1;
${context.alias_a_value} = ${ay}.pToken1.pvalue;
${context.alias_b_value} = ${bee}.pToken1.pvalue;
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
C -> << ${context.c_is_null} = ($$.valid) ? 0 : 1; >>
EOF
end
run_propane(language: language)

44
spec/test_named_optional_rule_component_tree.c
View File

@ -9,46 +9,42 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start->a == NULL);
assert(start->pToken2 != NULL);
assert_eq(TOKEN_b, start->pToken2->token);
assert(start->pR3 == NULL);
assert(start->pR == NULL);
assert(start->r == NULL);
Start start = p_result(context);
assert(!p_node_valid(p_Start_a(start)));
assert(p_node_valid(p_Start_pToken2(start)));
assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
assert(!p_node_valid(p_Start_pR3(start)));
assert(!p_node_valid(p_Start_pR(start)));
assert(!p_node_valid(p_Start_r(start)));
p_tree_delete(start);
p_context_delete(context);
input = "abcd";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start->a != NULL);
assert_eq(TOKEN_a, start->pToken1->token);
assert(start->pToken2 != NULL);
assert(start->pR3 != NULL);
assert(start->pR != NULL);
assert(start->r != NULL);
assert(start->pR == start->pR3);
assert(start->pR == start->r);
assert_eq(TOKEN_c, start->pR->pToken1->token);
assert(p_node_valid(p_Start_a(start)));
assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
assert(p_node_valid(p_Start_pToken2(start)));
assert(p_node_valid(p_Start_pR3(start)));
assert(p_node_valid(p_Start_pR(start)));
assert(p_node_valid(p_Start_r(start)));
assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_r(start)));
assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
p_tree_delete(start);
p_context_delete(context);
input = "bdc";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start->a == NULL);
assert(start->pToken2 != NULL);
assert(start->r != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token);
assert(!p_node_valid(p_Start_a(start)));
assert(p_node_valid(p_Start_pToken2(start)));
assert(p_node_valid(p_Start_r(start)));
assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
p_tree_delete(start);
p_context_delete(context);
return 0;
}

34
spec/test_named_optional_rule_component_tree.d
View File

@ -12,40 +12,40 @@ unittest
string input = "b";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start.pToken1 is null);
assert(start.pToken2 !is null);
Start start = p_result(context);
assert(!start.pToken1.valid);
assert(start.pToken2.valid);
assert_eq(TOKEN_b, start.pToken2.token);
assert(start.pR3 is null);
assert(start.pR is null);
assert(start.r is null);
assert(!start.pR3.valid);
assert(!start.pR.valid);
assert(!start.r.valid);
p_tree_delete(start);
p_context_delete(context);
input = "abcd";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start.pToken1 != null);
assert(start.pToken1.valid);
assert_eq(TOKEN_a, start.pToken1.token);
assert(start.pToken2 != null);
assert(start.pR3 != null);
assert(start.pR != null);
assert(start.r != null);
assert(start.pToken2.valid);
assert(start.pR3.valid);
assert(start.pR.valid);
assert(start.r.valid);
assert(start.pR == start.pR3);
assert(start.pR == start.r);
assert_eq(TOKEN_c, start.pR.pToken1.token);
p_tree_delete(start);
p_context_delete(context);
input = "bdc";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start.pToken1 is null);
assert(start.pToken2 !is null);
assert(start.pR !is null);
assert(!start.pToken1.valid);
assert(start.pToken2.valid);
assert(start.pR.valid);
assert_eq(TOKEN_d, start.pR.pToken1.token);
p_tree_delete(start);
p_context_delete(context);
}

38
spec/test_optional_rule_component_tree.c
View File

@ -9,43 +9,39 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start->pToken1 == NULL);
assert(start->pToken2 != NULL);
assert_eq(TOKEN_b, start->pToken2->token);
assert(start->pR3 == NULL);
assert(start->pR == NULL);
Start start = p_result(context);
assert(!p_node_valid(p_Start_pToken1(start)));
assert(p_node_valid(p_Start_pToken2(start)));
assert_eq(TOKEN_b, p_tree_walk_Start(start, pToken2, token));
assert(!p_node_valid(p_Start_pR3(start)));
assert(!p_node_valid(p_Start_pR(start)));
p_tree_delete(start);
p_context_delete(context);
input = "abcd";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start->pToken1 != NULL);
assert_eq(TOKEN_a, start->pToken1->token);
assert(start->pToken2 != NULL);
assert(start->pR3 != NULL);
assert(start->pR != NULL);
assert(start->pR == start->pR3);
assert_eq(TOKEN_c, start->pR->pToken1->token);
assert(p_node_valid(p_Start_pToken1(start)));
assert_eq(TOKEN_a, p_tree_walk_Start(start, pToken1, token));
assert(p_node_valid(p_Start_pToken2(start)));
assert(p_node_valid(p_Start_pR3(start)));
assert(p_node_valid(p_Start_pR(start)));
assert(p_node_id(p_Start_pR(start)) == p_node_id(p_Start_pR3(start)));
assert_eq(TOKEN_c, p_tree_walk_Start(start, pR, pToken1, token));
p_tree_delete(start);
p_context_delete(context);
input = "bdc";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start->pToken1 == NULL);
assert(start->pToken2 != NULL);
assert(start->pR != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token);
assert(!p_node_valid(p_Start_pToken1(start)));
assert(p_node_valid(p_Start_pToken2(start)));
assert(p_node_valid(p_Start_pR(start)));
assert_eq(TOKEN_d, p_tree_walk_Start(start, pR, pToken1, token));
p_tree_delete(start);
p_context_delete(context);
return 0;
}

30
spec/test_optional_rule_component_tree.d
View File

@ -12,37 +12,37 @@ unittest
string input = "b";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start.pToken1 is null);
assert(start.pToken2 !is null);
Start start = p_result(context);
assert(!start.pToken1.valid);
assert(start.pToken2.valid);
assert_eq(TOKEN_b, start.pToken2.token);
assert(start.pR3 is null);
assert(start.pR is null);
assert(!start.pR3.valid);
assert(!start.pR.valid);
p_tree_delete(start);
p_context_delete(context);
input = "abcd";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start.pToken1 != null);
assert(start.pToken1.valid);
assert_eq(TOKEN_a, start.pToken1.token);
assert(start.pToken2 != null);
assert(start.pR3 != null);
assert(start.pR != null);
assert(start.pToken2.valid);
assert(start.pR3.valid);
assert(start.pR.valid);
assert(start.pR == start.pR3);
assert_eq(TOKEN_c, start.pR.pToken1.token);
p_tree_delete(start);
p_context_delete(context);
input = "bdc";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
assert(start.pToken1 is null);
assert(start.pToken2 !is null);
assert(start.pR !is null);
assert(!start.pToken1.valid);
assert(start.pToken2.valid);
assert(start.pR.valid);
assert_eq(TOKEN_d, start.pR.pToken1.token);
p_tree_delete(start);
p_context_delete(context);
}

73
spec/test_parse_inner_nested_tree.c
View File

@ -3,21 +3,8 @@
#include <string.h>
#include "testutils.h"
/* Grammar (tree generation mode; parentheses handled by the lex function):
* tree;
* lex_fn mylexfn;
* token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/;
* Start -> Expr;
* Expr -> num;
* Expr -> Expr plus num;
*
* The same lexer-driven nested parse as test_parse_inner_nested, but in tree
* generation mode. Each "( ... )" group is parsed by a reentrant
* p_parse_inner_Start() call from the lex function; the resulting subtree is
* discarded and a single synthesized num token is handed to the outer parse.
* The synthesized token's position is set to span the whole group ('(' start
* through ')' end), so this verifies that positions survive the nested-parse
* boundary and land correctly in the outer tree. */
/* Grammar: see the D variant / spec. Tree generation mode; parentheses handled
* by the lex function. Tree nodes live in the context arena. */
size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
{
@ -36,8 +23,8 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
{
return inner_result;
}
Start * inner = p_result_Start(context);
assert_not_null(inner);
Start inner = p_result_Start(context);
assert(p_node_valid(inner));
/* p_parse_inner rewound the input so that ')' was not consumed; consume
* it now. */
p_token_info_t rparen_info;
@ -45,10 +32,11 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
assert(rparen_result == P_SUCCESS);
assert(rparen_info.token == TOKEN_rparen);
/* The subtree covers the region strictly between the parentheses. */
assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col);
assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col);
p_tree_delete_Start(inner);
/* Synthesize a num token spanning the entire "( ... )" group. */
assert_eq((size_t)(start_position.col + 1u), (size_t)p_node_position(inner).col);
assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)p_node_end_position(inner).col);
/* The inner subtree is discarded (the lexer synthesizes a num token in
* its place), but its nodes remain in the shared context arena and are
* freed with the context. */
out_token_info->token = TOKEN_num;
out_token_info->position = start_position;
out_token_info->end_position = rparen_info.end_position;
@ -64,40 +52,39 @@ int main()
p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * tree = p_result(context);
assert_not_null(tree);
Start tree = p_result(context);
assert(p_node_valid(tree));
/* Start -> Expr, where the top Expr is "Expr plus num". */
Expr * top = tree->pExpr;
assert_not_null(top);
assert_not_null(top->pExpr);
assert_not_null(top->pToken2);
assert_not_null(top->pToken3);
Expr top = p_Start_pExpr(tree);
assert(p_node_valid(top));
assert(p_node_valid(p_Expr_pExpr(top)));
assert(p_node_valid(p_Expr_pToken2(top)));
assert(p_node_valid(p_Expr_pToken3(top)));
/* The '+' joining the two groups is at column 9. */
assert_eq(1u, (size_t)top->pToken2->position.row);
assert_eq(9u, (size_t)top->pToken2->position.col);
assert_eq(1u, (size_t)p_node_position(p_Expr_pToken2(top)).row);
assert_eq(9u, (size_t)p_node_position(p_Expr_pToken2(top)).col);
/* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */
assert_eq(1u, (size_t)top->pToken3->position.row);
assert_eq(11u, (size_t)top->pToken3->position.col);
assert_eq(1u, (size_t)top->pToken3->end_position.row);
assert_eq(17u, (size_t)top->pToken3->end_position.col);
assert_eq(1u, (size_t)p_node_position(p_Expr_pToken3(top)).row);
assert_eq(11u, (size_t)p_node_position(p_Expr_pToken3(top)).col);
assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken3(top)).row);
assert_eq(17u, (size_t)p_node_end_position(p_Expr_pToken3(top)).col);
/* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
* columns 1..7. */
Expr * left = top->pExpr;
assert_not_null(left->pToken1);
assert_eq(1u, (size_t)left->pToken1->position.row);
assert_eq(1u, (size_t)left->pToken1->position.col);
assert_eq(1u, (size_t)left->pToken1->end_position.row);
assert_eq(7u, (size_t)left->pToken1->end_position.col);
Expr left = p_Expr_pExpr(top);
assert(p_node_valid(p_Expr_pToken1(left)));
assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).row);
assert_eq(1u, (size_t)p_node_position(p_Expr_pToken1(left)).col);
assert_eq(1u, (size_t)p_node_end_position(p_Expr_pToken1(left)).row);
assert_eq(7u, (size_t)p_node_end_position(p_Expr_pToken1(left)).col);
/* The whole tree spans columns 1..17. */
assert_eq(1u, (size_t)tree->position.col);
assert_eq(17u, (size_t)tree->end_position.col);
assert_eq(1u, (size_t)p_node_position(tree).col);
assert_eq(17u, (size_t)p_node_end_position(tree).col);
p_tree_delete(tree);
p_context_delete(context);
return 0;

27
spec/test_parse_inner_nested_tree.d
View File

@ -20,8 +20,8 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
{
return inner_result;
}
Start * inner = p_result_Start(context);
assert(inner !is null);
Start inner = p_result_Start(context);
assert(inner.valid);
/* p_parse_inner rewound the input so that ')' was not consumed; consume
* it now. */
p_token_info_t rparen_info;
@ -31,7 +31,9 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
/* The subtree covers the region strictly between the parentheses. */
assert_eq(start_position.col + 1u, inner.position.col);
assert_eq(rparen_info.position.col - 1u, inner.end_position.col);
p_tree_delete_Start(inner);
/* The inner subtree is discarded (the lexer synthesizes a num token in
* its place), but its nodes remain in the shared context arena and are
* freed with the context. */
/* Synthesize a num token spanning the entire "( ... )" group. */
out_token_info.token = TOKEN_num;
out_token_info.position = start_position;
@ -53,15 +55,15 @@ unittest
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * tree = p_result(context);
assert(tree !is null);
Start tree = p_result(context);
assert(tree.valid);
/* Start -> Expr, where the top Expr is "Expr plus num". */
Expr * top = tree.pExpr;
assert(top !is null);
assert(top.pExpr !is null);
assert(top.pToken2 !is null);
assert(top.pToken3 !is null);
Expr top = tree.pExpr;
assert(top.valid);
assert(top.pExpr.valid);
assert(top.pToken2.valid);
assert(top.pToken3.valid);
/* The '+' joining the two groups is at column 9. */
assert_eq(1u, top.pToken2.position.row);
@ -75,8 +77,8 @@ unittest
/* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning
* columns 1..7. */
Expr * left = top.pExpr;
assert(left.pToken1 !is null);
Expr left = top.pExpr;
assert(left.pToken1.valid);
assert_eq(1u, left.pToken1.position.row);
assert_eq(1u, left.pToken1.position.col);
assert_eq(1u, left.pToken1.end_position.row);
@ -86,6 +88,5 @@ unittest
assert_eq(1u, tree.position.col);
assert_eq(17u, tree.end_position.col);
p_tree_delete(tree);
p_context_delete(context);
}

67
spec/test_parse_inner_tree.c
View File

@ -5,24 +5,8 @@
int main()
{
/* Grammar (tree generation mode, shared reduce state):
* tree;
* token a; token b;
* start Start;
* start R1;
* Start -> R1 a;
* Start -> R2 b;
* R1 -> a b;
* R2 -> a b;
*
* Exercises p_parse_inner_R1() with a non-EOF follow token in tree
* generation mode. Verifies:
* * The reduced tree for R1 is well-formed after a follow-token
* completion.
* * The follow token is not consumed and remains available for a
* subsequent p_lex() call.
* * p_tree_delete_R1() cleans up the returned tree without leaks
* (verified in CI via valgrind). */
/* See the D variant / grammar comments for details. In tree mode the tree
* nodes live in the context arena and are freed with p_context_delete(). */
/* Baseline: p_parse_R1 works on "ab" and the returned tree is
* well-formed. */
@ -30,13 +14,12 @@ int main()
char const * input = "ab";
p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse_R1(context) == P_SUCCESS);
R1 * tree = p_result_R1(context);
assert_not_null(tree);
assert_not_null(tree->pToken1);
assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
assert_not_null(tree->pToken2);
assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
p_tree_delete_R1(tree);
R1 tree = p_result_R1(context);
assert(p_node_valid(tree));
assert(p_node_valid(p_R1_pToken1(tree)));
assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
assert(p_node_valid(p_R1_pToken2(tree)));
assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
p_context_delete(context);
}
@ -50,23 +33,23 @@ int main()
assert(p_parse_inner_R1(context, follow_tokens, 1u) == P_SUCCESS);
/* Tree is well-formed. */
R1 * tree = p_result_R1(context);
assert_not_null(tree);
assert_not_null(tree->pToken1);
assert_eq((size_t)TOKEN_a, (size_t)tree->pToken1->token);
assert_eq(1u, (size_t)tree->pToken1->position.row);
assert_eq(1u, (size_t)tree->pToken1->position.col);
assert_not_null(tree->pToken2);
assert_eq((size_t)TOKEN_b, (size_t)tree->pToken2->token);
assert_eq(1u, (size_t)tree->pToken2->position.row);
assert_eq(2u, (size_t)tree->pToken2->position.col);
R1 tree = p_result_R1(context);
assert(p_node_valid(tree));
assert(p_node_valid(p_R1_pToken1(tree)));
assert_eq((size_t)TOKEN_a, (size_t)p_tree_walk_R1(tree, pToken1, token));
assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).row);
assert_eq(1u, (size_t)p_node_position(p_R1_pToken1(tree)).col);
assert(p_node_valid(p_R1_pToken2(tree)));
assert_eq((size_t)TOKEN_b, (size_t)p_tree_walk_R1(tree, pToken2, token));
assert_eq(1u, (size_t)p_node_position(p_R1_pToken2(tree)).row);
assert_eq(2u, (size_t)p_node_position(p_R1_pToken2(tree)).col);
/* The R1 tree covers positions 1..2 — the third `b` at column 3 is
/* The R1 tree covers positions 1..2 - the third `b` at column 3 is
* the follow token and is not part of the tree. */
assert_eq(1u, (size_t)tree->position.row);
assert_eq(1u, (size_t)tree->position.col);
assert_eq(1u, (size_t)tree->end_position.row);
assert_eq(2u, (size_t)tree->end_position.col);
assert_eq(1u, (size_t)p_node_position(tree).row);
assert_eq(1u, (size_t)p_node_position(tree).col);
assert_eq(1u, (size_t)p_node_end_position(tree).row);
assert_eq(2u, (size_t)p_node_end_position(tree).col);
/* Follow token remains in the input. */
p_position_t pos = p_position(context);
@ -78,10 +61,6 @@ int main()
assert_eq(1u, (size_t)token_info.position.row);
assert_eq(3u, (size_t)token_info.position.col);
/* p_tree_delete_R1 must free every node reachable from the tree
* without leaking anything. valgrind (invoked by the spec runner on
* Linux) will detect any missed frees. */
p_tree_delete_R1(tree);
p_context_delete(context);
}

20
spec/test_parse_inner_tree.d
View File

@ -16,13 +16,13 @@ unittest
string input = "ab";
p_context_t * context = p_context_new(input);
assert(p_parse_R1(context) == P_SUCCESS);
R1 * tree = p_result_R1(context);
assert(tree !is null);
assert(tree.pToken1 !is null);
R1 tree = p_result_R1(context);
assert(tree.valid);
assert(tree.pToken1.valid);
assert(tree.pToken1.token == TOKEN_a);
assert(tree.pToken2 !is null);
assert(tree.pToken2.valid);
assert(tree.pToken2.token == TOKEN_b);
p_tree_delete_R1(tree);
p_context_delete(context);
}
/* Primary case: p_parse_inner_R1 with a non-EOF follow token completes
@ -35,13 +35,13 @@ unittest
assert(p_parse_inner_R1(context, follow_tokens) == P_SUCCESS);
/* Tree is well-formed. */
R1 * tree = p_result_R1(context);
assert(tree !is null);
assert(tree.pToken1 !is null);
R1 tree = p_result_R1(context);
assert(tree.valid);
assert(tree.pToken1.valid);
assert(tree.pToken1.token == TOKEN_a);
assert(tree.pToken1.position.row == 1);
assert(tree.pToken1.position.col == 1);
assert(tree.pToken2 !is null);
assert(tree.pToken2.valid);
assert(tree.pToken2.token == TOKEN_b);
assert(tree.pToken2.position.row == 1);
assert(tree.pToken2.position.col == 2);
@ -63,6 +63,6 @@ unittest
assert(token_info.position.row == 1);
assert(token_info.position.col == 3);
p_tree_delete_R1(tree);
p_context_delete(context);
}
}

9
spec/test_parser_user_code_tree.c
View File

@ -28,11 +28,10 @@ int main()
assert_eq(11, context->alias_a_value);
assert_eq(22, context->alias_b_value);
Start * start = p_result(context);
assert(start->pA != NULL);
assert(start->pB != NULL);
assert(start->pC == NULL);
p_tree_delete(start);
Start start = p_result(context);
assert(p_node_valid(p_Start_pA(start)));
assert(p_node_valid(p_Start_pB(start)));
assert(!p_node_valid(p_Start_pC(start)));
p_context_delete(context);
return 0;

10
spec/test_parser_user_code_tree.d
View File

@ -31,9 +31,9 @@ unittest
assert_eq(11, context.alias_a_value);
assert_eq(22, context.alias_b_value);
Start * start = p_result(context);
assert(start.pA !is null);
assert(start.pB !is null);
assert(start.pC is null);
p_tree_delete(start);
Start start = p_result(context);
assert(start.pA.valid);
assert(start.pB.valid);
assert(!start.pC.valid);
p_context_delete(context);
}

7
spec/test_start_rule_tree.c
View File

@ -9,11 +9,10 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
Top * top = p_result(context);
assert(top->pToken != NULL);
assert_eq(TOKEN_hi, top->pToken->token);
Top top = p_result(context);
assert(p_node_valid(p_Top_pToken(top)));
assert_eq(TOKEN_hi, p_tree_walk_Top(top, pToken, token));
p_tree_delete(top);
p_context_delete(context);
return 0;

4
spec/test_start_rule_tree.d
View File

@ -13,7 +13,7 @@ unittest
p_context_t * context;
context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
Top * top = p_result(context);
assert(top.pToken !is null);
Top top = p_result(context);
assert(top.pToken.valid);
assert_eq(TOKEN_hi, top.pToken.token);
}

29
spec/test_starting_rules_tree.c
View File

@ -9,31 +9,28 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert_not_null(start->bs);
assert_not_null(start->bs->b);
assert_not_null(start->bs->bs->b);
assert_not_null(start->bs->bs->bs->b);
assert_not_null(start->bs->bs->bs->bs->b);
p_tree_delete(start);
Start start = p_result(context);
assert(p_node_valid(p_Start_bs(start)));
assert(p_node_valid(p_tree_walk_Start(start, bs, b)));
assert(p_node_valid(p_tree_walk_Start(start, bs, bs, b)));
assert(p_node_valid(p_tree_walk_Start(start, bs, bs, bs, b)));
assert(p_node_valid(p_tree_walk_Start(start, bs, bs, bs, bs, b)));
p_context_delete(context);
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse_Bs(context) == P_SUCCESS);
Bs * bs = p_result_Bs(context);
assert_not_null(bs->b);
assert_not_null(bs->bs->b);
assert_not_null(bs->bs->bs->b);
assert_not_null(bs->bs->bs->bs->b);
p_tree_delete_Bs(bs);
Bs bs = p_result_Bs(context);
assert(p_node_valid(p_Bs_b(bs)));
assert(p_node_valid(p_tree_walk_Bs(bs, bs, b)));
assert(p_node_valid(p_tree_walk_Bs(bs, bs, bs, b)));
assert(p_node_valid(p_tree_walk_Bs(bs, bs, bs, bs, b)));
p_context_delete(context);
input = "c";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse_R(context) == P_SUCCESS);
R * r = p_result_R(context);
assert_not_null(r->c);
p_tree_delete_R(r);
R r = p_result_R(context);
assert(p_node_valid(p_R_c(r)));
p_context_delete(context);
return 0;

32
spec/test_starting_rules_tree.d
View File

@ -12,30 +12,30 @@ unittest
string input = "bbbb";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start.bs);
assert(start.bs.b);
assert(start.bs.bs.b);
assert(start.bs.bs.bs.b);
assert(start.bs.bs.bs.bs.b);
Start start = p_result(context);
assert(start.bs.valid);
assert(start.bs.b.valid);
assert(start.bs.bs.b.valid);
assert(start.bs.bs.bs.b.valid);
assert(start.bs.bs.bs.bs.b.valid);
p_tree_delete(start);
p_context_delete(context);
context = p_context_new(input);
assert(p_parse_Bs(context) == P_SUCCESS);
Bs * bs = p_result_Bs(context);
assert(bs.b);
assert(bs.bs.b);
assert(bs.bs.bs.b);
assert(bs.bs.bs.bs.b);
Bs bs = p_result_Bs(context);
assert(bs.b.valid);
assert(bs.bs.b.valid);
assert(bs.bs.bs.b.valid);
assert(bs.bs.bs.bs.b.valid);
p_tree_delete_Bs(bs);
p_context_delete(context);
input = "c";
context = p_context_new(input);
assert(p_parse_R(context) == P_SUCCESS);
R * r = p_result_R(context);
assert(r.c);
R r = p_result_R(context);
assert(r.c.valid);
p_tree_delete_R(r);
p_context_delete(context);
}

33
spec/test_token_user_fields.c
View File

@ -18,29 +18,22 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start->pIDs);
assert(start->pIDs->id);
#ifdef __cplusplus
assert(start->pIDs->id->comments == "# c1\n# c2\n");
#else
assert(start->pIDs->id->comments);
assert(strcmp(start->pIDs->id->comments, "# c1\n# c2\n") == 0);
#endif
assert(start->pIDs->pIDs);
assert(start->pIDs->pIDs->id);
#ifdef __cplusplus
assert(start->pIDs->pIDs->id->comments == "# s1\n# s2\n");
#else
assert(start->pIDs->pIDs->id->comments);
assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n# s2\n") == 0);
#endif
Start start = p_result(context);
IDs ids = p_Start_pIDs(start);
assert(p_node_valid(ids));
Token id0 = p_IDs_id(ids);
assert(p_node_valid(id0));
assert(p_node_data(id0)->comments);
assert(strcmp(p_node_data(id0)->comments, "# c1\n# c2\n") == 0);
IDs ids2 = p_IDs_pIDs(ids);
assert(p_node_valid(ids2));
Token id1 = p_IDs_id(ids2);
assert(p_node_valid(id1));
assert(p_node_data(id1)->comments);
assert(strcmp(p_node_data(id1)->comments, "# s1\n# s2\n") == 0);
#ifndef __cplusplus
free(context->comments);
#endif
p_context_delete(context);
p_tree_delete(start);
return 0;
}

31
spec/test_token_user_fields.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "testparser.h"
#include <cassert>
#include <cstring>
#include <string>
int main()
{
char const * input =
"# c1\n"
"# c2\n"
"\n"
"first\n"
"\n \n \n"
" # s1\n"
" # s2\n"
"second\n";
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start start = p_result(context);
assert(start.pIDs().valid());
assert(start.pIDs().id().valid());
assert(start.pIDs().id().data()->comments == "# c1\n# c2\n");
assert(start.pIDs().pIDs().valid());
assert(start.pIDs().pIDs().id().valid());
assert(start.pIDs().pIDs().id().data()->comments == "# s1\n# s2\n");
p_context_delete(context);
return 0;
}

12
spec/test_token_user_fields.d
View File

@ -19,13 +19,13 @@ unittest
"second\n";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
assert(start.pIDs);
assert(start.pIDs.id);
Start start = p_result(context);
assert(start.pIDs.valid);
assert(start.pIDs.id.valid);
assert(start.pIDs.id.comments == "# c1\n# c2\n");
assert(start.pIDs.pIDs);
assert(start.pIDs.pIDs.id);
assert(start.pIDs.pIDs.valid);
assert(start.pIDs.pIDs.id.valid);
assert(start.pIDs.pIDs.id.comments == "# s1\n# s2\n");
p_tree_delete(start);
p_context_delete(context);
}

65
spec/test_tree.c
View File

@ -9,55 +9,52 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
Start * start = p_result(context);
assert(start->pItems1 != NULL);
assert(start->pItems != NULL);
Items * items = start->pItems;
assert(items->pItem != NULL);
assert(items->pItem->pToken1 != NULL);
assert_eq(TOKEN_a, items->pItem->pToken1->token);
assert_eq(11, items->pItem->pToken1->pvalue);
assert(items->pItemsMore != NULL);
ItemsMore * itemsmore = items->pItemsMore;
assert(itemsmore->pItem != NULL);
assert(itemsmore->pItem->pItem != NULL);
assert(itemsmore->pItem->pItem->pItem != NULL);
assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore != NULL);
itemsmore = itemsmore->pItemsMore;
assert(itemsmore->pItem != NULL);
assert(itemsmore->pItem->pToken1 != NULL);
assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL);
Start start = p_result(context);
assert(p_node_valid(p_Start_pItems1(start)));
assert(p_node_valid(p_Start_pItems(start)));
Items items = p_Start_pItems(start);
assert(p_node_valid(p_Items_pItem(items)));
assert(p_node_valid(p_tree_walk_Items(items, pItem, pToken1)));
assert_eq(TOKEN_a, p_tree_walk_Items(items, pItem, pToken1, token));
assert_eq(11, p_tree_walk_Items(items, pItem, pToken1, pvalue));
assert(p_node_valid(p_Items_pItemsMore(items)));
ItemsMore itemsmore = p_Items_pItemsMore(items);
assert(p_node_valid(p_ItemsMore_pItem(itemsmore)));
assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem)));
assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem)));
assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1)));
assert_eq(TOKEN_b, p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1, token));
assert_eq(22, p_tree_walk_ItemsMore(itemsmore, pItem, pItem, pItem, pToken1, pvalue));
assert(p_node_valid(p_ItemsMore_pItemsMore(itemsmore)));
itemsmore = p_ItemsMore_pItemsMore(itemsmore);
assert(p_node_valid(p_ItemsMore_pItem(itemsmore)));
assert(p_node_valid(p_tree_walk_ItemsMore(itemsmore, pItem, pToken1)));
assert_eq(TOKEN_b, p_tree_walk_ItemsMore(itemsmore, pItem, pToken1, token));
assert_eq(22, p_tree_walk_ItemsMore(itemsmore, pItem, pToken1, pvalue));
assert(!p_node_valid(p_ItemsMore_pItemsMore(itemsmore)));
p_tree_delete(start);
p_context_delete(context);
input = "";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start->pItems == NULL);
assert(!p_node_valid(p_Start_pItems(start)));
p_tree_delete(start);
p_context_delete(context);
input = "2 1";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start->pItems != NULL);
assert(start->pItems->pItem != NULL);
assert(start->pItems->pItem->pDual != NULL);
assert(start->pItems->pItem->pDual->pTwo1 != NULL);
assert(start->pItems->pItem->pDual->pOne2 != NULL);
assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL);
assert(p_node_valid(p_Start_pItems(start)));
assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem)));
assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual)));
assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pTwo1)));
assert(p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pOne2)));
assert(!p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pTwo2)));
assert(!p_node_valid(p_tree_walk_Start(start, pItems, pItem, pDual, pOne1)));
p_tree_delete(start);
p_context_delete(context);
return 0;

61
spec/test_tree.cpp Normal file
View File

@ -0,0 +1,61 @@
#include "testparser.h"
#include <cassert>
#include <cstring>
#include "testutils.h"
int main()
{
char const * input = "a, ((b)), b";
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
Start start = p_result(context);
assert(start.pItems1().valid());
assert(start.pItems().valid());
Items items = start.pItems();
assert(items.pItem().valid());
assert(items.pItem().pToken1().valid());
assert_eq(TOKEN_a, items.pItem().pToken1().token());
assert_eq(11, items.pItem().pToken1().pvalue());
assert(items.pItemsMore().valid());
ItemsMore itemsmore = items.pItemsMore();
assert(itemsmore.pItem().valid());
assert(itemsmore.pItem().pItem().valid());
assert(itemsmore.pItem().pItem().pItem().valid());
assert(itemsmore.pItem().pItem().pItem().pToken1().valid());
assert_eq(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
assert_eq(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
assert(itemsmore.pItemsMore().valid());
itemsmore = itemsmore.pItemsMore();
assert(itemsmore.pItem().valid());
assert(itemsmore.pItem().pToken1().valid());
assert_eq(TOKEN_b, itemsmore.pItem().pToken1().token());
assert_eq(22, itemsmore.pItem().pToken1().pvalue());
assert(!itemsmore.pItemsMore().valid());
p_context_delete(context);
input = "";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(!start.pItems().valid());
p_context_delete(context);
input = "2 1";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start.pItems().valid());
assert(start.pItems().pItem().valid());
assert(start.pItems().pItem().pDual().valid());
assert(start.pItems().pItem().pDual().pTwo1().valid());
assert(start.pItems().pItem().pDual().pOne2().valid());
assert(!start.pItems().pItem().pDual().pTwo2().valid());
assert(!start.pItems().pItem().pDual().pOne1().valid());
p_context_delete(context);
return 0;
}

54
spec/test_tree.d
View File

@ -12,51 +12,51 @@ unittest
string input = "a, ((b)), b";
p_context_t * context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
Start * start = p_result(context);
assert(start.pItems1 !is null);
assert(start.pItems !is null);
Items * items = start.pItems;
assert(items.pItem !is null);
assert(items.pItem.pToken1 !is null);
Start start = p_result(context);
assert(start.pItems1.valid);
assert(start.pItems.valid);
Items items = start.pItems;
assert(items.pItem.valid);
assert(items.pItem.pToken1.valid);
assert_eq(TOKEN_a, items.pItem.pToken1.token);
assert_eq(11, items.pItem.pToken1.pvalue);
assert(items.pItemsMore !is null);
ItemsMore * itemsmore = items.pItemsMore;
assert(itemsmore.pItem !is null);
assert(itemsmore.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
assert(items.pItemsMore.valid);
ItemsMore itemsmore = items.pItemsMore;
assert(itemsmore.pItem.valid);
assert(itemsmore.pItem.pItem.valid);
assert(itemsmore.pItem.pItem.pItem.valid);
assert(itemsmore.pItem.pItem.pItem.pToken1.valid);
assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
assert(itemsmore.pItemsMore !is null);
assert(itemsmore.pItemsMore.valid);
itemsmore = itemsmore.pItemsMore;
assert(itemsmore.pItem !is null);
assert(itemsmore.pItem.pToken1 !is null);
assert(itemsmore.pItem.valid);
assert(itemsmore.pItem.pToken1.valid);
assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
assert_eq(22, itemsmore.pItem.pToken1.pvalue);
assert(itemsmore.pItemsMore is null);
assert(!itemsmore.pItemsMore.valid);
p_tree_delete(start);
p_context_delete(context);
input = "";
context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start.pItems is null);
assert(!start.pItems.valid);
p_tree_delete(start);
p_context_delete(context);
input = "2 1";
context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start.pItems !is null);
assert(start.pItems.pItem !is null);
assert(start.pItems.pItem.pDual !is null);
assert(start.pItems.pItem.pDual.pTwo1 !is null);
assert(start.pItems.pItem.pDual.pOne2 !is null);
assert(start.pItems.pItem.pDual.pTwo2 is null);
assert(start.pItems.pItem.pDual.pOne1 is null);
assert(start.pItems.valid);
assert(start.pItems.pItem.valid);
assert(start.pItems.pItem.pDual.valid);
assert(start.pItems.pItem.pDual.pTwo1.valid);
assert(start.pItems.pItem.pDual.pOne2.valid);
assert(!start.pItems.pItem.pDual.pTwo2.valid);
assert(!start.pItems.pItem.pDual.pOne1.valid);
p_tree_delete(start);
p_context_delete(context);
}

11
spec/test_tree_delete_token_node_memory.c
View File

@ -9,12 +9,11 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
Start * start = p_result(context);
assert(start->a != NULL);
assert(*start->a->pvalue == 1);
assert(start->b != NULL);
assert(*start->b->pvalue == 2);
Start start = p_result(context);
assert(p_node_valid(p_Start_a(start)));
assert(*p_tree_walk_Start(start, a, pvalue) == 1);
assert(p_node_valid(p_Start_b(start)));
assert(*p_tree_walk_Start(start, b, pvalue) == 2);
p_tree_delete(start);
p_context_delete(context);
}

9
spec/test_tree_field_aliases.c
View File

@ -9,13 +9,12 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
assert_eq(TOKEN_a, start->first->pToken->token);
assert_eq(TOKEN_b, start->second->pToken->token);
assert_eq(TOKEN_c, start->third->pToken->token);
assert_eq(TOKEN_a, p_tree_walk_Start(start, first, pToken, token));
assert_eq(TOKEN_b, p_tree_walk_Start(start, second, pToken, token));
assert_eq(TOKEN_c, p_tree_walk_Start(start, third, pToken, token));
p_tree_delete(start);
p_context_delete(context);
return 0;

4
spec/test_tree_field_aliases.d
View File

@ -12,11 +12,11 @@ unittest
string input = "\na\nb\nc";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
assert_eq(TOKEN_a, start.first.pToken.token);
assert_eq(TOKEN_b, start.second.pToken.token);
assert_eq(TOKEN_c, start.third.pToken.token);
p_tree_delete(start);
p_context_delete(context);
}

146
spec/test_tree_invalid_positions.c
View File

@ -9,105 +9,113 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
T t1 = p_Start_pT1(start);
Token k1 = p_T_pToken(t1);
A a1 = p_T_pA(t1);
assert_eq(2, start->pT1->pToken->position.row);
assert_eq(1, start->pT1->pToken->position.col);
assert_eq(2, start->pT1->pToken->end_position.row);
assert_eq(1, start->pT1->pToken->end_position.col);
assert(p_position_valid(start->pT1->pA->position));
assert_eq(3, start->pT1->pA->position.row);
assert_eq(3, start->pT1->pA->position.col);
assert_eq(3, start->pT1->pA->end_position.row);
assert_eq(8, start->pT1->pA->end_position.col);
assert_eq(2, start->pT1->position.row);
assert_eq(1, start->pT1->position.col);
assert_eq(3, start->pT1->end_position.row);
assert_eq(8, start->pT1->end_position.col);
assert_eq(2, p_node_position(k1).row);
assert_eq(1, p_node_position(k1).col);
assert_eq(2, p_node_end_position(k1).row);
assert_eq(1, p_node_end_position(k1).col);
assert(p_position_valid(p_node_position(a1)));
assert_eq(3, p_node_position(a1).row);
assert_eq(3, p_node_position(a1).col);
assert_eq(3, p_node_end_position(a1).row);
assert_eq(8, p_node_end_position(a1).col);
assert_eq(2, p_node_position(t1).row);
assert_eq(1, p_node_position(t1).col);
assert_eq(3, p_node_end_position(t1).row);
assert_eq(8, p_node_end_position(t1).col);
assert_eq(2, start->position.row);
assert_eq(1, start->position.col);
assert_eq(3, start->end_position.row);
assert_eq(8, start->end_position.col);
assert_eq(2, p_node_position(start).row);
assert_eq(1, p_node_position(start).col);
assert_eq(3, p_node_end_position(start).row);
assert_eq(8, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
input = "a\nbb";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
t1 = p_Start_pT1(start);
k1 = p_T_pToken(t1);
a1 = p_T_pA(t1);
assert_eq(1, start->pT1->pToken->position.row);
assert_eq(1, start->pT1->pToken->position.col);
assert_eq(1, start->pT1->pToken->end_position.row);
assert_eq(1, start->pT1->pToken->end_position.col);
assert(p_position_valid(start->pT1->pA->position));
assert_eq(2, start->pT1->pA->position.row);
assert_eq(1, start->pT1->pA->position.col);
assert_eq(2, start->pT1->pA->end_position.row);
assert_eq(2, start->pT1->pA->end_position.col);
assert_eq(1, start->pT1->position.row);
assert_eq(1, start->pT1->position.col);
assert_eq(2, start->pT1->end_position.row);
assert_eq(2, start->pT1->end_position.col);
assert_eq(1, p_node_position(k1).row);
assert_eq(1, p_node_position(k1).col);
assert_eq(1, p_node_end_position(k1).row);
assert_eq(1, p_node_end_position(k1).col);
assert(p_position_valid(p_node_position(a1)));
assert_eq(2, p_node_position(a1).row);
assert_eq(1, p_node_position(a1).col);
assert_eq(2, p_node_end_position(a1).row);
assert_eq(2, p_node_end_position(a1).col);
assert_eq(1, p_node_position(t1).row);
assert_eq(1, p_node_position(t1).col);
assert_eq(2, p_node_end_position(t1).row);
assert_eq(2, p_node_end_position(t1).col);
assert_eq(1, start->position.row);
assert_eq(1, start->position.col);
assert_eq(2, start->end_position.row);
assert_eq(2, start->end_position.col);
assert_eq(1, p_node_position(start).row);
assert_eq(1, p_node_position(start).col);
assert_eq(2, p_node_end_position(start).row);
assert_eq(2, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
input = "a\nc\nc";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
t1 = p_Start_pT1(start);
k1 = p_T_pToken(t1);
a1 = p_T_pA(t1);
assert_eq(1, start->pT1->pToken->position.row);
assert_eq(1, start->pT1->pToken->position.col);
assert_eq(1, start->pT1->pToken->end_position.row);
assert_eq(1, start->pT1->pToken->end_position.col);
assert(p_position_valid(start->pT1->pA->position));
assert_eq(2, start->pT1->pA->position.row);
assert_eq(1, start->pT1->pA->position.col);
assert_eq(3, start->pT1->pA->end_position.row);
assert_eq(1, start->pT1->pA->end_position.col);
assert_eq(1, start->pT1->position.row);
assert_eq(1, start->pT1->position.col);
assert_eq(3, start->pT1->end_position.row);
assert_eq(1, start->pT1->end_position.col);
assert_eq(1, p_node_position(k1).row);
assert_eq(1, p_node_position(k1).col);
assert_eq(1, p_node_end_position(k1).row);
assert_eq(1, p_node_end_position(k1).col);
assert(p_position_valid(p_node_position(a1)));
assert_eq(2, p_node_position(a1).row);
assert_eq(1, p_node_position(a1).col);
assert_eq(3, p_node_end_position(a1).row);
assert_eq(1, p_node_end_position(a1).col);
assert_eq(1, p_node_position(t1).row);
assert_eq(1, p_node_position(t1).col);
assert_eq(3, p_node_end_position(t1).row);
assert_eq(1, p_node_end_position(t1).col);
assert_eq(1, start->position.row);
assert_eq(1, start->position.col);
assert_eq(3, start->end_position.row);
assert_eq(1, start->end_position.col);
assert_eq(1, p_node_position(start).row);
assert_eq(1, p_node_position(start).col);
assert_eq(3, p_node_end_position(start).row);
assert_eq(1, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
input = "a";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
t1 = p_Start_pT1(start);
k1 = p_T_pToken(t1);
a1 = p_T_pA(t1);
assert_eq(1, start->pT1->pToken->position.row);
assert_eq(1, start->pT1->pToken->position.col);
assert_eq(1, start->pT1->pToken->end_position.row);
assert_eq(1, start->pT1->pToken->end_position.col);
assert(!p_position_valid(start->pT1->pA->position));
assert_eq(1, start->pT1->position.row);
assert_eq(1, start->pT1->position.col);
assert_eq(1, start->pT1->end_position.row);
assert_eq(1, start->pT1->end_position.col);
assert_eq(1, p_node_position(k1).row);
assert_eq(1, p_node_position(k1).col);
assert_eq(1, p_node_end_position(k1).row);
assert_eq(1, p_node_end_position(k1).col);
assert(!p_position_valid(p_node_position(a1)));
assert_eq(1, p_node_position(t1).row);
assert_eq(1, p_node_position(t1).col);
assert_eq(1, p_node_end_position(t1).row);
assert_eq(1, p_node_end_position(t1).col);
assert_eq(1, start->position.row);
assert_eq(1, start->position.col);
assert_eq(1, start->end_position.row);
assert_eq(1, start->end_position.col);
assert_eq(1, p_node_position(start).row);
assert_eq(1, p_node_position(start).col);
assert_eq(1, p_node_end_position(start).row);
assert_eq(1, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
return 0;

10
spec/test_tree_invalid_positions.d
View File

@ -12,7 +12,7 @@ unittest
string input = "\na\n bb ccc";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
assert_eq(2, start.pT1.pToken.position.row);
assert_eq(1, start.pT1.pToken.position.col);
@ -33,7 +33,7 @@ unittest
assert_eq(3, start.end_position.row);
assert_eq(8, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
input = "a\nbb";
context = p_context_new(input);
@ -59,7 +59,7 @@ unittest
assert_eq(2, start.end_position.row);
assert_eq(2, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
input = "a\nc\nc";
context = p_context_new(input);
@ -85,7 +85,7 @@ unittest
assert_eq(3, start.end_position.row);
assert_eq(1, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
input = "a";
context = p_context_new(input);
@ -107,5 +107,5 @@ unittest
assert_eq(1, start.end_position.row);
assert_eq(1, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
}

35
spec/test_tree_node_memory_remains.c
View File

@ -373,46 +373,45 @@ int main(int argc, char * argv[])
context = p_context_new((const uint8_t *)input, strlen(input));
size_t result = p_parse(context);
assert_eq(P_SUCCESS, result);
PModule * pmod = p_result(context);
PModuleItems * pmis = pmod->pModuleItems;
PFunctionDefinition ** pfds;
PModule pmod = p_result(context);
PModuleItems pmis = p_PModule_pModuleItems(pmod);
PFunctionDefinition * pfds;
size_t n_pfds = 0u;
while (pmis != NULL)
while (p_node_valid(pmis))
{
PModuleItem * pmi = pmis->pModuleItem;
if (pmi->pFunctionDefinition != NULL)
PModuleItem pmi = p_PModuleItems_pModuleItem(pmis);
if (p_node_valid(p_PModuleItem_pFunctionDefinition(pmi)))
{
n_pfds++;
}
pmis = pmis->pModuleItems;
pmis = p_PModuleItems_pModuleItems(pmis);
}
pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
pmis = pmod->pModuleItems;
pfds = (PFunctionDefinition *)malloc(n_pfds * sizeof(PFunctionDefinition));
pmis = p_PModule_pModuleItems(pmod);
size_t pfd_i = n_pfds;
while (pmis != NULL)
while (p_node_valid(pmis))
{
PModuleItem * pmi = pmis->pModuleItem;
PFunctionDefinition * pfd = pmi->pFunctionDefinition;
if (pfd != NULL)
PModuleItem pmi = p_PModuleItems_pModuleItem(pmis);
PFunctionDefinition pfd = p_PModuleItem_pFunctionDefinition(pmi);
if (p_node_valid(pfd))
{
pfd_i--;
assert(pfd_i < n_pfds);
pfds[pfd_i] = pfd;
}
pmis = pmis->pModuleItems;
pmis = p_PModuleItems_pModuleItems(pmis);
}
assert_eq(51, n_pfds);
for (size_t i = 0; i < n_pfds; i++)
{
if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
(expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
if (strncmp(expected[i].name, (const char *)p_node_data(p_PFunctionDefinition_name(pfds[i]))->pvalue.s, strlen(expected[i].name)) != 0 ||
(expected[i].token != p_tree_walk_PFunctionDefinition(pfds[i], returntype, pType, pTypeBase, pToken1, token)))
{
fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, p_tree_walk_PFunctionDefinition(pfds[i], returntype, pType, pTypeBase, pToken1, token));
}
}
free(pfds);
p_tree_delete(pmod);
p_context_delete(context);
return 0;

18
spec/test_tree_node_memory_remains.d
View File

@ -378,19 +378,19 @@ def main() -> int
context = p_context_new(input);
size_t result = p_parse(context);
assert_eq(P_SUCCESS, result);
PModule * pmod = p_result(context);
PModuleItems * pmis = pmod.pModuleItems;
PFunctionDefinition *[] pfds;
while (pmis !is null)
PModule pmod = p_result(context);
PModuleItems pmis = pmod.pModuleItems;
PFunctionDefinition[] pfds;
while (pmis.valid)
{
PModuleItem * pmi = pmis.pModuleItem;
if (pmi is null)
PModuleItem pmi = pmis.pModuleItem;
if (!pmi.valid)
{
stderr.writeln("pmi is null!!!?");
assert(0);
}
PFunctionDefinition * pfd = pmi.pFunctionDefinition;
if (pfd !is null)
PFunctionDefinition pfd = pmi.pFunctionDefinition;
if (pfd.valid)
{
pfds = [pfd] ~ pfds;
}
@ -405,5 +405,5 @@ def main() -> int
stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
}
}
p_tree_delete(pmod);
p_context_delete(context);
}

65
spec/test_tree_ps.c
View File

@ -9,55 +9,52 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
PStartS * start = p_result(context);
assert(start->pItems1 != NULL);
assert(start->pItems != NULL);
PItemsS * items = start->pItems;
assert(items->pItem != NULL);
assert(items->pItem->pToken1 != NULL);
assert_eq(TOKEN_a, items->pItem->pToken1->token);
assert_eq(11, items->pItem->pToken1->pvalue);
assert(items->pItemsMore != NULL);
PItemsMoreS * itemsmore = items->pItemsMore;
assert(itemsmore->pItem != NULL);
assert(itemsmore->pItem->pItem != NULL);
assert(itemsmore->pItem->pItem->pItem != NULL);
assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore != NULL);
itemsmore = itemsmore->pItemsMore;
assert(itemsmore->pItem != NULL);
assert(itemsmore->pItem->pToken1 != NULL);
assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL);
PStartS start = p_result(context);
assert(p_node_valid(p_PStartS_pItems1(start)));
assert(p_node_valid(p_PStartS_pItems(start)));
PItemsS items = p_PStartS_pItems(start);
assert(p_node_valid(p_PItemsS_pItem(items)));
assert(p_node_valid(p_tree_walk_PItemsS(items, pItem, pToken1)));
assert_eq(TOKEN_a, p_tree_walk_PItemsS(items, pItem, pToken1, token));
assert_eq(11, p_tree_walk_PItemsS(items, pItem, pToken1, pvalue));
assert(p_node_valid(p_PItemsS_pItemsMore(items)));
PItemsMoreS itemsmore = p_PItemsS_pItemsMore(items);
assert(p_node_valid(p_PItemsMoreS_pItem(itemsmore)));
assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem)));
assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem)));
assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1)));
assert_eq(TOKEN_b, p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1, token));
assert_eq(22, p_tree_walk_PItemsMoreS(itemsmore, pItem, pItem, pItem, pToken1, pvalue));
assert(p_node_valid(p_PItemsMoreS_pItemsMore(itemsmore)));
itemsmore = p_PItemsMoreS_pItemsMore(itemsmore);
assert(p_node_valid(p_PItemsMoreS_pItem(itemsmore)));
assert(p_node_valid(p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1)));
assert_eq(TOKEN_b, p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1, token));
assert_eq(22, p_tree_walk_PItemsMoreS(itemsmore, pItem, pToken1, pvalue));
assert(!p_node_valid(p_PItemsMoreS_pItemsMore(itemsmore)));
p_tree_delete(start);
p_context_delete(context);
input = "";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start->pItems == NULL);
assert(!p_node_valid(p_PStartS_pItems(start)));
p_tree_delete(start);
p_context_delete(context);
input = "2 1";
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start->pItems != NULL);
assert(start->pItems->pItem != NULL);
assert(start->pItems->pItem->pDual != NULL);
assert(start->pItems->pItem->pDual->pTwo1 != NULL);
assert(start->pItems->pItem->pDual->pOne2 != NULL);
assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL);
assert(p_node_valid(p_PStartS_pItems(start)));
assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem)));
assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual)));
assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pTwo1)));
assert(p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pOne2)));
assert(!p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pTwo2)));
assert(!p_node_valid(p_tree_walk_PStartS(start, pItems, pItem, pDual, pOne1)));
p_tree_delete(start);
p_context_delete(context);
return 0;

54
spec/test_tree_ps.d
View File

@ -12,51 +12,51 @@ unittest
string input = "a, ((b)), b";
p_context_t * context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
PStartS * start = p_result(context);
assert(start.pItems1 !is null);
assert(start.pItems !is null);
PItemsS * items = start.pItems;
assert(items.pItem !is null);
assert(items.pItem.pToken1 !is null);
PStartS start = p_result(context);
assert(start.pItems1.valid);
assert(start.pItems.valid);
PItemsS items = start.pItems;
assert(items.pItem.valid);
assert(items.pItem.pToken1.valid);
assert_eq(TOKEN_a, items.pItem.pToken1.token);
assert_eq(11, items.pItem.pToken1.pvalue);
assert(items.pItemsMore !is null);
PItemsMoreS * itemsmore = items.pItemsMore;
assert(itemsmore.pItem !is null);
assert(itemsmore.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
assert(items.pItemsMore.valid);
PItemsMoreS itemsmore = items.pItemsMore;
assert(itemsmore.pItem.valid);
assert(itemsmore.pItem.pItem.valid);
assert(itemsmore.pItem.pItem.pItem.valid);
assert(itemsmore.pItem.pItem.pItem.pToken1.valid);
assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
assert(itemsmore.pItemsMore !is null);
assert(itemsmore.pItemsMore.valid);
itemsmore = itemsmore.pItemsMore;
assert(itemsmore.pItem !is null);
assert(itemsmore.pItem.pToken1 !is null);
assert(itemsmore.pItem.valid);
assert(itemsmore.pItem.pToken1.valid);
assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
assert_eq(22, itemsmore.pItem.pToken1.pvalue);
assert(itemsmore.pItemsMore is null);
assert(!itemsmore.pItemsMore.valid);
p_tree_delete(start);
p_context_delete(context);
input = "";
context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start.pItems is null);
assert(!start.pItems.valid);
p_tree_delete(start);
p_context_delete(context);
input = "2 1";
context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
start = p_result(context);
assert(start.pItems !is null);
assert(start.pItems.pItem !is null);
assert(start.pItems.pItem.pDual !is null);
assert(start.pItems.pItem.pDual.pTwo1 !is null);
assert(start.pItems.pItem.pDual.pOne2 !is null);
assert(start.pItems.pItem.pDual.pTwo2 is null);
assert(start.pItems.pItem.pDual.pOne1 is null);
assert(start.pItems.valid);
assert(start.pItems.pItem.valid);
assert(start.pItems.pItem.pDual.valid);
assert(start.pItems.pItem.pDual.pTwo1.valid);
assert(start.pItems.pItem.pDual.pOne2.valid);
assert(!start.pItems.pItem.pDual.pTwo2.valid);
assert(!start.pItems.pItem.pDual.pOne1.valid);
p_tree_delete(start);
p_context_delete(context);
}

128
spec/test_tree_token_positions.c
View File

@ -9,81 +9,91 @@ int main()
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
T t1 = p_Start_pT1(start);
T t2 = p_Start_pT2(start);
T t3 = p_Start_pT3(start);
Token k1 = p_T_pToken(t1);
Token k2 = p_T_pToken(t2);
Token k3 = p_T_pToken(t3);
assert_eq(1, start->pT1->pToken->position.row);
assert_eq(1, start->pT1->pToken->position.col);
assert_eq(1, start->pT1->pToken->end_position.row);
assert_eq(1, start->pT1->pToken->end_position.col);
assert_eq(1, start->pT1->position.row);
assert_eq(1, start->pT1->position.col);
assert_eq(1, start->pT1->end_position.row);
assert_eq(1, start->pT1->end_position.col);
assert_eq(1, p_node_position(k1).row);
assert_eq(1, p_node_position(k1).col);
assert_eq(1, p_node_end_position(k1).row);
assert_eq(1, p_node_end_position(k1).col);
assert_eq(1, p_node_position(t1).row);
assert_eq(1, p_node_position(t1).col);
assert_eq(1, p_node_end_position(t1).row);
assert_eq(1, p_node_end_position(t1).col);
assert_eq(1, start->pT2->pToken->position.row);
assert_eq(2, start->pT2->pToken->position.col);
assert_eq(1, start->pT2->pToken->end_position.row);
assert_eq(3, start->pT2->pToken->end_position.col);
assert_eq(1, start->pT2->position.row);
assert_eq(2, start->pT2->position.col);
assert_eq(1, start->pT2->end_position.row);
assert_eq(3, start->pT2->end_position.col);
assert_eq(1, p_node_position(k2).row);
assert_eq(2, p_node_position(k2).col);
assert_eq(1, p_node_end_position(k2).row);
assert_eq(3, p_node_end_position(k2).col);
assert_eq(1, p_node_position(t2).row);
assert_eq(2, p_node_position(t2).col);
assert_eq(1, p_node_end_position(t2).row);
assert_eq(3, p_node_end_position(t2).col);
assert_eq(1, start->pT3->pToken->position.row);
assert_eq(4, start->pT3->pToken->position.col);
assert_eq(1, start->pT3->pToken->end_position.row);
assert_eq(6, start->pT3->pToken->end_position.col);
assert_eq(1, start->pT3->position.row);
assert_eq(4, start->pT3->position.col);
assert_eq(1, start->pT3->end_position.row);
assert_eq(6, start->pT3->end_position.col);
assert_eq(1, p_node_position(k3).row);
assert_eq(4, p_node_position(k3).col);
assert_eq(1, p_node_end_position(k3).row);
assert_eq(6, p_node_end_position(k3).col);
assert_eq(1, p_node_position(t3).row);
assert_eq(4, p_node_position(t3).col);
assert_eq(1, p_node_end_position(t3).row);
assert_eq(6, p_node_end_position(t3).col);
assert_eq(1, start->position.row);
assert_eq(1, start->position.col);
assert_eq(1, start->end_position.row);
assert_eq(6, start->end_position.col);
assert_eq(1, p_node_position(start).row);
assert_eq(1, p_node_position(start).col);
assert_eq(1, p_node_end_position(start).row);
assert_eq(6, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
input = "\n\n bb\nc\ncc\n\n a";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
start = p_result(context);
t1 = p_Start_pT1(start);
t2 = p_Start_pT2(start);
t3 = p_Start_pT3(start);
k1 = p_T_pToken(t1);
k2 = p_T_pToken(t2);
k3 = p_T_pToken(t3);
assert_eq(3, start->pT1->pToken->position.row);
assert_eq(3, start->pT1->pToken->position.col);
assert_eq(3, start->pT1->pToken->end_position.row);
assert_eq(4, start->pT1->pToken->end_position.col);
assert_eq(3, start->pT1->position.row);
assert_eq(3, start->pT1->position.col);
assert_eq(3, start->pT1->end_position.row);
assert_eq(4, start->pT1->end_position.col);
assert_eq(3, p_node_position(k1).row);
assert_eq(3, p_node_position(k1).col);
assert_eq(3, p_node_end_position(k1).row);
assert_eq(4, p_node_end_position(k1).col);
assert_eq(3, p_node_position(t1).row);
assert_eq(3, p_node_position(t1).col);
assert_eq(3, p_node_end_position(t1).row);
assert_eq(4, p_node_end_position(t1).col);
assert_eq(4, start->pT2->pToken->position.row);
assert_eq(1, start->pT2->pToken->position.col);
assert_eq(5, start->pT2->pToken->end_position.row);
assert_eq(2, start->pT2->pToken->end_position.col);
assert_eq(4, start->pT2->position.row);
assert_eq(1, start->pT2->position.col);
assert_eq(5, start->pT2->end_position.row);
assert_eq(2, start->pT2->end_position.col);
assert_eq(4, p_node_position(k2).row);
assert_eq(1, p_node_position(k2).col);
assert_eq(5, p_node_end_position(k2).row);
assert_eq(2, p_node_end_position(k2).col);
assert_eq(4, p_node_position(t2).row);
assert_eq(1, p_node_position(t2).col);
assert_eq(5, p_node_end_position(t2).row);
assert_eq(2, p_node_end_position(t2).col);
assert_eq(7, start->pT3->pToken->position.row);
assert_eq(6, start->pT3->pToken->position.col);
assert_eq(7, start->pT3->pToken->end_position.row);
assert_eq(6, start->pT3->pToken->end_position.col);
assert_eq(7, start->pT3->position.row);
assert_eq(6, start->pT3->position.col);
assert_eq(7, start->pT3->end_position.row);
assert_eq(6, start->pT3->end_position.col);
assert_eq(7, p_node_position(k3).row);
assert_eq(6, p_node_position(k3).col);
assert_eq(7, p_node_end_position(k3).row);
assert_eq(6, p_node_end_position(k3).col);
assert_eq(7, p_node_position(t3).row);
assert_eq(6, p_node_position(t3).col);
assert_eq(7, p_node_end_position(t3).row);
assert_eq(6, p_node_end_position(t3).col);
assert_eq(3, start->position.row);
assert_eq(3, start->position.col);
assert_eq(7, start->end_position.row);
assert_eq(6, start->end_position.col);
assert_eq(3, p_node_position(start).row);
assert_eq(3, p_node_position(start).col);
assert_eq(7, p_node_end_position(start).row);
assert_eq(6, p_node_end_position(start).col);
p_tree_delete(start);
p_context_delete(context);
return 0;

6
spec/test_tree_token_positions.d
View File

@ -12,7 +12,7 @@ unittest
string input = "abbccc";
p_context_t * context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
Start * start = p_result(context);
Start start = p_result(context);
assert_eq(1, start.pT1.pToken.position.row);
assert_eq(1, start.pT1.pToken.position.col);
@ -46,7 +46,7 @@ unittest
assert_eq(1, start.end_position.row);
assert_eq(6, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
input = "\n\n bb\nc\ncc\n\n a";
context = p_context_new(input);
@ -85,5 +85,5 @@ unittest
assert_eq(7, start.end_position.row);
assert_eq(6, start.end_position.col);
p_tree_delete(start);
p_context_delete(context);
}