josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Rename AST generation mode to tree generation mode

Browse Source
This commit is contained in:
Josh Holtrop 2026-02-09 21:59:00 -05:00
parent 6a87bb2d56
commit 77ec7c9de4
34 changed files with 279 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -7,7 +7,7 @@ Propane is a LALR Parser Generator (LPG) which:
* supports UTF-8 lexer inputs * supports UTF-8 lexer inputs
* generates a table-driven shift/reduce parser to parse input in linear time * generates a table-driven shift/reduce parser to parse input in linear time
* targets C, C++, or D language outputs * targets C, C++, or D language outputs
* optionally supports automatic full AST generation * optionally supports automatic full parse tree generation
* is MIT-licensed * is MIT-licensed
* is distributable as a standalone Ruby script * is distributable as a standalone Ruby script

98
assets/parser.c.erb
View File

@ -638,7 +638,7 @@ typedef struct
* reduce action. * reduce action.
*/ */
parser_state_id_t n_states; parser_state_id_t n_states;
<% if @grammar.ast %> <% if @grammar.tree %>
/** /**
* Map of rule components to rule set child fields. * Map of rule components to rule set child fields.
@ -646,7 +646,7 @@ typedef struct
uint16_t const * rule_set_node_field_index_map; uint16_t const * rule_set_node_field_index_map;
/** /**
* Number of rule set AST node fields. * Number of rule set tree node fields.
*/ */
uint16_t rule_set_node_field_array_size; uint16_t rule_set_node_field_array_size;
@ -688,21 +688,21 @@ typedef struct
/** Parser value from this state. */ /** Parser value from this state. */
<%= @grammar.prefix %>value_t pvalue; <%= @grammar.prefix %>value_t pvalue;
<% if @grammar.ast %> <% if @grammar.tree %>
/** AST node. */ /** tree node. */
void * ast_node; void * tree_node;
<% end %> <% end %>
} state_value_t; } state_value_t;
/** Common AST node structure. */ /** Common tree node structure. */
typedef struct ASTNode_s typedef struct TreeNode_s
{ {
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields; uint16_t n_fields;
uint8_t is_token; uint8_t is_token;
struct ASTNode_s * fields[]; struct TreeNode_s * fields[];
} ASTNode; } TreeNode;
/** Parser shift table. */ /** Parser shift table. */
static const shift_t parser_shift_table[] = { static const shift_t parser_shift_table[] = {
@ -711,7 +711,7 @@ static const shift_t parser_shift_table[] = {
<% end %> <% end %>
}; };
<% if @grammar.ast %> <% if @grammar.tree %>
<% @grammar.rules.each do |rule| %> <% @grammar.rules.each do |rule| %>
<% unless rule.flat_rule_set_node_field_index_map? %> <% unless rule.flat_rule_set_node_field_index_map? %>
const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>}; const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
@ -726,14 +726,14 @@ static const reduce_t parser_reduce_table[] = {
<%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */ <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
<%= reduce[:rule_id] %>u, /* Rule ID */ <%= reduce[:rule_id] %>u, /* Rule ID */
<%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */ <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
<% if @grammar.ast %> <% if @grammar.tree %>
<%= reduce[:n_states] %>u, /* Number of states */ <%= reduce[:n_states] %>u, /* Number of states */
<% if reduce[:rule].flat_rule_set_node_field_index_map? %> <% if reduce[:rule].flat_rule_set_node_field_index_map? %>
NULL, /* No rule set node field index map (flat map) */ NULL, /* No rule set node field index map (flat map) */
<% else %> <% else %>
&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */ &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
<% end %> <% end %>
<%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */ <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
<%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */ <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
<% else %> <% else %>
<%= reduce[:n_states] %>u}, <%= reduce[:n_states] %>u},
@ -841,7 +841,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
free(stack->entries); free(stack->entries);
} }
<% unless @grammar.ast %> <% unless @grammar.tree %>
/** /**
* Execute user code associated with a parser rule. * Execute user code associated with a parser rule.
* *
@ -944,7 +944,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID; <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
state_values_stack_t statevalues; state_values_stack_t statevalues;
size_t reduced_rule_set = INVALID_ID; size_t reduced_rule_set = INVALID_ID;
<% if @grammar.ast %> <% if @grammar.tree %>
void * reduced_parser_node; void * reduced_parser_node;
<% else %> <% else %>
<%= @grammar.prefix %>value_t reduced_parser_value; <%= @grammar.prefix %>value_t reduced_parser_value;
@ -976,8 +976,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF)) if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
{ {
/* Successful parse. */ /* Successful parse. */
<% if @grammar.ast %> <% if @grammar.tree %>
context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node; context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
<% else %> <% else %>
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue; context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
<% end %> <% end %>
@ -993,15 +993,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
if (reduced_rule_set == INVALID_ID) if (reduced_rule_set == INVALID_ID)
{ {
/* We shifted a token, mark it consumed. */ /* We shifted a token, mark it consumed. */
<% if @grammar.ast %> <% if @grammar.tree %>
<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>)); <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = (<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *)malloc(sizeof(<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>));
token_ast_node->position = token_info.position; token_tree_node->position = token_info.position;
token_ast_node->end_position = token_info.end_position; token_tree_node->end_position = token_info.end_position;
token_ast_node->n_fields = 0u; token_tree_node->n_fields = 0u;
token_ast_node->is_token = 1u; token_tree_node->is_token = 1u;
token_ast_node->token = token; token_tree_node->token = token;
token_ast_node->pvalue = token_info.pvalue; token_tree_node->pvalue = token_info.pvalue;
state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node; state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
<% else %> <% else %>
state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue; state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
<% end %> <% end %>
@ -1010,8 +1010,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
else else
{ {
/* We shifted a RuleSet. */ /* We shifted a RuleSet. */
<% if @grammar.ast %> <% if @grammar.tree %>
state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node; state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
<% else %> <% else %>
state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value; state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
<%= @grammar.prefix %>value_t new_parse_result; <%= @grammar.prefix %>value_t new_parse_result;
@ -1027,16 +1027,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
if (reduce_index != INVALID_ID) if (reduce_index != INVALID_ID)
{ {
/* We have something to reduce. */ /* We have something to reduce. */
<% if @grammar.ast %> <% if @grammar.tree %>
if (parser_reduce_table[reduce_index].propagate_optional_target) if (parser_reduce_table[reduce_index].propagate_optional_target)
{ {
reduced_parser_node = state_values_stack_index(&statevalues, -1)->ast_node; reduced_parser_node = state_values_stack_index(&statevalues, -1)->tree_node;
} }
else if (parser_reduce_table[reduce_index].n_states > 0) else if (parser_reduce_table[reduce_index].n_states > 0)
{ {
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size; size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *); size_t bytes = sizeof(TreeNode) + n_fields * sizeof(void *);
ASTNode * node = (ASTNode *)malloc(bytes); TreeNode * node = (TreeNode *)malloc(bytes);
memset(node, 0, bytes); memset(node, 0, bytes);
node->position = INVALID_POSITION; node->position = INVALID_POSITION;
node->end_position = INVALID_POSITION; node->end_position = INVALID_POSITION;
@ -1045,20 +1045,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{ {
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++) for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{ {
node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node; node->fields[i] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
} }
} }
else else
{ {
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++) for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{ {
node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node; node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (TreeNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->tree_node;
} }
} }
bool position_found = false; bool position_found = false;
for (size_t i = 0; i < n_fields; i++) for (size_t i = 0; i < n_fields; i++)
{ {
ASTNode * child = node->fields[i]; TreeNode * child = node->fields[i];
if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position)) if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
{ {
if (!position_found) if (!position_found)
@ -1125,15 +1125,15 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
* *
* @return Parse result value. * @return Parse result value.
*/ */
<% if @grammar.ast %> <% if @grammar.tree %>
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{ {
return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result; return (<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *) context->parse_result;
} }
<% @grammar.start_rules.each_with_index do |start_rule, i| %> <% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context) <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{ {
return (<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *) context->parse_result; return (<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *) context->parse_result;
} }
<% end %> <% end %>
<% else %> <% else %>
@ -1184,14 +1184,14 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
{ {
return context->token; return context->token;
} }
<% if @grammar.ast %> <% if @grammar.tree %>
static void free_ast_node(ASTNode * node) static void free_tree_node(TreeNode * node)
{ {
if (node->is_token) if (node->is_token)
{ {
<% if @grammar.free_token_node %> <% if @grammar.free_token_node %>
<%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node); <%= @grammar.free_token_node %>((<%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> *) node);
<% end %> <% end %>
/* TODO: free value_t */ /* TODO: free value_t */
} }
@ -1201,7 +1201,7 @@ static void free_ast_node(ASTNode * node)
{ {
if (node->fields[i] != NULL) if (node->fields[i] != NULL)
{ {
free_ast_node(node->fields[i]); free_tree_node(node->fields[i]);
} }
} }
} }
@ -1209,20 +1209,20 @@ static void free_ast_node(ASTNode * node)
} }
/** /**
* Free all AST node memory. * Free all tree node memory.
*/ */
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast) void <%= @grammar.prefix %>free_tree(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
{ {
free_ast_node((ASTNode *)ast); free_tree_node((TreeNode *)tree);
} }
<% @grammar.start_rules.each_with_index do |start_rule, i| %> <% @grammar.start_rules.each_with_index do |start_rule, i| %>
/** /**
* Free all AST node memory. * Free all tree node memory.
*/ */
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast) void <%= @grammar.prefix %>free_tree_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
{ {
free_ast_node((ASTNode *)ast); free_tree_node((TreeNode *)tree);
} }
<% end %> <% end %>
<% end %> <% end %>

78
assets/parser.d.erb
View File

@ -75,7 +75,7 @@ public struct <%= @grammar.prefix %>position_t
} }
} }
<% if @grammar.ast %> <% if @grammar.tree %>
/** Parser values type. */ /** Parser values type. */
public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>; public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
<% else %> <% else %>
@ -88,19 +88,19 @@ public union <%= @grammar.prefix %>value_t
} }
<% end %> <% end %>
<% if @grammar.ast %> <% if @grammar.tree %>
/** Common AST node structure. */ /** Common tree node structure. */
private struct ASTNode private struct TreeNode
{ {
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
void *[0] fields; void *[0] fields;
} }
/** AST node types. @{ */ /** Tree node types. @{ */
public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
{ {
/* ASTNode fields must be present in the same order here. */ /* TreeNode fields must be present in the same order here. */
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
<%= @grammar.prefix %>token_t token; <%= @grammar.prefix %>token_t token;
@ -110,11 +110,11 @@ public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
<% @parser.rule_sets.each do |name, rule_set| %> <% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %> <% next if name.start_with?("$") %>
<% next if rule_set.optional? %> <% next if rule_set.optional? %>
public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %> public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
{ {
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
<% rule_set.ast_fields.each do |fields| %> <% rule_set.tree_fields.each do |fields| %>
union union
{ {
<% fields.each do |field_name, type| %> <% fields.each do |field_name, type| %>
@ -172,7 +172,7 @@ public struct <%= @grammar.prefix %>context_t
/* Parser context data. */ /* Parser context data. */
/** Parse result value. */ /** Parse result value. */
<% if @grammar.ast %> <% if @grammar.tree %>
void * parse_result; void * parse_result;
<% else %> <% else %>
<%= @grammar.prefix %>value_t parse_result; <%= @grammar.prefix %>value_t parse_result;
@ -797,7 +797,7 @@ private struct reduce_t
* reduce action. * reduce action.
*/ */
parser_state_id_t n_states; parser_state_id_t n_states;
<% if @grammar.ast %> <% if @grammar.tree %>
/** /**
* Map of rule components to rule set child fields. * Map of rule components to rule set child fields.
@ -805,7 +805,7 @@ private struct reduce_t
immutable(ushort) * rule_set_node_field_index_map; immutable(ushort) * rule_set_node_field_index_map;
/** /**
* Number of rule set AST node fields. * Number of rule set tree node fields.
*/ */
ushort rule_set_node_field_array_size; ushort rule_set_node_field_array_size;
@ -847,9 +847,9 @@ private struct state_value_t
/** Parser value from this state. */ /** Parser value from this state. */
<%= @grammar.prefix %>value_t pvalue; <%= @grammar.prefix %>value_t pvalue;
<% if @grammar.ast %> <% if @grammar.tree %>
/** AST node. */ /** Tree node. */
void * ast_node; void * tree_node;
<% end %> <% end %>
this(size_t state_id) this(size_t state_id)
@ -865,7 +865,7 @@ private immutable shift_t[] parser_shift_table = [
<% end %> <% end %>
]; ];
<% if @grammar.ast %> <% if @grammar.tree %>
<% @grammar.rules.each do |rule| %> <% @grammar.rules.each do |rule| %>
<% unless rule.flat_rule_set_node_field_index_map? %> <% unless rule.flat_rule_set_node_field_index_map? %>
immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>]; immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
@ -880,14 +880,14 @@ private immutable reduce_t[] parser_reduce_table = [
<%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */ <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
<%= reduce[:rule_id] %>u, /* Rule ID */ <%= reduce[:rule_id] %>u, /* Rule ID */
<%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */ <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
<% if @grammar.ast %> <% if @grammar.tree %>
<%= reduce[:n_states] %>u, /* Number of states */ <%= reduce[:n_states] %>u, /* Number of states */
<% if reduce[:rule].flat_rule_set_node_field_index_map? %> <% if reduce[:rule].flat_rule_set_node_field_index_map? %>
null, /* No rule set node field index map (flat map) */ null, /* No rule set node field index map (flat map) */
<% else %> <% else %>
&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */ &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
<% end %> <% end %>
<%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */ <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
<%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */ <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
<% else %> <% else %>
<%= reduce[:n_states] %>u), /* Number of states */ <%= reduce[:n_states] %>u), /* Number of states */
@ -902,7 +902,7 @@ private immutable parser_state_t[] parser_state_table = [
<% end %> <% end %>
]; ];
<% unless @grammar.ast %> <% unless @grammar.tree %>
/** /**
* Execute user code associated with a parser rule. * Execute user code associated with a parser rule.
* *
@ -1006,7 +1006,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
state_value_t[] statevalues = new state_value_t[](1); state_value_t[] statevalues = new state_value_t[](1);
statevalues[0].state_id = start_state_id; statevalues[0].state_id = start_state_id;
size_t reduced_rule_set = INVALID_ID; size_t reduced_rule_set = INVALID_ID;
<% if @grammar.ast %> <% if @grammar.tree %>
void * reduced_parser_node; void * reduced_parser_node;
<% else %> <% else %>
<%= @grammar.prefix %>value_t reduced_parser_value; <%= @grammar.prefix %>value_t reduced_parser_value;
@ -1033,8 +1033,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF)) if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
{ {
/* Successful parse. */ /* Successful parse. */
<% if @grammar.ast %> <% if @grammar.tree %>
context.parse_result = statevalues[$-1].ast_node; context.parse_result = statevalues[$-1].tree_node;
<% else %> <% else %>
context.parse_result = statevalues[$-1].pvalue; context.parse_result = statevalues[$-1].pvalue;
<% end %> <% end %>
@ -1048,9 +1048,9 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
if (reduced_rule_set == INVALID_ID) if (reduced_rule_set == INVALID_ID)
{ {
/* We shifted a token, mark it consumed. */ /* We shifted a token, mark it consumed. */
<% if @grammar.ast %> <% if @grammar.tree %>
<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue); <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
statevalues[$-1].ast_node = token_ast_node; statevalues[$-1].tree_node = token_tree_node;
<% else %> <% else %>
statevalues[$-1].pvalue = token_info.pvalue; statevalues[$-1].pvalue = token_info.pvalue;
<% end %> <% end %>
@ -1059,8 +1059,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
else else
{ {
/* We shifted a RuleSet. */ /* We shifted a RuleSet. */
<% if @grammar.ast %> <% if @grammar.tree %>
statevalues[$-1].ast_node = reduced_parser_node; statevalues[$-1].tree_node = reduced_parser_node;
<% else %> <% else %>
statevalues[$-1].pvalue = reduced_parser_value; statevalues[$-1].pvalue = reduced_parser_value;
<%= @grammar.prefix %>value_t new_parse_result; <%= @grammar.prefix %>value_t new_parse_result;
@ -1075,16 +1075,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
if (reduce_index != INVALID_ID) if (reduce_index != INVALID_ID)
{ {
/* We have something to reduce. */ /* We have something to reduce. */
<% if @grammar.ast %> <% if @grammar.tree %>
if (parser_reduce_table[reduce_index].propagate_optional_target) if (parser_reduce_table[reduce_index].propagate_optional_target)
{ {
reduced_parser_node = statevalues[$ - 1].ast_node; reduced_parser_node = statevalues[$ - 1].tree_node;
} }
else if (parser_reduce_table[reduce_index].n_states > 0) else if (parser_reduce_table[reduce_index].n_states > 0)
{ {
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size; size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
size_t node_size = ASTNode.sizeof + n_fields * (void *).sizeof; size_t node_size = TreeNode.sizeof + n_fields * (void *).sizeof;
ASTNode * node = cast(ASTNode *)malloc(node_size); TreeNode * node = cast(TreeNode *)malloc(node_size);
GC.addRange(node, node_size); GC.addRange(node, node_size);
node.position = <%= @grammar.prefix %>position_t.INVALID; node.position = <%= @grammar.prefix %>position_t.INVALID;
node.end_position = <%= @grammar.prefix %>position_t.INVALID; node.end_position = <%= @grammar.prefix %>position_t.INVALID;
@ -1096,20 +1096,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{ {
foreach (i; 0..parser_reduce_table[reduce_index].n_states) foreach (i; 0..parser_reduce_table[reduce_index].n_states)
{ {
node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node; node.fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
} }
} }
else else
{ {
foreach (i; 0..parser_reduce_table[reduce_index].n_states) foreach (i; 0..parser_reduce_table[reduce_index].n_states)
{ {
node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node; node.fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].tree_node;
} }
} }
bool position_found = false; bool position_found = false;
foreach (i; 0..n_fields) foreach (i; 0..n_fields)
{ {
ASTNode * child = cast(ASTNode *)node.fields[i]; TreeNode * child = cast(TreeNode *)node.fields[i];
if (child && child.position.valid) if (child && child.position.valid)
{ {
if (!position_found) if (!position_found)
@ -1171,15 +1171,15 @@ public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix
* *
* @return Parse result value. * @return Parse result value.
*/ */
<% if @grammar.ast %> <% if @grammar.tree %>
public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{ {
return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result; return cast(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> *)context.parse_result;
} }
<% @grammar.start_rules.each_with_index do |start_rule, i| %> <% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context) public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{ {
return cast(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *)context.parse_result; return cast(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> *)context.parse_result;
} }
<% end %> <% end %>
<% else %> <% else %>

34
assets/parser.h.erb
View File

@ -58,7 +58,7 @@ typedef struct
/** User header code blocks. */ /** User header code blocks. */
<%= @grammar.code_blocks.fetch("header", "") %> <%= @grammar.code_blocks.fetch("header", "") %>
<% if @grammar.ast %> <% if @grammar.tree %>
/** Parser values type. */ /** Parser values type. */
typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t; typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
<% else %> <% else %>
@ -71,18 +71,18 @@ typedef union
} <%= @grammar.prefix %>value_t; } <%= @grammar.prefix %>value_t;
<% end %> <% end %>
<% if @grammar.ast %> <% if @grammar.tree %>
/** AST node types. @{ */ /** Tree node types. @{ */
typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
{ {
<% # ASTNode fields must be present in the same order here. # %> <% # TreeNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields; uint16_t n_fields;
uint8_t is_token; uint8_t is_token;
<%= @grammar.prefix %>token_t token; <%= @grammar.prefix %>token_t token;
<%= @grammar.prefix %>value_t pvalue; <%= @grammar.prefix %>value_t pvalue;
} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>; } <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>;
<% @parser.rule_sets.each do |name, rule_set| %> <% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %> <% next if name.start_with?("$") %>
@ -93,14 +93,14 @@ struct <%= name %>;
<% @parser.rule_sets.each do |name, rule_set| %> <% @parser.rule_sets.each do |name, rule_set| %>
<% next if name.start_with?("$") %> <% next if name.start_with?("$") %>
<% next if rule_set.optional? %> <% next if rule_set.optional? %>
typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %> typedef struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
{ {
<% # ASTNode fields must be present in the same order here. # %> <% # TreeNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields; uint16_t n_fields;
uint8_t is_token; uint8_t is_token;
<% rule_set.ast_fields.each do |fields| %> <% rule_set.tree_fields.each do |fields| %>
union union
{ {
<% fields.each do |field_name, type| %> <% fields.each do |field_name, type| %>
@ -108,7 +108,7 @@ typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
<% end %> <% end %>
}; };
<% end %> <% end %>
} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>; } <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
<% end %> <% end %>
/** @} */ /** @} */
@ -161,7 +161,7 @@ typedef struct
/* Parser context data. */ /* Parser context data. */
/** Parse result value. */ /** Parse result value. */
<% if @grammar.ast %> <% if @grammar.tree %>
void * parse_result; void * parse_result;
<% else %> <% else %>
<%= @grammar.prefix %>value_t parse_result; <%= @grammar.prefix %>value_t parse_result;
@ -193,10 +193,10 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context); size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %> <% end %>
<% if @grammar.ast %> <% if @grammar.tree %>
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context); <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %> <% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context); <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %> <% end %>
<% else %> <% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context); <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
@ -205,10 +205,10 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
<% end %> <% end %>
<% end %> <% end %>
<% if @grammar.ast %> <% if @grammar.tree %>
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast); void <%= @grammar.prefix %>free_tree(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree);
<% @grammar.start_rules.each_with_index do |start_rule, i| %> <% @grammar.start_rules.each_with_index do |start_rule, i| %>
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast); void <%= @grammar.prefix %>free_tree_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree);
<% end %> <% end %>
<% end %> <% end %>

88
doc/user_guide.md
View File

@ -14,7 +14,7 @@ Propane is a LALR Parser Generator (LPG) which:
* supports UTF-8 lexer inputs * supports UTF-8 lexer inputs
* generates a table-driven shift/reduce parser to parse input in linear time * generates a table-driven shift/reduce parser to parse input in linear time
* targets C, C++, or D language outputs * targets C, C++, or D language outputs
* optionally supports automatic full AST generation * optionally supports automatic full parse tree generation
* tracks input text start and end positions for all matched tokens/rules * tracks input text start and end positions for all matched tokens/rules
* is MIT-licensed * is MIT-licensed
* is distributable as a standalone Ruby script * is distributable as a standalone Ruby script
@ -189,21 +189,21 @@ rule.
Parser values for the rules or tokens in the rule pattern can be accessed Parser values for the rules or tokens in the rule pattern can be accessed
positionally with tokens `$1`, `$2`, `$3`, etc... positionally with tokens `$1`, `$2`, `$3`, etc...
Parser rule code blocks are not available in AST generation mode. Parser rule code blocks are not available in tree generation mode.
In AST generation mode, a full parse tree is automatically constructed in In tree generation mode, a full parse tree is automatically constructed in
memory for user code to traverse after parsing is complete. memory for user code to traverse after parsing is complete.
##> AST generation mode - the `ast` statement ##> Tree generation mode - the `tree` statement
To activate AST generation mode, place the `ast` statement in your grammar file: To activate tree generation mode, place the `tree` statement in your grammar file:
``` ```
ast; tree;
``` ```
It is recommended to place this statement early in the grammar. It is recommended to place this statement early in the grammar.
In AST generation mode various aspects of propane's behavior are changed: In tree generation mode various aspects of propane's behavior are changed:
* Only one `ptype` is allowed. * Only one `ptype` is allowed.
* Parser user code blocks are not supported. * Parser user code blocks are not supported.
@ -214,10 +214,10 @@ In AST generation mode various aspects of propane's behavior are changed:
with the `start` grammar statement, the name of the start struct will be with the `start` grammar statement, the name of the start struct will be
given by the user-specified start rule instead of `Start`. given by the user-specified start rule instead of `Start`.
Example AST generation grammar: Example tree generation grammar:
``` ```
ast; tree;
ptype int; ptype int;
@ -284,23 +284,23 @@ assert_eq(22, itemsmore.item.pToken1.pvalue);
assert(itemsmore.pItemsMore is null); assert(itemsmore.pItemsMore is null);
``` ```
## `ast_prefix` and `ast_suffix` statements ## `tree_prefix` and `tree_suffix` statements
In AST generation mode, structure types are defined and named based on the In tree generation mode, structure types are defined and named based on the
rules in the grammar. rules in the grammar.
Additionally, a structure type called `Token` is generated to hold parsed Additionally, a structure type called `Token` is generated to hold parsed
token information. token information.
These structure names can be modified by using the `ast_prefix` or `ast_suffix` These structure names can be modified by using the `tree_prefix` or `tree_suffix`
statements in the grammar file. statements in the grammar file.
The field names that point to instances of the structures are not affected by The field names that point to instances of the structures are not affected by
the `ast_prefix` or `ast_suffix` values. the `tree_prefix` or `tree_suffix` values.
For example, if the following two lines were added to the example above: For example, if the following two lines were added to the example above:
``` ```
ast_prefix ABC; tree_prefix ABC;
ast_suffix XYZ; tree_suffix XYZ;
``` ```
Then the types would be used as such instead: Then the types would be used as such instead:
@ -330,7 +330,7 @@ assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
If user lexer code block allocates memory to store in a token node's `pvalue`, If user lexer code block allocates memory to store in a token node's `pvalue`,
the `free_token_node` grammar statement can be used to specify the name of a the `free_token_node` grammar statement can be used to specify the name of a
function which will be called during the `p_free_ast()` call to free the memory function which will be called during the `p_free_tree()` call to free the memory
associated with a token node. associated with a token node.
Example: Example:
@ -342,7 +342,7 @@ static void free_token(Token * token)
free(token->pvalue); free(token->pvalue);
} }
>> >>
ast; tree;
free_token_node free_token; free_token_node free_token;
ptype int *; ptype int *;
token a << token a <<
@ -641,7 +641,7 @@ In this example:
* a reduced `Values`'s parser value has a type of `Value[]`. * a reduced `Values`'s parser value has a type of `Value[]`.
* a reduced `KeyValue`'s parser value has a type of `Value[string]`. * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
When AST generation mode is active, the `ptype` functionality works differently. When tree generation mode is active, the `ptype` functionality works differently.
In this mode, only one `ptype` is used by the parser. In this mode, only one `ptype` is used by the parser.
Lexer user code blocks may assign a parse value to the generated `Token` node Lexer user code blocks may assign a parse value to the generated `Token` node
by assigning to `$$` within a lexer code block. by assigning to `$$` within a lexer code block.
@ -700,8 +700,8 @@ A field can be immediately followed by a `?` character to signify that it is
optional. optional.
A field can optionally be followed by a `:` and then a field alias name. A field can optionally be followed by a `:` and then a field alias name.
If present, the field alias name is used to refer to the field value in user If present, the field alias name is used to refer to the field value in user
code blocks, or if AST mode is active, the field alias name is used as the code blocks, or if tree generation mode is active, the field alias name is used
field name in the generated AST node structure. as the field name in the generated tree node structure.
An optional and named field must use the format `field?:name`. An optional and named field must use the format `field?:name`.
Example: Example:
@ -725,7 +725,7 @@ The `$$` symbol accesses the output parser value for this rule.
The above examples demonstrate how the parser values for the rule components The above examples demonstrate how the parser values for the rule components
can be used to produce the parser value for the accepted rule. can be used to produce the parser value for the accepted rule.
Parser rule code blocks are not allowed and not used when AST generation mode Parser rule code blocks are not allowed and not used when tree generation mode
is active. is active.
##> Specifying the parser start rule name - the `start` statement ##> Specifying the parser start rule name - the `start` statement
@ -747,12 +747,12 @@ start Module ModuleItem Statement Expression;
``` ```
When multiple start rules are specified, multiple `p_parse_*()` functions, When multiple start rules are specified, multiple `p_parse_*()` functions,
`p_result_*()`, and `p_free_ast_*()` functions (in AST mode) are generated. `p_result_*()`, and `p_free_tree_*()` functions (in tree mode) are generated.
A default `p_parse()`, `p_result()`, `p_free_ast()` are generated corresponding A default `p_parse()`, `p_result()`, `p_free_tree()` are generated corresponding
to the first start rule. to the first start rule.
Additionally, each start rule causes the generation of another version of each Additionally, each start rule causes the generation of another version of each
of these functions, for example `p_parse_Statement()`, `p_result_Statement()`, of these functions, for example `p_parse_Statement()`, `p_result_Statement()`,
and `p_free_ast_Statement()`. and `p_free_tree_Statement()`.
##> Specifying the parser module name - the `module` statement ##> Specifying the parser module name - the `module` statement
@ -873,17 +873,17 @@ The `p_token_info_t` structure contains the following fields:
* `token` (`p_token_t`) holds the token ID of the lexed token * `token` (`p_token_t`) holds the token ID of the lexed token
* `pvalue` (`p_value_t`) holds the parser value associated with the token. * `pvalue` (`p_value_t`) holds the parser value associated with the token.
### AST Node Types ### Tree Node Types
If AST generation mode is enabled, a structure type for each rule will be If tree generation mode is enabled, a structure type for each rule will be
generated. generated.
The name of the structure type is given by the name of the rule. The name of the structure type is given by the name of the rule.
Additionally a structure type called `Token` is generated to represent an Additionally a structure type called `Token` is generated to represent a
AST node which refers to a raw parser token rather than a composite rule. tree node which refers to a raw parser token rather than a composite rule.
#### AST Node Fields #### Tree Node Fields
All AST nodes have a `position` field specifying the text position of the All tree nodes have a `position` field specifying the text position of the
beginning of the matched token or rule, and an `end_position` field specifying beginning of the matched token or rule, and an `end_position` field specifying
the text position of the end of the matched token or rule. the text position of the end of the matched token or rule.
Each of these fields are instances of the `p_position_t` structure. Each of these fields are instances of the `p_position_t` structure.
@ -902,7 +902,7 @@ A `Token` node has the following additional fields:
* `pvalue` which specifies the parser value for the token. If a lexer user * `pvalue` which specifies the parser value for the token. If a lexer user
code block assigned to `$$`, the assigned value will be stored here. code block assigned to `$$`, the assigned value will be stored here.
AST node structures for rules contain generated fields based on the Tree node structures for rules contain generated fields based on the
right hand side components specified for all rules of a given name. right hand side components specified for all rules of a given name.
In this example: In this example:
@ -925,10 +925,10 @@ The `Items` structure will have fields:
* `pItemsMore` and `pItemsMore2` which point to the parsed `ItemsMore` structure. * `pItemsMore` and `pItemsMore2` which point to the parsed `ItemsMore` structure.
If a rule can be empty (for example in the second `Items` rule above), then If a rule can be empty (for example in the second `Items` rule above), then
an instance of a pointer to that rule's generated AST node will be null if the an instance of a pointer to that rule's generated tree node will be null if the
parser matches the empty rule pattern. parser matches the empty rule pattern.
The non-positional AST node field pointer will not be generated if there are The non-positional tree node field pointer will not be generated if there are
multiple positions in which an instance of the node it points to could be multiple positions in which an instance of the node it points to could be
present. present.
For example, in the below rules: For example, in the below rules:
@ -948,7 +948,7 @@ If the first rule is matched, then `pOne1` and `pTwo2` will be non-null while
If the second rule is matched instead, then the opposite would be the case. If the second rule is matched instead, then the opposite would be the case.
If a field alias is present in a rule definition, an additional field will be If a field alias is present in a rule definition, an additional field will be
generated in the AST node with the field alias name. generated in the tree node with the field alias name.
For example: For example:
``` ```
@ -1079,8 +1079,8 @@ if (p_parse(&context) == P_SUCCESS)
} }
``` ```
If AST generation mode is active, then the `p_result()` function returns a If tree generation mode is active, then the `p_result()` function returns a
`Start *` pointing to the `Start` AST structure. `Start *` pointing to the `Start` tree node structure.
When multiple start rules are specified, a separate result function is generated When multiple start rules are specified, a separate result function is generated
for each which returns the parse result for the corresponding rule. for each which returns the parse result for the corresponding rule.
@ -1171,29 +1171,29 @@ assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u); assert(code_point_length == 4u);
``` ```
### `p_free_ast` ### `p_free_tree`
The `p_free_ast()` function can be used to free the memory used by the AST. The `p_free_tree()` function can be used to free the memory used by the tree.
It should be passed the same value that is returned by `p_result()`. It should be passed the same value that is returned by `p_result()`.
The `p_free_ast()` function is only available for C/C++ output targets. The `p_free_tree()` function is only available for C/C++ output targets.
Note that if any lexer user code block allocates memory to store in a token's Note that if any lexer user code block allocates memory to store in a token's
`pvalue`, in order to properly free this memory a `free_token_node` function `pvalue`, in order to properly free this memory a `free_token_node` function
should be specified in the grammar file. should be specified in the grammar file.
If specified, the `free_token_node` function will be called during the If specified, the `free_token_node` function will be called during the
`p_free_ast()` process to allow user code to free any memory associated with `p_free_tree()` process to allow user code to free any memory associated with
a token node's `pvalue`. a token node's `pvalue`.
When multiple start rules are specified, a separate `p_free_ast` function is When multiple start rules are specified, a separate `p_free_tree` function is
generated for each which frees the AST resulting from parsing the given rule. generated for each which frees the tree resulting from parsing the given rule.
For example, if `Statement` is specified as a start rule: For example, if `Statement` is specified as a start rule:
``` ```
p_free_ast_Statement(statement_ast); p_free_tree_Statement(statement_tree);
``` ```
In this case, Propane will free a `Statement` AST structure returned by the In this case, Propane will free a `Statement` tree structure returned by the
`p_parse_Statement(&context)` function. `p_parse_Statement(&context)` function.
##> Data ##> Data

2
extra/vim/syntax/propane.vim
View File

@ -20,7 +20,7 @@ syn match propaneOperator "->"
syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
syn match propaneFieldOperator ":" contained syn match propaneFieldOperator ":" contained
syn match propaneOperator "?" syn match propaneOperator "?"
syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid syn keyword propaneKeyword drop free_token_node module prefix ptype start token tokenid tree tree_prefix tree_suffix
syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/" syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"

2
lib/propane/generator.rb
View File

@ -298,7 +298,7 @@ class Propane
end end
else else
code = code.gsub(/\$\$/) do |match| code = code.gsub(/\$\$/) do |match|
if @grammar.ast if @grammar.tree
case @language case @language
when "c" when "c"
"out_token_info->pvalue" "out_token_info->pvalue"

58
lib/propane/grammar.rb
View File

@ -5,9 +5,9 @@ class Propane
# Reserve identifiers beginning with a double-underscore for internal use. # Reserve identifiers beginning with a double-underscore for internal use.
IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/ IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
attr_reader :ast attr_reader :tree
attr_reader :ast_prefix attr_reader :tree_prefix
attr_reader :ast_suffix attr_reader :tree_suffix
attr_reader :free_token_node attr_reader :free_token_node
attr_reader :modulename attr_reader :modulename
attr_reader :patterns attr_reader :patterns
@ -30,9 +30,9 @@ class Propane
@input = input.gsub("\r\n", "\n") @input = input.gsub("\r\n", "\n")
@ptypes = {"default" => "void *"} @ptypes = {"default" => "void *"}
@prefix = "p_" @prefix = "p_"
@ast = false @tree = false
@ast_prefix = "" @tree_prefix = ""
@ast_suffix = "" @tree_suffix = ""
@free_token_node = nil @free_token_node = nil
parse_grammar! parse_grammar!
@start_rules << "Start" if @start_rules.empty? @start_rules << "Start" if @start_rules.empty?
@ -62,9 +62,9 @@ class Propane
if parse_white_space! if parse_white_space!
elsif parse_comment_line! elsif parse_comment_line!
elsif @modeline.nil? && parse_mode_label! elsif @modeline.nil? && parse_mode_label!
elsif parse_ast_statement! elsif parse_tree_statement!
elsif parse_ast_prefix_statement! elsif parse_tree_prefix_statement!
elsif parse_ast_suffix_statement! elsif parse_tree_suffix_statement!
elsif parse_free_token_node_statement! elsif parse_free_token_node_statement!
elsif parse_module_statement! elsif parse_module_statement!
elsif parse_ptype_statement! elsif parse_ptype_statement!
@ -98,21 +98,21 @@ class Propane
consume!(/#.*\n/) consume!(/#.*\n/)
end end
def parse_ast_statement! def parse_tree_statement!
if consume!(/ast\s*;/) if consume!(/tree\s*;/)
@ast = true @tree = true
end end
end end
def parse_ast_prefix_statement! def parse_tree_prefix_statement!
if md = consume!(/ast_prefix\s+(\w+)\s*;/) if md = consume!(/tree_prefix\s+(\w+)\s*;/)
@ast_prefix = md[1] @tree_prefix = md[1]
end end
end end
def parse_ast_suffix_statement! def parse_tree_suffix_statement!
if md = consume!(/ast_suffix\s+(\w+)\s*;/) if md = consume!(/tree_suffix\s+(\w+)\s*;/)
@ast_suffix = md[1] @tree_suffix = md[1]
end end
end end
@ -136,8 +136,8 @@ class Propane
if consume!(/ptype\s+/) if consume!(/ptype\s+/)
name = "default" name = "default"
if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/) if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
if @ast if @tree
raise Error.new("Multiple ptypes are unsupported in AST mode") raise Error.new("Multiple ptypes are unsupported in tree mode")
end end
name = md[1] name = md[1]
end end
@ -151,8 +151,8 @@ class Propane
md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name") md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
name = md[1] name = md[1]
if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/) if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
if @ast if @tree
raise Error.new("Multiple ptypes are unsupported in AST mode") raise Error.new("Multiple ptypes are unsupported in tree mode")
end end
ptypename = md[1] ptypename = md[1]
end end
@ -175,8 +175,8 @@ class Propane
md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name") md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
name = md[1] name = md[1]
if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/) if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
if @ast if @tree
raise Error.new("Multiple ptypes are unsupported in AST mode") raise Error.new("Multiple ptypes are unsupported in tree mode")
end end
ptypename = md[1] ptypename = md[1]
end end
@ -205,12 +205,12 @@ class Propane
def parse_rule_statement! def parse_rule_statement!
if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/) if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
rule_name, ptypename = *md[1, 2] rule_name, ptypename = *md[1, 2]
if @ast && ptypename if @tree && ptypename
raise Error.new("Multiple ptypes are unsupported in AST mode") raise Error.new("Multiple ptypes are unsupported in tree mode")
end end
md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list") md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
components = md[1].strip.split(/\s+/) components = md[1].strip.split(/\s+/)
if @ast if @tree
consume!(/;/, "expected `;'") consume!(/;/, "expected `;'")
else else
unless code = parse_code_block! unless code = parse_code_block!
@ -227,8 +227,8 @@ class Propane
if pattern = parse_pattern! if pattern = parse_pattern!
consume!(/\s+/) consume!(/\s+/)
if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/) if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
if @ast if @tree
raise Error.new("Multiple ptypes are unsupported in AST mode") raise Error.new("Multiple ptypes are unsupported in tree mode")
end end
ptypename = md[1] ptypename = md[1]
end end

2
lib/propane/rule.rb
View File

@ -36,7 +36,7 @@ class Propane
# @return [Array<Integer>] # @return [Array<Integer>]
# Map this rule's components to their positions in the parent RuleSet's # Map this rule's components to their positions in the parent RuleSet's
# node field pointer array. This is used for AST construction. # node field pointer array. This is used for tree construction.
attr_accessor :rule_set_node_field_index_map attr_accessor :rule_set_node_field_index_map
# Construct a Rule. # Construct a Rule.

46
lib/propane/rule_set.rb
View File

@ -4,8 +4,8 @@ class Propane
class RuleSet class RuleSet
# @return [Array<Hash>] # @return [Array<Hash>]
# AST fields. # tree fields.
attr_reader :ast_fields attr_reader :tree_fields
# @return [Integer] # @return [Integer]
# ID of the RuleSet. # ID of the RuleSet.
@ -100,28 +100,28 @@ class Propane
# Finalize a RuleSet after adding all Rules to it. # Finalize a RuleSet after adding all Rules to it.
def finalize(grammar) def finalize(grammar)
if grammar.ast if grammar.tree
build_ast_fields(grammar) build_tree_fields(grammar)
end end
end end
private private
# Build the set of AST fields for this RuleSet. # Build the set of tree fields for this RuleSet.
# #
# This is an Array of Hashes. Each entry in the Array corresponds to a # This is an Array of Hashes. Each entry in the Array corresponds to a
# field location in the AST node. The entry is a Hash. It could have one or # field location in the tree node. The entry is a Hash. It could have one or
# two keys. It will always have the field name with a positional suffix as # two keys. It will always have the field name with a positional suffix as
# a key. It may also have the field name without the positional suffix if # a key. It may also have the field name without the positional suffix if
# that field only exists in one position across all Rules in the RuleSet. # that field only exists in one position across all Rules in the RuleSet.
# #
# @return [void] # @return [void]
def build_ast_fields(grammar) def build_tree_fields(grammar)
field_ast_node_indexes = {} field_tree_node_indexes = {}
field_indexes_across_all_rules = {} field_indexes_across_all_rules = {}
# Stores the index into @ast_fields by field alias name. # Stores the index into @tree_fields by field alias name.
field_aliases = {} field_aliases = {}
@ast_fields = [] @tree_fields = []
@rules.each do |rule| @rules.each do |rule|
rule.components.each_with_index do |component, i| rule.components.each_with_index do |component, i|
if component.is_a?(RuleSet) && component.optional? if component.is_a?(RuleSet) && component.optional?
@ -132,25 +132,25 @@ class Propane
else else
node_name = component.name node_name = component.name
end end
struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}" struct_name = "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
field_name = "p#{node_name}#{i + 1}" field_name = "p#{node_name}#{i + 1}"
unless field_ast_node_indexes[field_name] unless field_tree_node_indexes[field_name]
field_ast_node_indexes[field_name] = @ast_fields.size field_tree_node_indexes[field_name] = @tree_fields.size
@ast_fields << {field_name => struct_name} @tree_fields << {field_name => struct_name}
end end
rule.aliases.each do |alias_name, index| rule.aliases.each do |alias_name, index|
if index == i if index == i
alias_ast_fields_index = field_ast_node_indexes[field_name] alias_tree_fields_index = field_tree_node_indexes[field_name]
if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index if field_aliases[alias_name] && field_aliases[alias_name] != alias_tree_fields_index
raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}") raise Error.new("Error: conflicting tree node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
end end
field_aliases[alias_name] = alias_ast_fields_index field_aliases[alias_name] = alias_tree_fields_index
@ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1] @tree_fields[alias_tree_fields_index][alias_name] = @tree_fields[alias_tree_fields_index].first[1]
end end
end end
field_indexes_across_all_rules[node_name] ||= Set.new field_indexes_across_all_rules[node_name] ||= Set.new
field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name] field_indexes_across_all_rules[node_name] << field_tree_node_indexes[field_name]
rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name] rule.rule_set_node_field_index_map[i] = field_tree_node_indexes[field_name]
end end
end end
field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules| field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
@ -158,8 +158,8 @@ class Propane
# If this field was only seen in one position across all rules, # If this field was only seen in one position across all rules,
# then add an alias to the positional field name that does not # then add an alias to the positional field name that does not
# include the position. # include the position.
@ast_fields[indexes_across_all_rules.first]["p#{node_name}"] = @tree_fields[indexes_across_all_rules.first]["p#{node_name}"] =
"#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}" "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
end end
end end
end end

90
spec/propane_spec.rb
View File

@ -245,20 +245,20 @@ EOF
expect(results.status).to_not eq 0 expect(results.status).to_not eq 0
end end
it "errors when an alias is in different positions for different rules in a rule set when AST mode is enabled" do it "errors when an alias is in different positions for different rules in a rule set when tree mode is enabled" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token a; token a;
token b; token b;
Start -> a:foo b; Start -> a:foo b;
Start -> b b:foo; Start -> b b:foo;
EOF EOF
results = run_propane(extra_args: %w[-w], capture: true) results = run_propane(extra_args: %w[-w], capture: true)
expect(results.stderr).to match %r{Error: conflicting AST node field positions for alias `foo`} expect(results.stderr).to match %r{Error: conflicting tree node field positions for alias `foo`}
expect(results.status).to_not eq 0 expect(results.status).to_not eq 0
end end
it "does not error when an alias is in different positions for different rules in a rule set when AST mode is not enabled" do it "does not error when an alias is in different positions for different rules in a rule set when tree mode is not enabled" do
write_grammar <<EOF write_grammar <<EOF
token a; token a;
token b; token b;
@ -996,9 +996,9 @@ EOF
run_propane(language: language) run_propane(language: language)
end end
it "generates an AST" do it "generates a tree" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
ptype int; ptype int;
@ -1032,17 +1032,17 @@ One -> one;
Two -> two; Two -> two;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast.#{language}", language: language) compile("spec/test_tree.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "supports AST node prefix and suffix" do it "supports tree node prefix and suffix" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
ast_prefix P ; tree_prefix P ;
ast_suffix S; tree_suffix S;
ptype int; ptype int;
@ -1076,7 +1076,7 @@ One -> one;
Two -> two; Two -> two;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_ps.#{language}", language: language) compile("spec/test_tree_ps.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
@ -1092,15 +1092,15 @@ EOF
compile("spec/test_start_rule.#{language}", language: language) compile("spec/test_start_rule.#{language}", language: language)
end end
it "allows specifying a different start rule with AST generation" do it "allows specifying a different start rule with tree generation" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token hi; token hi;
start Top; start Top;
Top -> hi; Top -> hi;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_start_rule_ast.#{language}", language: language) compile("spec/test_start_rule_tree.#{language}", language: language)
end end
it "allows marking a rule component as optional" do it "allows marking a rule component as optional" do
@ -1167,10 +1167,10 @@ EOF
]) ])
end end
it "allows marking a rule component as optional in AST generation mode" do it "allows marking a rule component as optional in tree generation mode" do
if language == "d" if language == "d"
write_grammar <<EOF write_grammar <<EOF
ast; tree;
<< <<
import std.stdio; import std.stdio;
@ -1186,7 +1186,7 @@ R -> d c;
EOF EOF
else else
write_grammar <<EOF write_grammar <<EOF
ast; tree;
<< <<
#include <stdio.h> #include <stdio.h>
@ -1202,16 +1202,16 @@ R -> d c;
EOF EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_optional_rule_component_ast.#{language}", language: language) compile("spec/test_optional_rule_component_tree.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "allows naming an optional rule component in AST generation mode" do it "allows naming an optional rule component in tree generation mode" do
if language == "d" if language == "d"
write_grammar <<EOF write_grammar <<EOF
ast; tree;
<< <<
import std.stdio; import std.stdio;
@ -1227,7 +1227,7 @@ R -> d c;
EOF EOF
else else
write_grammar <<EOF write_grammar <<EOF
ast; tree;
<< <<
#include <stdio.h> #include <stdio.h>
@ -1243,15 +1243,15 @@ R -> d c;
EOF EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_named_optional_rule_component_ast.#{language}", language: language) compile("spec/test_named_optional_rule_component_tree.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "stores token and rule positions in AST nodes" do it "stores token and rule positions in tree nodes" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token a; token a;
token bb; token bb;
@ -1263,7 +1263,7 @@ T -> bb;
T -> c; T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_token_positions.#{language}", language: language) compile("spec/test_tree_token_positions.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
@ -1271,7 +1271,7 @@ EOF
it "stores invalid positions for empty rule matches" do it "stores invalid positions for empty rule matches" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token a; token a;
token bb; token bb;
@ -1283,15 +1283,15 @@ T -> a A;
A -> bb? c?; A -> bb? c?;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_invalid_positions.#{language}", language: language) compile("spec/test_tree_invalid_positions.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "allows specifying field aliases in AST mode" do it "allows specifying field aliases in tree mode" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token a; token a;
token b; token b;
@ -1303,15 +1303,15 @@ T -> b;
T -> c; T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language) compile("spec/test_tree_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "aliases the correct field when multiple rules are in a rule set in AST mode" do it "aliases the correct field when multiple rules are in a rule set in tree mode" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
token a; token a;
token b; token b;
@ -1326,13 +1326,13 @@ T -> b;
T -> c; T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language) compile("spec/test_tree_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "allows specifying field aliases when AST mode is not enabled" do it "allows specifying field aliases when tree mode is not enabled" do
if language == "d" if language == "d"
write_grammar <<EOF write_grammar <<EOF
<< <<
@ -1378,7 +1378,7 @@ EOF
expect(results.stdout).to match /first is foo1.*second is bar2/m expect(results.stdout).to match /first is foo1.*second is bar2/m
end end
it "aliases the correct field when multiple rules are in a rule set when AST mode is not enabled" do it "aliases the correct field when multiple rules are in a rule set when tree mode is not enabled" do
if language == "d" if language == "d"
write_grammar <<EOF write_grammar <<EOF
<< <<
@ -1430,11 +1430,11 @@ EOF
expect(results.stdout).to match /first is foo1.*second is bar2/m expect(results.stdout).to match /first is foo1.*second is bar2/m
end end
it "does not free memory allocated for AST nodes" do it "does not free memory allocated for tree nodes" do
ext = language == "cpp" ? "c" : language ext = language == "cpp" ? "c" : language
write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane")) write_grammar(File.read("spec/tree_node_memory_remains.#{ext}.propane"))
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_node_memory_remains.#{language}", language: language) compile("spec/test_tree_node_memory_remains.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
@ -1460,9 +1460,9 @@ EOF
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "allows multiple starting rules in AST mode" do it "allows multiple starting rules in tree mode" do
write_grammar <<EOF write_grammar <<EOF
ast; tree;
ptype int; ptype int;
token a << $$ = 1; >> token a << $$ = 1; >>
token b << $$ = 2; >> token b << $$ = 2; >>
@ -1475,14 +1475,14 @@ Bs -> b:b Bs:bs;
start Start R Bs; start Start R Bs;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_starting_rules_ast.#{language}", language: language) compile("spec/test_starting_rules_tree.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
if %w[c cpp].include?(language) if %w[c cpp].include?(language)
it "allows a user function to free token node memory in AST mode" do it "allows a user function to free token node memory in tree mode" do
write_grammar <<EOF write_grammar <<EOF
<< <<
static void free_token(Token * token) static void free_token(Token * token)
@ -1490,7 +1490,7 @@ static void free_token(Token * token)
free(token->pvalue); free(token->pvalue);
} }
>> >>
ast; tree;
free_token_node free_token; free_token_node free_token;
ptype int *; ptype int *;
token a << token a <<
@ -1504,7 +1504,7 @@ token b <<
Start -> a:a b:b; Start -> a:a b:b;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_free_ast_token_node_memory.#{language}", language: language) compile("spec/test_free_tree_token_node_memory.#{language}", language: language)
results = run_test(language: language) results = run_test(language: language)
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0

2
spec/test_free_ast_token_node_memory.c → spec/test_free_tree_token_node_memory.c
View File

@ -15,5 +15,5 @@ int main()
assert(start->b != NULL); assert(start->b != NULL);
assert(*start->b->pvalue == 2); assert(*start->b->pvalue == 2);
p_free_ast(start); p_free_tree(start);
} }

6
spec/test_named_optional_rule_component_ast.c → spec/test_named_optional_rule_component_tree.c
View File

@ -17,7 +17,7 @@ int main()
assert(start->pR == NULL); assert(start->pR == NULL);
assert(start->r == NULL); assert(start->r == NULL);
p_free_ast(start); p_free_tree(start);
input = "abcd"; input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -33,7 +33,7 @@ int main()
assert(start->pR == start->r); assert(start->pR == start->r);
assert_eq(TOKEN_c, start->pR->pToken1->token); assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start); p_free_tree(start);
input = "bdc"; input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -44,7 +44,7 @@ int main()
assert(start->r != NULL); assert(start->r != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token); assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_named_optional_rule_component_ast.d → spec/test_named_optional_rule_component_tree.d
View File

6
spec/test_optional_rule_component_ast.c → spec/test_optional_rule_component_tree.c
View File

@ -16,7 +16,7 @@ int main()
assert(start->pR3 == NULL); assert(start->pR3 == NULL);
assert(start->pR == NULL); assert(start->pR == NULL);
p_free_ast(start); p_free_tree(start);
input = "abcd"; input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -30,7 +30,7 @@ int main()
assert(start->pR == start->pR3); assert(start->pR == start->pR3);
assert_eq(TOKEN_c, start->pR->pToken1->token); assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start); p_free_tree(start);
input = "bdc"; input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -41,7 +41,7 @@ int main()
assert(start->pR != NULL); assert(start->pR != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token); assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_optional_rule_component_ast.d → spec/test_optional_rule_component_tree.d
View File

2
spec/test_start_rule_ast.c → spec/test_start_rule_tree.c
View File

@ -13,7 +13,7 @@ int main()
assert(top->pToken != NULL); assert(top->pToken != NULL);
assert_eq(TOKEN_hi, top->pToken->token); assert_eq(TOKEN_hi, top->pToken->token);
p_free_ast(top); p_free_tree(top);
return 0; return 0;
} }

0
spec/test_start_rule_ast.d → spec/test_start_rule_tree.d
View File

6
spec/test_starting_rules_ast.c → spec/test_starting_rules_tree.c
View File

@ -15,7 +15,7 @@ int main()
assert_not_null(start->bs->bs->b); assert_not_null(start->bs->bs->b);
assert_not_null(start->bs->bs->bs->b); assert_not_null(start->bs->bs->bs->b);
assert_not_null(start->bs->bs->bs->bs->b); assert_not_null(start->bs->bs->bs->bs->b);
p_free_ast(start); p_free_tree(start);
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_Bs(&context) == P_SUCCESS); assert(p_parse_Bs(&context) == P_SUCCESS);
@ -24,14 +24,14 @@ int main()
assert_not_null(bs->bs->b); assert_not_null(bs->bs->b);
assert_not_null(bs->bs->bs->b); assert_not_null(bs->bs->bs->b);
assert_not_null(bs->bs->bs->bs->b); assert_not_null(bs->bs->bs->bs->b);
p_free_ast_Bs(bs); p_free_tree_Bs(bs);
input = "c"; input = "c";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_R(&context) == P_SUCCESS); assert(p_parse_R(&context) == P_SUCCESS);
R * r = p_result_R(&context); R * r = p_result_R(&context);
assert_not_null(r->c); assert_not_null(r->c);
p_free_ast_R(r); p_free_tree_R(r);
return 0; return 0;
} }

0
spec/test_starting_rules_ast.d → spec/test_starting_rules_tree.d
View File

6
spec/test_ast.c → spec/test_tree.c
View File

@ -33,7 +33,7 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue); assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL); assert(itemsmore->pItemsMore == NULL);
p_free_ast(start); p_free_tree(start);
input = ""; input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -41,7 +41,7 @@ int main()
start = p_result(&context); start = p_result(&context);
assert(start->pItems == NULL); assert(start->pItems == NULL);
p_free_ast(start); p_free_tree(start);
input = "2 1"; input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -55,7 +55,7 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL); assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL); assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_ast.d → spec/test_tree.d
View File

2
spec/test_ast_field_aliases.c → spec/test_tree_field_aliases.c
View File

@ -15,7 +15,7 @@ int main()
assert_eq(TOKEN_b, start->second->pToken->token); assert_eq(TOKEN_b, start->second->pToken->token);
assert_eq(TOKEN_c, start->third->pToken->token); assert_eq(TOKEN_c, start->third->pToken->token);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_ast_field_aliases.d → spec/test_tree_field_aliases.d
View File

8
spec/test_ast_invalid_positions.c → spec/test_tree_invalid_positions.c
View File

@ -30,7 +30,7 @@ int main()
assert_eq(3, start->end_position.row); assert_eq(3, start->end_position.row);
assert_eq(8, start->end_position.col); assert_eq(8, start->end_position.col);
p_free_ast(start); p_free_tree(start);
input = "a\nbb"; input = "a\nbb";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -56,7 +56,7 @@ int main()
assert_eq(2, start->end_position.row); assert_eq(2, start->end_position.row);
assert_eq(2, start->end_position.col); assert_eq(2, start->end_position.col);
p_free_ast(start); p_free_tree(start);
input = "a\nc\nc"; input = "a\nc\nc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -82,7 +82,7 @@ int main()
assert_eq(3, start->end_position.row); assert_eq(3, start->end_position.row);
assert_eq(1, start->end_position.col); assert_eq(1, start->end_position.col);
p_free_ast(start); p_free_tree(start);
input = "a"; input = "a";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -104,7 +104,7 @@ int main()
assert_eq(1, start->end_position.row); assert_eq(1, start->end_position.row);
assert_eq(1, start->end_position.col); assert_eq(1, start->end_position.col);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_ast_invalid_positions.d → spec/test_tree_invalid_positions.d
View File

2
spec/test_ast_node_memory_remains.c → spec/test_tree_node_memory_remains.c
View File

@ -412,7 +412,7 @@ int main(int argc, char * argv[])
} }
free(pfds); free(pfds);
p_free_ast(pmod); p_free_tree(pmod);
return 0; return 0;
} }

0
spec/test_ast_node_memory_remains.d → spec/test_tree_node_memory_remains.d
View File

6
spec/test_ast_ps.c → spec/test_tree_ps.c
View File

@ -33,7 +33,7 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue); assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL); assert(itemsmore->pItemsMore == NULL);
p_free_ast(start); p_free_tree(start);
input = ""; input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -41,7 +41,7 @@ int main()
start = p_result(&context); start = p_result(&context);
assert(start->pItems == NULL); assert(start->pItems == NULL);
p_free_ast(start); p_free_tree(start);
input = "2 1"; input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -55,7 +55,7 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL); assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL); assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_ast_ps.d → spec/test_tree_ps.d
View File

4
spec/test_ast_token_positions.c → spec/test_tree_token_positions.c
View File

@ -43,7 +43,7 @@ int main()
assert_eq(1, start->end_position.row); assert_eq(1, start->end_position.row);
assert_eq(6, start->end_position.col); assert_eq(6, start->end_position.col);
p_free_ast(start); p_free_tree(start);
input = "\n\n bb\nc\ncc\n\n a"; input = "\n\n bb\nc\ncc\n\n a";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
@ -82,7 +82,7 @@ int main()
assert_eq(7, start->end_position.row); assert_eq(7, start->end_position.row);
assert_eq(6, start->end_position.col); assert_eq(6, start->end_position.col);
p_free_ast(start); p_free_tree(start);
return 0; return 0;
} }

0
spec/test_ast_token_positions.d → spec/test_tree_token_positions.d
View File

4
spec/ast_node_memory_remains.c.propane → spec/tree_node_memory_remains.c.propane
View File

@ -1,5 +1,5 @@
ast; tree;
ast_prefix P; tree_prefix P;
<<header <<header
#include <stdio.h> #include <stdio.h>

4
spec/ast_node_memory_remains.d.propane → spec/tree_node_memory_remains.d.propane
View File

@ -1,5 +1,5 @@
ast; tree;
ast_prefix P; tree_prefix P;
<< <<
import std.bigint; import std.bigint;