josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Compare commits

base: josh:multiple-start-rules-wip
Branches Tags
josh:master
josh:multiple-start-rules-wip
josh:wip
josh:c-backend
josh:sumtype
josh:v1.5.1
josh:v1.5.0
josh:v1.4.0
josh:v1.3.0
josh:v1.2.0
josh:v1.1.0
josh:v1.0.0
..
compare: josh:master
Branches Tags
josh:multiple-start-rules-wip
josh:master
josh:wip
josh:c-backend
josh:sumtype
josh:v1.5.1
josh:v1.5.0
josh:v1.4.0
josh:v1.3.0
josh:v1.2.0
josh:v1.1.0
josh:v1.0.0

No commits in common. "multiple-start-rules-wip" and "master" have entirely different histories.
multiple-s ... master

24 changed files with 86 additions and 636 deletions
Show Stats Expand all files Collapse all files

109
assets/parser.c.erb
View File

@ -695,13 +695,11 @@ typedef struct
} state_value_t; } state_value_t;
/** Common AST node structure. */ /** Common AST node structure. */
typedef struct ASTNode_s typedef struct
{ {
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields; void * fields[];
uint8_t is_token;
struct ASTNode_s * fields[];
} ASTNode; } ASTNode;
/** Parser shift table. */ /** Parser shift table. */
@ -924,8 +922,6 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
* *
* @param context * @param context
* Lexer/parser context structure. * Lexer/parser context structure.
* @start_state_id
* ID of the state in which to start.
* *
* @retval P_SUCCESS * @retval P_SUCCESS
* The parser successfully matched the input text. The parse result value * The parser successfully matched the input text. The parse result value
@ -938,7 +934,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
* @reval P_UNEXPECTED_INPUT * @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern. * Input text does not match any lexer pattern.
*/ */
static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id) size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{ {
<%= @grammar.prefix %>token_info_t token_info; <%= @grammar.prefix %>token_info_t token_info;
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID; <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
@ -951,7 +947,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
<% end %> <% end %>
state_values_stack_init(&statevalues); state_values_stack_init(&statevalues);
state_values_stack_push(&statevalues); state_values_stack_push(&statevalues);
state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
size_t result; size_t result;
for (;;) for (;;)
{ {
@ -977,7 +972,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{ {
/* Successful parse. */ /* Successful parse. */
<% if @grammar.ast %> <% if @grammar.ast %>
context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node; context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
<% else %> <% else %>
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue; context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
<% end %> <% end %>
@ -997,8 +992,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>)); <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
token_ast_node->position = token_info.position; token_ast_node->position = token_info.position;
token_ast_node->end_position = token_info.end_position; token_ast_node->end_position = token_info.end_position;
token_ast_node->n_fields = 0u;
token_ast_node->is_token = 1u;
token_ast_node->token = token; token_ast_node->token = token;
token_ast_node->pvalue = token_info.pvalue; token_ast_node->pvalue = token_info.pvalue;
state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node; state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@ -1035,30 +1028,31 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
else if (parser_reduce_table[reduce_index].n_states > 0) else if (parser_reduce_table[reduce_index].n_states > 0)
{ {
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size; size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *); ASTNode * node = (ASTNode *)malloc(sizeof(ASTNode) + n_fields * sizeof(void *));
ASTNode * node = (ASTNode *)malloc(bytes);
memset(node, 0, bytes);
node->position = INVALID_POSITION; node->position = INVALID_POSITION;
node->end_position = INVALID_POSITION; node->end_position = INVALID_POSITION;
node->n_fields = n_fields; for (size_t i = 0; i < n_fields; i++)
{
node->fields[i] = NULL;
}
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL) if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
{ {
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++) for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{ {
node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node; node->fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
} }
} }
else else
{ {
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++) for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{ {
node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node; node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
} }
} }
bool position_found = false; bool position_found = false;
for (size_t i = 0; i < n_fields; i++) for (size_t i = 0; i < n_fields; i++)
{ {
ASTNode * child = node->fields[i]; ASTNode * child = (ASTNode *)node->fields[i];
if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position)) if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
{ {
if (!position_found) if (!position_found)
@ -1080,7 +1074,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2)); memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED) if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
{ {
state_values_stack_free(&statevalues);
return P_USER_TERMINATED; return P_USER_TERMINATED;
} }
reduced_parser_value = reduced_parser_value2; reduced_parser_value = reduced_parser_value2;
@ -1104,19 +1097,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
return result; return result;
} }
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, 0u);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, <%= i %>u);
}
<% end %>
/** /**
* Get the parse result value. * Get the parse result value.
* *
@ -1126,28 +1106,17 @@ size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>conte
* @return Parse result value. * @return Parse result value.
*/ */
<% if @grammar.ast %> <% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *) context->parse_result;
}
<% end %>
<% else %> <% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<% end %>
{ {
<% if @grammar.ast %>
return context->parse_result;
<% else %>
return context->parse_result.v_<%= start_rule_type[0] %>; return context->parse_result.v_<%= start_rule_type[0] %>;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return context->parse_result.v_<%= start_rule_type(i)[0] %>;
}
<% end %>
<% end %> <% end %>
}
/** /**
* Get the current text input position. * Get the current text input position.
@ -1184,45 +1153,3 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
{ {
return context->token; return context->token;
} }
<% if @grammar.ast %>
static void free_ast_node(ASTNode * node)
{
if (node->is_token)
{
<% if @grammar.free_token_node %>
<%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
<% end %>
/* TODO: free value_t */
}
else if (node->n_fields > 0u)
{
for (size_t i = 0u; i < node->n_fields; i++)
{
if (node->fields[i] != NULL)
{
free_ast_node(node->fields[i]);
}
}
}
free(node);
}
/**
* Free all AST node memory.
*/
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast)
{
free_ast_node((ASTNode *)ast);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
/**
* Free all AST node memory.
*/
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast)
{
free_ast_node((ASTNode *)ast);
}
<% end %>
<% end %>

45
assets/parser.d.erb
View File

@ -173,7 +173,7 @@ public struct <%= @grammar.prefix %>context_t
/** Parse result value. */ /** Parse result value. */
<% if @grammar.ast %> <% if @grammar.ast %>
void * parse_result; <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
<% else %> <% else %>
<%= @grammar.prefix %>value_t parse_result; <%= @grammar.prefix %>value_t parse_result;
<% end %> <% end %>
@ -985,8 +985,6 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
* *
* @param context * @param context
* Lexer/parser context structure. * Lexer/parser context structure.
* @start_state_id
* ID of the state in which to start.
* *
* @retval P_SUCCESS * @retval P_SUCCESS
* The parser successfully matched the input text. The parse result value * The parser successfully matched the input text. The parse result value
@ -999,12 +997,11 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
* @reval P_UNEXPECTED_INPUT * @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern. * Input text does not match any lexer pattern.
*/ */
private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id) public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{ {
<%= @grammar.prefix %>token_info_t token_info; <%= @grammar.prefix %>token_info_t token_info;
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID; <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
state_value_t[] statevalues = new state_value_t[](1); state_value_t[] statevalues = new state_value_t[](1);
statevalues[0].state_id = start_state_id;
size_t reduced_rule_set = INVALID_ID; size_t reduced_rule_set = INVALID_ID;
<% if @grammar.ast %> <% if @grammar.ast %>
void * reduced_parser_node; void * reduced_parser_node;
@ -1034,7 +1031,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{ {
/* Successful parse. */ /* Successful parse. */
<% if @grammar.ast %> <% if @grammar.ast %>
context.parse_result = statevalues[$-1].ast_node; context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
<% else %> <% else %>
context.parse_result = statevalues[$-1].pvalue; context.parse_result = statevalues[$-1].pvalue;
<% end %> <% end %>
@ -1150,19 +1147,6 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
} }
} }
public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, 0u);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, <%= i %>u);
}
<% end %>
/** /**
* Get the parse result value. * Get the parse result value.
* *
@ -1172,28 +1156,17 @@ public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix
* @return Parse result value. * @return Parse result value.
*/ */
<% if @grammar.ast %> <% if @grammar.ast %>
public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *)context.parse_result;
}
<% end %>
<% else %> <% else %>
public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context) public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<% end %>
{ {
<% if @grammar.ast %>
return context.parse_result;
<% else %>
return context.parse_result.v_<%= start_rule_type[0] %>; return context.parse_result.v_<%= start_rule_type[0] %>;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return context.parse_result.v_<%= start_rule_type(i)[0] %>;
}
<% end %>
<% end %> <% end %>
}
/** /**
* Get the current text input position. * Get the current text input position.

27
assets/parser.h.erb
View File

@ -75,11 +75,9 @@ typedef union
/** AST node types. @{ */ /** AST node types. @{ */
typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
{ {
<% # ASTNode fields must be present in the same order here. # %> /* ASTNode fields must be present in the same order here. */
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
<%= @grammar.prefix %>token_t token; <%= @grammar.prefix %>token_t token;
<%= @grammar.prefix %>value_t pvalue; <%= @grammar.prefix %>value_t pvalue;
} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>; } <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
@ -95,11 +93,8 @@ struct <%= name %>;
<% next if rule_set.optional? %> <% next if rule_set.optional? %>
typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %> typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
{ {
<% # ASTNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position; <%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position; <%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
<% rule_set.ast_fields.each do |fields| %> <% rule_set.ast_fields.each do |fields| %>
union union
{ {
@ -162,7 +157,7 @@ typedef struct
/** Parse result value. */ /** Parse result value. */
<% if @grammar.ast %> <% if @grammar.ast %>
void * parse_result; <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
<% else %> <% else %>
<%= @grammar.prefix %>value_t parse_result; <%= @grammar.prefix %>value_t parse_result;
<% end %> <% end %>
@ -189,27 +184,11 @@ size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t inp
size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info); size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context); size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% if @grammar.ast %> <% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context); <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% else %> <% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context); <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% end %>
<% if @grammar.ast %>
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast);
<% end %>
<% end %> <% end %>
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context); <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);

98
doc/user_guide.md
View File

@ -326,36 +326,6 @@ assert(itemsmore.pItem.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem.pToken1 !is null); assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
``` ```
## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
If user lexer code block allocates memory to store in a token node's `pvalue`,
the `free_token_node` grammar statement can be used to specify the name of a
function which will be called during the `p_free_ast()` call to free the memory
associated with a token node.
Example:
```
<<
static void free_token(Token * token)
{
free(token->pvalue);
}
>>
ast;
free_token_node free_token;
ptype int *;
token a <<
$$ = (int *)malloc(sizeof(int));
*$$ = 1;
>>
token b <<
$$ = (int *)malloc(sizeof(int));
*$$ = 2;
>>
Start -> a:a b:b;
```
##> Specifying tokens - the `token` statement ##> Specifying tokens - the `token` statement
The `token` statement allows defining a lexer token and a pattern to match that The `token` statement allows defining a lexer token and a pattern to match that
@ -738,22 +708,6 @@ Example:
start MyStartRule; start MyStartRule;
``` ```
Multiple start rules can be specified, either with multiple `start` statements
or one `start` statement listing multiple start rules.
Example:
```
start Module ModuleItem Statement Expression;
```
When multiple start rules are specified, multiple `p_parse_*()` functions,
`p_result_*()`, and `p_free_ast_*()` functions (in AST mode) are generated.
A default `p_parse()`, `p_result()`, `p_free_ast()` are generated corresponding
to the first start rule.
Additionally, each start rule causes the generation of another version of each
of these functions, for example `p_parse_Statement()`, `p_result_Statement()`,
and `p_free_ast_Statement()`.
##> Specifying the parser module name - the `module` statement ##> Specifying the parser module name - the `module` statement
The `module` statement can be used to specify the module name for a generated The `module` statement can be used to specify the module name for a generated
@ -1034,16 +988,6 @@ p_context_init(&context, input, input_length);
size_t result = p_parse(&context); size_t result = p_parse(&context);
``` ```
When multiple start rules are specified, a separate parse function is generated
for each which starts parsing at the given rule.
For example, if `Statement` is specified as a start rule:
```
size_t result = p_parse_Statement(&context);
```
In this case, the parser will start parsing with the `Statement` rule.
### `p_position_valid` ### `p_position_valid`
The `p_position_valid()` function is only generated for C targets. The `p_position_valid()` function is only generated for C targets.
@ -1082,23 +1026,6 @@ if (p_parse(&context) == P_SUCCESS)
If AST generation mode is active, then the `p_result()` function returns a If AST generation mode is active, then the `p_result()` function returns a
`Start *` pointing to the `Start` AST structure. `Start *` pointing to the `Start` AST structure.
When multiple start rules are specified, a separate result function is generated
for each which returns the parse result for the corresponding rule.
For example, if `Statement` is specified as a start rule:
```
p_context_t context;
p_context_init(&context, input, input_length);
size_t result = p_parse(&context);
if (p_parse_Statement(&context) == P_SUCCESS)
{
result = p_result_Statement(&context);
}
```
In this case, the parser will start parsing with the `Statement` rule and the
parse result from the `Statement` rule will be returned.
### `p_position` ### `p_position`
The `p_position()` function can be used to retrieve the parser position where The `p_position()` function can be used to retrieve the parser position where
@ -1171,31 +1098,6 @@ assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u); assert(code_point_length == 4u);
``` ```
### `p_free_ast`
The `p_free_ast()` function can be used to free the memory used by the AST.
It should be passed the same value that is returned by `p_result()`.
The `p_free_ast()` function is only available for C/C++ output targets.
Note that if any lexer user code block allocates memory to store in a token's
`pvalue`, in order to properly free this memory a `free_token_node` function
should be specified in the grammar file.
If specified, the `free_token_node` function will be called during the
`p_free_ast()` process to allow user code to free any memory associated with
a token node's `pvalue`.
When multiple start rules are specified, a separate `p_free_ast` function is
generated for each which frees the AST resulting from parsing the given rule.
For example, if `Statement` is specified as a start rule:
```
p_free_ast_Statement(statement_ast);
```
In this case, Propane will free a `Statement` AST structure returned by the
`p_parse_Statement(&context)` function.
##> Data ##> Data
### `p_token_names` ### `p_token_names`

17
lib/propane/generator.rb
View File

@ -71,15 +71,12 @@ class Propane
end end
tokens_by_name[token.name] = token tokens_by_name[token.name] = token
end end
# Create real start rule(s). # Check for user start rule.
real_start_rules = @grammar.start_rules.map do |start_rule| unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
unless @grammar.rules.find {|rule| rule.name == start_rule} raise Error.new("Start rule `#{@grammar.start_rule}` not found")
raise Error.new("Start rule `#{start_rule}` not found")
end end
Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil) # Add "real" start rule.
end @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
# Add real start rules before user-given rules.
@grammar.rules = real_start_rules + @grammar.rules
# Generate and add rules for optional components. # Generate and add rules for optional components.
generate_optional_component_rules!(tokens_by_name) generate_optional_component_rules!(tokens_by_name)
# Build rule sets. # Build rule sets.
@ -335,9 +332,9 @@ class Propane
# #
# @return [Array<String>] # @return [Array<String>]
# Start rule parser value type name and type string. # Start rule parser value type name and type string.
def start_rule_type(start_rule_index = 0) def start_rule_type
start_rule = @grammar.rules.find do |rule| start_rule = @grammar.rules.find do |rule|
rule.name == @grammar.start_rules[start_rule_index] rule.name == @grammar.start_rule
end end
[start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]] [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
end end

23
lib/propane/grammar.rb
View File

@ -8,11 +8,10 @@ class Propane
attr_reader :ast attr_reader :ast
attr_reader :ast_prefix attr_reader :ast_prefix
attr_reader :ast_suffix attr_reader :ast_suffix
attr_reader :free_token_node
attr_reader :modulename attr_reader :modulename
attr_reader :patterns attr_reader :patterns
attr_accessor :rules attr_reader :rules
attr_reader :start_rules attr_reader :start_rule
attr_reader :tokens attr_reader :tokens
attr_reader :code_blocks attr_reader :code_blocks
attr_reader :ptypes attr_reader :ptypes
@ -20,7 +19,7 @@ class Propane
def initialize(input) def initialize(input)
@patterns = [] @patterns = []
@start_rules = [] @start_rule = "Start"
@tokens = [] @tokens = []
@rules = [] @rules = []
@code_blocks = {} @code_blocks = {}
@ -33,9 +32,7 @@ class Propane
@ast = false @ast = false
@ast_prefix = "" @ast_prefix = ""
@ast_suffix = "" @ast_suffix = ""
@free_token_node = nil
parse_grammar! parse_grammar!
@start_rules << "Start" if @start_rules.empty?
end end
def ptype def ptype
@ -65,7 +62,6 @@ class Propane
elsif parse_ast_statement! elsif parse_ast_statement!
elsif parse_ast_prefix_statement! elsif parse_ast_prefix_statement!
elsif parse_ast_suffix_statement! elsif parse_ast_suffix_statement!
elsif parse_free_token_node_statement!
elsif parse_module_statement! elsif parse_module_statement!
elsif parse_ptype_statement! elsif parse_ptype_statement!
elsif parse_pattern_statement! elsif parse_pattern_statement!
@ -116,12 +112,6 @@ class Propane
end end
end end
def parse_free_token_node_statement!
if md = consume!(/free_token_node\s+(\w+)\s*;/)
@free_token_node = md[1]
end
end
def parse_module_statement! def parse_module_statement!
if consume!(/module\s+/) if consume!(/module\s+/)
md = consume!(/([\w.]+)\s*/, "expected module name") md = consume!(/([\w.]+)\s*/, "expected module name")
@ -242,11 +232,8 @@ class Propane
end end
def parse_start_statement! def parse_start_statement!
if md = consume!(/start\s+([\w\s]*);/) if md = consume!(/start\s+(\w+)\s*;/)
start_rules = md[1].split(/\s+/).map(&:strip) @start_rule = md[1]
start_rules.each do |start_rule|
@start_rules << start_rule unless @start_rules.include?(start_rule)
end
end end
end end

15
lib/propane/parser.rb
View File

@ -16,20 +16,11 @@ class Propane
@warnings = Set.new @warnings = Set.new
@errors = Set.new @errors = Set.new
@options = options @options = options
start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule| start_item = Item.new(grammar.rules.first, 0)
Item.new(start_rule, 0) eval_item_sets = Set[ItemSet.new([start_item])]
end
start_item_sets = start_items.map {|item| ItemSet.new([item])}
eval_item_sets = Set[*start_item_sets]
while eval_item_sets.size > 0 while eval_item_sets.size > 0
item_set = item_set = eval_item_sets.first
if start_item_sets.size > 0
# Ensure we evaluate start_item_sets first in order
start_item_sets.slice!(0)
else
eval_item_sets.first
end
eval_item_sets.delete(item_set) eval_item_sets.delete(item_set)
unless @item_sets_set.include?(item_set) unless @item_sets_set.include?(item_set)
item_set.id = @item_sets.size item_set.id = @item_sets.size

161
spec/propane_spec.rb
View File

@ -78,9 +78,9 @@ EOF
end end
case options[:language] case options[:language]
when "c" when "c"
command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"] command = [*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
when "cpp" when "cpp"
command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"] command = [*%w[g++ -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
when "d" when "d"
command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"] command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
end end
@ -88,24 +88,12 @@ EOF
expect(result).to be_truthy expect(result).to be_truthy
end end
def run_test(options) def run_test
stdout, stderr, status = Open3.capture3("spec/run/testparser") stdout, stderr, status = Open3.capture3("spec/run/testparser")
File.binwrite("spec/run/.stderr", stderr) File.binwrite("spec/run/.stderr", stderr)
File.binwrite("spec/run/.stdout", stdout) File.binwrite("spec/run/.stdout", stdout)
stderr.sub!(/^.*modules passed unittests\n/, "") stderr.sub!(/^.*modules passed unittests\n/, "")
results = Results.new(stdout, stderr, status) Results.new(stdout, stderr, status)
if %w[c cpp].include?(options[:language])
stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
vgout = stdout + stderr
File.binwrite("spec/run/.vgout", vgout)
vgout.scan(/lost: (\d+) bytes/) do |match|
bytes = $1.to_i
if bytes > 0
raise "Valgrind detected memory leak"
end
end
end
results
end end
def lines(str) def lines(str)
@ -298,7 +286,7 @@ Foo -> plus <<>>
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer.#{language}", language: language) compile("spec/test_lexer.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -336,7 +324,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer_unknown_character.#{language}", language: language) compile("spec/test_lexer_unknown_character.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -430,7 +418,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_basic_math_grammar.#{language}", language: language) compile("spec/test_basic_math_grammar.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -456,7 +444,7 @@ R2 -> a b;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language) compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -471,7 +459,7 @@ R1 -> b;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language) compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -506,7 +494,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_user_code.#{language}", language: language) compile("spec/test_user_code.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"abc!", "abc!",
@ -542,7 +530,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_pattern.#{language}", language: language) compile("spec/test_pattern.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"def!", "def!",
@ -584,7 +572,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_return_token_from_pattern.#{language}", language: language) compile("spec/test_return_token_from_pattern.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"def!", "def!",
@ -642,7 +630,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer_modes.#{language}", language: language) compile("spec/test_lexer_modes.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"begin string mode", "begin string mode",
@ -700,7 +688,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer_multiple_modes.#{language}", language: language) compile("spec/test_lexer_multiple_modes.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"ident: d", "ident: d",
@ -737,7 +725,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_parser_rule_user_code.#{language}", language: language) compile("spec/test_parser_rule_user_code.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"A!", "A!",
@ -756,7 +744,7 @@ As -> As a << $$ = $1 + 1u; >>
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_parsing_lists.#{language}", language: language) compile("spec/test_parsing_lists.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
end end
@ -810,7 +798,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer_match_text.#{language}", language: language) compile("spec/test_lexer_match_text.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
"Matched token is identifier_123", "Matched token is identifier_123",
@ -843,7 +831,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_lexer_result_value.#{language}", language: language) compile("spec/test_lexer_result_value.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -858,7 +846,7 @@ Start -> a num;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_error_positions.#{language}", language: language) compile("spec/test_error_positions.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -887,7 +875,7 @@ Start -> b c b;
EOF EOF
run_propane(name: "myp2", language: language) run_propane(name: "myp2", language: language)
compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language) compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -906,7 +894,7 @@ Any -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_user_terminate_lexer.#{language}", language: language) compile("spec/test_user_terminate_lexer.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -924,7 +912,7 @@ Any -> ;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_user_terminate.#{language}", language: language) compile("spec/test_user_terminate.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -968,7 +956,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_match_backslashes.#{language}", language: language) compile("spec/test_match_backslashes.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
@ -1033,7 +1021,7 @@ Two -> two;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast.#{language}", language: language) compile("spec/test_ast.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1077,7 +1065,7 @@ Two -> two;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_ps.#{language}", language: language) compile("spec/test_ast_ps.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1151,7 +1139,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_optional_rule_component.#{language}", language: language) compile("spec/test_optional_rule_component.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
verify_lines(results.stdout, [ verify_lines(results.stdout, [
@ -1203,7 +1191,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_optional_rule_component_ast.#{language}", language: language) compile("spec/test_optional_rule_component_ast.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1244,7 +1232,7 @@ EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_named_optional_rule_component_ast.#{language}", language: language) compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1264,7 +1252,7 @@ T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_token_positions.#{language}", language: language) compile("spec/test_ast_token_positions.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1284,7 +1272,7 @@ A -> bb? c?;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_invalid_positions.#{language}", language: language) compile("spec/test_ast_invalid_positions.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1304,7 +1292,7 @@ T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language) compile("spec/test_ast_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1327,7 +1315,7 @@ T -> c;
EOF EOF
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language) compile("spec/test_ast_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
@ -1354,7 +1342,7 @@ EOF
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
>> >>
ptype char *; ptype char const *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ << token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1); char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length); strncpy(s, (char const *)match, match_length);
@ -1365,14 +1353,12 @@ drop /\\s+/;
Start -> id:first id:second << Start -> id:first id:second <<
printf("first is %s\\n", ${first}); printf("first is %s\\n", ${first});
printf("second is %s\\n", ${second}); printf("second is %s\\n", ${second});
free(${first});
free(${second});
>> >>
EOF EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_field_aliases.#{language}", language: language) compile("spec/test_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
expect(results.stdout).to match /first is foo1.*second is bar2/m expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1403,7 +1389,7 @@ EOF
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
>> >>
ptype char *; ptype char const *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ << token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1); char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length); strncpy(s, (char const *)match, match_length);
@ -1416,15 +1402,13 @@ Start -> Foo;
Start -> id:first id:second << Start -> id:first id:second <<
printf("first is %s\\n", ${first}); printf("first is %s\\n", ${first});
printf("second is %s\\n", ${second}); printf("second is %s\\n", ${second});
free(${first});
free(${second});
>> >>
Foo -> ; Foo -> ;
EOF EOF
end end
run_propane(language: language) run_propane(language: language)
compile("spec/test_field_aliases.#{language}", language: language) compile("spec/test_field_aliases.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
expect(results.stdout).to match /first is foo1.*second is bar2/m expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1435,81 +1419,10 @@ EOF
write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane")) write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane"))
run_propane(language: language) run_propane(language: language)
compile("spec/test_ast_node_memory_remains.#{language}", language: language) compile("spec/test_ast_node_memory_remains.#{language}", language: language)
results = run_test(language: language) results = run_test
expect(results.stderr).to eq "" expect(results.stderr).to eq ""
expect(results.status).to eq 0 expect(results.status).to eq 0
end end
it "allows multiple starting rules" do
write_grammar <<EOF
ptype int;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
Start -> a b R;
Start -> Bs:bs << $$ = $1; >>
R -> c:c << $$ = $1; >>
Bs -> << $$ = 0; >>
Bs -> b:b Bs:bs << $$ = $1 + $2; >>
start Start R Bs;
EOF
run_propane(language: language)
compile("spec/test_starting_rules.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
it "allows multiple starting rules in AST mode" do
write_grammar <<EOF
ast;
ptype int;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
Start -> a b R;
Start -> Bs:bs;
R -> c:c;
Bs -> ;
Bs -> b:b Bs:bs;
start Start R Bs;
EOF
run_propane(language: language)
compile("spec/test_starting_rules_ast.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
if %w[c cpp].include?(language)
it "allows a user function to free token node memory in AST mode" do
write_grammar <<EOF
<<
static void free_token(Token * token)
{
free(token->pvalue);
}
>>
ast;
free_token_node free_token;
ptype int *;
token a <<
$$ = (int *)malloc(sizeof(int));
*$$ = 1;
>>
token b <<
$$ = (int *)malloc(sizeof(int));
*$$ = 2;
>>
Start -> a:a b:b;
EOF
run_propane(language: language)
compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
end
end end
end end
end end

6
spec/test_ast.c
View File

@ -33,16 +33,12 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue); assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL); assert(itemsmore->pItemsMore == NULL);
p_free_ast(start);
input = ""; input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context)); assert_eq(P_SUCCESS, p_parse(&context));
start = p_result(&context); start = p_result(&context);
assert(start->pItems == NULL); assert(start->pItems == NULL);
p_free_ast(start);
input = "2 1"; input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context)); assert_eq(P_SUCCESS, p_parse(&context));
@ -55,7 +51,5 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL); assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL); assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start);
return 0; return 0;
} }

2
spec/test_ast_field_aliases.c
View File

@ -15,7 +15,5 @@ int main()
assert_eq(TOKEN_b, start->second->pToken->token); assert_eq(TOKEN_b, start->second->pToken->token);
assert_eq(TOKEN_c, start->third->pToken->token); assert_eq(TOKEN_c, start->third->pToken->token);
p_free_ast(start);
return 0; return 0;
} }

8
spec/test_ast_invalid_positions.c
View File

@ -30,8 +30,6 @@ int main()
assert_eq(3, start->end_position.row); assert_eq(3, start->end_position.row);
assert_eq(8, start->end_position.col); assert_eq(8, start->end_position.col);
p_free_ast(start);
input = "a\nbb"; input = "a\nbb";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -56,8 +54,6 @@ int main()
assert_eq(2, start->end_position.row); assert_eq(2, start->end_position.row);
assert_eq(2, start->end_position.col); assert_eq(2, start->end_position.col);
p_free_ast(start);
input = "a\nc\nc"; input = "a\nc\nc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -82,8 +78,6 @@ int main()
assert_eq(3, start->end_position.row); assert_eq(3, start->end_position.row);
assert_eq(1, start->end_position.col); assert_eq(1, start->end_position.col);
p_free_ast(start);
input = "a"; input = "a";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -104,7 +98,5 @@ int main()
assert_eq(1, start->end_position.row); assert_eq(1, start->end_position.row);
assert_eq(1, start->end_position.col); assert_eq(1, start->end_position.col);
p_free_ast(start);
return 0; return 0;
} }

3
spec/test_ast_node_memory_remains.c
View File

@ -411,8 +411,5 @@ int main(int argc, char * argv[])
} }
} }
free(pfds);
p_free_ast(pmod);
return 0; return 0;
} }

6
spec/test_ast_ps.c
View File

@ -33,16 +33,12 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue); assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL); assert(itemsmore->pItemsMore == NULL);
p_free_ast(start);
input = ""; input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context)); assert_eq(P_SUCCESS, p_parse(&context));
start = p_result(&context); start = p_result(&context);
assert(start->pItems == NULL); assert(start->pItems == NULL);
p_free_ast(start);
input = "2 1"; input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context)); assert_eq(P_SUCCESS, p_parse(&context));
@ -55,7 +51,5 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL); assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL); assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start);
return 0; return 0;
} }

4
spec/test_ast_token_positions.c
View File

@ -43,8 +43,6 @@ int main()
assert_eq(1, start->end_position.row); assert_eq(1, start->end_position.row);
assert_eq(6, start->end_position.col); assert_eq(6, start->end_position.col);
p_free_ast(start);
input = "\n\n bb\nc\ncc\n\n a"; input = "\n\n bb\nc\ncc\n\n a";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -82,7 +80,5 @@ int main()
assert_eq(7, start->end_position.row); assert_eq(7, start->end_position.row);
assert_eq(6, start->end_position.col); assert_eq(6, start->end_position.col);
p_free_ast(start);
return 0; return 0;
} }

19
spec/test_free_ast_token_node_memory.c
View File

@ -1,19 +0,0 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "ab";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
Start * start = p_result(&context);
assert(start->a != NULL);
assert(*start->a->pvalue == 1);
assert(start->b != NULL);
assert(*start->b->pvalue == 2);
p_free_ast(start);
}

6
spec/test_named_optional_rule_component_ast.c
View File

@ -17,8 +17,6 @@ int main()
assert(start->pR == NULL); assert(start->pR == NULL);
assert(start->r == NULL); assert(start->r == NULL);
p_free_ast(start);
input = "abcd"; input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -33,8 +31,6 @@ int main()
assert(start->pR == start->r); assert(start->pR == start->r);
assert_eq(TOKEN_c, start->pR->pToken1->token); assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start);
input = "bdc"; input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -44,8 +40,6 @@ int main()
assert(start->r != NULL); assert(start->r != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token); assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start);
return 0; return 0;
} }

6
spec/test_optional_rule_component_ast.c
View File

@ -16,8 +16,6 @@ int main()
assert(start->pR3 == NULL); assert(start->pR3 == NULL);
assert(start->pR == NULL); assert(start->pR == NULL);
p_free_ast(start);
input = "abcd"; input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -30,8 +28,6 @@ int main()
assert(start->pR == start->pR3); assert(start->pR == start->pR3);
assert_eq(TOKEN_c, start->pR->pToken1->token); assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start);
input = "bdc"; input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input)); p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
@ -41,8 +37,6 @@ int main()
assert(start->pR != NULL); assert(start->pR != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token); assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start);
return 0; return 0;
} }

2
spec/test_start_rule_ast.c
View File

@ -13,7 +13,5 @@ int main()
assert(top->pToken != NULL); assert(top->pToken != NULL);
assert_eq(TOKEN_hi, top->pToken->token); assert_eq(TOKEN_hi, top->pToken->token);
p_free_ast(top);
return 0; return 0;
} }

27
spec/test_starting_rules.c
View File

@ -1,27 +0,0 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "bbbb";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
int result = p_result(&context);
assert_eq(8, result);
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_Bs(&context) == P_SUCCESS);
result = p_result_Bs(&context);
assert_eq(8, result);
input = "c";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_R(&context) == P_SUCCESS);
result = p_result_R(&context);
assert_eq(3, result);
return 0;
}

29
spec/test_starting_rules.d
View File

@ -1,29 +0,0 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "bbbb";
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
int result = p_result(&context);
assert(result == 8);
p_context_init(&context, input);
assert(p_parse_Bs(&context) == P_SUCCESS);
result = p_result_Bs(&context);
assert(result == 8);
input = "c";
p_context_init(&context, input);
assert(p_parse_R(&context) == P_SUCCESS);
result = p_result_R(&context);
assert(result == 3);
}

37
spec/test_starting_rules_ast.c
View File

@ -1,37 +0,0 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "bbbb";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
Start * start = p_result(&context);
assert_not_null(start->bs);
assert_not_null(start->bs->b);
assert_not_null(start->bs->bs->b);
assert_not_null(start->bs->bs->bs->b);
assert_not_null(start->bs->bs->bs->bs->b);
p_free_ast(start);
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_Bs(&context) == P_SUCCESS);
Bs * bs = p_result_Bs(&context);
assert_not_null(bs->b);
assert_not_null(bs->bs->b);
assert_not_null(bs->bs->bs->b);
assert_not_null(bs->bs->bs->bs->b);
p_free_ast_Bs(bs);
input = "c";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_R(&context) == P_SUCCESS);
R * r = p_result_R(&context);
assert_not_null(r->c);
p_free_ast_R(r);
return 0;
}

36
spec/test_starting_rules_ast.d
View File

@ -1,36 +0,0 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "bbbb";
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
Start * start = p_result(&context);
assert(start.bs);
assert(start.bs.b);
assert(start.bs.bs.b);
assert(start.bs.bs.bs.b);
assert(start.bs.bs.bs.bs.b);
p_context_init(&context, input);
assert(p_parse_Bs(&context) == P_SUCCESS);
Bs * bs = p_result_Bs(&context);
assert(bs.b);
assert(bs.bs.b);
assert(bs.bs.bs.b);
assert(bs.bs.bs.bs.b);
input = "c";
p_context_init(&context, input);
assert(p_parse_R(&context) == P_SUCCESS);
R * r = p_result_R(&context);
assert(r.c);
}

18
spec/testutils.c
View File

@ -14,24 +14,6 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
} }
} }
void assert_ne_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
{
if (expected == actual)
{
fprintf(stderr, "%s:%lu: expected not %lu, got %lu\n", file, line, expected, actual);
assert(false);
}
}
void assert_not_null_i(void * ptr, char const * file, size_t line)
{
if (ptr == NULL)
{
fprintf(stderr, "%s:%lu: expected not NULL\n", file, line);
assert(false);
}
}
void str_init(str_t * str, char const * cs) void str_init(str_t * str, char const * cs)
{ {
size_t length = strlen(cs); size_t length = strlen(cs);

10
spec/testutils.h
View File

@ -5,16 +5,6 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
#define assert_eq(expected, actual) \ #define assert_eq(expected, actual) \
assert_eq_size_t_i(expected, actual, __FILE__, __LINE__) assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
void assert_ne_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
#define assert_ne(expected, actual) \
assert_ne_size_t_i(expected, actual, __FILE__, __LINE__)
void assert_not_null_i(void * ptr, char const * file, size_t line);
#define assert_not_null(ptr) \
assert_not_null_i(ptr, __FILE__, __LINE__)
typedef struct typedef struct
{ {
char * cs; char * cs;