Compare commits

...

7 Commits

24 changed files with 632 additions and 82 deletions

View File

@ -695,11 +695,13 @@ typedef struct
} state_value_t;
/** Common AST node structure. */
typedef struct
typedef struct ASTNode_s
{
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
void * fields[];
uint16_t n_fields;
uint8_t is_token;
struct ASTNode_s * fields[];
} ASTNode;
/** Parser shift table. */
@ -922,6 +924,8 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
*
* @param context
* Lexer/parser context structure.
* @start_state_id
* ID of the state in which to start.
*
* @retval P_SUCCESS
* The parser successfully matched the input text. The parse result value
@ -934,7 +938,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
*/
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
{
<%= @grammar.prefix %>token_info_t token_info;
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
@ -947,6 +951,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
<% end %>
state_values_stack_init(&statevalues);
state_values_stack_push(&statevalues);
state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
size_t result;
for (;;)
{
@ -972,7 +977,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{
/* Successful parse. */
<% if @grammar.ast %>
context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
context->parse_result = state_values_stack_index(&statevalues, -1)->ast_node;
<% else %>
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
<% end %>
@ -992,6 +997,8 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = (<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *)malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
token_ast_node->position = token_info.position;
token_ast_node->end_position = token_info.end_position;
token_ast_node->n_fields = 0u;
token_ast_node->is_token = 1u;
token_ast_node->token = token;
token_ast_node->pvalue = token_info.pvalue;
state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@ -1028,31 +1035,30 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
else if (parser_reduce_table[reduce_index].n_states > 0)
{
size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
ASTNode * node = (ASTNode *)malloc(sizeof(ASTNode) + n_fields * sizeof(void *));
size_t bytes = sizeof(ASTNode) + n_fields * sizeof(void *);
ASTNode * node = (ASTNode *)malloc(bytes);
memset(node, 0, bytes);
node->position = INVALID_POSITION;
node->end_position = INVALID_POSITION;
for (size_t i = 0; i < n_fields; i++)
{
node->fields[i] = NULL;
}
node->n_fields = n_fields;
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
{
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{
node->fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
node->fields[i] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
}
}
else
{
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
{
node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
node->fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = (ASTNode *)state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
}
}
bool position_found = false;
for (size_t i = 0; i < n_fields; i++)
{
ASTNode * child = (ASTNode *)node->fields[i];
ASTNode * child = node->fields[i];
if ((child != NULL) && <%= @grammar.prefix %>position_valid(child->position))
{
if (!position_found)
@ -1074,6 +1080,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
{
state_values_stack_free(&statevalues);
return P_USER_TERMINATED;
}
reduced_parser_value = reduced_parser_value2;
@ -1097,6 +1104,19 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
return result;
}
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, 0u);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, <%= i %>u);
}
<% end %>
/**
* Get the parse result value.
*
@ -1106,17 +1126,28 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
* @return Parse result value.
*/
<% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *) context->parse_result;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return (<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *) context->parse_result;
}
<% end %>
<% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<% end %>
{
<% if @grammar.ast %>
return context->parse_result;
<% else %>
return context->parse_result.v_<%= start_rule_type[0] %>;
<% end %>
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return context->parse_result.v_<%= start_rule_type(i)[0] %>;
}
<% end %>
<% end %>
/**
* Get the current text input position.
@ -1153,3 +1184,45 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
{
return context->token;
}
<% if @grammar.ast %>
static void free_ast_node(ASTNode * node)
{
if (node->is_token)
{
<% if @grammar.free_token_node %>
<%= @grammar.free_token_node %>((<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> *) node);
<% end %>
/* TODO: free value_t */
}
else if (node->n_fields > 0u)
{
for (size_t i = 0u; i < node->n_fields; i++)
{
if (node->fields[i] != NULL)
{
free_ast_node(node->fields[i]);
}
}
}
free(node);
}
/**
* Free all AST node memory.
*/
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast)
{
free_ast_node((ASTNode *)ast);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
/**
* Free all AST node memory.
*/
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast)
{
free_ast_node((ASTNode *)ast);
}
<% end %>
<% end %>

View File

@ -173,7 +173,7 @@ public struct <%= @grammar.prefix %>context_t
/** Parse result value. */
<% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
void * parse_result;
<% else %>
<%= @grammar.prefix %>value_t parse_result;
<% end %>
@ -985,6 +985,8 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
*
* @param context
* Lexer/parser context structure.
* @start_state_id
* ID of the state in which to start.
*
* @retval P_SUCCESS
* The parser successfully matched the input text. The parse result value
@ -997,11 +999,12 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
*/
public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
{
<%= @grammar.prefix %>token_info_t token_info;
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
state_value_t[] statevalues = new state_value_t[](1);
statevalues[0].state_id = start_state_id;
size_t reduced_rule_set = INVALID_ID;
<% if @grammar.ast %>
void * reduced_parser_node;
@ -1031,7 +1034,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
{
/* Successful parse. */
<% if @grammar.ast %>
context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
context.parse_result = statevalues[$-1].ast_node;
<% else %>
context.parse_result = statevalues[$-1].pvalue;
<% end %>
@ -1147,6 +1150,19 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
}
}
public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, 0u);
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return parse_from(context, <%= i %>u);
}
<% end %>
/**
* Get the parse result value.
*
@ -1156,17 +1172,28 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
* @return Parse result value.
*/
<% if @grammar.ast %>
public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
public <%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> *)context.parse_result;
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return cast(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> *)context.parse_result;
}
<% end %>
<% else %>
public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
<% end %>
{
<% if @grammar.ast %>
return context.parse_result;
<% else %>
return context.parse_result.v_<%= start_rule_type[0] %>;
<% end %>
}
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
{
return context.parse_result.v_<%= start_rule_type(i)[0] %>;
}
<% end %>
<% end %>
/**
* Get the current text input position.

View File

@ -75,9 +75,11 @@ typedef union
/** AST node types. @{ */
typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
{
/* ASTNode fields must be present in the same order here. */
<% # ASTNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
<%= @grammar.prefix %>token_t token;
<%= @grammar.prefix %>value_t pvalue;
} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
@ -93,8 +95,11 @@ struct <%= name %>;
<% next if rule_set.optional? %>
typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
{
<% # ASTNode fields must be present in the same order here. # %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
uint16_t n_fields;
uint8_t is_token;
<% rule_set.ast_fields.each do |fields| %>
union
{
@ -157,7 +162,7 @@ typedef struct
/** Parse result value. */
<% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
void * parse_result;
<% else %>
<%= @grammar.prefix %>value_t parse_result;
<% end %>
@ -184,11 +189,27 @@ size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t inp
size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% if @grammar.ast %>
<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% else %>
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
<% end %>
<% end %>
<% if @grammar.ast %>
void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast);
<% end %>
<% end %>
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);

View File

@ -326,6 +326,36 @@ assert(itemsmore.pItem.pItem.pItem !is null);
assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
```
## Freeing user-allocated memory in token node `pvalue`: the `free_token_node` statement
If user lexer code block allocates memory to store in a token node's `pvalue`,
the `free_token_node` grammar statement can be used to specify the name of a
function which will be called during the `p_free_ast()` call to free the memory
associated with a token node.
Example:
```
<<
static void free_token(Token * token)
{
free(token->pvalue);
}
>>
ast;
free_token_node free_token;
ptype int *;
token a <<
$$ = (int *)malloc(sizeof(int));
*$$ = 1;
>>
token b <<
$$ = (int *)malloc(sizeof(int));
*$$ = 2;
>>
Start -> a:a b:b;
```
##> Specifying tokens - the `token` statement
The `token` statement allows defining a lexer token and a pattern to match that
@ -708,6 +738,22 @@ Example:
start MyStartRule;
```
Multiple start rules can be specified, either with multiple `start` statements
or one `start` statement listing multiple start rules.
Example:
```
start Module ModuleItem Statement Expression;
```
When multiple start rules are specified, multiple `p_parse_*()` functions,
`p_result_*()`, and `p_free_ast_*()` functions (in AST mode) are generated.
A default `p_parse()`, `p_result()`, `p_free_ast()` are generated corresponding
to the first start rule.
Additionally, each start rule causes the generation of another version of each
of these functions, for example `p_parse_Statement()`, `p_result_Statement()`,
and `p_free_ast_Statement()`.
##> Specifying the parser module name - the `module` statement
The `module` statement can be used to specify the module name for a generated
@ -988,6 +1034,16 @@ p_context_init(&context, input, input_length);
size_t result = p_parse(&context);
```
When multiple start rules are specified, a separate parse function is generated
for each which starts parsing at the given rule.
For example, if `Statement` is specified as a start rule:
```
size_t result = p_parse_Statement(&context);
```
In this case, the parser will start parsing with the `Statement` rule.
### `p_position_valid`
The `p_position_valid()` function is only generated for C targets.
@ -1026,6 +1082,23 @@ if (p_parse(&context) == P_SUCCESS)
If AST generation mode is active, then the `p_result()` function returns a
`Start *` pointing to the `Start` AST structure.
When multiple start rules are specified, a separate result function is generated
for each which returns the parse result for the corresponding rule.
For example, if `Statement` is specified as a start rule:
```
p_context_t context;
p_context_init(&context, input, input_length);
size_t result = p_parse(&context);
if (p_parse_Statement(&context) == P_SUCCESS)
{
result = p_result_Statement(&context);
}
```
In this case, the parser will start parsing with the `Statement` rule and the
parse result from the `Statement` rule will be returned.
### `p_position`
The `p_position()` function can be used to retrieve the parser position where
@ -1098,6 +1171,31 @@ assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u);
```
### `p_free_ast`
The `p_free_ast()` function can be used to free the memory used by the AST.
It should be passed the same value that is returned by `p_result()`.
The `p_free_ast()` function is only available for C/C++ output targets.
Note that if any lexer user code block allocates memory to store in a token's
`pvalue`, in order to properly free this memory a `free_token_node` function
should be specified in the grammar file.
If specified, the `free_token_node` function will be called during the
`p_free_ast()` process to allow user code to free any memory associated with
a token node's `pvalue`.
When multiple start rules are specified, a separate `p_free_ast` function is
generated for each which frees the AST resulting from parsing the given rule.
For example, if `Statement` is specified as a start rule:
```
p_free_ast_Statement(statement_ast);
```
In this case, Propane will free a `Statement` AST structure returned by the
`p_parse_Statement(&context)` function.
##> Data
### `p_token_names`

View File

@ -71,12 +71,15 @@ class Propane
end
tokens_by_name[token.name] = token
end
# Check for user start rule.
unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
raise Error.new("Start rule `#{@grammar.start_rule}` not found")
# Create real start rule(s).
real_start_rules = @grammar.start_rules.map do |start_rule|
unless @grammar.rules.find {|rule| rule.name == start_rule}
raise Error.new("Start rule `#{start_rule}` not found")
end
Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil)
end
# Add "real" start rule.
@grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
# Add real start rules before user-given rules.
@grammar.rules = real_start_rules + @grammar.rules
# Generate and add rules for optional components.
generate_optional_component_rules!(tokens_by_name)
# Build rule sets.
@ -332,9 +335,9 @@ class Propane
#
# @return [Array<String>]
# Start rule parser value type name and type string.
def start_rule_type
def start_rule_type(start_rule_index = 0)
start_rule = @grammar.rules.find do |rule|
rule.name == @grammar.start_rule
rule.name == @grammar.start_rules[start_rule_index]
end
[start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
end

View File

@ -8,10 +8,11 @@ class Propane
attr_reader :ast
attr_reader :ast_prefix
attr_reader :ast_suffix
attr_reader :free_token_node
attr_reader :modulename
attr_reader :patterns
attr_reader :rules
attr_reader :start_rule
attr_accessor :rules
attr_reader :start_rules
attr_reader :tokens
attr_reader :code_blocks
attr_reader :ptypes
@ -19,7 +20,7 @@ class Propane
def initialize(input)
@patterns = []
@start_rule = "Start"
@start_rules = []
@tokens = []
@rules = []
@code_blocks = {}
@ -32,7 +33,9 @@ class Propane
@ast = false
@ast_prefix = ""
@ast_suffix = ""
@free_token_node = nil
parse_grammar!
@start_rules << "Start" if @start_rules.empty?
end
def ptype
@ -62,6 +65,7 @@ class Propane
elsif parse_ast_statement!
elsif parse_ast_prefix_statement!
elsif parse_ast_suffix_statement!
elsif parse_free_token_node_statement!
elsif parse_module_statement!
elsif parse_ptype_statement!
elsif parse_pattern_statement!
@ -112,6 +116,12 @@ class Propane
end
end
def parse_free_token_node_statement!
if md = consume!(/free_token_node\s+(\w+)\s*;/)
@free_token_node = md[1]
end
end
def parse_module_statement!
if consume!(/module\s+/)
md = consume!(/([\w.]+)\s*/, "expected module name")
@ -232,8 +242,11 @@ class Propane
end
def parse_start_statement!
if md = consume!(/start\s+(\w+)\s*;/)
@start_rule = md[1]
if md = consume!(/start\s+([\w\s]*);/)
start_rules = md[1].split(/\s+/).map(&:strip)
start_rules.each do |start_rule|
@start_rules << start_rule unless @start_rules.include?(start_rule)
end
end
end

View File

@ -16,11 +16,20 @@ class Propane
@warnings = Set.new
@errors = Set.new
@options = options
start_item = Item.new(grammar.rules.first, 0)
eval_item_sets = Set[ItemSet.new([start_item])]
start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule|
Item.new(start_rule, 0)
end
start_item_sets = start_items.map {|item| ItemSet.new([item])}
eval_item_sets = Set[*start_item_sets]
while eval_item_sets.size > 0
item_set = eval_item_sets.first
item_set =
if start_item_sets.size > 0
# Ensure we evaluate start_item_sets first in order
start_item_sets.slice!(0)
else
eval_item_sets.first
end
eval_item_sets.delete(item_set)
unless @item_sets_set.include?(item_set)
item_set.id = @item_sets.size

View File

@ -78,9 +78,9 @@ EOF
end
case options[:language]
when "c"
command = [*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
command = [*%w[gcc -g -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
when "cpp"
command = [*%w[g++ -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
when "d"
command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
end
@ -88,12 +88,24 @@ EOF
expect(result).to be_truthy
end
def run_test
def run_test(options)
stdout, stderr, status = Open3.capture3("spec/run/testparser")
File.binwrite("spec/run/.stderr", stderr)
File.binwrite("spec/run/.stdout", stdout)
stderr.sub!(/^.*modules passed unittests\n/, "")
Results.new(stdout, stderr, status)
results = Results.new(stdout, stderr, status)
if %w[c cpp].include?(options[:language])
stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
vgout = stdout + stderr
File.binwrite("spec/run/.vgout", vgout)
vgout.scan(/lost: (\d+) bytes/) do |match|
bytes = $1.to_i
if bytes > 0
raise "Valgrind detected memory leak"
end
end
end
results
end
def lines(str)
@ -286,7 +298,7 @@ Foo -> plus <<>>
EOF
run_propane(language: language)
compile("spec/test_lexer.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -324,7 +336,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_lexer_unknown_character.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -418,7 +430,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_basic_math_grammar.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -444,7 +456,7 @@ R2 -> a b;
EOF
run_propane(language: language)
compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
end
@ -459,7 +471,7 @@ R1 -> b;
EOF
run_propane(language: language)
compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
end
@ -494,7 +506,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_user_code.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"abc!",
@ -530,7 +542,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_pattern.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"def!",
@ -572,7 +584,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_return_token_from_pattern.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"def!",
@ -630,7 +642,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_lexer_modes.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"begin string mode",
@ -688,7 +700,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_lexer_multiple_modes.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"ident: d",
@ -725,7 +737,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_parser_rule_user_code.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"A!",
@ -744,7 +756,7 @@ As -> As a << $$ = $1 + 1u; >>
EOF
run_propane(language: language)
compile("spec/test_parsing_lists.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
expect(results.stderr).to eq ""
end
@ -798,7 +810,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_lexer_match_text.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.status).to eq 0
verify_lines(results.stdout, [
"Matched token is identifier_123",
@ -831,7 +843,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_lexer_result_value.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -846,7 +858,7 @@ Start -> a num;
EOF
run_propane(language: language)
compile("spec/test_error_positions.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -875,7 +887,7 @@ Start -> b c b;
EOF
run_propane(name: "myp2", language: language)
compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -894,7 +906,7 @@ Any -> c;
EOF
run_propane(language: language)
compile("spec/test_user_terminate_lexer.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -912,7 +924,7 @@ Any -> ;
EOF
run_propane(language: language)
compile("spec/test_user_terminate.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -956,7 +968,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_match_backslashes.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
verify_lines(results.stdout, [
@ -1021,7 +1033,7 @@ Two -> two;
EOF
run_propane(language: language)
compile("spec/test_ast.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1065,7 +1077,7 @@ Two -> two;
EOF
run_propane(language: language)
compile("spec/test_ast_ps.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1139,7 +1151,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_optional_rule_component.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
verify_lines(results.stdout, [
@ -1191,7 +1203,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_optional_rule_component_ast.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1232,7 +1244,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1252,7 +1264,7 @@ T -> c;
EOF
run_propane(language: language)
compile("spec/test_ast_token_positions.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1272,7 +1284,7 @@ A -> bb? c?;
EOF
run_propane(language: language)
compile("spec/test_ast_invalid_positions.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1292,7 +1304,7 @@ T -> c;
EOF
run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1315,7 +1327,7 @@ T -> c;
EOF
run_propane(language: language)
compile("spec/test_ast_field_aliases.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
@ -1342,7 +1354,7 @@ EOF
#include <stdio.h>
#include <string.h>
>>
ptype char const *;
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
@ -1353,12 +1365,14 @@ drop /\\s+/;
Start -> id:first id:second <<
printf("first is %s\\n", ${first});
printf("second is %s\\n", ${second});
free(${first});
free(${second});
>>
EOF
end
run_propane(language: language)
compile("spec/test_field_aliases.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1389,7 +1403,7 @@ EOF
#include <stdio.h>
#include <string.h>
>>
ptype char const *;
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
@ -1402,13 +1416,15 @@ Start -> Foo;
Start -> id:first id:second <<
printf("first is %s\\n", ${first});
printf("second is %s\\n", ${second});
free(${first});
free(${second});
>>
Foo -> ;
EOF
end
run_propane(language: language)
compile("spec/test_field_aliases.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
expect(results.stdout).to match /first is foo1.*second is bar2/m
@ -1419,10 +1435,81 @@ EOF
write_grammar(File.read("spec/ast_node_memory_remains.#{ext}.propane"))
run_propane(language: language)
compile("spec/test_ast_node_memory_remains.#{language}", language: language)
results = run_test
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
it "allows multiple starting rules" do
write_grammar <<EOF
ptype int;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
Start -> a b R;
Start -> Bs:bs << $$ = $1; >>
R -> c:c << $$ = $1; >>
Bs -> << $$ = 0; >>
Bs -> b:b Bs:bs << $$ = $1 + $2; >>
start Start R Bs;
EOF
run_propane(language: language)
compile("spec/test_starting_rules.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
it "allows multiple starting rules in AST mode" do
write_grammar <<EOF
ast;
ptype int;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
Start -> a b R;
Start -> Bs:bs;
R -> c:c;
Bs -> ;
Bs -> b:b Bs:bs;
start Start R Bs;
EOF
run_propane(language: language)
compile("spec/test_starting_rules_ast.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
if %w[c cpp].include?(language)
it "allows a user function to free token node memory in AST mode" do
write_grammar <<EOF
<<
static void free_token(Token * token)
{
free(token->pvalue);
}
>>
ast;
free_token_node free_token;
ptype int *;
token a <<
$$ = (int *)malloc(sizeof(int));
*$$ = 1;
>>
token b <<
$$ = (int *)malloc(sizeof(int));
*$$ = 2;
>>
Start -> a:a b:b;
EOF
run_propane(language: language)
compile("spec/test_free_ast_token_node_memory.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
end
end
end
end

View File

@ -33,12 +33,16 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL);
p_free_ast(start);
input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
start = p_result(&context);
assert(start->pItems == NULL);
p_free_ast(start);
input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
@ -51,5 +55,7 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start);
return 0;
}

View File

@ -15,5 +15,7 @@ int main()
assert_eq(TOKEN_b, start->second->pToken->token);
assert_eq(TOKEN_c, start->third->pToken->token);
p_free_ast(start);
return 0;
}

View File

@ -30,6 +30,8 @@ int main()
assert_eq(3, start->end_position.row);
assert_eq(8, start->end_position.col);
p_free_ast(start);
input = "a\nbb";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -54,6 +56,8 @@ int main()
assert_eq(2, start->end_position.row);
assert_eq(2, start->end_position.col);
p_free_ast(start);
input = "a\nc\nc";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -78,6 +82,8 @@ int main()
assert_eq(3, start->end_position.row);
assert_eq(1, start->end_position.col);
p_free_ast(start);
input = "a";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -98,5 +104,7 @@ int main()
assert_eq(1, start->end_position.row);
assert_eq(1, start->end_position.col);
p_free_ast(start);
return 0;
}

View File

@ -411,5 +411,8 @@ int main(int argc, char * argv[])
}
}
free(pfds);
p_free_ast(pmod);
return 0;
}

View File

@ -33,12 +33,16 @@ int main()
assert_eq(22, itemsmore->pItem->pToken1->pvalue);
assert(itemsmore->pItemsMore == NULL);
p_free_ast(start);
input = "";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
start = p_result(&context);
assert(start->pItems == NULL);
p_free_ast(start);
input = "2 1";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
@ -51,5 +55,7 @@ int main()
assert(start->pItems->pItem->pDual->pTwo2 == NULL);
assert(start->pItems->pItem->pDual->pOne1 == NULL);
p_free_ast(start);
return 0;
}

View File

@ -43,6 +43,8 @@ int main()
assert_eq(1, start->end_position.row);
assert_eq(6, start->end_position.col);
p_free_ast(start);
input = "\n\n bb\nc\ncc\n\n a";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -80,5 +82,7 @@ int main()
assert_eq(7, start->end_position.row);
assert_eq(6, start->end_position.col);
p_free_ast(start);
return 0;
}

View File

@ -0,0 +1,19 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "ab";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(&context));
Start * start = p_result(&context);
assert(start->a != NULL);
assert(*start->a->pvalue == 1);
assert(start->b != NULL);
assert(*start->b->pvalue == 2);
p_free_ast(start);
}

View File

@ -17,6 +17,8 @@ int main()
assert(start->pR == NULL);
assert(start->r == NULL);
p_free_ast(start);
input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -31,6 +33,8 @@ int main()
assert(start->pR == start->r);
assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start);
input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -40,6 +44,8 @@ int main()
assert(start->r != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start);
return 0;
}

View File

@ -16,6 +16,8 @@ int main()
assert(start->pR3 == NULL);
assert(start->pR == NULL);
p_free_ast(start);
input = "abcd";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -28,6 +30,8 @@ int main()
assert(start->pR == start->pR3);
assert_eq(TOKEN_c, start->pR->pToken1->token);
p_free_ast(start);
input = "bdc";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
@ -37,6 +41,8 @@ int main()
assert(start->pR != NULL);
assert_eq(TOKEN_d, start->pR->pToken1->token);
p_free_ast(start);
return 0;
}

View File

@ -13,5 +13,7 @@ int main()
assert(top->pToken != NULL);
assert_eq(TOKEN_hi, top->pToken->token);
p_free_ast(top);
return 0;
}

View File

@ -0,0 +1,27 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "bbbb";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
int result = p_result(&context);
assert_eq(8, result);
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_Bs(&context) == P_SUCCESS);
result = p_result_Bs(&context);
assert_eq(8, result);
input = "c";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_R(&context) == P_SUCCESS);
result = p_result_R(&context);
assert_eq(3, result);
return 0;
}

View File

@ -0,0 +1,29 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "bbbb";
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
int result = p_result(&context);
assert(result == 8);
p_context_init(&context, input);
assert(p_parse_Bs(&context) == P_SUCCESS);
result = p_result_Bs(&context);
assert(result == 8);
input = "c";
p_context_init(&context, input);
assert(p_parse_R(&context) == P_SUCCESS);
result = p_result_R(&context);
assert(result == 3);
}

View File

@ -0,0 +1,37 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "bbbb";
p_context_t context;
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse(&context) == P_SUCCESS);
Start * start = p_result(&context);
assert_not_null(start->bs);
assert_not_null(start->bs->b);
assert_not_null(start->bs->bs->b);
assert_not_null(start->bs->bs->bs->b);
assert_not_null(start->bs->bs->bs->bs->b);
p_free_ast(start);
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_Bs(&context) == P_SUCCESS);
Bs * bs = p_result_Bs(&context);
assert_not_null(bs->b);
assert_not_null(bs->bs->b);
assert_not_null(bs->bs->bs->b);
assert_not_null(bs->bs->bs->bs->b);
p_free_ast_Bs(bs);
input = "c";
p_context_init(&context, (uint8_t const *)input, strlen(input));
assert(p_parse_R(&context) == P_SUCCESS);
R * r = p_result_R(&context);
assert_not_null(r->c);
p_free_ast_R(r);
return 0;
}

View File

@ -0,0 +1,36 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "bbbb";
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
Start * start = p_result(&context);
assert(start.bs);
assert(start.bs.b);
assert(start.bs.bs.b);
assert(start.bs.bs.bs.b);
assert(start.bs.bs.bs.bs.b);
p_context_init(&context, input);
assert(p_parse_Bs(&context) == P_SUCCESS);
Bs * bs = p_result_Bs(&context);
assert(bs.b);
assert(bs.bs.b);
assert(bs.bs.bs.b);
assert(bs.bs.bs.bs.b);
input = "c";
p_context_init(&context, input);
assert(p_parse_R(&context) == P_SUCCESS);
R * r = p_result_R(&context);
assert(r.c);
}

View File

@ -14,6 +14,24 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
}
}
void assert_ne_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
{
if (expected == actual)
{
fprintf(stderr, "%s:%lu: expected not %lu, got %lu\n", file, line, expected, actual);
assert(false);
}
}
void assert_not_null_i(void * ptr, char const * file, size_t line)
{
if (ptr == NULL)
{
fprintf(stderr, "%s:%lu: expected not NULL\n", file, line);
assert(false);
}
}
void str_init(str_t * str, char const * cs)
{
size_t length = strlen(cs);

View File

@ -5,6 +5,16 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
#define assert_eq(expected, actual) \
assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
void assert_ne_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
#define assert_ne(expected, actual) \
assert_ne_size_t_i(expected, actual, __FILE__, __LINE__)
void assert_not_null_i(void * ptr, char const * file, size_t line);
#define assert_not_null(ptr) \
assert_not_null_i(ptr, __FILE__, __LINE__)
typedef struct
{
char * cs;