josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Allow parser user code blocks in tree mode

Browse Source
This commit is contained in:
Josh Holtrop 2026-07-06 23:27:44 -04:00
parent b04c0ad205
commit 98390429a8
9 changed files with 281 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,3 +1,9 @@
## Unreleased
### New Features
- Support parser rule user code blocks in tree generation mode.
## v4.6.0
### New Features

11
assets/parser.c.erb
View File

@ -923,7 +923,9 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
memset(&empty_pos, 0, sizeof(empty_pos));
return empty_pos;
}
<% end %>
<% if !@grammar.tree || @grammar.parser_user_code_used? %>
/**
* Execute user code associated with a parser rule.
*
@ -934,7 +936,7 @@ static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t *
* @retval P_USER_TERMINATED
* User requested to terminate parsing.
*/
static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
static size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
{
switch (rule)
{
@ -1169,6 +1171,13 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
reduced_parser_node = NULL;
}
<% if @grammar.parser_user_code_used? %>
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
{
state_values_stack_free(&statevalues);
return P_USER_TERMINATED;
}
<% end %>
<% else %>
<%= @grammar.prefix %>value_t reduced_parser_value2;
memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));

10
assets/parser.d.erb
View File

@ -994,7 +994,9 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
}
return <%= @grammar.prefix %>position_t.INVALID;
}
<% end %>
<% if !@grammar.tree || @grammar.parser_user_code_used? %>
/**
* Execute user code associated with a parser rule.
*
@ -1005,7 +1007,7 @@ private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] state
* @retval P_USER_TERMINATED
* User requested to terminate parsing.
*/
private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
private size_t parser_user_code(<%= @grammar.tree ? "void" : "#{@grammar.prefix}value_t" %> * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
{
switch (rule)
{
@ -1227,6 +1229,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
{
reduced_parser_node = null;
}
<% if @grammar.parser_user_code_used? %>
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
{
return P_USER_TERMINATED;
}
<% end %>
<% else %>
<%= @grammar.prefix %>value_t reduced_parser_value2;
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)

43
doc/user_guide.md
View File

@ -270,9 +270,38 @@ For C targets this can be accomplished with
`if (p_position_valid(${$.position}))` and for D targets this can be
accomplished with `if (${$.position}.valid)`.
Parser rule code blocks are not available in tree generation mode.
In tree generation mode, a full parse tree is automatically constructed in
memory for user code to traverse after parsing is complete.
Parser rule code blocks are still supported in tree generation mode, but they
behave differently than when tree generation mode is not active.
The code block for a rule is executed after the rule has been matched and its
tree node has been fully formed.
Within the code block, `$$` refers to the tree node for the reduced rule, typed
as a pointer to that rule's generated tree node structure.
The tree nodes for the rule components are accessed positionally with `$1`,
`$2`, `$3`, etc..., each typed as a pointer to the generated tree node structure
for that component (a rule node or a `Token` node).
Field aliases (see the "Specifying parser rules" section) may also be used to
reference a component tree node by name; a field alias behaves identically to
the positional reference for that component.
The positional position expansions (`${$.position}`, `${N.position}`, etc...)
are not available in tree generation mode; the `position` and `end_position`
fields of the tree nodes can be accessed directly instead.
Example:
```
tree;
Assignment -> ident equals Expr <<
/* $$ is the Assignment tree node, $1 is the ident Token node, and $3 is
* the Expr rule node. */
printf("assignment on row %d, col %d\n",
$$->position.row, $$->position.col);
printf("target identifier ends on row %d, col %d\n",
$1->end_position.row, $1->end_position.col);
>>
```
##> `context_user_fields` statement - adding custom fields to the context
@ -710,7 +739,9 @@ It is recommended to place this statement early in the grammar.
In tree generation mode various aspects of propane's behavior are changed:
* Only one `ptype` is allowed.
* Parser user code blocks are not supported.
* Parser user code blocks execute after the rule's tree node has been formed
and access the tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser rule
code blocks" section).
* Structure types are generated to represent the parsed tokens and rules as
defined in the grammar.
* The parse result from `p_result()` points to a `Start` struct containing
@ -1059,8 +1090,12 @@ The `$$` symbol accesses the output parser value for this rule.
The above examples demonstrate how the parser values for the rule components
can be used to produce the parser value for the accepted rule.
Parser rule code blocks are not allowed and not used when tree generation mode
is active.
In tree generation mode, parser rule code blocks access the reduced rule tree
node and its component tree nodes via `$$`, `$1`, `$2`, etc... (see the "Parser
rule code blocks" section).
Field aliases may still be used in tree generation mode to reference a component
tree node by name, behaving identically to the corresponding positional
reference.
##> User termination of the lexer or parser

80
lib/propane/generator.rb
View File

@ -185,7 +185,8 @@ class Propane
end
end
@grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
@grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
optcode = @grammar.tree ? nil : "$$ = $1;\n"
@grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
optional_rules_added << component
end
end
@ -296,21 +297,24 @@ class Propane
end
if parser
code = code.gsub(/\$\$/) do |match|
case @language
when "c"
"_pvalue->v_#{rule.ptypename}"
when "d"
"_pvalue.v_#{rule.ptypename}"
if @grammar.tree
case @language
when "c"
"((#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
when "d"
"(cast(#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix} *)_pvalue)"
end
else
case @language
when "c"
"_pvalue->v_#{rule.ptypename}"
when "d"
"_pvalue.v_#{rule.ptypename}"
end
end
end
code = code.gsub(/\$(\d+)/) do |match|
index = $1.to_i
case @language
when "c"
"state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
when "d"
"statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
end
parser_component_reference(rule, $1.to_i)
end
code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
index = $1.to_i
@ -323,12 +327,11 @@ class Propane
code = code.gsub(/\$\{(\w+)\}/) do |match|
aliasname = $1
if index = rule.aliases[aliasname]
case @language
when "c"
"state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
when "d"
"statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
end
# Field aliases are just a named reference to a positional rule
# component, so reuse the same expansion as `$1', `$2', etc. Note
# that rule.aliases stores a 0-based component index, so add 1 to
# convert it to the 1-based index used for positional references.
parser_component_reference(rule, index + 1)
else
raise Error.new("Field alias '#{aliasname}' not found")
end
@ -384,6 +387,45 @@ class Propane
code
end
# Expand a positional reference to a parser rule component.
#
# This is used to expand `$1', `$2', etc. as well as field aliases (which
# are just named references to a positional rule component).
#
# @param rule [Rule]
# The Rule containing the user code.
# @param index [Integer]
# 1-based index of the rule component to reference.
#
# @return [String]
# Expanded rule component reference.
def parser_component_reference(rule, index)
component = rule.components[index - 1]
if @grammar.tree
# In tree mode a component reference yields a pointer to that
# component's tree node. An optional component propagates its target
# node (or null), so use the optional target's node type.
if component.is_a?(RuleSet) && component.optional?
component = component.option_target
end
node_name = component.is_a?(Token) ? "Token" : component.name
typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
case @language
when "c"
"((#{typename} *)state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->tree_node)"
when "d"
"(cast(#{typename} *)statevalues[$-1-n_states+#{index}].tree_node)"
end
else
case @language
when "c"
"state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
when "d"
"statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
end
end
end
# Get the lex function to use.
#
# @return [String]

12
lib/propane/grammar.rb
View File

@ -58,6 +58,10 @@ class Propane
@tokens.size + 1
end
def parser_user_code_used?
@rules.any? {|r| r.code}
end
private
def parse_grammar!
@ -263,12 +267,8 @@ class Propane
end
md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
components = md[1].strip.split(/\s+/)
if @tree
consume!(/;/, "expected `;'")
else
unless code = parse_code_block!
consume!(/;/, "expected `;' or code block")
end
unless code = parse_code_block!
consume!(/;/, "expected `;' or code block")
end
@rules << Rule.new(rule_name, components, code, ptypename, @line_number)
@modeline = nil

72
spec/propane_spec.rb
View File

@ -745,6 +745,78 @@ EOF
])
end
it "executes user code associated with a parser rule in tree mode" do
case language
when "c", "cpp"
write_grammar <<EOF
tree;
context_user_fields <<
int start_n_fields;
int start_a_value;
int a_value;
int b_value;
p_token_t b_token;
int c_is_null;
int c_field_is_null;
int alias_a_value;
int alias_b_value;
>>
ptype int;
token a << $$ = 11; >>
token b << $$ = 22; >>
Start -> A:ay B:bee C <<
${context.start_n_fields} = $$->n_fields;
${context.start_a_value} = $$->pA->pToken1->pvalue;
${context.a_value} = $1->pToken1->pvalue;
${context.b_value} = $2->pToken1->pvalue;
${context.b_token} = $2->pToken1->token;
${context.c_field_is_null} = ($$->pC == NULL) ? 1 : 0;
${context.alias_a_value} = ${ay}->pToken1->pvalue;
${context.alias_b_value} = ${bee}->pToken1->pvalue;
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = ($$ == NULL) ? 1 : 0; >>
EOF
when "d"
write_grammar <<EOF
tree;
context_user_fields <<
int start_n_fields;
int start_a_value;
int a_value;
int b_value;
p_token_t b_token;
int c_is_null;
int c_field_is_null;
int alias_a_value;
int alias_b_value;
>>
ptype int;
token a << $$ = 11; >>
token b << $$ = 22; >>
Start -> A:ay B:bee C <<
${context.start_n_fields} = $$.n_fields;
${context.start_a_value} = $$.pA.pToken1.pvalue;
${context.a_value} = $1.pToken1.pvalue;
${context.b_value} = $2.pToken1.pvalue;
${context.b_token} = $2.pToken1.token;
${context.c_field_is_null} = ($$.pC is null) ? 1 : 0;
${context.alias_a_value} = ${ay}.pToken1.pvalue;
${context.alias_b_value} = ${bee}.pToken1.pvalue;
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = ($$ is null) ? 1 : 0; >>
EOF
end
run_propane(language: language)
compile("spec/test_parser_user_code_tree.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
it "parses lists" do
write_grammar <<EOF
ptype #{language == "d" ? "uint" : "uint32_t"};

39
spec/test_parser_user_code_tree.c Normal file
View File

@ -0,0 +1,39 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include "testutils.h"
int main()
{
char const * input = "ab";
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
/* The parser user code recorded values accessed via $$, $1, and $2 while
* the tree node for the Start rule was being formed. */
assert_eq(3, context->start_n_fields);
assert_eq(11, context->start_a_value);
assert_eq(11, context->a_value);
assert_eq(22, context->b_value);
assert_eq(TOKEN_b, context->b_token);
/* The empty-matched rule C has a null $$ tree node, and its field in the
* Start node is null as well. */
assert_eq(1, context->c_is_null);
assert_eq(1, context->c_field_is_null);
/* Field aliases reference the same component tree nodes as the positional
* references. */
assert_eq(11, context->alias_a_value);
assert_eq(22, context->alias_b_value);
Start * start = p_result(context);
assert(start->pA != NULL);
assert(start->pB != NULL);
assert(start->pC == NULL);
p_tree_delete(start);
p_context_delete(context);
return 0;
}

39
spec/test_parser_user_code_tree.d Normal file
View File

@ -0,0 +1,39 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "ab";
p_context_t * context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
/* The parser user code recorded values accessed via $$, $1, and $2 while
* the tree node for the Start rule was being formed. */
assert_eq(3, context.start_n_fields);
assert_eq(11, context.start_a_value);
assert_eq(11, context.a_value);
assert_eq(22, context.b_value);
assert_eq(TOKEN_b, context.b_token);
/* The empty-matched rule C has a null $$ tree node, and its field in the
* Start node is null as well. */
assert_eq(1, context.c_is_null);
assert_eq(1, context.c_field_is_null);
/* Field aliases reference the same component tree nodes as the positional
* references. */
assert_eq(11, context.alias_a_value);
assert_eq(22, context.alias_b_value);
Start * start = p_result(context);
assert(start.pA !is null);
assert(start.pB !is null);
assert(start.pC is null);
p_tree_delete(start);
}