josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Track rule component positions - close #41

Browse Source
This commit is contained in:
Josh Holtrop 2026-07-02 16:33:10 -04:00
parent 7942a3be97
commit 1729546d69
7 changed files with 365 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
assets/parser.c.erb
View File

@ -711,6 +711,8 @@ typedef struct
/** tree node. */
void * tree_node;
<% else %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
/** Parser value from this state. */
<%= @grammar.prefix %>value_t pvalue;
<% end %>
@ -866,6 +868,57 @@ static void state_values_stack_free(state_values_stack_t * stack)
}
<% unless @grammar.tree %>
/**
* Get the rule position (start or end) for the currently matched rule.
*/
static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
{
if (n_states > 0u)
{
if (i == 0u)
{
if (get_end)
{
int stack_index = -1;
for (size_t j = 0u; j < n_states; j++)
{
state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
if (<%= @grammar.prefix %>position_valid(sv->end_position))
{
return sv->end_position;
}
}
}
else
{
int stack_index = -(int)n_states;
for (size_t j = 0u; j < n_states; j++)
{
state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
if (<%= @grammar.prefix %>position_valid(sv->position))
{
return sv->position;
}
}
}
}
else
{
if (get_end)
{
return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
}
else
{
return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
}
}
}
<%= @grammar.prefix %>position_t empty_pos;
memset(&empty_pos, 0, sizeof(empty_pos));
return empty_pos;
}
/**
* Execute user code associated with a parser rule.
*
@ -971,6 +1024,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
<% if @grammar.tree %>
void * reduced_parser_node;
<% else %>
<%= @grammar.prefix %>position_t reduced_position;
<%= @grammar.prefix %>position_t reduced_end_position;
<%= @grammar.prefix %>value_t reduced_parser_value;
<% end %>
state_values_stack_init(&statevalues);
@ -1013,7 +1068,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
/* We have something to shift. */
state_values_stack_push(&statevalues);
state_values_stack_index(&statevalues, -1)->state_id = shift_state;
state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
new_state_info->state_id = shift_state;
if (reduced_rule_set == INVALID_ID)
{
/* We shifted a token, mark it consumed. */
@ -1030,9 +1086,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
token_tree_node->token = token;
token_tree_node->pvalue = token_info.pvalue;
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
new_state_info->tree_node = token_tree_node;
<% else %>
state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
new_state_info->position = token_info.position;
new_state_info->end_position = token_info.end_position;
new_state_info->pvalue = token_info.pvalue;
<% end %>
token = INVALID_TOKEN_ID;
}
@ -1040,9 +1098,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
{
/* We shifted a RuleSet. */
<% if @grammar.tree %>
state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
new_state_info->tree_node = reduced_parser_node;
<% else %>
state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
new_state_info->pvalue = reduced_parser_value;
new_state_info->position = reduced_position;
new_state_info->end_position = reduced_end_position;
<%= @grammar.prefix %>value_t new_parse_result;
memset(&new_parse_result, 0, sizeof(new_parse_result));
reduced_parser_value = new_parse_result;
@ -1113,6 +1173,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
return P_USER_TERMINATED;
}
reduced_parser_value = reduced_parser_value2;
if (parser_reduce_table[reduce_index].n_states > 0u)
{
reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
}
else
{
memset(&reduced_position, 0, sizeof(reduced_position));
memset(&reduced_end_position, 0, sizeof(reduced_end_position));
}
<% end %>
reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);

65
assets/parser.d.erb
View File

@ -886,6 +886,8 @@ private struct state_value_t
/** Tree node. */
void * tree_node;
<% else %>
<%= @grammar.prefix %>position_t position;
<%= @grammar.prefix %>position_t end_position;
/** Parser value from this state. */
<%= @grammar.prefix %>value_t pvalue;
<% end %>
@ -941,6 +943,53 @@ private immutable parser_state_t[] parser_state_table = [
];
<% unless @grammar.tree %>
/**
* Get the rule position (start or end) for the currently matched rule.
*/
private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
{
if (n_states > 0u)
{
if (i == 0u)
{
if (get_end)
{
for (size_t j = 0u; j < n_states; j++)
{
state_value_t * sv = &statevalues[$-1-j];
if (sv.end_position.valid)
{
return sv.end_position;
}
}
}
else
{
for (size_t j = 0u; j < n_states; j++)
{
state_value_t * sv = &statevalues[$-n_states+j];
if (sv.position.valid)
{
return sv.position;
}
}
}
}
else
{
if (get_end)
{
return statevalues[$-1-n_states+i].end_position;
}
else
{
return statevalues[$-1-n_states+i].position;
}
}
}
return <%= @grammar.prefix %>position_t.INVALID;
}
/**
* Execute user code associated with a parser rule.
*
@ -1047,6 +1096,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
<% if @grammar.tree %>
void * reduced_parser_node;
<% else %>
<%= @grammar.prefix %>position_t reduced_position;
<%= @grammar.prefix %>position_t reduced_end_position;
<%= @grammar.prefix %>value_t reduced_parser_value;
<% end %>
for (;;)
@ -1091,6 +1142,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
statevalues[$-1].tree_node = token_tree_node;
<% else %>
statevalues[$-1].position = token_info.position;
statevalues[$-1].end_position = token_info.end_position;
statevalues[$-1].pvalue = token_info.pvalue;
<% end %>
token = INVALID_TOKEN_ID;
@ -1102,6 +1155,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
statevalues[$-1].tree_node = reduced_parser_node;
<% else %>
statevalues[$-1].pvalue = reduced_parser_value;
statevalues[$-1].position = reduced_position;
statevalues[$-1].end_position = reduced_end_position;
<%= @grammar.prefix %>value_t new_parse_result;
reduced_parser_value = new_parse_result;
<% end %>
@ -1174,6 +1229,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
return P_USER_TERMINATED;
}
reduced_parser_value = reduced_parser_value2;
if (parser_reduce_table[reduce_index].n_states > 0u)
{
reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
}
else
{
reduced_position = <%= @grammar.prefix %>position_t.INVALID;
reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
}
<% end %>
reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
statevalues.length -= parser_reduce_table[reduce_index].n_states;

35
doc/user_guide.md
View File

@ -217,6 +217,41 @@ rule.
Parser values for the rules or tokens in the rule pattern can be accessed
positionally with tokens `$1`, `$2`, `$3`, etc...
The input text positions for the reduced rule and for the individual rule
components can also be accessed from within a parser rule code block.
Each of these positions is an instance of the `p_position_t` structure (see
`${#p_position_t}`), which contains 1-based `row` and `col` fields.
The start position of the overall reduced rule is accessed with
`${$.position}`, and the end position of the overall reduced rule is accessed
with `${$.end_position}`.
The start and end positions of an individual rule component are accessed
positionally with `${N.position}` and `${N.end_position}`, where `N` is the
1-based index of the component (`${1.position}` for the first component,
`${2.position}` for the second, and so on).
Example:
```
Assignment -> ident equals Expr <<
printf("assignment on row %d, col %d\n",
${$.position}.row, ${$.position}.col);
printf("target identifier ends on row %d, col %d\n",
${1.end_position}.row, ${1.end_position}.col);
printf("expression starts on row %d, col %d\n",
${3.position}.row, ${3.position}.col);
>>
```
A rule or rule component that allows for an empty match may not have valid
positions.
In this case the position should be checked for validity before its `row` and
`col` fields are used (see `${#p_position_valid}`).
For C targets this can be accomplished with
`if (p_position_valid(${$.position}))` and for D targets this can be
accomplished with `if (${$.position}.valid)`.
Parser rule code blocks are not available in tree generation mode.
In tree generation mode, a full parse tree is automatically constructed in
memory for user code to traverse after parsing is complete.

8
lib/propane/generator.rb
View File

@ -312,6 +312,14 @@ class Propane
"statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
end
end
code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
index = $1.to_i
"get_rule_position(statevalues, #{index}, n_states, false)"
end
code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
index = $1.to_i
"get_rule_position(statevalues, #{index}, n_states, true)"
end
code = code.gsub(/\$\{(\w+)\}/) do |match|
aliasname = $1
if index = rule.aliases[aliasname]

126
spec/propane_spec.rb
View File

@ -1814,6 +1814,132 @@ EOF
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
it "allows accessing rule and component text positions" do
if language == "d"
write_grammar <<EOF
<<
import std.stdio;
>>
drop /\\s+/;
token tok1;
token tok2;
token ident /[a-zA-Z_]\\w*/;
token num /\\d+/;
Num -> num;
Start -> ident Num <<
writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col);
writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col);
writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col);
writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
>>
R -> Empty tok2 <<
writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col);
writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col);
writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col);
writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
>>
R -> tok1 Empty <<
writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col);
writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col);
writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col);
writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
>>
Empty -> ;
Start -> R <<
writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col);
writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
>>
Start -> Empty <<
writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col);
writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
>>
EOF
else
write_grammar <<EOF
<<
#include <stdio.h>
>>
drop /\\s+/;
token tok1;
token tok2;
token ident /[a-zA-Z_]\\w*/;
token num /\\d+/;
Num -> num;
token pct /%/;
Start -> ident Num <<
printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
>>
R -> Empty tok2 <<
printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
>>
R -> tok1 Empty <<
printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
>>
Empty -> ;
Start -> R <<
printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
>>
Start -> Empty <<
printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
>>
EOF
end
run_propane(language: language)
compile("spec/test_positions.#{language}", language: language)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
expect(results.stdout).to eq <<EOF
ident start: 1, 5
ident end: 1, 9
Num start: 3, 9
Num end: 3, 12
Start start: 1, 5
Start end: 3, 12
Empty start: 0, 0
Empty end: 0, 0
tok2 start: 2, 2
tok2 end: 2, 5
R start: 2, 2
R end: 2, 5
StartR start: 2, 2
StartR end: 2, 5
tok1 start: 1, 3
tok1 end: 1, 6
Empty start: 0, 0
Empty end: 0, 0
R2 start: 1, 3
R2 end: 1, 6
StartR start: 1, 3
StartR end: 1, 6
EOF
end
end
end
end

29
spec/test_positions.c Normal file
View File

@ -0,0 +1,29 @@
#include "testparser.h"
#include <assert.h>
#include <string.h>
#include <stdio.h>
int main()
{
char const * input = " Hello\n\n 4200\n";
p_context_t * context;
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
p_context_delete(context);
printf("\n");
input = "\n tok2";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
p_context_delete(context);
printf("\n");
input = " tok1";
context = p_context_new((uint8_t const *)input, strlen(input));
assert(p_parse(context) == P_SUCCESS);
p_context_delete(context);
return 0;
}

27
spec/test_positions.d Normal file
View File

@ -0,0 +1,27 @@
import testparser;
import std.stdio;
int main()
{
return 0;
}
unittest
{
string input = " Hello\n\n 4200\n";
p_context_t * context;
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
writeln();
input = "\n tok2";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
writeln();
input = " tok1";
context = p_context_new(input);
assert(p_parse(context) == P_SUCCESS);
}