From 1729546d692ff8b95c1cb3c6ffbbf1cc089a8548 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Thu, 2 Jul 2026 16:33:10 -0400 Subject: [PATCH] Track rule component positions - close #41 --- assets/parser.c.erb | 80 +++++++++++++++++++++++-- assets/parser.d.erb | 65 ++++++++++++++++++++ doc/user_guide.md | 35 +++++++++++ lib/propane/generator.rb | 8 +++ spec/propane_spec.rb | 126 +++++++++++++++++++++++++++++++++++++++ spec/test_positions.c | 29 +++++++++ spec/test_positions.d | 27 +++++++++ 7 files changed, 365 insertions(+), 5 deletions(-) create mode 100644 spec/test_positions.c create mode 100644 spec/test_positions.d diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 724b5e9..9294ba4 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -711,6 +711,8 @@ typedef struct /** tree node. */ void * tree_node; <% else %> + <%= @grammar.prefix %>position_t position; + <%= @grammar.prefix %>position_t end_position; /** Parser value from this state. */ <%= @grammar.prefix %>value_t pvalue; <% end %> @@ -866,6 +868,57 @@ static void state_values_stack_free(state_values_stack_t * stack) } <% unless @grammar.tree %> +/** + * Get the rule position (start or end) for the currently matched rule. + */ +static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end) +{ + if (n_states > 0u) + { + if (i == 0u) + { + if (get_end) + { + int stack_index = -1; + for (size_t j = 0u; j < n_states; j++) + { + state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j); + if (<%= @grammar.prefix %>position_valid(sv->end_position)) + { + return sv->end_position; + } + } + } + else + { + int stack_index = -(int)n_states; + for (size_t j = 0u; j < n_states; j++) + { + state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j); + if (<%= @grammar.prefix %>position_valid(sv->position)) + { + return sv->position; + } + } + } + } + else + { + if (get_end) + { + return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position; + } + else + { + return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position; + } + } + } + <%= @grammar.prefix %>position_t empty_pos; + memset(&empty_pos, 0, sizeof(empty_pos)); + return empty_pos; +} + /** * Execute user code associated with a parser rule. * @@ -971,6 +1024,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start <% if @grammar.tree %> void * reduced_parser_node; <% else %> + <%= @grammar.prefix %>position_t reduced_position; + <%= @grammar.prefix %>position_t reduced_end_position; <%= @grammar.prefix %>value_t reduced_parser_value; <% end %> state_values_stack_init(&statevalues); @@ -1013,7 +1068,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start { /* We have something to shift. */ state_values_stack_push(&statevalues); - state_values_stack_index(&statevalues, -1)->state_id = shift_state; + state_value_t * new_state_info = state_values_stack_index(&statevalues, -1); + new_state_info->state_id = shift_state; if (reduced_rule_set == INVALID_ID) { /* We shifted a token, mark it consumed. */ @@ -1030,9 +1086,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start token_tree_node->token = token; token_tree_node->pvalue = token_info.pvalue; <%= expand_code(@grammar.on_token_node, false, nil, nil) %> - state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node; + new_state_info->tree_node = token_tree_node; <% else %> - state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue; + new_state_info->position = token_info.position; + new_state_info->end_position = token_info.end_position; + new_state_info->pvalue = token_info.pvalue; <% end %> token = INVALID_TOKEN_ID; } @@ -1040,9 +1098,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start { /* We shifted a RuleSet. */ <% if @grammar.tree %> - state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node; + new_state_info->tree_node = reduced_parser_node; <% else %> - state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value; + new_state_info->pvalue = reduced_parser_value; + new_state_info->position = reduced_position; + new_state_info->end_position = reduced_end_position; <%= @grammar.prefix %>value_t new_parse_result; memset(&new_parse_result, 0, sizeof(new_parse_result)); reduced_parser_value = new_parse_result; @@ -1113,6 +1173,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start return P_USER_TERMINATED; } reduced_parser_value = reduced_parser_value2; + if (parser_reduce_table[reduce_index].n_states > 0u) + { + reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false); + reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true); + } + else + { + memset(&reduced_position, 0, sizeof(reduced_position)); + memset(&reduced_end_position, 0, sizeof(reduced_end_position)); + } <% end %> reduced_rule_set = parser_reduce_table[reduce_index].rule_set; state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states); diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 46da513..5154937 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -886,6 +886,8 @@ private struct state_value_t /** Tree node. */ void * tree_node; <% else %> + <%= @grammar.prefix %>position_t position; + <%= @grammar.prefix %>position_t end_position; /** Parser value from this state. */ <%= @grammar.prefix %>value_t pvalue; <% end %> @@ -941,6 +943,53 @@ private immutable parser_state_t[] parser_state_table = [ ]; <% unless @grammar.tree %> +/** + * Get the rule position (start or end) for the currently matched rule. + */ +private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end) +{ + if (n_states > 0u) + { + if (i == 0u) + { + if (get_end) + { + for (size_t j = 0u; j < n_states; j++) + { + state_value_t * sv = &statevalues[$-1-j]; + if (sv.end_position.valid) + { + return sv.end_position; + } + } + } + else + { + for (size_t j = 0u; j < n_states; j++) + { + state_value_t * sv = &statevalues[$-n_states+j]; + if (sv.position.valid) + { + return sv.position; + } + } + } + } + else + { + if (get_end) + { + return statevalues[$-1-n_states+i].end_position; + } + else + { + return statevalues[$-1-n_states+i].position; + } + } + } + return <%= @grammar.prefix %>position_t.INVALID; +} + /** * Execute user code associated with a parser rule. * @@ -1047,6 +1096,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star <% if @grammar.tree %> void * reduced_parser_node; <% else %> + <%= @grammar.prefix %>position_t reduced_position; + <%= @grammar.prefix %>position_t reduced_end_position; <%= @grammar.prefix %>value_t reduced_parser_value; <% end %> for (;;) @@ -1091,6 +1142,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star <%= expand_code(@grammar.on_token_node, false, nil, nil) %> statevalues[$-1].tree_node = token_tree_node; <% else %> + statevalues[$-1].position = token_info.position; + statevalues[$-1].end_position = token_info.end_position; statevalues[$-1].pvalue = token_info.pvalue; <% end %> token = INVALID_TOKEN_ID; @@ -1102,6 +1155,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star statevalues[$-1].tree_node = reduced_parser_node; <% else %> statevalues[$-1].pvalue = reduced_parser_value; + statevalues[$-1].position = reduced_position; + statevalues[$-1].end_position = reduced_end_position; <%= @grammar.prefix %>value_t new_parse_result; reduced_parser_value = new_parse_result; <% end %> @@ -1174,6 +1229,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star return P_USER_TERMINATED; } reduced_parser_value = reduced_parser_value2; + if (parser_reduce_table[reduce_index].n_states > 0u) + { + reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false); + reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true); + } + else + { + reduced_position = <%= @grammar.prefix %>position_t.INVALID; + reduced_end_position = <%= @grammar.prefix %>position_t.INVALID; + } <% end %> reduced_rule_set = parser_reduce_table[reduce_index].rule_set; statevalues.length -= parser_reduce_table[reduce_index].n_states; diff --git a/doc/user_guide.md b/doc/user_guide.md index f2d8b61..3225fe0 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -217,6 +217,41 @@ rule. Parser values for the rules or tokens in the rule pattern can be accessed positionally with tokens `$1`, `$2`, `$3`, etc... +The input text positions for the reduced rule and for the individual rule +components can also be accessed from within a parser rule code block. +Each of these positions is an instance of the `p_position_t` structure (see +`${#p_position_t}`), which contains 1-based `row` and `col` fields. + +The start position of the overall reduced rule is accessed with +`${$.position}`, and the end position of the overall reduced rule is accessed +with `${$.end_position}`. + +The start and end positions of an individual rule component are accessed +positionally with `${N.position}` and `${N.end_position}`, where `N` is the +1-based index of the component (`${1.position}` for the first component, +`${2.position}` for the second, and so on). + +Example: + +``` +Assignment -> ident equals Expr << + printf("assignment on row %d, col %d\n", + ${$.position}.row, ${$.position}.col); + printf("target identifier ends on row %d, col %d\n", + ${1.end_position}.row, ${1.end_position}.col); + printf("expression starts on row %d, col %d\n", + ${3.position}.row, ${3.position}.col); +>> +``` + +A rule or rule component that allows for an empty match may not have valid +positions. +In this case the position should be checked for validity before its `row` and +`col` fields are used (see `${#p_position_valid}`). +For C targets this can be accomplished with +`if (p_position_valid(${$.position}))` and for D targets this can be +accomplished with `if (${$.position}.valid)`. + Parser rule code blocks are not available in tree generation mode. In tree generation mode, a full parse tree is automatically constructed in memory for user code to traverse after parsing is complete. diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 03b2a8a..2d88ed6 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -312,6 +312,14 @@ class Propane "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}" end end + code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match| + index = $1.to_i + "get_rule_position(statevalues, #{index}, n_states, false)" + end + code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match| + index = $1.to_i + "get_rule_position(statevalues, #{index}, n_states, true)" + end code = code.gsub(/\$\{(\w+)\}/) do |match| aliasname = $1 if index = rule.aliases[aliasname] diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index c068e9b..0e4ad53 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1814,6 +1814,132 @@ EOF expect(results.stderr).to eq "" expect(results.status).to eq 0 end + + it "allows accessing rule and component text positions" do + if language == "d" + write_grammar <> +drop /\\s+/; +token tok1; +token tok2; +token ident /[a-zA-Z_]\\w*/; +token num /\\d+/; +Num -> num; +Start -> ident Num << + writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col); + writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col); + writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col); + writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col); + writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col); + writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col); +>> +R -> Empty tok2 << + writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col); + writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col); + writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col); + writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col); + writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col); + writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col); +>> +R -> tok1 Empty << + writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col); + writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col); + writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col); + writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col); + writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col); + writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col); +>> +Empty -> ; +Start -> R << + writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col); + writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col); +>> +Start -> Empty << + writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col); + writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col); +>> +EOF + else + write_grammar < +>> +drop /\\s+/; +token tok1; +token tok2; +token ident /[a-zA-Z_]\\w*/; +token num /\\d+/; +Num -> num; +token pct /%/; +Start -> ident Num << + printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col); + printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col); + printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col); + printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col); + printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col); + printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col); +>> +R -> Empty tok2 << + printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col); + printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col); + printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col); + printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col); + printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col); + printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col); +>> +R -> tok1 Empty << + printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col); + printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col); + printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col); + printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col); + printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col); + printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col); +>> +Empty -> ; +Start -> R << + printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col); + printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col); +>> +Start -> Empty << + printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col); + printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col); +>> +EOF + end + run_propane(language: language) + compile("spec/test_positions.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + expect(results.stdout).to eq < +#include +#include + +int main() +{ + char const * input = " Hello\n\n 4200\n"; + p_context_t * context; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + p_context_delete(context); + + printf("\n"); + + input = "\n tok2"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + p_context_delete(context); + + printf("\n"); + + input = " tok1"; + context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_positions.d b/spec/test_positions.d new file mode 100644 index 0000000..b4aa45e --- /dev/null +++ b/spec/test_positions.d @@ -0,0 +1,27 @@ +import testparser; +import std.stdio; + +int main() +{ + return 0; +} + +unittest +{ + string input = " Hello\n\n 4200\n"; + p_context_t * context; + context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); + + writeln(); + + input = "\n tok2"; + context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); + + writeln(); + + input = " tok1"; + context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); +}