diff --git a/spec/parse_inner_nested.c.propane b/spec/parse_inner_nested.c.propane new file mode 100644 index 0000000..e0a5576 --- /dev/null +++ b/spec/parse_inner_nested.c.propane @@ -0,0 +1,19 @@ +<< +#include +#include +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info); +>> + +ptype int; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >> + +Start -> Expr << $$ = $1; >> +Expr -> num << $$ = $1; >> +Expr -> Expr plus num << $$ = $1 + $3; >> diff --git a/spec/parse_inner_nested.d.propane b/spec/parse_inner_nested.d.propane new file mode 100644 index 0000000..2faf472 --- /dev/null +++ b/spec/parse_inner_nested.d.propane @@ -0,0 +1,17 @@ +<< +import test_parse_inner_nested; +>> + +ptype int; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >> + +Start -> Expr << $$ = $1; >> +Expr -> num << $$ = $1; >> +Expr -> Expr plus num << $$ = $1 + $3; >> diff --git a/spec/parse_inner_nested_tree.c.propane b/spec/parse_inner_nested_tree.c.propane new file mode 100644 index 0000000..df98be3 --- /dev/null +++ b/spec/parse_inner_nested_tree.c.propane @@ -0,0 +1,17 @@ +<< +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info); +>> + +tree; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/; + +Start -> Expr; +Expr -> num; +Expr -> Expr plus num; diff --git a/spec/parse_inner_nested_tree.d.propane b/spec/parse_inner_nested_tree.d.propane new file mode 100644 index 0000000..7bfac2b --- /dev/null +++ b/spec/parse_inner_nested_tree.d.propane @@ -0,0 +1,17 @@ +<< +import test_parse_inner_nested_tree; +>> + +tree; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/; + +Start -> Expr; +Expr -> num; +Expr -> Expr plus num; diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index e6ff43a..a7ec903 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1686,6 +1686,26 @@ EOF expect(results.status).to eq 0 end + it "supports a reentrant nested parse driven from a custom lex function" do + ext = language == "cpp" ? "c" : language + write_grammar(File.read("spec/parse_inner_nested.#{ext}.propane")) + run_propane(language: language) + compile("spec/test_parse_inner_nested.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + + it "tracks positions across a lex-function nested parse in tree mode" do + ext = language == "cpp" ? "c" : language + write_grammar(File.read("spec/parse_inner_nested_tree.#{ext}.propane")) + run_propane(language: language) + compile("spec/test_parse_inner_nested_tree.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "allows multiple starting rules in tree mode" do write_grammar < +#include +#include "testutils.h" + +/* Grammar (integer evaluator; parentheses handled by the lex function): + * ptype int; + * lex_fn mylexfn; + * token lparen /\(/; token rparen /\)/; token plus /\+/; + * token num /\d+/ << ... atoi ... >> + * Start -> Expr << $$ = $1; >> + * Expr -> num << $$ = $1; >> + * Expr -> Expr plus num << $$ = $1 + $3; >> + * + * The tokens lparen and rparen appear in no grammar rule. Instead, when the + * lex function lexes a '(', it performs a nested parse (p_parse_inner_Start) + * of the parenthesized sub-expression -- reentrantly, while the outer parse is + * still suspended in this callback -- reads the computed value with + * p_result_Start, consumes the ')' that p_parse_inner deliberately left in the + * input, and hands a single synthesized num token carrying that value back to + * the outer parse. Nested groups recurse this process to arbitrary depth. */ + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + if (out_token_info->token == TOKEN_lparen) + { + /* Nested parse of the parenthesized sub-expression, stopping at the + * closing ')' follow token. This re-enters the parser while the outer + * parse is suspended in this lex callback. */ + p_token_t follow_tokens[] = { TOKEN_rparen }; + size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u); + if (inner_result != P_SUCCESS) + { + return inner_result; + } + int value = p_result_Start(context); + /* p_parse_inner rewound the input so that ')' was not consumed; consume + * it now. */ + p_token_info_t rparen_info; + size_t rparen_result = p_lex(context, &rparen_info); + assert(rparen_result == P_SUCCESS); + assert(rparen_info.token == TOKEN_rparen); + /* Replace the '(' token with a synthesized num carrying the nested + * parse result. */ + out_token_info->token = TOKEN_num; + out_token_info->pvalue.v_default = value; + } + return P_SUCCESS; +} + +static int eval(char const * input) +{ + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + int value = p_result(context); + p_context_delete(context); + return value; +} + +int main() +{ + /* No parentheses: plain outer parse. */ + assert_eq(5u, (size_t)eval("2 + 3")); + /* A single group evaluated by the nested parse. */ + assert_eq(3u, (size_t)eval("(1 + 2)")); + /* A group in the middle of an outer expression. */ + assert_eq(14u, (size_t)eval("2 + (3 + 4) + 5")); + /* Nested groups: the nested parse re-enters itself. */ + assert_eq(37u, (size_t)eval("2 + (10 + (20 + 5))")); + assert_eq(15u, (size_t)eval("(1 + 2) + (3 + (4 + 5))")); + + return 0; +} diff --git a/spec/test_parse_inner_nested.d b/spec/test_parse_inner_nested.d new file mode 100644 index 0000000..3abc008 --- /dev/null +++ b/spec/test_parse_inner_nested.d @@ -0,0 +1,64 @@ +import testparser; +import testutils; + +/* Grammar: see test_parse_inner_nested.c. */ + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + if (out_token_info.token == TOKEN_lparen) + { + /* Nested parse of the parenthesized sub-expression, stopping at the + * closing ')' follow token. This re-enters the parser while the outer + * parse is suspended in this lex callback. */ + p_token_t[] follow_tokens = [TOKEN_rparen]; + size_t inner_result = p_parse_inner_Start(context, follow_tokens); + if (inner_result != P_SUCCESS) + { + return inner_result; + } + int value = p_result_Start(context); + /* p_parse_inner rewound the input so that ')' was not consumed; consume + * it now. */ + p_token_info_t rparen_info; + size_t rparen_result = p_lex(context, &rparen_info); + assert(rparen_result == P_SUCCESS); + assert(rparen_info.token == TOKEN_rparen); + /* Replace the '(' token with a synthesized num carrying the nested + * parse result. */ + out_token_info.token = TOKEN_num; + out_token_info.pvalue.v_default = value; + } + return P_SUCCESS; +} + +int eval(string input) +{ + p_context_t * context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); + int value = p_result(context); + p_context_delete(context); + return value; +} + +int main() +{ + return 0; +} + +unittest +{ + /* No parentheses: plain outer parse. */ + assert_eq(5, eval("2 + 3")); + /* A single group evaluated by the nested parse. */ + assert_eq(3, eval("(1 + 2)")); + /* A group in the middle of an outer expression. */ + assert_eq(14, eval("2 + (3 + 4) + 5")); + /* Nested groups: the nested parse re-enters itself. */ + assert_eq(37, eval("2 + (10 + (20 + 5))")); + assert_eq(15, eval("(1 + 2) + (3 + (4 + 5))")); +} diff --git a/spec/test_parse_inner_nested_tree.c b/spec/test_parse_inner_nested_tree.c new file mode 100644 index 0000000..713aadd --- /dev/null +++ b/spec/test_parse_inner_nested_tree.c @@ -0,0 +1,104 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +/* Grammar (tree generation mode; parentheses handled by the lex function): + * tree; + * lex_fn mylexfn; + * token lparen /\(/; token rparen /\)/; token plus /\+/; token num /\d+/; + * Start -> Expr; + * Expr -> num; + * Expr -> Expr plus num; + * + * The same lexer-driven nested parse as test_parse_inner_nested, but in tree + * generation mode. Each "( ... )" group is parsed by a reentrant + * p_parse_inner_Start() call from the lex function; the resulting subtree is + * discarded and a single synthesized num token is handed to the outer parse. + * The synthesized token's position is set to span the whole group ('(' start + * through ')' end), so this verifies that positions survive the nested-parse + * boundary and land correctly in the outer tree. */ + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + if (out_token_info->token == TOKEN_lparen) + { + p_position_t start_position = out_token_info->position; + /* Reentrant nested parse of the parenthesized sub-expression. */ + p_token_t follow_tokens[] = { TOKEN_rparen }; + size_t inner_result = p_parse_inner_Start(context, follow_tokens, 1u); + if (inner_result != P_SUCCESS) + { + return inner_result; + } + Start * inner = p_result_Start(context); + assert_not_null(inner); + /* p_parse_inner rewound the input so that ')' was not consumed; consume + * it now. */ + p_token_info_t rparen_info; + size_t rparen_result = p_lex(context, &rparen_info); + assert(rparen_result == P_SUCCESS); + assert(rparen_info.token == TOKEN_rparen); + /* The subtree covers the region strictly between the parentheses. */ + assert_eq((size_t)(start_position.col + 1u), (size_t)inner->position.col); + assert_eq((size_t)(rparen_info.position.col - 1u), (size_t)inner->end_position.col); + p_tree_delete_Start(inner); + /* Synthesize a num token spanning the entire "( ... )" group. */ + out_token_info->token = TOKEN_num; + out_token_info->position = start_position; + out_token_info->end_position = rparen_info.end_position; + } + return P_SUCCESS; +} + +int main() +{ + /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the + * lexer into a single num token spanning its group. */ + char const * input = "(3 + 4) + (5 + 6)"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + + Start * tree = p_result(context); + assert_not_null(tree); + + /* Start -> Expr, where the top Expr is "Expr plus num". */ + Expr * top = tree->pExpr; + assert_not_null(top); + assert_not_null(top->pExpr); + assert_not_null(top->pToken2); + assert_not_null(top->pToken3); + + /* The '+' joining the two groups is at column 9. */ + assert_eq(1u, (size_t)top->pToken2->position.row); + assert_eq(9u, (size_t)top->pToken2->position.col); + + /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */ + assert_eq(1u, (size_t)top->pToken3->position.row); + assert_eq(11u, (size_t)top->pToken3->position.col); + assert_eq(1u, (size_t)top->pToken3->end_position.row); + assert_eq(17u, (size_t)top->pToken3->end_position.col); + + /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning + * columns 1..7. */ + Expr * left = top->pExpr; + assert_not_null(left->pToken1); + assert_eq(1u, (size_t)left->pToken1->position.row); + assert_eq(1u, (size_t)left->pToken1->position.col); + assert_eq(1u, (size_t)left->pToken1->end_position.row); + assert_eq(7u, (size_t)left->pToken1->end_position.col); + + /* The whole tree spans columns 1..17. */ + assert_eq(1u, (size_t)tree->position.col); + assert_eq(17u, (size_t)tree->end_position.col); + + p_tree_delete(tree); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_parse_inner_nested_tree.d b/spec/test_parse_inner_nested_tree.d new file mode 100644 index 0000000..f1005b6 --- /dev/null +++ b/spec/test_parse_inner_nested_tree.d @@ -0,0 +1,91 @@ +import testparser; +import testutils; + +/* Grammar: see test_parse_inner_nested_tree.c. */ + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + if (out_token_info.token == TOKEN_lparen) + { + p_position_t start_position = out_token_info.position; + /* Reentrant nested parse of the parenthesized sub-expression. */ + p_token_t[] follow_tokens = [TOKEN_rparen]; + size_t inner_result = p_parse_inner_Start(context, follow_tokens); + if (inner_result != P_SUCCESS) + { + return inner_result; + } + Start * inner = p_result_Start(context); + assert(inner !is null); + /* p_parse_inner rewound the input so that ')' was not consumed; consume + * it now. */ + p_token_info_t rparen_info; + size_t rparen_result = p_lex(context, &rparen_info); + assert(rparen_result == P_SUCCESS); + assert(rparen_info.token == TOKEN_rparen); + /* The subtree covers the region strictly between the parentheses. */ + assert_eq(start_position.col + 1u, inner.position.col); + assert_eq(rparen_info.position.col - 1u, inner.end_position.col); + p_tree_delete_Start(inner); + /* Synthesize a num token spanning the entire "( ... )" group. */ + out_token_info.token = TOKEN_num; + out_token_info.position = start_position; + out_token_info.end_position = rparen_info.end_position; + } + return P_SUCCESS; +} + +int main() +{ + return 0; +} + +unittest +{ + /* "(3 + 4) + (5 + 6)": two parenthesized groups, each collapsed by the + * lexer into a single num token spanning its group. */ + string input = "(3 + 4) + (5 + 6)"; + p_context_t * context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); + + Start * tree = p_result(context); + assert(tree !is null); + + /* Start -> Expr, where the top Expr is "Expr plus num". */ + Expr * top = tree.pExpr; + assert(top !is null); + assert(top.pExpr !is null); + assert(top.pToken2 !is null); + assert(top.pToken3 !is null); + + /* The '+' joining the two groups is at column 9. */ + assert_eq(1u, top.pToken2.position.row); + assert_eq(9u, top.pToken2.position.col); + + /* Right operand: synthesized num for "(5 + 6)", spanning columns 11..17. */ + assert_eq(1u, top.pToken3.position.row); + assert_eq(11u, top.pToken3.position.col); + assert_eq(1u, top.pToken3.end_position.row); + assert_eq(17u, top.pToken3.end_position.col); + + /* Left operand: Expr -> num, the synthesized num for "(3 + 4)", spanning + * columns 1..7. */ + Expr * left = top.pExpr; + assert(left.pToken1 !is null); + assert_eq(1u, left.pToken1.position.row); + assert_eq(1u, left.pToken1.position.col); + assert_eq(1u, left.pToken1.end_position.row); + assert_eq(7u, left.pToken1.end_position.col); + + /* The whole tree spans columns 1..17. */ + assert_eq(1u, tree.position.col); + assert_eq(17u, tree.end_position.col); + + p_tree_delete(tree); + p_context_delete(context); +}