diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a2dd1..f0d7bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ context. Useful for setting the initial text position to something other than `(1, 1)` for a nested parse operation. - Add `p_input_index()` API to get the current input text byte offset. +- Add `p_set_input_index()` API to set the current input text byte offset. + Useful together with `p_set_position()` to rewind the input part-way through + a parse in order to re-read an earlier section of the input. ## v4.7.0 diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 6c3aab9..0f126ea 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1377,6 +1377,27 @@ size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * conte return context->input_index; } +/** + * Set the current input text byte offset. + * + * This moves the lexer's read cursor to the given byte offset (measured from + * the start of the input text passed to <%= @grammar.prefix %>context_new()). + * It can be used together with <%= @grammar.prefix %>set_position() to rewind + * the input part-way through a parse in order to re-read an earlier section of + * the input. The byte offset is not validated; the caller is responsible for + * providing an offset within the bounds of the input text. A value previously + * returned by <%= @grammar.prefix %>input_index() is a suitable argument. + * + * @param context + * Lexer/parser context structure. + * @param input_index + * Input text byte offset to set. + */ +void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index) +{ + context->input_index = input_index; +} + /** * Get the user terminate code. * diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 7f9eb6e..f8f102b 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -1456,6 +1456,27 @@ public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t return context.input_index; } +/** + * Set the current input text byte offset. + * + * This moves the lexer's read cursor to the given byte offset (measured from + * the start of the input text passed to <%= @grammar.prefix %>context_new()). + * It can be used together with <%= @grammar.prefix %>set_position() to rewind + * the input part-way through a parse in order to re-read an earlier section of + * the input. The byte offset is not validated; the caller is responsible for + * providing an offset within the bounds of the input text. A value previously + * returned by <%= @grammar.prefix %>input_index() is a suitable argument. + * + * @param context + * Lexer/parser context structure. + * @param input_index + * Input text byte offset to set. + */ +public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index) +{ + context.input_index = input_index; +} + /** * Get the user terminate code. * diff --git a/assets/parser.h.erb b/assets/parser.h.erb index afe4b87..966127f 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -241,6 +241,8 @@ void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * contex size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context); +void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index); + size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context); <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context); diff --git a/doc/user_guide.md b/doc/user_guide.md index cc57e30..0a71dd9 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -1544,6 +1544,28 @@ size_t offset = p_input_index(context); /* Remaining input starts at `input + offset`. */ ``` +### `p_set_input_index` + +The `p_set_input_index()` function sets the current input text byte offset, +measured from the start of the input text passed to `p_context_new()`. +This moves the lexer's read cursor, which can be used together with +`p_set_position()` to rewind the input part-way through a parse in order to +re-read an earlier section of the input. +The byte offset is not validated; the caller is responsible for providing an +offset within the bounds of the input text. +A value previously returned by `p_input_index()` is a suitable argument. + +Example: + +``` +/* Save the cursor and text position at the start of a section. */ +size_t saved_index = p_input_index(context); +p_position_t saved_position = p_position(context); +/* ... later, rewind to re-read that section. */ +p_set_input_index(context, saved_index); +p_set_position(context, saved_position); +``` + ### `p_user_terminate_code` The `p_user_terminate_code()` function can be used to retrieve the user diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index a7ec903..348f023 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -966,6 +966,16 @@ EOF expect(results.status).to eq 0 end + it "allows rewinding the input to repeat a section during a parse" do + ext = language == "cpp" ? "c" : language + write_grammar(File.read("spec/rewind.#{ext}.propane")) + run_propane(language: language) + compile("spec/test_rewind.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "allows creating a JSON parser" do ext = language == "cpp" ? "c" : language write_grammar(File.read("spec/json_parser.#{ext}.propane")) diff --git a/spec/rewind.c.propane b/spec/rewind.c.propane new file mode 100644 index 0000000..9e536c5 --- /dev/null +++ b/spec/rewind.c.propane @@ -0,0 +1,23 @@ +<< +#include +#include +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info); +void record(int value); +>> + +ptype int; + +lex_fn mylexfn; + +drop /\s+/; +token repeat /repeat/; +token lbrace /\{/; +token rbrace /\}/; +token plus /\+/; +token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >> + +Start -> Statements; +Statements -> ; +Statements -> Statement Statements; +Statement -> Add; +Add -> num plus num << record($1 + $3); >> diff --git a/spec/rewind.d.propane b/spec/rewind.d.propane new file mode 100644 index 0000000..8cc4fc6 --- /dev/null +++ b/spec/rewind.d.propane @@ -0,0 +1,20 @@ +<< +import test_rewind; +>> + +ptype int; + +lex_fn mylexfn; + +drop /\s+/; +token repeat /repeat/; +token lbrace /\{/; +token rbrace /\}/; +token plus /\+/; +token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >> + +Start -> Statements; +Statements -> ; +Statements -> Statement Statements; +Statement -> Add; +Add -> num plus num << record($1 + $3); >> diff --git a/spec/test_parse_inner_nested.c b/spec/test_parse_inner_nested.c index 5f5c01a..df06e5e 100644 --- a/spec/test_parse_inner_nested.c +++ b/spec/test_parse_inner_nested.c @@ -48,7 +48,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) /* Replace the '(' token with a synthesized num carrying the nested * parse result. */ out_token_info->token = TOKEN_num; - out_token_info->pvalue.v_default = value; + out_token_info->pvalue = p_value(value); } return P_SUCCESS; } diff --git a/spec/test_parse_inner_nested.d b/spec/test_parse_inner_nested.d index 3abc008..7fa16b4 100644 --- a/spec/test_parse_inner_nested.d +++ b/spec/test_parse_inner_nested.d @@ -31,7 +31,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) /* Replace the '(' token with a synthesized num carrying the nested * parse result. */ out_token_info.token = TOKEN_num; - out_token_info.pvalue.v_default = value; + out_token_info.pvalue = p_value(value); } return P_SUCCESS; } diff --git a/spec/test_rewind.c b/spec/test_rewind.c new file mode 100644 index 0000000..18d52a8 --- /dev/null +++ b/spec/test_rewind.c @@ -0,0 +1,124 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +/* Grammar (statement list of additions; a "repeat" directive handled entirely + * by the lex function): + * ptype int; + * lex_fn mylexfn; + * token repeat /repeat/; token lbrace /\{/; token rbrace /\}/; + * token plus /\+/; token num /\d+/ << ... atoi ... >> + * Start -> Statements; + * Statements -> ; + * Statements -> Statement Statements; + * Statement -> Add; + * Add -> num plus num << record($1 + $3); >> + * + * Scenario: a "repeat { }" directive that expands its body + * times, similar to loop unrolling in a configuration DSL. The tokens + * repeat, lbrace, and rbrace appear in no grammar rule; the lex function + * interprets the directive and feeds the body's tokens to the parser + * times. Rather than buffering the body tokens, the lex function records the + * input byte offset and text position at the start of the body (with + * p_input_index() and p_position()) and, each time it reaches the closing '}', + * rewinds the lexer back to that point (with p_set_input_index() and + * p_set_position()) to re-read the body from the original input. Rewinding the + * text position as well as the byte offset means each expansion reports the + * same token positions as the first. */ + +static int nums[16]; +static size_t n_nums; +static uint32_t num_cols[16]; +static size_t n_num_cols; + +void record(int value) +{ + nums[n_nums++] = value; +} + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + static int remaining; + static size_t body_index; + static p_position_t body_position; + + for (;;) + { + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + + if (out_token_info->token == TOKEN_repeat) + { + /* Consume "repeat {" and remember where the body begins. */ + p_token_info_t count_info; + size_t count_result = p_lex(context, &count_info); + assert(count_result == P_SUCCESS); + assert(count_info.token == TOKEN_num); + p_token_info_t brace_info; + size_t brace_result = p_lex(context, &brace_info); + assert(brace_result == P_SUCCESS); + assert(brace_info.token == TOKEN_lbrace); + remaining = p_value_get(&count_info.pvalue); + body_index = p_input_index(context); + body_position = p_position(context); + continue; + } + if (out_token_info->token == TOKEN_rbrace) + { + /* End of the body. If more expansions remain, rewind the lexer to + * the start of the body and re-read it; otherwise fall through to + * the input following the '}'. */ + if (remaining > 1) + { + remaining--; + p_set_input_index(context, body_index); + p_set_position(context, body_position); + continue; + } + remaining = 0; + continue; + } + if (out_token_info->token == TOKEN_num) + { + num_cols[n_num_cols++] = out_token_info->position.col; + } + return result; + } +} + +int main() +{ + /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three + * times (recording 30 each time), followed by "5 + 5" (recording 10). */ + char const * input = "repeat 3 { 10 + 20 } 5 + 5"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse(context) == P_SUCCESS); + p_context_delete(context); + + /* The additions were recorded once per body expansion, then once more for + * the trailing statement. */ + assert_eq(4u, n_nums); + assert_eq(30u, (size_t)nums[0]); + assert_eq(30u, (size_t)nums[1]); + assert_eq(30u, (size_t)nums[2]); + assert_eq(10u, (size_t)nums[3]); + + /* Each body expansion reported the same columns for its num tokens (12 and + * 17), because the text position was rewound along with the byte offset. + * The trailing statement's nums are at columns 22 and 26. */ + assert_eq(8u, n_num_cols); + assert_eq(12u, (size_t)num_cols[0]); + assert_eq(17u, (size_t)num_cols[1]); + assert_eq(12u, (size_t)num_cols[2]); + assert_eq(17u, (size_t)num_cols[3]); + assert_eq(12u, (size_t)num_cols[4]); + assert_eq(17u, (size_t)num_cols[5]); + assert_eq(22u, (size_t)num_cols[6]); + assert_eq(26u, (size_t)num_cols[7]); + + return 0; +} diff --git a/spec/test_rewind.d b/spec/test_rewind.d new file mode 100644 index 0000000..0e2bef9 --- /dev/null +++ b/spec/test_rewind.d @@ -0,0 +1,101 @@ +import testparser; +import testutils; + +/* Grammar and scenario: see test_rewind.c. */ + +int[16] nums; +size_t n_nums; +uint[16] num_cols; +size_t n_num_cols; + +void record(int value) +{ + nums[n_nums++] = value; +} + +size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + static int remaining; + static size_t body_index; + static p_position_t body_position; + + for (;;) + { + size_t result = p_lex(context, out_token_info); + if (result != P_SUCCESS) + { + return result; + } + + if (out_token_info.token == TOKEN_repeat) + { + /* Consume "repeat {" and remember where the body begins. */ + p_token_info_t count_info; + size_t count_result = p_lex(context, &count_info); + assert(count_result == P_SUCCESS); + assert(count_info.token == TOKEN_num); + p_token_info_t brace_info; + size_t brace_result = p_lex(context, &brace_info); + assert(brace_result == P_SUCCESS); + assert(brace_info.token == TOKEN_lbrace); + remaining = p_value_get(&count_info.pvalue); + body_index = p_input_index(context); + body_position = p_position(context); + continue; + } + if (out_token_info.token == TOKEN_rbrace) + { + /* End of the body. If more expansions remain, rewind the lexer to + * the start of the body and re-read it; otherwise fall through to + * the input following the '}'. */ + if (remaining > 1) + { + remaining--; + p_set_input_index(context, body_index); + p_set_position(context, body_position); + continue; + } + remaining = 0; + continue; + } + if (out_token_info.token == TOKEN_num) + { + num_cols[n_num_cols++] = out_token_info.position.col; + } + return result; + } +} + +int main() +{ + return 0; +} + +unittest +{ + /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three + * times (recording 30 each time), followed by "5 + 5" (recording 10). */ + string input = "repeat 3 { 10 + 20 } 5 + 5"; + p_context_t * context = p_context_new(input); + assert(p_parse(context) == P_SUCCESS); + p_context_delete(context); + + assert_eq(4u, n_nums); + assert_eq(30, nums[0]); + assert_eq(30, nums[1]); + assert_eq(30, nums[2]); + assert_eq(10, nums[3]); + + /* Each body expansion reported the same columns for its num tokens (12 and + * 17), because the text position was rewound along with the byte offset. + * The trailing statement's nums are at columns 22 and 26. */ + assert_eq(8u, n_num_cols); + assert_eq(12u, num_cols[0]); + assert_eq(17u, num_cols[1]); + assert_eq(12u, num_cols[2]); + assert_eq(17u, num_cols[3]); + assert_eq(12u, num_cols[4]); + assert_eq(17u, num_cols[5]); + assert_eq(22u, num_cols[6]); + assert_eq(26u, num_cols[7]); +} diff --git a/spec/test_set_position.c b/spec/test_set_position.c index 9da510e..d817b8c 100644 --- a/spec/test_set_position.c +++ b/spec/test_set_position.c @@ -77,5 +77,33 @@ int main() p_context_delete(context); } + /* p_set_input_index() rewinds the lexer's byte cursor. Combined with + * p_set_position(), it re-reads an earlier section of the input: both + * tokens are lexed, then the cursor and text position are rewound to the + * start so that the same tokens are produced again with the same reported + * positions. */ + { + char const * input = "ab"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_info_t token_info; + size_t start_index = p_input_index(context); + p_position_t start_position = p_position(context); + assert_eq(0u, start_index); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_a, (size_t)token_info.token); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_b, (size_t)token_info.token); + assert_eq(2u, p_input_index(context)); + /* Rewind and re-read from the start. */ + p_set_input_index(context, start_index); + p_set_position(context, start_position); + assert_eq(0u, p_input_index(context)); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_a, (size_t)token_info.token); + assert_eq(1u, (size_t)token_info.position.row); + assert_eq(1u, (size_t)token_info.position.col); + p_context_delete(context); + } + return 0; } diff --git a/spec/test_set_position.d b/spec/test_set_position.d index 1858df8..f7ba916 100644 --- a/spec/test_set_position.d +++ b/spec/test_set_position.d @@ -63,4 +63,28 @@ unittest assert(err_pos.row == 10); assert(err_pos.col == 3); } + + /* p_set_input_index() rewinds the lexer's byte cursor. Combined with + * p_set_position(), it re-reads an earlier section of the input. */ + { + string input = "ab"; + p_context_t * context = p_context_new(input); + p_token_info_t token_info; + size_t start_index = p_input_index(context); + p_position_t start_position = p_position(context); + assert(start_index == 0); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_a); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_b); + assert(p_input_index(context) == 2); + /* Rewind and re-read from the start. */ + p_set_input_index(context, start_index); + p_set_position(context, start_position); + assert(p_input_index(context) == 0); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_a); + assert(token_info.position.row == 1); + assert(token_info.position.col == 1); + } }