diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f65657..76a2dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Add `p_set_position()` API to set the current text position stored in the context. Useful for setting the initial text position to something other than `(1, 1)` for a nested parse operation. +- Add `p_input_index()` API to get the current input text byte offset. ## v4.7.0 diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 0aaf133..6c3aab9 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -1363,6 +1363,20 @@ void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * contex context->text_position = position; } +/** + * Get the current input text byte offset. + * + * @param context + * Lexer/parser context structure. + * + * @return Current input text byte offset (measured from the start of the + * input text passed to <%= @grammar.prefix %>context_new()). + */ +size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context) +{ + return context->input_index; +} + /** * Get the user terminate code. * diff --git a/assets/parser.d.erb b/assets/parser.d.erb index eb36c35..7f9eb6e 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -1442,6 +1442,20 @@ public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context.text_position = position; } +/** + * Get the current input text byte offset. + * + * @param context + * Lexer/parser context structure. + * + * @return Current input text byte offset (measured from the start of the + * input text passed to <%= @grammar.prefix %>context_new()). + */ +public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context) +{ + return context.input_index; +} + /** * Get the user terminate code. * diff --git a/assets/parser.h.erb b/assets/parser.h.erb index a4803e3..afe4b87 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -239,6 +239,8 @@ void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefi void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position); +size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context); + size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context); <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context); diff --git a/doc/user_guide.md b/doc/user_guide.md index 1c3bd02..03f1a6a 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -1493,6 +1493,22 @@ p_set_position(context, start); p_parse(context); ``` +### `p_input_index` + +The `p_input_index()` function returns the current input text byte offset, +measured from the start of the input text passed to `p_context_new()`. +This is useful for slicing out the remaining input after a partial parse, +or for locating tokens in the original input buffer. + +Example: + +``` +p_context_t * context = p_context_new(input, input_length); +p_parse_inner_Statement(context, follow_tokens, n_follow_tokens); +size_t offset = p_input_index(context); +/* Remaining input starts at `input + offset`. */ +``` + ### `p_user_terminate_code` The `p_user_terminate_code()` function can be used to retrieve the user diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 696b53a..438f633 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -951,6 +951,21 @@ EOF expect(results.status).to eq 0 end + it "exposes the current input byte offset via p_input_index()" do + write_grammar < a b; +EOF + run_propane(language: language) + compile("spec/test_input_index.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "allows creating a JSON parser" do ext = language == "cpp" ? "c" : language write_grammar(File.read("spec/json_parser.#{ext}.propane")) diff --git a/spec/test_input_index.c b/spec/test_input_index.c new file mode 100644 index 0000000..ea5f027 --- /dev/null +++ b/spec/test_input_index.c @@ -0,0 +1,60 @@ +#include "testparser.h" +#include +#include +#include "testutils.h" + +int main() +{ + /* Grammar (simple): + * drop /\\s+/; + * token a; token b; + * Start -> a b; + * + * Verifies that p_input_index() reports the parser/lexer's current byte + * offset into the input text. */ + + /* Fresh context: input_index starts at 0. */ + { + char const * input = "ab"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(0u, p_input_index(context)); + p_context_delete(context); + } + + /* After each successful lex the byte offset advances past the token. */ + { + char const * input = "a b"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_info_t token_info; + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_a, (size_t)token_info.token); + assert_eq(1u, p_input_index(context)); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert_eq((size_t)TOKEN_b, (size_t)token_info.token); + /* The dropped space between `a` and `b` advances input_index too. */ + assert_eq(3u, p_input_index(context)); + p_context_delete(context); + } + + /* After a full successful parse, input_index has reached the end. */ + { + char const * input = "ab"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert(p_parse_Start(context) == P_SUCCESS); + assert_eq(2u, p_input_index(context)); + p_context_delete(context); + } + + /* When parse_inner completes via a follow token, the follow token is not + * consumed, so input_index points at the start of the follow token. */ + { + char const * input = "abb"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_t follow_tokens[] = { TOKEN_b }; + assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS); + assert_eq(2u, p_input_index(context)); + p_context_delete(context); + } + + return 0; +} diff --git a/spec/test_input_index.d b/spec/test_input_index.d new file mode 100644 index 0000000..ab84f9a --- /dev/null +++ b/spec/test_input_index.d @@ -0,0 +1,51 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + /* See test_input_index.c for details on the grammar and cases. */ + + /* Fresh context: input_index starts at 0. */ + { + string input = "ab"; + p_context_t * context = p_context_new(input); + assert(p_input_index(context) == 0); + } + + /* After each successful lex the byte offset advances past the token. */ + { + string input = "a b"; + p_context_t * context = p_context_new(input); + p_token_info_t token_info; + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_a); + assert(p_input_index(context) == 1); + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_b); + assert(p_input_index(context) == 3); + } + + /* After a full successful parse, input_index has reached the end. */ + { + string input = "ab"; + p_context_t * context = p_context_new(input); + assert(p_parse_Start(context) == P_SUCCESS); + assert(p_input_index(context) == 2); + } + + /* When parse_inner completes via a follow token, the follow token is not + * consumed, so input_index points at the start of the follow token. */ + { + string input = "abb"; + p_context_t * context = p_context_new(input); + p_token_t[] follow_tokens = [TOKEN_b]; + assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS); + assert(p_input_index(context) == 2); + } +}