Add p_input_index()

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Add `p_set_position()` API to set the current text position stored in the
   context. Useful for setting the initial text position to something other
   than `(1, 1)` for a nested parse operation.
+- Add `p_input_index()` API to get the current input text byte offset.
 
 ## v4.7.0
 

assets/parser.c.erb

@@ -1363,6 +1363,20 @@ void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * contex
     context->text_position = position;
 }
 
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context->input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -1442,6 +1442,20 @@ public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t *
     context.text_position = position;
 }
 
+/**
+ * Get the current input text byte offset.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return Current input text byte offset (measured from the start of the
+ *   input text passed to <%= @grammar.prefix %>context_new()).
+ */
+public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
+{
+    return context.input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -239,6 +239,8 @@ void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefi
 
 void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
 
+size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -1493,6 +1493,22 @@ p_set_position(context, start);
 p_parse(context);
 ```
 
+### `p_input_index`
+
+The `p_input_index()` function returns the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This is useful for slicing out the remaining input after a partial parse,
+or for locating tokens in the original input buffer.
+
+Example:
+
+```
+p_context_t * context = p_context_new(input, input_length);
+p_parse_inner_Statement(context, follow_tokens, n_follow_tokens);
+size_t offset = p_input_index(context);
+/* Remaining input starts at `input + offset`. */
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user

spec/propane_spec.rb

@@ -951,6 +951,21 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "exposes the current input byte offset via p_input_index()" do
+        write_grammar <<EOF
+drop /\\s+/;
+token a;
+token b;
+start Start;
+Start -> a b;
+EOF
+        run_propane(language: language)
+        compile("spec/test_input_index.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))

spec/test_input_index.c

@@ -0,0 +1,60 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    /* Grammar (simple):
+     *   drop /\\s+/;
+     *   token a; token b;
+     *   Start -> a b;
+     *
+     * Verifies that p_input_index() reports the parser/lexer's current byte
+     * offset into the input text. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert_eq(0u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        char const * input = "a b";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        /* The dropped space between `a` and `b` advances input_index too. */
+        assert_eq(3u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        char const * input = "abb";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_t follow_tokens[] = { TOKEN_b };
+        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
+        assert_eq(2u, p_input_index(context));
+        p_context_delete(context);
+    }
+
+    return 0;
+}

spec/test_input_index.d

@@ -0,0 +1,51 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* See test_input_index.c for details on the grammar and cases. */
+
+    /* Fresh context: input_index starts at 0. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_input_index(context) == 0);
+    }
+
+    /* After each successful lex the byte offset advances past the token. */
+    {
+        string input = "a b";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_input_index(context) == 1);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 3);
+    }
+
+    /* After a full successful parse, input_index has reached the end. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        assert(p_parse_Start(context) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+
+    /* When parse_inner completes via a follow token, the follow token is not
+     * consumed, so input_index points at the start of the follow token. */
+    {
+        string input = "abb";
+        p_context_t * context = p_context_new(input);
+        p_token_t[] follow_tokens = [TOKEN_b];
+        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
+        assert(p_input_index(context) == 2);
+    }
+}