Add p_set_input_index()

CHANGELOG.md

@@ -11,6 +11,9 @@
   context. Useful for setting the initial text position to something other
   than `(1, 1)` for a nested parse operation.
 - Add `p_input_index()` API to get the current input text byte offset.
+- Add `p_set_input_index()` API to set the current input text byte offset.
+  Useful together with `p_set_position()` to rewind the input part-way through
+  a parse in order to re-read an earlier section of the input.
 
 ## v4.7.0
 

assets/parser.c.erb

@@ -1377,6 +1377,27 @@ size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * conte
     return context->input_index;
 }
 
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context->input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -1456,6 +1456,27 @@ public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t
     return context.input_index;
 }
 
+/**
+ * Set the current input text byte offset.
+ *
+ * This moves the lexer's read cursor to the given byte offset (measured from
+ * the start of the input text passed to <%= @grammar.prefix %>context_new()).
+ * It can be used together with <%= @grammar.prefix %>set_position() to rewind
+ * the input part-way through a parse in order to re-read an earlier section of
+ * the input. The byte offset is not validated; the caller is responsible for
+ * providing an offset within the bounds of the input text. A value previously
+ * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ * @param input_index
+ *   Input text byte offset to set.
+ */
+public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
+{
+    context.input_index = input_index;
+}
+
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -241,6 +241,8 @@ void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * contex
 
 size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
 
+void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
+
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -1544,6 +1544,28 @@ size_t offset = p_input_index(context);
 /* Remaining input starts at `input + offset`. */
 ```
 
+### `p_set_input_index`
+
+The `p_set_input_index()` function sets the current input text byte offset,
+measured from the start of the input text passed to `p_context_new()`.
+This moves the lexer's read cursor, which can be used together with
+`p_set_position()` to rewind the input part-way through a parse in order to
+re-read an earlier section of the input.
+The byte offset is not validated; the caller is responsible for providing an
+offset within the bounds of the input text.
+A value previously returned by `p_input_index()` is a suitable argument.
+
+Example:
+
+```
+/* Save the cursor and text position at the start of a section. */
+size_t saved_index = p_input_index(context);
+p_position_t saved_position = p_position(context);
+/* ... later, rewind to re-read that section. */
+p_set_input_index(context, saved_index);
+p_set_position(context, saved_position);
+```
+
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user

spec/propane_spec.rb

@@ -966,6 +966,16 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows rewinding the input to repeat a section during a parse" do
+        ext = language == "cpp" ? "c" : language
+        write_grammar(File.read("spec/rewind.#{ext}.propane"))
+        run_propane(language: language)
+        compile("spec/test_rewind.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))

spec/rewind.c.propane

@@ -0,0 +1,23 @@
+<<
+#include <stdlib.h>
+#include <string.h>
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
+void record(int value);
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -0,0 +1,20 @@
+<<
+import test_rewind;
+>>
+
+ptype int;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << record($1 + $3); >>

spec/test_parse_inner_nested.c

@@ -48,7 +48,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         /* Replace the '(' token with a synthesized num carrying the nested
          * parse result. */
         out_token_info->token = TOKEN_num;
-        out_token_info->pvalue.v_default = value;
+        out_token_info->pvalue = p_value(value);
     }
     return P_SUCCESS;
 }

spec/test_parse_inner_nested.d

@@ -31,7 +31,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         /* Replace the '(' token with a synthesized num carrying the nested
          * parse result. */
         out_token_info.token = TOKEN_num;
-        out_token_info.pvalue.v_default = value;
+        out_token_info.pvalue = p_value(value);
     }
     return P_SUCCESS;
 }

spec/test_rewind.c

@@ -0,0 +1,124 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+/* Grammar (statement list of additions; a "repeat" directive handled entirely
+ * by the lex function):
+ *   ptype int;
+ *   lex_fn mylexfn;
+ *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
+ *   token plus /\+/; token num /\d+/ << ... atoi ... >>
+ *   Start      -> Statements;
+ *   Statements -> ;
+ *   Statements -> Statement Statements;
+ *   Statement  -> Add;
+ *   Add        -> num plus num  << record($1 + $3); >>
+ *
+ * Scenario: a "repeat <count> { <body> }" directive that expands its body
+ * <count> times, similar to loop unrolling in a configuration DSL. The tokens
+ * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
+ * interprets the directive and feeds the body's tokens to the parser <count>
+ * times. Rather than buffering the body tokens, the lex function records the
+ * input byte offset and text position at the start of the body (with
+ * p_input_index() and p_position()) and, each time it reaches the closing '}',
+ * rewinds the lexer back to that point (with p_set_input_index() and
+ * p_set_position()) to re-read the body from the original input. Rewinding the
+ * text position as well as the byte offset means each expansion reports the
+ * same token positions as the first. */
+
+static int nums[16];
+static size_t n_nums;
+static uint32_t num_cols[16];
+static size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info->token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info->token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info->token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info->position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    /* The additions were recorded once per body expansion, then once more for
+     * the trailing statement. */
+    assert_eq(4u, n_nums);
+    assert_eq(30u, (size_t)nums[0]);
+    assert_eq(30u, (size_t)nums[1]);
+    assert_eq(30u, (size_t)nums[2]);
+    assert_eq(10u, (size_t)nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, (size_t)num_cols[0]);
+    assert_eq(17u, (size_t)num_cols[1]);
+    assert_eq(12u, (size_t)num_cols[2]);
+    assert_eq(17u, (size_t)num_cols[3]);
+    assert_eq(12u, (size_t)num_cols[4]);
+    assert_eq(17u, (size_t)num_cols[5]);
+    assert_eq(22u, (size_t)num_cols[6]);
+    assert_eq(26u, (size_t)num_cols[7]);
+
+    return 0;
+}

spec/test_rewind.d

@@ -0,0 +1,101 @@
+import testparser;
+import testutils;
+
+/* Grammar and scenario: see test_rewind.c. */
+
+int[16] nums;
+size_t n_nums;
+uint[16] num_cols;
+size_t n_num_cols;
+
+void record(int value)
+{
+    nums[n_nums++] = value;
+}
+
+size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static int remaining;
+    static size_t body_index;
+    static p_position_t body_position;
+
+    for (;;)
+    {
+        size_t result = p_lex(context, out_token_info);
+        if (result != P_SUCCESS)
+        {
+            return result;
+        }
+
+        if (out_token_info.token == TOKEN_repeat)
+        {
+            /* Consume "repeat <count> {" and remember where the body begins. */
+            p_token_info_t count_info;
+            size_t count_result = p_lex(context, &count_info);
+            assert(count_result == P_SUCCESS);
+            assert(count_info.token == TOKEN_num);
+            p_token_info_t brace_info;
+            size_t brace_result = p_lex(context, &brace_info);
+            assert(brace_result == P_SUCCESS);
+            assert(brace_info.token == TOKEN_lbrace);
+            remaining = p_value_get(&count_info.pvalue);
+            body_index = p_input_index(context);
+            body_position = p_position(context);
+            continue;
+        }
+        if (out_token_info.token == TOKEN_rbrace)
+        {
+            /* End of the body. If more expansions remain, rewind the lexer to
+             * the start of the body and re-read it; otherwise fall through to
+             * the input following the '}'. */
+            if (remaining > 1)
+            {
+                remaining--;
+                p_set_input_index(context, body_index);
+                p_set_position(context, body_position);
+                continue;
+            }
+            remaining = 0;
+            continue;
+        }
+        if (out_token_info.token == TOKEN_num)
+        {
+            num_cols[n_num_cols++] = out_token_info.position.col;
+        }
+        return result;
+    }
+}
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    string input = "repeat 3 { 10 + 20 } 5 + 5";
+    p_context_t * context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    assert_eq(4u, n_nums);
+    assert_eq(30, nums[0]);
+    assert_eq(30, nums[1]);
+    assert_eq(30, nums[2]);
+    assert_eq(10, nums[3]);
+
+    /* Each body expansion reported the same columns for its num tokens (12 and
+     * 17), because the text position was rewound along with the byte offset.
+     * The trailing statement's nums are at columns 22 and 26. */
+    assert_eq(8u, n_num_cols);
+    assert_eq(12u, num_cols[0]);
+    assert_eq(17u, num_cols[1]);
+    assert_eq(12u, num_cols[2]);
+    assert_eq(17u, num_cols[3]);
+    assert_eq(12u, num_cols[4]);
+    assert_eq(17u, num_cols[5]);
+    assert_eq(22u, num_cols[6]);
+    assert_eq(26u, num_cols[7]);
+}

spec/test_set_position.c

@@ -77,5 +77,33 @@ int main()
         p_context_delete(context);
     }
 
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input: both
+     * tokens are lexed, then the cursor and text position are rewound to the
+     * start so that the same tokens are produced again with the same reported
+     * positions. */
+    {
+        char const * input = "ab";
+        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert_eq(0u, start_index);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
+        assert_eq(2u, p_input_index(context));
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert_eq(0u, p_input_index(context));
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
+        assert_eq(1u, (size_t)token_info.position.row);
+        assert_eq(1u, (size_t)token_info.position.col);
+        p_context_delete(context);
+    }
+
     return 0;
 }

spec/test_set_position.d

@@ -63,4 +63,28 @@ unittest
         assert(err_pos.row == 10);
         assert(err_pos.col == 3);
     }
+
+    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
+     * p_set_position(), it re-reads an earlier section of the input. */
+    {
+        string input = "ab";
+        p_context_t * context = p_context_new(input);
+        p_token_info_t token_info;
+        size_t start_index = p_input_index(context);
+        p_position_t start_position = p_position(context);
+        assert(start_index == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_b);
+        assert(p_input_index(context) == 2);
+        /* Rewind and re-read from the start. */
+        p_set_input_index(context, start_index);
+        p_set_position(context, start_position);
+        assert(p_input_index(context) == 0);
+        assert(p_lex(context, &token_info) == P_SUCCESS);
+        assert(token_info.token == TOKEN_a);
+        assert(token_info.position.row == 1);
+        assert(token_info.position.col == 1);
+    }
 }