CHANGELOG.md

@@ -1,10 +1,4 @@
-## v4.8.1
+## Unreleased
-
-### Fixes
-
-- Fix tree node struct type forward-declarations for C/C++
-
-## v4.8.0
 
 ### New Features
 
@@ -17,9 +11,6 @@
   context. Useful for setting the initial text position to something other
   than `(1, 1)` for a nested parse operation.
 - Add `p_input_index()` API to get the current input text byte offset.
-- Add `p_set_input_index()` API to set the current input text byte offset.
-  Useful together with `p_set_position()` to rewind the input part-way through
-  a parse in order to re-read an earlier section of the input.
 
 ## v4.7.0
 

assets/parser.c.erb

@@ -1377,27 +1377,6 @@ size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * conte
     return context->input_index;
 }
 
-/**
- * Set the current input text byte offset.
- *
- * This moves the lexer's read cursor to the given byte offset (measured from
- * the start of the input text passed to <%= @grammar.prefix %>context_new()).
- * It can be used together with <%= @grammar.prefix %>set_position() to rewind
- * the input part-way through a parse in order to re-read an earlier section of
- * the input. The byte offset is not validated; the caller is responsible for
- * providing an offset within the bounds of the input text. A value previously
- * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param input_index
- *   Input text byte offset to set.
- */
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
-{
-    context->input_index = input_index;
-}
-
 /**
  * Get the user terminate code.
  *

assets/parser.d.erb

@@ -1456,27 +1456,6 @@ public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t
     return context.input_index;
 }
 
-/**
- * Set the current input text byte offset.
- *
- * This moves the lexer's read cursor to the given byte offset (measured from
- * the start of the input text passed to <%= @grammar.prefix %>context_new()).
- * It can be used together with <%= @grammar.prefix %>set_position() to rewind
- * the input part-way through a parse in order to re-read an earlier section of
- * the input. The byte offset is not validated; the caller is responsible for
- * providing an offset within the bounds of the input text. A value previously
- * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param input_index
- *   Input text byte offset to set.
- */
-public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
-{
-    context.input_index = input_index;
-}
-
 /**
  * Get the user terminate code.
  *

assets/parser.h.erb

@@ -104,7 +104,7 @@ typedef struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
 <%     next if rule_set.optional? %>
-struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>;
+struct <%= name %>;
 <%   end %>
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -241,8 +241,6 @@ void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * contex
 
 size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
 
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
-
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -1544,28 +1544,6 @@ size_t offset = p_input_index(context);
 /* Remaining input starts at `input + offset`. */
 ```
 
-### `p_set_input_index`
-
-The `p_set_input_index()` function sets the current input text byte offset,
-measured from the start of the input text passed to `p_context_new()`.
-This moves the lexer's read cursor, which can be used together with
-`p_set_position()` to rewind the input part-way through a parse in order to
-re-read an earlier section of the input.
-The byte offset is not validated; the caller is responsible for providing an
-offset within the bounds of the input text.
-A value previously returned by `p_input_index()` is a suitable argument.
-
-Example:
-
-```
-/* Save the cursor and text position at the start of a section. */
-size_t saved_index = p_input_index(context);
-p_position_t saved_position = p_position(context);
-/* ... later, rewind to re-read that section. */
-p_set_input_index(context, saved_index);
-p_set_position(context, saved_position);
-```
-
 ### `p_user_terminate_code`
 
 The `p_user_terminate_code()` function can be used to retrieve the user

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.8.1"
+  VERSION = "4.7.0"
 end

spec/propane_spec.rb

@@ -966,16 +966,6 @@ EOF
         expect(results.status).to eq 0
       end
 
-      it "allows rewinding the input to repeat a section during a parse" do
-        ext = language == "cpp" ? "c" : language
-        write_grammar(File.read("spec/rewind.#{ext}.propane"))
-        run_propane(language: language)
-        compile("spec/test_rewind.#{language}", language: language)
-        results = run_test(language: language)
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
-
       it "allows creating a JSON parser" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/json_parser.#{ext}.propane"))

spec/rewind.c.propane

@@ -1,23 +0,0 @@
-<<
-#include <stdlib.h>
-#include <string.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
-void record(int value);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -1,20 +0,0 @@
-<<
-import test_rewind;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/test_macros.c

@@ -3,7 +3,6 @@
 #include <string.h>
 #include <assert.h>
 #include <stddef.h>
-#include <stdbool.h>
 
 static p_context_t * context;
 size_t n_tokens;

spec/test_parse_inner_nested.c

@@ -48,7 +48,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         /* Replace the '(' token with a synthesized num carrying the nested
          * parse result. */
         out_token_info->token = TOKEN_num;
-        out_token_info->pvalue = p_value(value);
+        out_token_info->pvalue.v_default = value;
     }
     return P_SUCCESS;
 }

spec/test_parse_inner_nested.d

@@ -31,7 +31,7 @@ size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
         /* Replace the '(' token with a synthesized num carrying the nested
          * parse result. */
         out_token_info.token = TOKEN_num;
-        out_token_info.pvalue = p_value(value);
+        out_token_info.pvalue.v_default = value;
     }
     return P_SUCCESS;
 }

spec/test_rewind.c

@@ -1,124 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-/* Grammar (statement list of additions; a "repeat" directive handled entirely
- * by the lex function):
- *   ptype int;
- *   lex_fn mylexfn;
- *   token repeat /repeat/; token lbrace /\{/; token rbrace /\}/;
- *   token plus /\+/; token num /\d+/ << ... atoi ... >>
- *   Start      -> Statements;
- *   Statements -> ;
- *   Statements -> Statement Statements;
- *   Statement  -> Add;
- *   Add        -> num plus num  << record($1 + $3); >>
- *
- * Scenario: a "repeat <count> { <body> }" directive that expands its body
- * <count> times, similar to loop unrolling in a configuration DSL. The tokens
- * repeat, lbrace, and rbrace appear in no grammar rule; the lex function
- * interprets the directive and feeds the body's tokens to the parser <count>
- * times. Rather than buffering the body tokens, the lex function records the
- * input byte offset and text position at the start of the body (with
- * p_input_index() and p_position()) and, each time it reaches the closing '}',
- * rewinds the lexer back to that point (with p_set_input_index() and
- * p_set_position()) to re-read the body from the original input. Rewinding the
- * text position as well as the byte offset means each expansion reports the
- * same token positions as the first. */
-
-static int nums[16];
-static size_t n_nums;
-static uint32_t num_cols[16];
-static size_t n_num_cols;
-
-void record(int value)
-{
-    nums[n_nums++] = value;
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static int remaining;
-    static size_t body_index;
-    static p_position_t body_position;
-
-    for (;;)
-    {
-        size_t result = p_lex(context, out_token_info);
-        if (result != P_SUCCESS)
-        {
-            return result;
-        }
-
-        if (out_token_info->token == TOKEN_repeat)
-        {
-            /* Consume "repeat <count> {" and remember where the body begins. */
-            p_token_info_t count_info;
-            size_t count_result = p_lex(context, &count_info);
-            assert(count_result == P_SUCCESS);
-            assert(count_info.token == TOKEN_num);
-            p_token_info_t brace_info;
-            size_t brace_result = p_lex(context, &brace_info);
-            assert(brace_result == P_SUCCESS);
-            assert(brace_info.token == TOKEN_lbrace);
-            remaining = p_value_get(&count_info.pvalue);
-            body_index = p_input_index(context);
-            body_position = p_position(context);
-            continue;
-        }
-        if (out_token_info->token == TOKEN_rbrace)
-        {
-            /* End of the body. If more expansions remain, rewind the lexer to
-             * the start of the body and re-read it; otherwise fall through to
-             * the input following the '}'. */
-            if (remaining > 1)
-            {
-                remaining--;
-                p_set_input_index(context, body_index);
-                p_set_position(context, body_position);
-                continue;
-            }
-            remaining = 0;
-            continue;
-        }
-        if (out_token_info->token == TOKEN_num)
-        {
-            num_cols[n_num_cols++] = out_token_info->position.col;
-        }
-        return result;
-    }
-}
-
-int main()
-{
-    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
-     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
-    char const * input = "repeat 3 { 10 + 20 } 5 + 5";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    /* The additions were recorded once per body expansion, then once more for
-     * the trailing statement. */
-    assert_eq(4u, n_nums);
-    assert_eq(30u, (size_t)nums[0]);
-    assert_eq(30u, (size_t)nums[1]);
-    assert_eq(30u, (size_t)nums[2]);
-    assert_eq(10u, (size_t)nums[3]);
-
-    /* Each body expansion reported the same columns for its num tokens (12 and
-     * 17), because the text position was rewound along with the byte offset.
-     * The trailing statement's nums are at columns 22 and 26. */
-    assert_eq(8u, n_num_cols);
-    assert_eq(12u, (size_t)num_cols[0]);
-    assert_eq(17u, (size_t)num_cols[1]);
-    assert_eq(12u, (size_t)num_cols[2]);
-    assert_eq(17u, (size_t)num_cols[3]);
-    assert_eq(12u, (size_t)num_cols[4]);
-    assert_eq(17u, (size_t)num_cols[5]);
-    assert_eq(22u, (size_t)num_cols[6]);
-    assert_eq(26u, (size_t)num_cols[7]);
-
-    return 0;
-}

spec/test_rewind.d

@@ -1,101 +0,0 @@
-import testparser;
-import testutils;
-
-/* Grammar and scenario: see test_rewind.c. */
-
-int[16] nums;
-size_t n_nums;
-uint[16] num_cols;
-size_t n_num_cols;
-
-void record(int value)
-{
-    nums[n_nums++] = value;
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static int remaining;
-    static size_t body_index;
-    static p_position_t body_position;
-
-    for (;;)
-    {
-        size_t result = p_lex(context, out_token_info);
-        if (result != P_SUCCESS)
-        {
-            return result;
-        }
-
-        if (out_token_info.token == TOKEN_repeat)
-        {
-            /* Consume "repeat <count> {" and remember where the body begins. */
-            p_token_info_t count_info;
-            size_t count_result = p_lex(context, &count_info);
-            assert(count_result == P_SUCCESS);
-            assert(count_info.token == TOKEN_num);
-            p_token_info_t brace_info;
-            size_t brace_result = p_lex(context, &brace_info);
-            assert(brace_result == P_SUCCESS);
-            assert(brace_info.token == TOKEN_lbrace);
-            remaining = p_value_get(&count_info.pvalue);
-            body_index = p_input_index(context);
-            body_position = p_position(context);
-            continue;
-        }
-        if (out_token_info.token == TOKEN_rbrace)
-        {
-            /* End of the body. If more expansions remain, rewind the lexer to
-             * the start of the body and re-read it; otherwise fall through to
-             * the input following the '}'. */
-            if (remaining > 1)
-            {
-                remaining--;
-                p_set_input_index(context, body_index);
-                p_set_position(context, body_position);
-                continue;
-            }
-            remaining = 0;
-            continue;
-        }
-        if (out_token_info.token == TOKEN_num)
-        {
-            num_cols[n_num_cols++] = out_token_info.position.col;
-        }
-        return result;
-    }
-}
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
-     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
-    string input = "repeat 3 { 10 + 20 } 5 + 5";
-    p_context_t * context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    assert_eq(4u, n_nums);
-    assert_eq(30, nums[0]);
-    assert_eq(30, nums[1]);
-    assert_eq(30, nums[2]);
-    assert_eq(10, nums[3]);
-
-    /* Each body expansion reported the same columns for its num tokens (12 and
-     * 17), because the text position was rewound along with the byte offset.
-     * The trailing statement's nums are at columns 22 and 26. */
-    assert_eq(8u, n_num_cols);
-    assert_eq(12u, num_cols[0]);
-    assert_eq(17u, num_cols[1]);
-    assert_eq(12u, num_cols[2]);
-    assert_eq(17u, num_cols[3]);
-    assert_eq(12u, num_cols[4]);
-    assert_eq(17u, num_cols[5]);
-    assert_eq(22u, num_cols[6]);
-    assert_eq(26u, num_cols[7]);
-}

spec/test_set_position.c

@@ -77,33 +77,5 @@ int main()
         p_context_delete(context);
     }
 
-    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
-     * p_set_position(), it re-reads an earlier section of the input: both
-     * tokens are lexed, then the cursor and text position are rewound to the
-     * start so that the same tokens are produced again with the same reported
-     * positions. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_info_t token_info;
-        size_t start_index = p_input_index(context);
-        p_position_t start_position = p_position(context);
-        assert_eq(0u, start_index);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        assert_eq(2u, p_input_index(context));
-        /* Rewind and re-read from the start. */
-        p_set_input_index(context, start_index);
-        p_set_position(context, start_position);
-        assert_eq(0u, p_input_index(context));
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(1u, (size_t)token_info.position.row);
-        assert_eq(1u, (size_t)token_info.position.col);
-        p_context_delete(context);
-    }
-
     return 0;
 }

spec/test_set_position.d

@@ -63,28 +63,4 @@ unittest
         assert(err_pos.row == 10);
         assert(err_pos.col == 3);
     }
-
-    /* p_set_input_index() rewinds the lexer's byte cursor. Combined with
-     * p_set_position(), it re-reads an earlier section of the input. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        p_token_info_t token_info;
-        size_t start_index = p_input_index(context);
-        p_position_t start_position = p_position(context);
-        assert(start_index == 0);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(p_input_index(context) == 2);
-        /* Rewind and re-read from the start. */
-        p_set_input_index(context, start_index);
-        p_set_position(context, start_position);
-        assert(p_input_index(context) == 0);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(token_info.position.row == 1);
-        assert(token_info.position.col == 1);
-    }
 }