Add p_parse_inner_XXX() APIs

assets/parser.c.erb

@@ -1008,8 +1008,17 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-NULL, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional array of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be NULL for a standard parse.
+ * @param n_follow_tokens
+ *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1022,12 +1031,15 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1051,6 +1063,18 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        for (size_t i = 0u; i < n_follow_tokens; i++)
+        {
+            if (token == follow_tokens[i])
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1070,10 +1094,36 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 result = P_SUCCESS;
                 break;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. */
+<% if @grammar.tree %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->tree_node;
+<% else %>
+                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
+                    result = P_SUCCESS;
+                    break;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
             state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
             new_state_info->state_id = shift_state;
@@ -1120,6 +1170,15 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1219,14 +1278,20 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
+}
+
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
 }
 <% end %>
 

assets/parser.d.erb

@@ -1079,8 +1079,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *
  * @param context
  *   Lexer/parser context structure.
- * @start_state_id
+ * @param start_state_id
  *   ID of the state in which to start.
+ * @param start_rule_set_id
+ *   Rule set ID for the requested start rule. Only used when
+ *   @p follow_tokens is non-empty, to gate follow-token shift success.
+ * @param follow_tokens
+ *   Optional slice of caller-provided follow tokens (tokens expected to
+ *   appear immediately after the start rule in some outer context). Used to
+ *   drive the "parse inner" retry logic. May be null/empty for a standard
+ *   parse.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
@@ -1093,13 +1101,16 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id)
+private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+    size_t start_rule_set_id,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     statevalues[0].state_id = start_state_id;
     size_t reduced_rule_set = INVALID_ID;
+    size_t last_shifted_rule_set_id = INVALID_ID;
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
@@ -1118,6 +1129,18 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             }
             token = token_info.token;
         }
+        /* For a "parse inner" operation, determine once per iteration whether
+         * the current token is a member of the caller-provided follow token
+         * set. Used by both the shift-side and reduce-side retries below. */
+        bool token_is_follow = false;
+        foreach (eof_token; follow_tokens)
+        {
+            if (token == eof_token)
+            {
+                token_is_follow = true;
+                break;
+            }
+        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1136,10 +1159,35 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% end %>
                 return P_SUCCESS;
             }
+            if ((shift_state == INVALID_ID) && token_is_follow)
+            {
+                /* For a "parse inner" operation, if the incoming token is one
+                 * of the caller's follow tokens, retry the shift as
+                 * TOKEN___EOF. Only consider the parse complete if the reduced
+                 * start rule is the only thing on the parse stack (i.e. the
+                 * initial state plus a single shifted start rule set entry). */
+                size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
+                if ((retry_shift_state != INVALID_ID) &&
+                    (statevalues.length == 2u) &&
+                    (last_shifted_rule_set_id == start_rule_set_id))
+                {
+                    /* Successful parse via follow token. */
+<% if @grammar.tree %>
+                    context.parse_result = statevalues[$-1].tree_node;
+<% else %>
+                    context.parse_result = statevalues[$-1].pvalue;
+<% end %>
+                    return P_SUCCESS;
+                }
+            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. */
+            /* We have something to shift. Track the last shifted rule set ID
+             * (INVALID_ID if we just shifted a token) so the follow-token
+             * shift retry can gate success on the reduced start rule being the
+             * only thing on top of the initial state. */
+            last_shifted_rule_set_id = reduced_rule_set;
             statevalues ~= state_value_t(shift_state);
             if (reduced_rule_set == INVALID_ID)
             {
@@ -1173,6 +1221,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
         }
 
         size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
+        if ((reduce_index == INVALID_ID) && token_is_follow)
+        {
+            /* For a "parse inner" operation, if the incoming token is one of
+             * the caller's follow tokens, retry the reduce lookup as
+             * TOKEN___EOF. Whatever reduce_index results (if any) is used
+             * regardless of which rule set it reduces to; this allows chains
+             * of reductions leading up to the start rule. */
+            reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
+        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
@@ -1271,14 +1328,20 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 
 public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, 0u);
+    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
 }
 
 <% @grammar.start_rules.each_with_index do |start_rule, i| %>
 
 public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
 {
-    return parse_from(context, <%= i %>u);
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
+}
+
+public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+    const(<%= @grammar.prefix %>token_t)[] follow_tokens)
+{
+    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
 }
 <% end %>
 

assets/parser.h.erb

@@ -212,6 +212,8 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%   @grammar.start_rules.each_with_index do |start_rule, i| %>
 size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
+size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
+        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
 <%   end %>
 
 <% if @grammar.tree %>