Josh Holtrop
5fc712c6ee
Store tree nodes in congruent, compact arena array. Define handle types to refer to tree nodes rather than pointers to structure instances. Free tree with context.
1513 lines
52 KiB
Plaintext
1513 lines
52 KiB
Plaintext
#include "<%= File.basename(output_file).sub(%r{\.[a-z]+$}, "") %>.h"
|
|
#include <stdbool.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
/**************************************************************************
|
|
* Public data
|
|
*************************************************************************/
|
|
|
|
/** Token names. */
|
|
const char * <%= @grammar.prefix %>token_names[] = {
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
"<%= token.name %>",
|
|
<% end %>
|
|
};
|
|
|
|
/**************************************************************************
|
|
* User code blocks
|
|
*************************************************************************/
|
|
|
|
<%= @grammar.code_blocks.fetch("", "") %>
|
|
|
|
/**************************************************************************
|
|
* Private types
|
|
*************************************************************************/
|
|
|
|
<% if @grammar.prefix.upcase != "P_" %>
|
|
/* Result codes. */
|
|
#define P_SUCCESS 0u
|
|
#define P_DECODE_ERROR 1u
|
|
#define P_UNEXPECTED_INPUT 2u
|
|
#define P_UNEXPECTED_TOKEN 3u
|
|
#define P_DROP 4u
|
|
#define P_EOF 5u
|
|
#define P_USER_TERMINATED 6u
|
|
<% end %>
|
|
|
|
/* An invalid ID value. */
|
|
#define INVALID_ID ((size_t)-1)
|
|
|
|
/**************************************************************************
|
|
* State initialization
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Allocate and initialize lexer/parser context structure.
|
|
*
|
|
* Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
|
|
*
|
|
* @param input
|
|
* Text input.
|
|
* @param input_length
|
|
* Text input length.
|
|
*
|
|
* @return Context structure for lexer/parser.
|
|
*/
|
|
<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length)
|
|
{
|
|
<% if @cpp %>
|
|
<%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t();
|
|
<% else %>
|
|
<%= @grammar.prefix %>context_t * context = (<%= @grammar.prefix %>context_t *)calloc(1, sizeof(<%= @grammar.prefix %>context_t));
|
|
<% end %>
|
|
|
|
/* Lexer initialization. */
|
|
context->input = input;
|
|
context->input_length = input_length;
|
|
context->text_position.row = 1u;
|
|
context->text_position.col = 1u;
|
|
context->mode = <%= @lexer.mode_id("default") %>;
|
|
<% if @grammar.tree %>
|
|
|
|
/* Reserve node ID 0 as the null tree node. */
|
|
<% if @cpp %>
|
|
context-><%= @grammar.prefix %>tree_nodes.resize(1);
|
|
<% else %>
|
|
context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
|
|
context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
|
|
memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
|
|
context-><%= @grammar.prefix %>tree_nodes_length = 1u;
|
|
<% end %>
|
|
<% end %>
|
|
|
|
return context;
|
|
}
|
|
|
|
/**
|
|
* Deinitialize and deallocate lexer/parser context structure.
|
|
*
|
|
* For C++, destructors will be called for any context user fields. However, if
|
|
* pointers are used to store allocated resources, the user should free them
|
|
* before calling this function.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
|
|
*/
|
|
void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
<% if @grammar.tree && @grammar.free_token_node != "" %>
|
|
<% if @cpp %>
|
|
for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
|
|
<% else %>
|
|
for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
|
|
<% end %>
|
|
{
|
|
if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
|
|
{
|
|
<%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
|
|
<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
|
|
}
|
|
}
|
|
<% end %>
|
|
<% if @cpp %>
|
|
delete context;
|
|
<% else %>
|
|
<% if @grammar.tree %>
|
|
free(context-><%= @grammar.prefix %>tree_nodes);
|
|
free(context-><%= @grammar.prefix %>tree_children);
|
|
<% end %>
|
|
free(context);
|
|
<% end %>
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Decoder
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Decode a UTF-8 code point.
|
|
*
|
|
* @param input
|
|
* Text input to decode.
|
|
* @param input_length
|
|
* Input text length.
|
|
* @param[out] out_code_point
|
|
* The decoded code point is stored here if the return value is P_SUCCESS.
|
|
* @param[out] out_code_point_length
|
|
* The number of bytes the code point used is stored here if the return value
|
|
* is P_SUCCESS.
|
|
*
|
|
* @retval P_SUCCESS on a successful code point decode
|
|
* @retval P_DECODE_ERROR when an encoding error is observed
|
|
* @retval P_EOF when the end of the text input is reached
|
|
*/
|
|
size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
|
|
<%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length)
|
|
{
|
|
if (input_length == 0u)
|
|
{
|
|
return P_EOF;
|
|
}
|
|
char c = input[0];
|
|
<%= @grammar.prefix %>code_point_t code_point;
|
|
uint8_t code_point_length;
|
|
if ((c & 0x80u) == 0u)
|
|
{
|
|
code_point = c;
|
|
code_point_length = 1u;
|
|
}
|
|
else
|
|
{
|
|
uint8_t following_bytes;
|
|
if ((c & 0xE0u) == 0xC0u)
|
|
{
|
|
code_point = c & 0x1Fu;
|
|
following_bytes = 1u;
|
|
}
|
|
else if ((c & 0xF0u) == 0xE0u)
|
|
{
|
|
code_point = c & 0x0Fu;
|
|
following_bytes = 2u;
|
|
}
|
|
else if ((c & 0xF8u) == 0xF0u)
|
|
{
|
|
code_point = c & 0x07u;
|
|
following_bytes = 3u;
|
|
}
|
|
else if ((c & 0xFCu) == 0xF8u)
|
|
{
|
|
code_point = c & 0x03u;
|
|
following_bytes = 4u;
|
|
}
|
|
else if ((c & 0xFEu) == 0xFCu)
|
|
{
|
|
code_point = c & 0x01u;
|
|
following_bytes = 5u;
|
|
}
|
|
else
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
if (input_length <= following_bytes)
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point_length = (uint8_t)(following_bytes + 1u);
|
|
for (size_t i = 0u; i < following_bytes; i++)
|
|
{
|
|
char b = input[i + 1u];
|
|
if ((b & 0xC0u) != 0x80u)
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point = (code_point << 6u) | (b & 0x3Fu);
|
|
}
|
|
}
|
|
*out_code_point = code_point;
|
|
*out_code_point_length = code_point_length;
|
|
return P_SUCCESS;
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Lexer
|
|
*************************************************************************/
|
|
|
|
/** Lexer state ID type. */
|
|
typedef <%= get_type_for(@lexer.state_table.size) %> lexer_state_id_t;
|
|
|
|
/** Invalid lexer state ID. */
|
|
#define INVALID_LEXER_STATE_ID <%= @lexer.state_table.size %>u
|
|
|
|
/** Lexer user code ID type. */
|
|
<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
|
|
typedef <%= get_type_for(user_code_id_count) %> lexer_user_code_id_t;
|
|
|
|
/** Invalid lexer user code ID. */
|
|
#define INVALID_USER_CODE_ID <%= user_code_id_count %>u
|
|
|
|
/**
|
|
* Lexer transition table entry.
|
|
*
|
|
* An incoming code point matching the range for a transition entry will cause
|
|
* the lexer to progress to the destination state.
|
|
*/
|
|
typedef struct
|
|
{
|
|
/** First code point in the range for this transition. */
|
|
<%= @grammar.prefix %>code_point_t first;
|
|
|
|
/** Last code point in the range for this transition. */
|
|
<%= @grammar.prefix %>code_point_t last;
|
|
|
|
/** Destination lexer state ID for this transition. */
|
|
lexer_state_id_t destination_state;
|
|
} lexer_transition_t;
|
|
|
|
/** Lexer state table entry. */
|
|
typedef struct
|
|
{
|
|
/** Index to the transition table for this state. */
|
|
<%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
|
|
|
|
/** Number of transition table entries for this state. */
|
|
<%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
|
|
|
|
/** Lexer token formed at this state. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/** Lexer user code ID to execute at this state. */
|
|
lexer_user_code_id_t code_id;
|
|
|
|
/** Whether this state matches a lexer pattern. */
|
|
bool accepts;
|
|
} lexer_state_t;
|
|
|
|
/** Lexer mode table entry. */
|
|
typedef struct
|
|
{
|
|
/** Offset in the state table to be used for this mode. */
|
|
uint32_t state_table_offset;
|
|
} lexer_mode_t;
|
|
|
|
/**
|
|
* Lexer match info structure.
|
|
*
|
|
* This structure holds output values from the lexer upon a successful pattern
|
|
* match.
|
|
*/
|
|
typedef struct
|
|
{
|
|
/** Number of bytes of input text used to match. */
|
|
size_t length;
|
|
|
|
/** Input text position delta to end of token. */
|
|
<%= @grammar.prefix %>position_t end_delta_position;
|
|
|
|
/** Input text position delta to next code point after token end. */
|
|
<%= @grammar.prefix %>position_t delta_position;
|
|
|
|
/** Accepting lexer state from the match. */
|
|
lexer_state_t const * accepting_state;
|
|
} lexer_match_info_t;
|
|
|
|
/** Lexer transition table. */
|
|
static lexer_transition_t lexer_transition_table[] = {
|
|
<% @lexer.transition_table.each do |transition_table_entry| %>
|
|
{<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u},
|
|
<% end %>
|
|
};
|
|
|
|
/** Lexer state table. */
|
|
static lexer_state_t lexer_state_table[] = {
|
|
<% @lexer.state_table.each do |state_table_entry| %>
|
|
{<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>},
|
|
<% end %>
|
|
};
|
|
|
|
/** Lexer mode table. */
|
|
static lexer_mode_t lexer_mode_table[] = {
|
|
<% @lexer.mode_table.each do |mode_table_entry| %>
|
|
{<%= mode_table_entry[:state_table_offset] %>},
|
|
<% end %>
|
|
};
|
|
|
|
/**
|
|
* Execute user code associated with a lexer pattern.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param code_id
|
|
* The ID of the user code block to execute.
|
|
* @param match
|
|
* Matched text for this pattern.
|
|
* @param match_length
|
|
* Matched text length.
|
|
* @param out_token_info
|
|
* Lexer token info in progress.
|
|
*
|
|
* @return Token to accept, or invalid token if the user code does
|
|
* not explicitly return a token.
|
|
*/
|
|
static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
|
|
lexer_user_code_id_t code_id, uint8_t const * match,
|
|
size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
switch (code_id)
|
|
{
|
|
<% @grammar.patterns.each do |pattern| %>
|
|
<% if pattern.code_id %>
|
|
case <%= pattern.code_id %>u: {
|
|
<%= expand_code(pattern.code, false, nil, pattern) %>
|
|
} break;
|
|
<% end %>
|
|
<% end %>
|
|
default: break;
|
|
}
|
|
|
|
return INVALID_TOKEN_ID;
|
|
}
|
|
|
|
/**
|
|
* Check if there is a transition from the current lexer state to another
|
|
* based on the given input code point.
|
|
*
|
|
* @param current_state
|
|
* Current lexer state.
|
|
* @param code_point
|
|
* Input code point.
|
|
*
|
|
* @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
|
|
*/
|
|
static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t code_point)
|
|
{
|
|
uint32_t transition_table_index = lexer_state_table[current_state].transition_table_index;
|
|
for (uint32_t i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
|
|
{
|
|
if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
|
|
(code_point <= lexer_transition_table[transition_table_index + i].last))
|
|
{
|
|
return lexer_transition_table[transition_table_index + i].destination_state;
|
|
}
|
|
}
|
|
return INVALID_LEXER_STATE_ID;
|
|
}
|
|
|
|
/**
|
|
* Find the longest lexer pattern match at the current position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_match_info
|
|
* The longest match information is stored here if the return value is
|
|
* P_SUCCESS or P_DECODE_ERROR.
|
|
* @param[out] out_unexpected_input_length
|
|
* The unexpected input length is stored here if the return value is
|
|
* P_UNEXPECTED_INPUT.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_EOF
|
|
* The end of the text input was reached.
|
|
*/
|
|
static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
|
|
lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
|
|
{
|
|
lexer_match_info_t longest_match;
|
|
memset(&longest_match, 0, sizeof(longest_match));
|
|
lexer_match_info_t attempt_match;
|
|
memset(&attempt_match, 0, sizeof(attempt_match));
|
|
*out_match_info = longest_match;
|
|
uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
|
|
for (;;)
|
|
{
|
|
size_t const input_index = context->input_index + attempt_match.length;
|
|
uint8_t const * input = &context->input[input_index];
|
|
size_t input_length = context->input_length - input_index;
|
|
<%= @grammar.prefix %>code_point_t code_point;
|
|
uint8_t code_point_length;
|
|
size_t result = <%= @grammar.prefix %>decode_code_point(input, input_length, &code_point, &code_point_length);
|
|
switch (result)
|
|
{
|
|
case P_SUCCESS:
|
|
{
|
|
lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
|
|
if (transition_state != INVALID_LEXER_STATE_ID)
|
|
{
|
|
attempt_match.length += code_point_length;
|
|
attempt_match.end_delta_position = attempt_match.delta_position;
|
|
if (code_point == '\n')
|
|
{
|
|
attempt_match.delta_position.row++;
|
|
attempt_match.delta_position.col = 1u;
|
|
}
|
|
else
|
|
{
|
|
attempt_match.delta_position.col++;
|
|
}
|
|
current_state = transition_state;
|
|
if (lexer_state_table[current_state].accepts)
|
|
{
|
|
attempt_match.accepting_state = &lexer_state_table[current_state];
|
|
longest_match = attempt_match;
|
|
}
|
|
}
|
|
else if (longest_match.length > 0)
|
|
{
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
}
|
|
else
|
|
{
|
|
*out_unexpected_input_length = attempt_match.length + code_point_length;
|
|
return P_UNEXPECTED_INPUT;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case P_EOF:
|
|
/* We hit EOF. */
|
|
if (longest_match.length > 0)
|
|
{
|
|
/* We have a match, so use it. */
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
}
|
|
else if (attempt_match.length != 0)
|
|
{
|
|
/* There is a partial match - error! */
|
|
*out_unexpected_input_length = attempt_match.length;
|
|
return P_UNEXPECTED_INPUT;
|
|
}
|
|
else
|
|
{
|
|
/* Valid EOF return. */
|
|
return P_EOF;
|
|
}
|
|
|
|
case P_DECODE_ERROR:
|
|
/* If we see a decode error, we may be partially in the middle of
|
|
* matching a pattern, so return the attempted match info so that
|
|
* the input text position can be updated. */
|
|
*out_match_info = attempt_match;
|
|
return result;
|
|
|
|
default:
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempt to lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_DROP
|
|
* A drop pattern was matched so the lexer should continue.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
<%= @grammar.prefix %>token_info_t token_info;
|
|
memset(&token_info, 0, sizeof(token_info));
|
|
token_info.position = context->text_position;
|
|
token_info.token = INVALID_TOKEN_ID;
|
|
lexer_match_info_t match_info;
|
|
size_t unexpected_input_length;
|
|
size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
|
|
switch (result)
|
|
{
|
|
case P_SUCCESS:
|
|
{
|
|
<%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
|
|
/* Calculate the token length and start/end positions before invoking
|
|
* the lexer user code so that the user code can access them. The
|
|
* context input text position tracking is not updated until after the
|
|
* user code has run so that it is left unchanged if the user code
|
|
* requests to terminate the lexer. */
|
|
token_info.length = match_info.length;
|
|
if (match_info.end_delta_position.row != 0u)
|
|
{
|
|
token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
|
|
token_info.end_position.col = match_info.end_delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
token_info.end_position.row = token_info.position.row;
|
|
token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
|
|
}
|
|
if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
|
|
{
|
|
uint8_t const * match = &context->input[context->input_index];
|
|
<%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
|
|
match_info.accepting_state->code_id, match, match_info.length, &token_info);
|
|
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
|
|
* that the user code is requesting to terminate the lexer. */
|
|
if (user_code_token == TERMINATE_TOKEN_ID)
|
|
{
|
|
return P_USER_TERMINATED;
|
|
}
|
|
/* An invalid token returned from lexer_user_code() means that the
|
|
* user code did not explicitly return a token. So only override
|
|
* the token to return if the user code does explicitly return a
|
|
* token. */
|
|
if (user_code_token != INVALID_TOKEN_ID)
|
|
{
|
|
token_to_accept = user_code_token;
|
|
}
|
|
}
|
|
|
|
/* Update the input position tracking. */
|
|
context->input_index += match_info.length;
|
|
context->text_position.row += match_info.delta_position.row;
|
|
if (match_info.delta_position.row != 0u)
|
|
{
|
|
context->text_position.col = match_info.delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
context->text_position.col += match_info.delta_position.col;
|
|
}
|
|
|
|
if (token_to_accept == INVALID_TOKEN_ID)
|
|
{
|
|
return P_DROP;
|
|
}
|
|
token_info.token = token_to_accept;
|
|
*out_token_info = token_info;
|
|
}
|
|
return P_SUCCESS;
|
|
|
|
case P_EOF:
|
|
token_info.token = TOKEN___EOF;
|
|
token_info.end_position = token_info.position;
|
|
*out_token_info = token_info;
|
|
return P_SUCCESS;
|
|
|
|
case P_DECODE_ERROR:
|
|
/* Update the input position tracking. */
|
|
context->input_index += match_info.length;
|
|
context->text_position.row += match_info.delta_position.row;
|
|
if (match_info.delta_position.row != 0u)
|
|
{
|
|
context->text_position.col = match_info.delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
context->text_position.col += match_info.delta_position.col;
|
|
}
|
|
return result;
|
|
|
|
default:
|
|
return result;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
for (;;)
|
|
{
|
|
size_t result = attempt_lex_token(context, out_token_info);
|
|
if (result != P_DROP)
|
|
{
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Parser
|
|
*************************************************************************/
|
|
|
|
/** Invalid position value. */
|
|
#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
|
|
|
|
/** Reduce ID type. */
|
|
typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
|
|
|
|
/**
|
|
* A symbol ID can hold either a token ID or a rule set ID.
|
|
*
|
|
* Token IDs and rule set IDs share the same namespace, with rule set IDs
|
|
* beginning after token IDs end.
|
|
*/
|
|
typedef <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %> symbol_id_t;
|
|
|
|
/** Parser state ID type. */
|
|
typedef <%= get_type_for(@parser.state_table.size) %> parser_state_id_t;
|
|
|
|
/** Parser rule ID type. */
|
|
typedef <%= get_type_for(@grammar.rules.size) %> rule_id_t;
|
|
|
|
/** Parser shift ID type. */
|
|
typedef <%= get_type_for(@parser.shift_table.size) %> shift_id_t;
|
|
|
|
/** Shift table entry. */
|
|
typedef struct
|
|
{
|
|
/** Token or rule set ID. */
|
|
symbol_id_t symbol_id;
|
|
|
|
/** Parser state to shift to. */
|
|
parser_state_id_t state_id;
|
|
} shift_t;
|
|
|
|
/** Reduce table entry. */
|
|
typedef struct
|
|
{
|
|
/** Lookahead token. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/**
|
|
* Rule ID.
|
|
*
|
|
* This is used to execute the parser user code block associated with a
|
|
* grammar rule.
|
|
*/
|
|
rule_id_t rule;
|
|
|
|
/**
|
|
* Rule set ID.
|
|
*
|
|
* This is used as the new top symbol ID of the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
symbol_id_t rule_set;
|
|
|
|
/**
|
|
* Number of states leading to this reduce action.
|
|
*
|
|
* This is the number of entries popped from the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
parser_state_id_t n_states;
|
|
<% if @grammar.tree %>
|
|
|
|
/**
|
|
* Map of rule components to rule set child fields.
|
|
*/
|
|
uint16_t const * rule_set_node_field_index_map;
|
|
|
|
/**
|
|
* Number of rule set tree node fields.
|
|
*/
|
|
uint16_t rule_set_node_field_array_size;
|
|
|
|
/**
|
|
* Whether this rule was a generated optional rule that matched the
|
|
* optional target. In this case, propagate the matched target node up
|
|
* instead of making a new node for this rule.
|
|
*/
|
|
bool propagate_optional_target;
|
|
<% end %>
|
|
} reduce_t;
|
|
|
|
/** Parser state entry. */
|
|
typedef struct
|
|
{
|
|
/** First shift table entry for this parser state. */
|
|
shift_id_t shift_table_index;
|
|
|
|
/** Number of shift table entries for this parser state. */
|
|
shift_id_t n_shift_entries;
|
|
|
|
/** First reduce table entry for this parser state. */
|
|
reduce_id_t reduce_table_index;
|
|
|
|
/** Number of reduce table entries for this parser state. */
|
|
reduce_id_t n_reduce_entries;
|
|
} parser_state_t;
|
|
|
|
/**
|
|
* Structure to hold a state ID and value pair.
|
|
*
|
|
* A stack of these structures makes up the parse stack.
|
|
*/
|
|
typedef struct
|
|
{
|
|
/** Parser state ID. */
|
|
size_t state_id;
|
|
|
|
<% if @grammar.tree %>
|
|
/** Tree node ID. */
|
|
<%= @grammar.prefix %>node_id_t node_id;
|
|
<% else %>
|
|
<%= @grammar.prefix %>position_t position;
|
|
<%= @grammar.prefix %>position_t end_position;
|
|
/** Parser value from this state. */
|
|
<%= @grammar.prefix %>value_t pvalue;
|
|
<% end %>
|
|
} state_value_t;
|
|
|
|
/** Parser shift table. */
|
|
static const shift_t parser_shift_table[] = {
|
|
<% @parser.shift_table.each do |shift| %>
|
|
{<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u},
|
|
<% end %>
|
|
};
|
|
|
|
<% if @grammar.tree %>
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% unless rule.flat_rule_set_node_field_index_map? %>
|
|
const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
|
|
<% end %>
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/** Parser reduce table. */
|
|
static const reduce_t parser_reduce_table[] = {
|
|
<% @parser.reduce_table.each do |reduce| %>
|
|
{
|
|
<%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
|
|
<%= reduce[:rule_id] %>u, /* Rule ID */
|
|
<%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
|
|
<% if @grammar.tree %>
|
|
<%= reduce[:n_states] %>u, /* Number of states */
|
|
<% if reduce[:rule].flat_rule_set_node_field_index_map? %>
|
|
NULL, /* No rule set node field index map (flat map) */
|
|
<% else %>
|
|
&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
|
|
<% end %>
|
|
<%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
|
|
<%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
|
|
<% else %>
|
|
<%= reduce[:n_states] %>u},
|
|
<% end %>
|
|
<% end %>
|
|
};
|
|
|
|
/** Parser state table. */
|
|
static const parser_state_t parser_state_table[] = {
|
|
<% @parser.state_table.each do |state| %>
|
|
{<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u},
|
|
<% end %>
|
|
};
|
|
|
|
/* state_values stack functionality */
|
|
|
|
/** state_values stack type. */
|
|
typedef struct
|
|
{
|
|
size_t length;
|
|
size_t capacity;
|
|
state_value_t * entries;
|
|
} state_values_stack_t;
|
|
|
|
/**
|
|
* Initialize state_values stack structure.
|
|
*
|
|
* @param stack
|
|
* state_values stack structure.
|
|
*/
|
|
static void state_values_stack_init(state_values_stack_t * stack)
|
|
{
|
|
const size_t initial_capacity = 10u;
|
|
stack->length = 0u;
|
|
stack->capacity = initial_capacity;
|
|
stack->entries = (state_value_t *)malloc(initial_capacity * sizeof(state_value_t));
|
|
}
|
|
|
|
/**
|
|
* Index a state_values stack.
|
|
*
|
|
* @param stack
|
|
* state_values stack structure.
|
|
* @param index
|
|
* Index to the stack.
|
|
*
|
|
* @return Pointer to the state value structure at the given index.
|
|
*/
|
|
static state_value_t * state_values_stack_index(state_values_stack_t * stack, int index)
|
|
{
|
|
if (index >= 0)
|
|
{
|
|
return &stack->entries[index];
|
|
}
|
|
else
|
|
{
|
|
return &stack->entries[stack->length - (size_t)(unsigned int)(-index)];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Push a new state_value to the state_values stack.
|
|
*
|
|
* @param stack
|
|
* state_values stack structure.
|
|
*/
|
|
static void state_values_stack_push(state_values_stack_t * stack)
|
|
{
|
|
size_t const current_capacity = stack->capacity;
|
|
size_t const current_length = stack->length;
|
|
if (current_length >= current_capacity)
|
|
{
|
|
size_t const new_capacity = current_capacity * 2u;
|
|
state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
|
|
memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
|
|
free(stack->entries);
|
|
stack->capacity = new_capacity;
|
|
stack->entries = new_entries;
|
|
}
|
|
memset(&stack->entries[current_length], 0, sizeof(state_value_t));
|
|
stack->length = current_length + 1u;
|
|
}
|
|
|
|
/**
|
|
* Pop entries from a state_values stack.
|
|
*
|
|
* @param stack
|
|
* state_values stack structure.
|
|
* @param n
|
|
* Number of states to pop.
|
|
*/
|
|
static void state_values_stack_pop(state_values_stack_t * stack, size_t n)
|
|
{
|
|
stack->length -= n;
|
|
}
|
|
|
|
/**
|
|
* Free memory for a state_values stack structure.
|
|
*
|
|
* @param stack
|
|
* state_values stack structure.
|
|
*/
|
|
static void state_values_stack_free(state_values_stack_t * stack)
|
|
{
|
|
free(stack->entries);
|
|
}
|
|
|
|
<% if @grammar.tree %>
|
|
/* Tree arena helpers. */
|
|
|
|
/**
|
|
* Allocate a new (zeroed) tree node in the context arena.
|
|
*
|
|
* @return The new node ID.
|
|
*/
|
|
static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
<% if @cpp %>
|
|
<%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
|
|
context-><%= @grammar.prefix %>tree_nodes.emplace_back();
|
|
return id;
|
|
<% else %>
|
|
if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
|
|
{
|
|
size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
|
|
<%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
|
|
memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
|
|
free(context-><%= @grammar.prefix %>tree_nodes);
|
|
context-><%= @grammar.prefix %>tree_nodes = new_nodes;
|
|
context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
|
|
}
|
|
<%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
|
|
memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
|
|
context-><%= @grammar.prefix %>tree_nodes_length += 1u;
|
|
return id;
|
|
<% end %>
|
|
}
|
|
|
|
/**
|
|
* Reserve n contiguous (zeroed) child slots in the shared children array.
|
|
*
|
|
* @return The offset of the first reserved slot.
|
|
*/
|
|
static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
|
|
{
|
|
<% if @cpp %>
|
|
<%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
|
|
context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
|
|
return offset;
|
|
<% else %>
|
|
size_t offset = context-><%= @grammar.prefix %>tree_children_length;
|
|
size_t needed = offset + n;
|
|
if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
|
|
{
|
|
size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
|
|
while (new_capacity < needed)
|
|
{
|
|
new_capacity *= 2u;
|
|
}
|
|
<%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
|
|
if (context-><%= @grammar.prefix %>tree_children != NULL)
|
|
{
|
|
memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
|
|
free(context-><%= @grammar.prefix %>tree_children);
|
|
}
|
|
context-><%= @grammar.prefix %>tree_children = new_children;
|
|
context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
|
|
}
|
|
memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
|
|
context-><%= @grammar.prefix %>tree_children_length = needed;
|
|
return (<%= @grammar.prefix %>node_id_t)offset;
|
|
<% end %>
|
|
}
|
|
|
|
/* Tree node field accessor functions. */
|
|
<%= c_tree_accessor_defs %>
|
|
<% end %>
|
|
|
|
<% unless @grammar.tree %>
|
|
/**
|
|
* Get the rule position (start or end) for the currently matched rule.
|
|
*/
|
|
static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
|
|
{
|
|
if (n_states > 0u)
|
|
{
|
|
if (i == 0u)
|
|
{
|
|
if (get_end)
|
|
{
|
|
int stack_index = -1;
|
|
for (size_t j = 0u; j < n_states; j++)
|
|
{
|
|
state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
|
|
if (<%= @grammar.prefix %>position_valid(sv->end_position))
|
|
{
|
|
return sv->end_position;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
int stack_index = -(int)n_states;
|
|
for (size_t j = 0u; j < n_states; j++)
|
|
{
|
|
state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
|
|
if (<%= @grammar.prefix %>position_valid(sv->position))
|
|
{
|
|
return sv->position;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (get_end)
|
|
{
|
|
return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
|
|
}
|
|
else
|
|
{
|
|
return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
|
|
}
|
|
}
|
|
}
|
|
<%= @grammar.prefix %>position_t empty_pos;
|
|
memset(&empty_pos, 0, sizeof(empty_pos));
|
|
return empty_pos;
|
|
}
|
|
<% end %>
|
|
|
|
<% if !@grammar.tree || @grammar.parser_user_code_used? %>
|
|
/**
|
|
* Execute user code associated with a parser rule.
|
|
*
|
|
* @param rule The ID of the rule.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* Continue parsing.
|
|
* @retval P_USER_TERMINATED
|
|
* User requested to terminate parsing.
|
|
*/
|
|
static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
|
|
{
|
|
switch (rule)
|
|
{
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% if rule.code %>
|
|
case <%= rule.id %>u: {
|
|
<%= expand_code(rule.code, true, rule, nil) %>
|
|
} break;
|
|
<% end %>
|
|
<% end %>
|
|
default: break;
|
|
}
|
|
|
|
return P_SUCCESS;
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Check if the parser should shift to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param symbol_id
|
|
* Incoming token/rule set ID.
|
|
*
|
|
* @return State to shift to, or INVALID_ID if none.
|
|
*/
|
|
static size_t check_shift(size_t state_id, size_t symbol_id)
|
|
{
|
|
uint32_t start = parser_state_table[state_id].shift_table_index;
|
|
uint32_t end = start + parser_state_table[state_id].n_shift_entries;
|
|
for (uint32_t i = start; i < end; i++)
|
|
{
|
|
if (parser_shift_table[i].symbol_id == symbol_id)
|
|
{
|
|
return parser_shift_table[i].state_id;
|
|
}
|
|
}
|
|
return INVALID_ID;
|
|
}
|
|
|
|
/**
|
|
* Check if the parser should reduce to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param token
|
|
* Incoming token.
|
|
*
|
|
* @return State to reduce to, or INVALID_ID if none.
|
|
*/
|
|
static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
|
|
{
|
|
size_t start = parser_state_table[state_id].reduce_table_index;
|
|
size_t end = start + parser_state_table[state_id].n_reduce_entries;
|
|
for (size_t i = start; i < end; i++)
|
|
{
|
|
if ((parser_reduce_table[i].token == token) ||
|
|
(parser_reduce_table[i].token == INVALID_TOKEN_ID))
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
return INVALID_ID;
|
|
}
|
|
|
|
/**
|
|
* Run the parser.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param start_state_id
|
|
* ID of the state in which to start.
|
|
* @param start_rule_set_id
|
|
* Rule set ID for the requested start rule. Only used when
|
|
* @p follow_tokens is non-NULL, to gate follow-token shift success.
|
|
* @param follow_tokens
|
|
* Optional array of caller-provided follow tokens (tokens expected to
|
|
* appear immediately after the start rule in some outer context). Used to
|
|
* drive the "parse inner" retry logic. May be NULL for a standard parse.
|
|
* @param n_follow_tokens
|
|
* Number of entries in @p follow_tokens.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* The parser successfully matched the input text. The parse result value
|
|
* can be accessed with <%= @grammar.prefix %>result().
|
|
* @retval P_UNEXPECTED_TOKEN
|
|
* An unexpected token was encountered that does not match any grammar rule.
|
|
* The function p_token(&context) can be used to get the unexpected token.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
*/
|
|
static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
|
|
size_t start_rule_set_id,
|
|
<%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
|
|
{
|
|
<%= @grammar.prefix %>token_info_t token_info;
|
|
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
|
|
state_values_stack_t statevalues;
|
|
size_t reduced_rule_set = INVALID_ID;
|
|
size_t last_shifted_rule_set_id = INVALID_ID;
|
|
<% if @grammar.tree %>
|
|
<%= @grammar.prefix %>node_id_t reduced_parser_node;
|
|
<% else %>
|
|
<%= @grammar.prefix %>position_t reduced_position;
|
|
<%= @grammar.prefix %>position_t reduced_end_position;
|
|
<%= @grammar.prefix %>value_t reduced_parser_value;
|
|
<% end %>
|
|
state_values_stack_init(&statevalues);
|
|
state_values_stack_push(&statevalues);
|
|
state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
|
|
size_t result;
|
|
for (;;)
|
|
{
|
|
if (token == INVALID_TOKEN_ID)
|
|
{
|
|
size_t lexer_result = <%= lex_fn %>(context, &token_info);
|
|
if (lexer_result != P_SUCCESS)
|
|
{
|
|
result = lexer_result;
|
|
break;
|
|
}
|
|
token = token_info.token;
|
|
}
|
|
/* For a "parse inner" operation, determine once per iteration whether
|
|
* the current token is a member of the caller-provided follow token
|
|
* set. Used by both the shift-side and reduce-side retries below. */
|
|
bool token_is_follow = false;
|
|
for (size_t i = 0u; i < n_follow_tokens; i++)
|
|
{
|
|
if (token == follow_tokens[i])
|
|
{
|
|
token_is_follow = true;
|
|
break;
|
|
}
|
|
}
|
|
size_t shift_state = INVALID_ID;
|
|
if (reduced_rule_set != INVALID_ID)
|
|
{
|
|
shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, reduced_rule_set);
|
|
}
|
|
if (shift_state == INVALID_ID)
|
|
{
|
|
shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, token);
|
|
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
|
|
{
|
|
/* Successful parse. */
|
|
<% if @grammar.tree %>
|
|
context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
|
|
<% else %>
|
|
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
|
|
<% end %>
|
|
result = P_SUCCESS;
|
|
break;
|
|
}
|
|
if ((shift_state == INVALID_ID) && token_is_follow)
|
|
{
|
|
/* For a "parse inner" operation, if the incoming token is one
|
|
* of the caller's follow tokens, retry the shift as
|
|
* TOKEN___EOF. Only consider the parse complete if the reduced
|
|
* start rule is the only thing on the parse stack (i.e. the
|
|
* initial state plus a single shifted start rule set entry). */
|
|
size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
|
|
if ((retry_shift_state != INVALID_ID) &&
|
|
(statevalues.length == 2u) &&
|
|
(last_shifted_rule_set_id == start_rule_set_id))
|
|
{
|
|
/* Successful parse via follow token. Rewind the input
|
|
* position so that the follow token is not consumed from
|
|
* the input stream and remains available for a subsequent
|
|
* call to <%= @grammar.prefix %>lex() or a
|
|
* <%= @grammar.prefix %>parse*() function. */
|
|
context->input_index -= token_info.length;
|
|
context->text_position = token_info.position;
|
|
<% if @grammar.tree %>
|
|
context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
|
|
<% else %>
|
|
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
|
|
<% end %>
|
|
result = P_SUCCESS;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (shift_state != INVALID_ID)
|
|
{
|
|
/* We have something to shift. Track the last shifted rule set ID
|
|
* (INVALID_ID if we just shifted a token) so the follow-token
|
|
* shift retry can gate success on the reduced start rule being the
|
|
* only thing on top of the initial state. */
|
|
last_shifted_rule_set_id = reduced_rule_set;
|
|
state_values_stack_push(&statevalues);
|
|
state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
|
|
new_state_info->state_id = shift_state;
|
|
if (reduced_rule_set == INVALID_ID)
|
|
{
|
|
/* We shifted a token, mark it consumed. */
|
|
<% if @grammar.tree %>
|
|
<%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
|
|
<%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
|
|
token_tree_node->position = token_info.position;
|
|
token_tree_node->end_position = token_info.end_position;
|
|
token_tree_node->n_fields = 0u;
|
|
token_tree_node->is_token = 1u;
|
|
token_tree_node->token = token;
|
|
token_tree_node->pvalue = token_info.pvalue;
|
|
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
|
|
new_state_info->node_id = token_node_id;
|
|
<% else %>
|
|
new_state_info->position = token_info.position;
|
|
new_state_info->end_position = token_info.end_position;
|
|
new_state_info->pvalue = token_info.pvalue;
|
|
<% end %>
|
|
token = INVALID_TOKEN_ID;
|
|
}
|
|
else
|
|
{
|
|
/* We shifted a RuleSet. */
|
|
<% if @grammar.tree %>
|
|
new_state_info->node_id = reduced_parser_node;
|
|
<% else %>
|
|
new_state_info->pvalue = reduced_parser_value;
|
|
new_state_info->position = reduced_position;
|
|
new_state_info->end_position = reduced_end_position;
|
|
<%= @grammar.prefix %>value_t new_parse_result;
|
|
memset(&new_parse_result, 0, sizeof(new_parse_result));
|
|
reduced_parser_value = new_parse_result;
|
|
<% end %>
|
|
reduced_rule_set = INVALID_ID;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
|
|
if ((reduce_index == INVALID_ID) && token_is_follow)
|
|
{
|
|
/* For a "parse inner" operation, if the incoming token is one of
|
|
* the caller's follow tokens, retry the reduce lookup as
|
|
* TOKEN___EOF. Whatever reduce_index results (if any) is used
|
|
* regardless of which rule set it reduces to; this allows chains
|
|
* of reductions leading up to the start rule. */
|
|
reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
|
|
}
|
|
if (reduce_index != INVALID_ID)
|
|
{
|
|
/* We have something to reduce. */
|
|
<% if @grammar.tree %>
|
|
if (parser_reduce_table[reduce_index].propagate_optional_target)
|
|
{
|
|
reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
|
|
}
|
|
else if (parser_reduce_table[reduce_index].n_states > 0)
|
|
{
|
|
uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
|
|
/* Reserve child slots. New slots are zero-initialized
|
|
* (null node ID) so absent optional children remain null. */
|
|
<%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
|
|
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
|
|
{
|
|
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
|
|
{
|
|
context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
|
|
{
|
|
context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
|
|
}
|
|
}
|
|
<%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
|
|
<%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
|
|
node->position = INVALID_POSITION;
|
|
node->end_position = INVALID_POSITION;
|
|
node->child_offset = child_offset;
|
|
node->n_fields = n_fields;
|
|
node->is_token = 0u;
|
|
bool position_found = false;
|
|
for (uint16_t i = 0; i < n_fields; i++)
|
|
{
|
|
<%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
|
|
if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
|
|
{
|
|
if (!position_found)
|
|
{
|
|
node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
|
|
position_found = true;
|
|
}
|
|
node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
|
|
}
|
|
}
|
|
reduced_parser_node = node_id;
|
|
}
|
|
else
|
|
{
|
|
reduced_parser_node = 0u;
|
|
}
|
|
<% if @grammar.parser_user_code_used? %>
|
|
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
|
|
{
|
|
state_values_stack_free(&statevalues);
|
|
return P_USER_TERMINATED;
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
<%= @grammar.prefix %>value_t reduced_parser_value2;
|
|
memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
|
|
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
|
|
{
|
|
state_values_stack_free(&statevalues);
|
|
return P_USER_TERMINATED;
|
|
}
|
|
reduced_parser_value = reduced_parser_value2;
|
|
if (parser_reduce_table[reduce_index].n_states > 0u)
|
|
{
|
|
reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
|
|
reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
|
|
}
|
|
else
|
|
{
|
|
memset(&reduced_position, 0, sizeof(reduced_position));
|
|
memset(&reduced_end_position, 0, sizeof(reduced_end_position));
|
|
}
|
|
<% end %>
|
|
reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
|
|
state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
|
|
continue;
|
|
}
|
|
|
|
/* A token was successfully lexed, so the input text position was
|
|
* advanced. However, this is an unexpected token, so we want to reset
|
|
* the context text position to point to the token rather than the text
|
|
* after it, so that if the caller wants to report the error position,
|
|
* it will point to the correct position of the unexpected token. */
|
|
context->text_position = token_info.position;
|
|
context->token = token;
|
|
result = P_UNEXPECTED_TOKEN;
|
|
break;
|
|
}
|
|
state_values_stack_free(&statevalues);
|
|
return result;
|
|
}
|
|
|
|
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
|
|
}
|
|
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
|
|
size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
|
|
}
|
|
|
|
size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
|
|
<%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
|
|
{
|
|
return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Get the parse result value.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Parse result value.
|
|
*/
|
|
<% if @grammar.tree %>
|
|
<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->parse_result.v_<%= start_rule_type[0] %>;
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->parse_result.v_<%= start_rule_type(i)[0] %>;
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/**
|
|
* Get the current text input position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current text position.
|
|
*/
|
|
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->text_position;
|
|
}
|
|
|
|
/**
|
|
* Set the current text input position.
|
|
*
|
|
* This can be used to set the initial text position to something other than
|
|
* (1, 1) for a nested parse operation so that error positions reported by
|
|
* subsequent lexer/parser calls are relative to a larger enclosing document.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param position
|
|
* Text position to set.
|
|
*/
|
|
void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
|
|
{
|
|
context->text_position = position;
|
|
}
|
|
|
|
/**
|
|
* Get the current input text byte offset.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current input text byte offset (measured from the start of the
|
|
* input text passed to <%= @grammar.prefix %>context_new()).
|
|
*/
|
|
size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->input_index;
|
|
}
|
|
|
|
/**
|
|
* Set the current input text byte offset.
|
|
*
|
|
* This moves the lexer's read cursor to the given byte offset (measured from
|
|
* the start of the input text passed to <%= @grammar.prefix %>context_new()).
|
|
* It can be used together with <%= @grammar.prefix %>set_position() to rewind
|
|
* the input part-way through a parse in order to re-read an earlier section of
|
|
* the input. The byte offset is not validated; the caller is responsible for
|
|
* providing an offset within the bounds of the input text. A value previously
|
|
* returned by <%= @grammar.prefix %>input_index() is a suitable argument.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param input_index
|
|
* Input text byte offset to set.
|
|
*/
|
|
void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
|
|
{
|
|
context->input_index = input_index;
|
|
}
|
|
|
|
/**
|
|
* Get the user terminate code.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return User terminate code.
|
|
*/
|
|
size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->user_terminate_code;
|
|
}
|
|
|
|
/**
|
|
* Get the parse token.
|
|
*
|
|
* @return Parse token.
|
|
*/
|
|
<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context->token;
|
|
}
|