Josh Holtrop
5fc712c6ee
Store tree nodes in congruent, compact arena array. Define handle types to refer to tree nodes rather than pointers to structure instances. Free tree with context.
1567 lines
52 KiB
Plaintext
1567 lines
52 KiB
Plaintext
/**
|
|
* @file
|
|
*
|
|
* This file is generated by Propane.
|
|
*/
|
|
<% if @grammar.modulename %>
|
|
|
|
module <%= @grammar.modulename %>;
|
|
<% end %>
|
|
|
|
import core.memory;
|
|
import core.stdc.stdlib : malloc, free;
|
|
|
|
/**************************************************************************
|
|
* User code blocks
|
|
*************************************************************************/
|
|
|
|
<%= @grammar.code_blocks.fetch("", "") %>
|
|
|
|
/**************************************************************************
|
|
* Public types
|
|
*************************************************************************/
|
|
|
|
/* Result codes. */
|
|
public enum : size_t
|
|
{
|
|
<%= @grammar.prefix.upcase %>SUCCESS,
|
|
<%= @grammar.prefix.upcase %>DECODE_ERROR,
|
|
<%= @grammar.prefix.upcase %>UNEXPECTED_INPUT,
|
|
<%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN,
|
|
<%= @grammar.prefix.upcase %>DROP,
|
|
<%= @grammar.prefix.upcase %>EOF,
|
|
<%= @grammar.prefix.upcase %>USER_TERMINATED,
|
|
}
|
|
|
|
/** Token type. */
|
|
public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
|
|
|
|
/** Token IDs. */
|
|
public enum : <%= @grammar.prefix %>token_t
|
|
{
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
TOKEN_<%= token.code_name %> = <%= index %>,
|
|
<% unless token.id == index %>
|
|
<% raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
|
|
<% end %>
|
|
<% end %>
|
|
INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
|
|
TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
|
|
}
|
|
|
|
/** Code point type. */
|
|
public alias <%= @grammar.prefix %>code_point_t = uint;
|
|
|
|
/**
|
|
* A structure to keep track of input position.
|
|
*
|
|
* This is useful for reporting errors, etc...
|
|
*/
|
|
public struct <%= @grammar.prefix %>position_t
|
|
{
|
|
/** Input text row (1-based). */
|
|
uint row;
|
|
|
|
/** Input text column (1-based). */
|
|
uint col;
|
|
|
|
/** Invalid position value. */
|
|
enum INVALID = <%= @grammar.prefix %>position_t(0u, 0u);
|
|
|
|
/** Return whether the position is valid. */
|
|
public @property bool valid()
|
|
{
|
|
return row != 0u;
|
|
}
|
|
}
|
|
|
|
<% if @grammar.tree %>
|
|
/** Parser values type. */
|
|
public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
|
|
<% else %>
|
|
/** Parser values type(s). */
|
|
public union <%= @grammar.prefix %>value_t
|
|
{
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
<%= typestring %> v_<%= name %>;
|
|
<% end %>
|
|
}
|
|
|
|
/** Parser value constructor(s). */
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
|
|
{
|
|
return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
|
|
}
|
|
<% end %>
|
|
|
|
/** Parser value accessor(s). */
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
|
|
{
|
|
return pvalue.v_<%= name %>;
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
|
|
<% if @grammar.tree %>
|
|
/** Tree node ID type (index into the context node arena). ID 0 is null. */
|
|
public alias <%= @grammar.prefix %>node_id_t = uint;
|
|
|
|
/**
|
|
* Tree node record.
|
|
*
|
|
* All tree nodes are stored contiguously in the context node arena. Child
|
|
* links are stored in a shared children array: a node's children
|
|
* occupy children[child_offset .. child_offset + n_fields]. Token payload
|
|
* fields (token, pvalue, and any user fields) are only meaningful when
|
|
* is_token is true.
|
|
*/
|
|
private struct <%= @grammar.prefix %>node_data_t
|
|
{
|
|
<%= @grammar.prefix %>position_t position;
|
|
<%= @grammar.prefix %>position_t end_position;
|
|
<%= @grammar.prefix %>node_id_t child_offset;
|
|
ushort n_fields;
|
|
bool is_token;
|
|
<%= @grammar.prefix %>token_t token;
|
|
<%= @grammar.prefix %>value_t pvalue;
|
|
<%= @grammar.token_user_fields %>
|
|
}
|
|
|
|
/** Tree node handle types. @{ */
|
|
|
|
/** Token tree node handle. */
|
|
public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
|
|
{
|
|
private <%= @grammar.prefix %>context_t * __context;
|
|
private <%= @grammar.prefix %>node_id_t __id;
|
|
|
|
this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
|
|
{
|
|
this.__context = context;
|
|
this.__id = id;
|
|
}
|
|
|
|
/** Return whether this handle refers to a valid (non-null) node. */
|
|
@property bool valid()
|
|
{
|
|
return __id != 0u;
|
|
}
|
|
|
|
/** Access the underlying node record (token, pvalue, and user fields). */
|
|
@property ref <%= @grammar.prefix %>node_data_t __node()
|
|
{
|
|
return __context.<%= @grammar.prefix %>tree_nodes[__id];
|
|
}
|
|
|
|
alias __node this;
|
|
}
|
|
|
|
<% tree_node_rule_sets.each do |rule_set| %>
|
|
/** <%= rule_set.name %> tree node handle. */
|
|
public struct <%= @grammar.tree_prefix %><%= rule_set.name %><%= @grammar.tree_suffix %>
|
|
{
|
|
private <%= @grammar.prefix %>context_t * __context;
|
|
private <%= @grammar.prefix %>node_id_t __id;
|
|
|
|
this(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>node_id_t id)
|
|
{
|
|
this.__context = context;
|
|
this.__id = id;
|
|
}
|
|
|
|
/** Return whether this handle refers to a valid (non-null) node. */
|
|
@property bool valid()
|
|
{
|
|
return __id != 0u;
|
|
}
|
|
|
|
/** Text position of the first code point spanned by this node. */
|
|
@property <%= @grammar.prefix %>position_t position()
|
|
{
|
|
return __context.<%= @grammar.prefix %>tree_nodes[__id].position;
|
|
}
|
|
|
|
/** Text position of the last code point spanned by this node. */
|
|
@property <%= @grammar.prefix %>position_t end_position()
|
|
{
|
|
return __context.<%= @grammar.prefix %>tree_nodes[__id].end_position;
|
|
}
|
|
|
|
/** Number of child fields in this node. */
|
|
@property ushort n_fields()
|
|
{
|
|
return __id ? __context.<%= @grammar.prefix %>tree_nodes[__id].n_fields : cast(ushort)0u;
|
|
}
|
|
<% rule_set.tree_fields.each_with_index do |fields, i| %>
|
|
<% fields.each do |field_name, type| %>
|
|
|
|
/** Access the <%= field_name %> child node. */
|
|
@property <%= type %> <%= field_name %>()
|
|
{
|
|
if (__id == 0u)
|
|
{
|
|
return <%= type %>(__context, 0u);
|
|
}
|
|
return <%= type %>(__context, __context.<%= @grammar.prefix %>tree_children[__context.<%= @grammar.prefix %>tree_nodes[__id].child_offset + <%= i %>u]);
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
}
|
|
|
|
<% end %>
|
|
/** @} */
|
|
<% end %>
|
|
|
|
/** Lexed token information. */
|
|
public struct <%= @grammar.prefix %>token_info_t
|
|
{
|
|
/** Text position of first code point in token. */
|
|
<%= @grammar.prefix %>position_t position;
|
|
|
|
/** Text position of last code point in token. */
|
|
<%= @grammar.prefix %>position_t end_position;
|
|
|
|
/** Number of input bytes used by the token. */
|
|
size_t length;
|
|
|
|
/** Token that was lexed. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/** Parser value associated with the token. */
|
|
<%= @grammar.prefix %>value_t pvalue;
|
|
}
|
|
|
|
/**
|
|
* Lexer and parser context.
|
|
*
|
|
* The user must allocate an instance of this structure and pass it to any
|
|
* public API function.
|
|
*/
|
|
public struct <%= @grammar.prefix %>context_t
|
|
{
|
|
/* Lexer context data. */
|
|
|
|
/** Input text. */
|
|
string input;
|
|
|
|
/** Input text index (byte offset). */
|
|
size_t input_index;
|
|
|
|
/** Input text position (row/column). */
|
|
<%= @grammar.prefix %>position_t text_position;
|
|
|
|
/** Current lexer mode. */
|
|
size_t mode;
|
|
|
|
/* Parser context data. */
|
|
|
|
/** Parse result value. */
|
|
<% if @grammar.tree %>
|
|
<%= @grammar.prefix %>node_id_t parse_result;
|
|
|
|
/** Tree node arena. Node ID 0 is reserved as the null node. */
|
|
<%= @grammar.prefix %>node_data_t[] <%= @grammar.prefix %>tree_nodes;
|
|
|
|
/** Shared tree child links. */
|
|
<%= @grammar.prefix %>node_id_t[] <%= @grammar.prefix %>tree_children;
|
|
<% else %>
|
|
<%= @grammar.prefix %>value_t parse_result;
|
|
<% end %>
|
|
|
|
/** Unexpected token received. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/** User terminate code. */
|
|
size_t user_terminate_code;
|
|
|
|
<%= @grammar.context_user_fields %>
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Public data
|
|
*************************************************************************/
|
|
|
|
/** Token names. */
|
|
public immutable string[] <%= @grammar.prefix %>token_names = [
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
"<%= token.name %>",
|
|
<% end %>
|
|
];
|
|
|
|
/**************************************************************************
|
|
* Private types
|
|
*************************************************************************/
|
|
|
|
<% if @grammar.prefix.upcase != "P_" %>
|
|
/* Result codes. */
|
|
private enum : size_t
|
|
{
|
|
P_SUCCESS,
|
|
P_DECODE_ERROR,
|
|
P_UNEXPECTED_INPUT,
|
|
P_UNEXPECTED_TOKEN,
|
|
P_DROP,
|
|
P_EOF,
|
|
P_USER_TERMINATED,
|
|
}
|
|
<% end %>
|
|
|
|
/* An invalid ID value. */
|
|
private enum size_t INVALID_ID = cast(size_t)-1;
|
|
|
|
/**************************************************************************
|
|
* State initialization
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Allocate and initialize lexer/parser context structure.
|
|
*
|
|
* Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
|
|
*
|
|
* @param input
|
|
* Text input.
|
|
* @param input_length
|
|
* Text input length.
|
|
*
|
|
* @return Context structure for lexer/parser.
|
|
*/
|
|
<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(string input)
|
|
{
|
|
/* New default-initialized context structure. */
|
|
<%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t;
|
|
|
|
/* Lexer initialization. */
|
|
context.input = input;
|
|
context.text_position.row = 1u;
|
|
context.text_position.col = 1u;
|
|
context.mode = <%= @lexer.mode_id("default") %>;
|
|
<% if @grammar.tree %>
|
|
|
|
/* Reserve node ID 0 as the null tree node. */
|
|
context.<%= @grammar.prefix %>tree_nodes = new <%= @grammar.prefix %>node_data_t[](1);
|
|
<% end %>
|
|
|
|
return context;
|
|
}
|
|
|
|
/**
|
|
* Deinitialize and deallocate lexer/parser context structure.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
|
|
*/
|
|
void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
<% if @grammar.tree && @grammar.free_token_node != "" %>
|
|
foreach (ref node; context.<%= @grammar.prefix %>tree_nodes)
|
|
{
|
|
if (node.is_token)
|
|
{
|
|
<%= @grammar.prefix %>node_data_t * token_tree_node = &node;
|
|
<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
|
|
}
|
|
}
|
|
<% end %>
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Decoder
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Decode a UTF-8 code point.
|
|
*
|
|
* @param input
|
|
* Text input to decode.
|
|
* @param[out] out_code_point
|
|
* The decoded code point is stored here if the return value is P_SUCCESS.
|
|
* @param[out] out_code_point_length
|
|
* The number of bytes the code point used is stored here if the return value
|
|
* is P_SUCCESS.
|
|
*
|
|
* @retval P_SUCCESS on a successful code point decode
|
|
* @retval P_DECODE_ERROR when an encoding error is observed
|
|
* @retval P_EOF when the end of the text input is reached
|
|
*/
|
|
public size_t <%= @grammar.prefix %>decode_code_point(string input,
|
|
<%= @grammar.prefix %>code_point_t * out_code_point, ubyte * out_code_point_length)
|
|
{
|
|
if (input.length == 0u)
|
|
{
|
|
return P_EOF;
|
|
}
|
|
char c = input[0];
|
|
<%= @grammar.prefix %>code_point_t code_point;
|
|
ubyte code_point_length;
|
|
if ((c & 0x80u) == 0u)
|
|
{
|
|
code_point = c;
|
|
code_point_length = 1u;
|
|
}
|
|
else
|
|
{
|
|
ubyte following_bytes;
|
|
if ((c & 0xE0u) == 0xC0u)
|
|
{
|
|
code_point = c & 0x1Fu;
|
|
following_bytes = 1u;
|
|
}
|
|
else if ((c & 0xF0u) == 0xE0u)
|
|
{
|
|
code_point = c & 0x0Fu;
|
|
following_bytes = 2u;
|
|
}
|
|
else if ((c & 0xF8u) == 0xF0u)
|
|
{
|
|
code_point = c & 0x07u;
|
|
following_bytes = 3u;
|
|
}
|
|
else if ((c & 0xFCu) == 0xF8u)
|
|
{
|
|
code_point = c & 0x03u;
|
|
following_bytes = 4u;
|
|
}
|
|
else if ((c & 0xFEu) == 0xFCu)
|
|
{
|
|
code_point = c & 0x01u;
|
|
following_bytes = 5u;
|
|
}
|
|
else
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
if (input.length <= following_bytes)
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point_length = cast(ubyte)(following_bytes + 1u);
|
|
for (size_t i = 0u; i < following_bytes; i++)
|
|
{
|
|
char b = input[i + 1u];
|
|
if ((b & 0xC0u) != 0x80u)
|
|
{
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point = (code_point << 6u) | (b & 0x3Fu);
|
|
}
|
|
}
|
|
*out_code_point = code_point;
|
|
*out_code_point_length = code_point_length;
|
|
return P_SUCCESS;
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Lexer
|
|
*************************************************************************/
|
|
|
|
/** Lexer state ID type. */
|
|
private alias lexer_state_id_t = <%= get_type_for(@lexer.state_table.size) %>;
|
|
|
|
/** Invalid lexer state ID. */
|
|
private enum lexer_state_id_t INVALID_LEXER_STATE_ID = <%= @lexer.state_table.size %>u;
|
|
|
|
/** Lexer user code ID type. */
|
|
<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
|
|
private alias lexer_user_code_id_t = <%= get_type_for(user_code_id_count) %>;
|
|
|
|
/** Invalid lexer user code ID. */
|
|
private enum lexer_user_code_id_t INVALID_USER_CODE_ID = <%= user_code_id_count %>u;
|
|
|
|
/**
|
|
* Lexer transition table entry.
|
|
*
|
|
* An incoming code point matching the range for a transition entry will cause
|
|
* the lexer to progress to the destination state.
|
|
*/
|
|
private struct lexer_transition_t
|
|
{
|
|
/** First code point in the range for this transition. */
|
|
<%= @grammar.prefix %>code_point_t first;
|
|
|
|
/** Last code point in the range for this transition. */
|
|
<%= @grammar.prefix %>code_point_t last;
|
|
|
|
/** Destination lexer state ID for this transition. */
|
|
lexer_state_id_t destination_state;
|
|
}
|
|
|
|
/** Lexer state table entry. */
|
|
private struct lexer_state_t
|
|
{
|
|
/** Index to the transition table for this state. */
|
|
<%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
|
|
|
|
/** Number of transition table entries for this state. */
|
|
<%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
|
|
|
|
/** Lexer token formed at this state. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/** Lexer user code ID to execute at this state. */
|
|
lexer_user_code_id_t code_id;
|
|
|
|
/** Whether this state matches a lexer pattern. */
|
|
bool accepts;
|
|
}
|
|
|
|
/** Lexer mode table entry. */
|
|
private struct lexer_mode_t
|
|
{
|
|
/** Offset in the state table to be used for this mode. */
|
|
uint state_table_offset;
|
|
}
|
|
|
|
/**
|
|
* Lexer match info structure.
|
|
*
|
|
* This structure holds output values from the lexer upon a successful pattern
|
|
* match.
|
|
*/
|
|
private struct lexer_match_info_t
|
|
{
|
|
/** Number of bytes of input text used to match. */
|
|
size_t length;
|
|
|
|
/** Input text position delta to end of token. */
|
|
<%= @grammar.prefix %>position_t end_delta_position;
|
|
|
|
/** Input text position delta to next code point after token end. */
|
|
<%= @grammar.prefix %>position_t delta_position;
|
|
|
|
/** Accepting lexer state from the match. */
|
|
const(lexer_state_t) * accepting_state;
|
|
}
|
|
|
|
/** Lexer transition table. */
|
|
private immutable lexer_transition_t[] lexer_transition_table = [
|
|
<% @lexer.transition_table.each do |transition_table_entry| %>
|
|
lexer_transition_t(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
|
|
<% end %>
|
|
];
|
|
|
|
/** Lexer state table. */
|
|
private immutable lexer_state_t[] lexer_state_table = [
|
|
<% @lexer.state_table.each do |state_table_entry| %>
|
|
lexer_state_t(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>),
|
|
<% end %>
|
|
];
|
|
|
|
/** Lexer mode table. */
|
|
private immutable lexer_mode_t[] lexer_mode_table = [
|
|
<% @lexer.mode_table.each do |mode_table_entry| %>
|
|
lexer_mode_t(<%= mode_table_entry[:state_table_offset] %>),
|
|
<% end %>
|
|
];
|
|
|
|
/**
|
|
* Execute user code associated with a lexer pattern.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param code_id
|
|
* The ID of the user code block to execute.
|
|
* @param match
|
|
* Matched text for this pattern.
|
|
* @param out_token_info
|
|
* Lexer token info in progress.
|
|
*
|
|
* @return Token to accept, or invalid token if the user code does
|
|
* not explicitly return a token.
|
|
*/
|
|
private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
|
|
lexer_user_code_id_t code_id, string match,
|
|
<%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
switch (code_id)
|
|
{
|
|
<% @grammar.patterns.each do |pattern| %>
|
|
<% if pattern.code_id %>
|
|
case <%= pattern.code_id %>u: {
|
|
<%= expand_code(pattern.code, false, nil, pattern) %>
|
|
} break;
|
|
<% end %>
|
|
<% end %>
|
|
default: break;
|
|
}
|
|
|
|
return INVALID_TOKEN_ID;
|
|
}
|
|
|
|
/**
|
|
* Check if there is a transition from the current lexer state to another
|
|
* based on the given input code point.
|
|
*
|
|
* @param current_state
|
|
* Current lexer state.
|
|
* @param code_point
|
|
* Input code point.
|
|
*
|
|
* @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
|
|
*/
|
|
private lexer_state_id_t check_lexer_transition(uint current_state, uint code_point)
|
|
{
|
|
uint transition_table_index = lexer_state_table[current_state].transition_table_index;
|
|
for (uint i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
|
|
{
|
|
if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
|
|
(code_point <= lexer_transition_table[transition_table_index + i].last))
|
|
{
|
|
return lexer_transition_table[transition_table_index + i].destination_state;
|
|
}
|
|
}
|
|
return INVALID_LEXER_STATE_ID;
|
|
}
|
|
|
|
/**
|
|
* Find the longest lexer pattern match at the current position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_match_info
|
|
* The longest match information is stored here if the return value is
|
|
* P_SUCCESS or P_DECODE_ERROR.
|
|
* @param[out] out_unexpected_input_length
|
|
* The unexpected input length is stored here if the return value is
|
|
* P_UNEXPECTED_INPUT.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_EOF
|
|
* The end of the text input was reached.
|
|
*/
|
|
private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
|
|
lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
|
|
{
|
|
lexer_match_info_t longest_match;
|
|
lexer_match_info_t attempt_match;
|
|
*out_match_info = longest_match;
|
|
uint current_state = lexer_mode_table[context.mode].state_table_offset;
|
|
for (;;)
|
|
{
|
|
string input = context.input[(context.input_index + attempt_match.length)..(context.input.length)];
|
|
<%= @grammar.prefix %>code_point_t code_point;
|
|
ubyte code_point_length;
|
|
size_t result = <%= @grammar.prefix %>decode_code_point(input, &code_point, &code_point_length);
|
|
switch (result)
|
|
{
|
|
case P_SUCCESS:
|
|
lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
|
|
if (transition_state != INVALID_LEXER_STATE_ID)
|
|
{
|
|
attempt_match.length += code_point_length;
|
|
attempt_match.end_delta_position = attempt_match.delta_position;
|
|
if (code_point == '\n')
|
|
{
|
|
attempt_match.delta_position.row++;
|
|
attempt_match.delta_position.col = 1u;
|
|
}
|
|
else
|
|
{
|
|
attempt_match.delta_position.col++;
|
|
}
|
|
current_state = transition_state;
|
|
if (lexer_state_table[current_state].accepts)
|
|
{
|
|
attempt_match.accepting_state = &lexer_state_table[current_state];
|
|
longest_match = attempt_match;
|
|
}
|
|
}
|
|
else if (longest_match.length > 0)
|
|
{
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
}
|
|
else
|
|
{
|
|
*out_unexpected_input_length = attempt_match.length + code_point_length;
|
|
return P_UNEXPECTED_INPUT;
|
|
}
|
|
break;
|
|
|
|
case P_EOF:
|
|
/* We hit EOF. */
|
|
if (longest_match.length > 0)
|
|
{
|
|
/* We have a match, so use it. */
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
}
|
|
else if (attempt_match.length != 0)
|
|
{
|
|
/* There is a partial match - error! */
|
|
*out_unexpected_input_length = attempt_match.length;
|
|
return P_UNEXPECTED_INPUT;
|
|
}
|
|
else
|
|
{
|
|
/* Valid EOF return. */
|
|
return P_EOF;
|
|
}
|
|
|
|
case P_DECODE_ERROR:
|
|
/* If we see a decode error, we may be partially in the middle of
|
|
* matching a pattern, so return the attempted match info so that
|
|
* the input text position can be updated. */
|
|
*out_match_info = attempt_match;
|
|
return result;
|
|
|
|
default:
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempt to lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_DROP
|
|
* A drop pattern was matched so the lexer should continue.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
<%= @grammar.prefix %>token_info_t token_info;
|
|
token_info.position = context.text_position;
|
|
token_info.token = INVALID_TOKEN_ID;
|
|
lexer_match_info_t match_info;
|
|
size_t unexpected_input_length;
|
|
size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
|
|
switch (result)
|
|
{
|
|
case P_SUCCESS:
|
|
<%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
|
|
/* Calculate the token length and start/end positions before invoking
|
|
* the lexer user code so that the user code can access them. The
|
|
* context input text position tracking is not updated until after the
|
|
* user code has run so that it is left unchanged if the user code
|
|
* requests to terminate the lexer. */
|
|
token_info.length = match_info.length;
|
|
if (match_info.end_delta_position.row != 0u)
|
|
{
|
|
token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
|
|
token_info.end_position.col = match_info.end_delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
token_info.end_position.row = token_info.position.row;
|
|
token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
|
|
}
|
|
if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
|
|
{
|
|
string match = context.input[context.input_index..(context.input_index + match_info.length)];
|
|
<%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
|
|
match_info.accepting_state.code_id, match, &token_info);
|
|
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
|
|
* that the user code is requesting to terminate the lexer. */
|
|
if (user_code_token == TERMINATE_TOKEN_ID)
|
|
{
|
|
return P_USER_TERMINATED;
|
|
}
|
|
/* An invalid token returned from lexer_user_code() means that the
|
|
* user code did not explicitly return a token. So only override
|
|
* the token to return if the user code does explicitly return a
|
|
* token. */
|
|
if (user_code_token != INVALID_TOKEN_ID)
|
|
{
|
|
token_to_accept = user_code_token;
|
|
}
|
|
}
|
|
|
|
/* Update the input position tracking. */
|
|
context.input_index += match_info.length;
|
|
context.text_position.row += match_info.delta_position.row;
|
|
if (match_info.delta_position.row != 0u)
|
|
{
|
|
context.text_position.col = match_info.delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
context.text_position.col += match_info.delta_position.col;
|
|
}
|
|
|
|
if (token_to_accept == INVALID_TOKEN_ID)
|
|
{
|
|
return P_DROP;
|
|
}
|
|
token_info.token = token_to_accept;
|
|
*out_token_info = token_info;
|
|
return P_SUCCESS;
|
|
|
|
case P_EOF:
|
|
token_info.token = TOKEN___EOF;
|
|
token_info.end_position = token_info.position;
|
|
*out_token_info = token_info;
|
|
return P_SUCCESS;
|
|
|
|
case P_DECODE_ERROR:
|
|
/* Update the input position tracking. */
|
|
context.input_index += match_info.length;
|
|
context.text_position.row += match_info.delta_position.row;
|
|
if (match_info.delta_position.row != 0u)
|
|
{
|
|
context.text_position.col = match_info.delta_position.col;
|
|
}
|
|
else
|
|
{
|
|
context.text_position.col += match_info.delta_position.col;
|
|
}
|
|
return result;
|
|
|
|
default:
|
|
return result;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
|
{
|
|
for (;;)
|
|
{
|
|
size_t result = attempt_lex_token(context, out_token_info);
|
|
if (result != P_DROP)
|
|
{
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Parser
|
|
*************************************************************************/
|
|
|
|
/** Reduce ID type. */
|
|
private alias reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
|
|
|
|
/**
|
|
* A symbol ID can hold either a token ID or a rule set ID.
|
|
*
|
|
* Token IDs and rule set IDs share the same namespace, with rule set IDs
|
|
* beginning after token IDs end.
|
|
*/
|
|
private alias symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
|
|
|
|
/** Parser state ID type. */
|
|
private alias parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
|
|
|
|
/** Parser rule ID type. */
|
|
private alias rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
|
|
|
|
/** Parser shift ID type. */
|
|
private alias shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
|
|
|
|
/** Shift table entry. */
|
|
private struct shift_t
|
|
{
|
|
/** Token or rule set ID. */
|
|
symbol_id_t symbol_id;
|
|
|
|
/** Parser state to shift to. */
|
|
parser_state_id_t state_id;
|
|
}
|
|
|
|
/** Reduce table entry. */
|
|
private struct reduce_t
|
|
{
|
|
/** Lookahead token. */
|
|
<%= @grammar.prefix %>token_t token;
|
|
|
|
/**
|
|
* Rule ID.
|
|
*
|
|
* This is used to execute the parser user code block associated with a
|
|
* grammar rule.
|
|
*/
|
|
rule_id_t rule;
|
|
|
|
/**
|
|
* Rule set ID.
|
|
*
|
|
* This is used as the new top symbol ID of the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
symbol_id_t rule_set;
|
|
|
|
/**
|
|
* Number of states leading to this reduce action.
|
|
*
|
|
* This is the number of entries popped from the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
parser_state_id_t n_states;
|
|
<% if @grammar.tree %>
|
|
|
|
/**
|
|
* Map of rule components to rule set child fields.
|
|
*/
|
|
immutable(ushort) * rule_set_node_field_index_map;
|
|
|
|
/**
|
|
* Number of rule set tree node fields.
|
|
*/
|
|
ushort rule_set_node_field_array_size;
|
|
|
|
/**
|
|
* Whether this rule was a generated optional rule that matched the
|
|
* optional target. In this case, propagate the matched target node up
|
|
* instead of making a new node for this rule.
|
|
*/
|
|
bool propagate_optional_target;
|
|
<% end %>
|
|
}
|
|
|
|
/** Parser state entry. */
|
|
private struct parser_state_t
|
|
{
|
|
/** First shift table entry for this parser state. */
|
|
shift_id_t shift_table_index;
|
|
|
|
/** Number of shift table entries for this parser state. */
|
|
shift_id_t n_shift_entries;
|
|
|
|
/** First reduce table entry for this parser state. */
|
|
reduce_id_t reduce_table_index;
|
|
|
|
/** Number of reduce table entries for this parser state. */
|
|
reduce_id_t n_reduce_entries;
|
|
}
|
|
|
|
/**
|
|
* Structure to hold a state ID and value pair.
|
|
*
|
|
* A stack of these structures makes up the parse stack.
|
|
*/
|
|
private struct state_value_t
|
|
{
|
|
/** Parser state ID. */
|
|
size_t state_id;
|
|
|
|
<% if @grammar.tree %>
|
|
/** Tree node ID. */
|
|
<%= @grammar.prefix %>node_id_t node_id;
|
|
<% else %>
|
|
<%= @grammar.prefix %>position_t position;
|
|
<%= @grammar.prefix %>position_t end_position;
|
|
/** Parser value from this state. */
|
|
<%= @grammar.prefix %>value_t pvalue;
|
|
<% end %>
|
|
|
|
this(size_t state_id)
|
|
{
|
|
this.state_id = state_id;
|
|
}
|
|
}
|
|
|
|
/** Parser shift table. */
|
|
private immutable shift_t[] parser_shift_table = [
|
|
<% @parser.shift_table.each do |shift| %>
|
|
shift_t(<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u),
|
|
<% end %>
|
|
];
|
|
|
|
<% if @grammar.tree %>
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% unless rule.flat_rule_set_node_field_index_map? %>
|
|
immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
|
|
<% end %>
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/** Parser reduce table. */
|
|
private immutable reduce_t[] parser_reduce_table = [
|
|
<% @parser.reduce_table.each do |reduce| %>
|
|
reduce_t(
|
|
<%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
|
|
<%= reduce[:rule_id] %>u, /* Rule ID */
|
|
<%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
|
|
<% if @grammar.tree %>
|
|
<%= reduce[:n_states] %>u, /* Number of states */
|
|
<% if reduce[:rule].flat_rule_set_node_field_index_map? %>
|
|
null, /* No rule set node field index map (flat map) */
|
|
<% else %>
|
|
&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
|
|
<% end %>
|
|
<%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
|
|
<%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
|
|
<% else %>
|
|
<%= reduce[:n_states] %>u), /* Number of states */
|
|
<% end %>
|
|
<% end %>
|
|
];
|
|
|
|
/** Parser state table. */
|
|
private immutable parser_state_t[] parser_state_table = [
|
|
<% @parser.state_table.each do |state| %>
|
|
parser_state_t(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
|
|
<% end %>
|
|
];
|
|
|
|
<% unless @grammar.tree %>
|
|
/**
|
|
* Get the rule position (start or end) for the currently matched rule.
|
|
*/
|
|
private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
|
|
{
|
|
if (n_states > 0u)
|
|
{
|
|
if (i == 0u)
|
|
{
|
|
if (get_end)
|
|
{
|
|
for (size_t j = 0u; j < n_states; j++)
|
|
{
|
|
state_value_t * sv = &statevalues[$-1-j];
|
|
if (sv.end_position.valid)
|
|
{
|
|
return sv.end_position;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (size_t j = 0u; j < n_states; j++)
|
|
{
|
|
state_value_t * sv = &statevalues[$-n_states+j];
|
|
if (sv.position.valid)
|
|
{
|
|
return sv.position;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (get_end)
|
|
{
|
|
return statevalues[$-1-n_states+i].end_position;
|
|
}
|
|
else
|
|
{
|
|
return statevalues[$-1-n_states+i].position;
|
|
}
|
|
}
|
|
}
|
|
return <%= @grammar.prefix %>position_t.INVALID;
|
|
}
|
|
<% end %>
|
|
|
|
<% if !@grammar.tree || @grammar.parser_user_code_used? %>
|
|
/**
|
|
* Execute user code associated with a parser rule.
|
|
*
|
|
* @param rule The ID of the rule.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* Continue parsing.
|
|
* @retval P_USER_TERMINATED
|
|
* User requested to terminate parsing.
|
|
*/
|
|
private size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
|
|
{
|
|
switch (rule)
|
|
{
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% if rule.code %>
|
|
case <%= rule.id %>u: {
|
|
<%= expand_code(rule.code, true, rule, nil) %>
|
|
} break;
|
|
<% end %>
|
|
<% end %>
|
|
default: break;
|
|
}
|
|
|
|
return P_SUCCESS;
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Check if the parser should shift to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param symbol_id
|
|
* Incoming token/rule set ID.
|
|
*
|
|
* @return State to shift to, or INVALID_ID if none.
|
|
*/
|
|
private size_t check_shift(size_t state_id, size_t symbol_id)
|
|
{
|
|
uint start = parser_state_table[state_id].shift_table_index;
|
|
uint end = start + parser_state_table[state_id].n_shift_entries;
|
|
for (uint i = start; i < end; i++)
|
|
{
|
|
if (parser_shift_table[i].symbol_id == symbol_id)
|
|
{
|
|
return parser_shift_table[i].state_id;
|
|
}
|
|
}
|
|
return INVALID_ID;
|
|
}
|
|
|
|
/**
|
|
* Check if the parser should reduce to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param token
|
|
* Incoming token.
|
|
*
|
|
* @return State to reduce to, or INVALID_ID if none.
|
|
*/
|
|
private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
|
|
{
|
|
size_t start = parser_state_table[state_id].reduce_table_index;
|
|
size_t end = start + parser_state_table[state_id].n_reduce_entries;
|
|
for (size_t i = start; i < end; i++)
|
|
{
|
|
if ((parser_reduce_table[i].token == token) ||
|
|
(parser_reduce_table[i].token == INVALID_TOKEN_ID))
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
return INVALID_ID;
|
|
}
|
|
|
|
/**
|
|
* Run the parser.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param start_state_id
|
|
* ID of the state in which to start.
|
|
* @param start_rule_set_id
|
|
* Rule set ID for the requested start rule. Only used when
|
|
* @p follow_tokens is non-empty, to gate follow-token shift success.
|
|
* @param follow_tokens
|
|
* Optional slice of caller-provided follow tokens (tokens expected to
|
|
* appear immediately after the start rule in some outer context). Used to
|
|
* drive the "parse inner" retry logic. May be null/empty for a standard
|
|
* parse.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* The parser successfully matched the input text. The parse result value
|
|
* can be accessed with <%= @grammar.prefix %>result().
|
|
* @retval P_UNEXPECTED_TOKEN
|
|
* An unexpected token was encountered that does not match any grammar rule.
|
|
* The function p_token(&context) can be used to get the unexpected token.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
*/
|
|
private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
|
|
size_t start_rule_set_id,
|
|
const(<%= @grammar.prefix %>token_t)[] follow_tokens)
|
|
{
|
|
<%= @grammar.prefix %>token_info_t token_info;
|
|
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
|
|
state_value_t[] statevalues = new state_value_t[](1);
|
|
statevalues[0].state_id = start_state_id;
|
|
size_t reduced_rule_set = INVALID_ID;
|
|
size_t last_shifted_rule_set_id = INVALID_ID;
|
|
<% if @grammar.tree %>
|
|
<%= @grammar.prefix %>node_id_t reduced_parser_node;
|
|
<% else %>
|
|
<%= @grammar.prefix %>position_t reduced_position;
|
|
<%= @grammar.prefix %>position_t reduced_end_position;
|
|
<%= @grammar.prefix %>value_t reduced_parser_value;
|
|
<% end %>
|
|
for (;;)
|
|
{
|
|
if (token == INVALID_TOKEN_ID)
|
|
{
|
|
size_t lexer_result = <%= lex_fn %>(context, &token_info);
|
|
if (lexer_result != P_SUCCESS)
|
|
{
|
|
return lexer_result;
|
|
}
|
|
token = token_info.token;
|
|
}
|
|
/* For a "parse inner" operation, determine once per iteration whether
|
|
* the current token is a member of the caller-provided follow token
|
|
* set. Used by both the shift-side and reduce-side retries below. */
|
|
bool token_is_follow = false;
|
|
foreach (eof_token; follow_tokens)
|
|
{
|
|
if (token == eof_token)
|
|
{
|
|
token_is_follow = true;
|
|
break;
|
|
}
|
|
}
|
|
size_t shift_state = INVALID_ID;
|
|
if (reduced_rule_set != INVALID_ID)
|
|
{
|
|
shift_state = check_shift(statevalues[$-1].state_id, reduced_rule_set);
|
|
}
|
|
if (shift_state == INVALID_ID)
|
|
{
|
|
shift_state = check_shift(statevalues[$-1].state_id, token);
|
|
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
|
|
{
|
|
/* Successful parse. */
|
|
<% if @grammar.tree %>
|
|
context.parse_result = statevalues[$-1].node_id;
|
|
<% else %>
|
|
context.parse_result = statevalues[$-1].pvalue;
|
|
<% end %>
|
|
return P_SUCCESS;
|
|
}
|
|
if ((shift_state == INVALID_ID) && token_is_follow)
|
|
{
|
|
/* For a "parse inner" operation, if the incoming token is one
|
|
* of the caller's follow tokens, retry the shift as
|
|
* TOKEN___EOF. Only consider the parse complete if the reduced
|
|
* start rule is the only thing on the parse stack (i.e. the
|
|
* initial state plus a single shifted start rule set entry). */
|
|
size_t retry_shift_state = check_shift(statevalues[$-1].state_id, TOKEN___EOF);
|
|
if ((retry_shift_state != INVALID_ID) &&
|
|
(statevalues.length == 2u) &&
|
|
(last_shifted_rule_set_id == start_rule_set_id))
|
|
{
|
|
/* Successful parse via follow token. Rewind the input
|
|
* position so that the follow token is not consumed from
|
|
* the input stream and remains available for a subsequent
|
|
* call to <%= @grammar.prefix %>lex() or a
|
|
* <%= @grammar.prefix %>parse*() function. */
|
|
context.input_index -= token_info.length;
|
|
context.text_position = token_info.position;
|
|
<% if @grammar.tree %>
|
|
context.parse_result = statevalues[$-1].node_id;
|
|
<% else %>
|
|
context.parse_result = statevalues[$-1].pvalue;
|
|
<% end %>
|
|
return P_SUCCESS;
|
|
}
|
|
}
|
|
}
|
|
if (shift_state != INVALID_ID)
|
|
{
|
|
/* We have something to shift. Track the last shifted rule set ID
|
|
* (INVALID_ID if we just shifted a token) so the follow-token
|
|
* shift retry can gate success on the reduced start rule being the
|
|
* only thing on top of the initial state. */
|
|
last_shifted_rule_set_id = reduced_rule_set;
|
|
statevalues ~= state_value_t(shift_state);
|
|
if (reduced_rule_set == INVALID_ID)
|
|
{
|
|
/* We shifted a token, mark it consumed. */
|
|
<% if @grammar.tree %>
|
|
<%= @grammar.prefix %>node_id_t token_node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
|
|
context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
|
|
<%= @grammar.prefix %>node_data_t * token_tree_node = &context.<%= @grammar.prefix %>tree_nodes[token_node_id];
|
|
token_tree_node.position = token_info.position;
|
|
token_tree_node.end_position = token_info.end_position;
|
|
token_tree_node.n_fields = 0u;
|
|
token_tree_node.is_token = true;
|
|
token_tree_node.token = token;
|
|
token_tree_node.pvalue = token_info.pvalue;
|
|
<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
|
|
statevalues[$-1].node_id = token_node_id;
|
|
<% else %>
|
|
statevalues[$-1].position = token_info.position;
|
|
statevalues[$-1].end_position = token_info.end_position;
|
|
statevalues[$-1].pvalue = token_info.pvalue;
|
|
<% end %>
|
|
token = INVALID_TOKEN_ID;
|
|
}
|
|
else
|
|
{
|
|
/* We shifted a RuleSet. */
|
|
<% if @grammar.tree %>
|
|
statevalues[$-1].node_id = reduced_parser_node;
|
|
<% else %>
|
|
statevalues[$-1].pvalue = reduced_parser_value;
|
|
statevalues[$-1].position = reduced_position;
|
|
statevalues[$-1].end_position = reduced_end_position;
|
|
<%= @grammar.prefix %>value_t new_parse_result;
|
|
reduced_parser_value = new_parse_result;
|
|
<% end %>
|
|
reduced_rule_set = INVALID_ID;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
size_t reduce_index = check_reduce(statevalues[$-1].state_id, token);
|
|
if ((reduce_index == INVALID_ID) && token_is_follow)
|
|
{
|
|
/* For a "parse inner" operation, if the incoming token is one of
|
|
* the caller's follow tokens, retry the reduce lookup as
|
|
* TOKEN___EOF. Whatever reduce_index results (if any) is used
|
|
* regardless of which rule set it reduces to; this allows chains
|
|
* of reductions leading up to the start rule. */
|
|
reduce_index = check_reduce(statevalues[$-1].state_id, TOKEN___EOF);
|
|
}
|
|
if (reduce_index != INVALID_ID)
|
|
{
|
|
/* We have something to reduce. */
|
|
<% if @grammar.tree %>
|
|
if (parser_reduce_table[reduce_index].propagate_optional_target)
|
|
{
|
|
reduced_parser_node = statevalues[$ - 1].node_id;
|
|
}
|
|
else if (parser_reduce_table[reduce_index].n_states > 0)
|
|
{
|
|
ushort n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
|
|
/* Reserve child slots. New slots are zero-initialized
|
|
* (null node ID) so absent optional children remain null. */
|
|
<%= @grammar.prefix %>node_id_t child_offset = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_children.length;
|
|
context.<%= @grammar.prefix %>tree_children.length += n_fields;
|
|
if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
|
|
{
|
|
foreach (i; 0..parser_reduce_table[reduce_index].n_states)
|
|
{
|
|
context.<%= @grammar.prefix %>tree_children[child_offset + i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
foreach (i; 0..parser_reduce_table[reduce_index].n_states)
|
|
{
|
|
context.<%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].node_id;
|
|
}
|
|
}
|
|
<%= @grammar.prefix %>node_id_t node_id = cast(<%= @grammar.prefix %>node_id_t)context.<%= @grammar.prefix %>tree_nodes.length;
|
|
context.<%= @grammar.prefix %>tree_nodes ~= <%= @grammar.prefix %>node_data_t.init;
|
|
<%= @grammar.prefix %>node_data_t * node = &context.<%= @grammar.prefix %>tree_nodes[node_id];
|
|
node.position = <%= @grammar.prefix %>position_t.INVALID;
|
|
node.end_position = <%= @grammar.prefix %>position_t.INVALID;
|
|
node.child_offset = child_offset;
|
|
node.n_fields = n_fields;
|
|
node.is_token = false;
|
|
bool position_found = false;
|
|
foreach (i; 0..n_fields)
|
|
{
|
|
<%= @grammar.prefix %>node_id_t child_id = context.<%= @grammar.prefix %>tree_children[child_offset + i];
|
|
if (child_id != 0u && context.<%= @grammar.prefix %>tree_nodes[child_id].position.valid)
|
|
{
|
|
if (!position_found)
|
|
{
|
|
node.position = context.<%= @grammar.prefix %>tree_nodes[child_id].position;
|
|
position_found = true;
|
|
}
|
|
node.end_position = context.<%= @grammar.prefix %>tree_nodes[child_id].end_position;
|
|
}
|
|
}
|
|
reduced_parser_node = node_id;
|
|
}
|
|
else
|
|
{
|
|
reduced_parser_node = 0u;
|
|
}
|
|
<% if @grammar.parser_user_code_used? %>
|
|
if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
|
|
{
|
|
return P_USER_TERMINATED;
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
<%= @grammar.prefix %>value_t reduced_parser_value2;
|
|
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
|
|
{
|
|
return P_USER_TERMINATED;
|
|
}
|
|
reduced_parser_value = reduced_parser_value2;
|
|
if (parser_reduce_table[reduce_index].n_states > 0u)
|
|
{
|
|
reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
|
|
reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
|
|
}
|
|
else
|
|
{
|
|
reduced_position = <%= @grammar.prefix %>position_t.INVALID;
|
|
reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
|
|
}
|
|
<% end %>
|
|
reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
|
|
statevalues.length -= parser_reduce_table[reduce_index].n_states;
|
|
continue;
|
|
}
|
|
|
|
/* A token was successfully lexed, so the input text position was
|
|
* advanced. However, this is an unexpected token, so we want to reset
|
|
* the context text position to point to the token rather than the text
|
|
* after it, so that if the caller wants to report the error position,
|
|
* it will point to the correct position of the unexpected token. */
|
|
context.text_position = token_info.position;
|
|
context.token = token;
|
|
return P_UNEXPECTED_TOKEN;
|
|
}
|
|
}
|
|
|
|
public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, null);
|
|
}
|
|
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
|
|
public size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, null);
|
|
}
|
|
|
|
public size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
|
|
const(<%= @grammar.prefix %>token_t)[] follow_tokens)
|
|
{
|
|
return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens);
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Get the parse result value.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Parse result value.
|
|
*/
|
|
<% if @grammar.tree %>
|
|
public <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return <%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %>(context, context.parse_result);
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
public <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return <%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %>(context, context.parse_result);
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.parse_result.v_<%= start_rule_type[0] %>;
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.parse_result.v_<%= start_rule_type(i)[0] %>;
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
|
|
|
|
/**
|
|
* Get the current text input position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current text position.
|
|
*/
|
|
public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.text_position;
|
|
}
|
|
|
|
/**
|
|
* Set the current text input position.
|
|
*
|
|
* This can be used to set the initial text position to something other than
|
|
* (1, 1) for a nested parse operation so that error positions reported by
|
|
* subsequent lexer/parser calls are relative to a larger enclosing document.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param position
|
|
* Text position to set.
|
|
*/
|
|
public void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
|
|
{
|
|
context.text_position = position;
|
|
}
|
|
|
|
/**
|
|
* Get the current input text byte offset.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current input text byte offset (measured from the start of the
|
|
* input text passed to <%= @grammar.prefix %>context_new()).
|
|
*/
|
|
public size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.input_index;
|
|
}
|
|
|
|
/**
|
|
* Set the current input text byte offset.
|
|
*
|
|
* This moves the lexer's read cursor to the given byte offset (measured from
|
|
* the start of the input text passed to <%= @grammar.prefix %>context_new()).
|
|
* It can be used together with <%= @grammar.prefix %>set_position() to rewind
|
|
* the input part-way through a parse in order to re-read an earlier section of
|
|
* the input. The byte offset is not validated; the caller is responsible for
|
|
* providing an offset within the bounds of the input text. A value previously
|
|
* returned by <%= @grammar.prefix %>input_index() is a suitable argument.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param input_index
|
|
* Input text byte offset to set.
|
|
*/
|
|
public void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
|
|
{
|
|
context.input_index = input_index;
|
|
}
|
|
|
|
/**
|
|
* Get the user terminate code.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return User terminate code.
|
|
*/
|
|
public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.user_terminate_code;
|
|
}
|
|
|
|
/**
|
|
* Get the parse token.
|
|
*
|
|
* @return Parse token.
|
|
*/
|
|
public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
|
|
{
|
|
return context.token;
|
|
}
|