1401 lines
55 KiB
Plaintext
1401 lines
55 KiB
Plaintext
/*
|
|
* This file is generated by Propane.
|
|
*/
|
|
|
|
#![allow(non_camel_case_types)]
|
|
#![allow(non_snake_case)]
|
|
#![allow(non_upper_case_globals)]
|
|
#![allow(dead_code)]
|
|
#![allow(unused_variables)]
|
|
#![allow(unused_mut)]
|
|
#![allow(unused_parens)]
|
|
#![allow(unused_assignments)]
|
|
#![allow(unreachable_patterns)]
|
|
|
|
/**************************************************************************
|
|
* User code blocks
|
|
*************************************************************************/
|
|
|
|
<%= @grammar.code_blocks.fetch("", "") %>
|
|
|
|
/**************************************************************************
|
|
* Public types
|
|
*************************************************************************/
|
|
|
|
/* Result codes. */
|
|
pub const <%= @grammar.prefix.upcase %>SUCCESS: usize = 0;
|
|
pub const <%= @grammar.prefix.upcase %>DECODE_ERROR: usize = 1;
|
|
pub const <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT: usize = 2;
|
|
pub const <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN: usize = 3;
|
|
pub const <%= @grammar.prefix.upcase %>DROP: usize = 4;
|
|
pub const <%= @grammar.prefix.upcase %>EOF: usize = 5;
|
|
pub const <%= @grammar.prefix.upcase %>USER_TERMINATED: usize = 6;
|
|
|
|
/** Token type. */
|
|
pub type <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
|
|
|
|
/** Token IDs. */
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
pub const TOKEN_<%= token.code_name %>: <%= @grammar.prefix %>token_t = <%= index %>;
|
|
<% unless token.id == index %>
|
|
<% raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
|
|
<% end %>
|
|
<% end %>
|
|
pub const INVALID_TOKEN_ID: <%= @grammar.prefix %>token_t = <%= @grammar.invalid_token_id %>;
|
|
pub const TERMINATE_TOKEN_ID: <%= @grammar.prefix %>token_t = <%= @grammar.terminate_token_id %>;
|
|
|
|
/** Code point type. */
|
|
pub type <%= @grammar.prefix %>code_point_t = u32;
|
|
|
|
/**
|
|
* A structure to keep track of input position.
|
|
*
|
|
* This is useful for reporting errors, etc...
|
|
*/
|
|
#[derive(Clone, Copy, Default, PartialEq)]
|
|
pub struct <%= @grammar.prefix %>position_t {
|
|
/** Input text row (1-based). */
|
|
pub row: u32,
|
|
/** Input text column (1-based). */
|
|
pub col: u32,
|
|
}
|
|
|
|
impl <%= @grammar.prefix %>position_t {
|
|
/** Return whether the position is valid. */
|
|
pub fn valid(&self) -> bool {
|
|
self.row != 0
|
|
}
|
|
}
|
|
|
|
/** An invalid position value. */
|
|
const INVALID_POSITION: <%= @grammar.prefix %>position_t = <%= @grammar.prefix %>position_t { row: 0, col: 0 };
|
|
|
|
<% if @grammar.tree %>
|
|
/** Parser values type. */
|
|
pub type <%= @grammar.prefix %>value_t = <%= rust_ptype(@grammar.ptype) %>;
|
|
<% else %>
|
|
/** Parser values type(s). */
|
|
#[derive(Clone, Default)]
|
|
pub enum <%= @grammar.prefix %>value_t {
|
|
#[default]
|
|
__None,
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
v_<%= name %>(<%= rust_ptype(typestring) %>),
|
|
<% end %>
|
|
}
|
|
|
|
impl <%= @grammar.prefix %>value_t {
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
fn v_<%= name %>_mut(&mut self) -> &mut <%= rust_ptype(typestring) %> {
|
|
match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v, _ => unreachable!() }
|
|
}
|
|
fn get_v_<%= name %>(&self) -> <%= rust_ptype(typestring) %> {
|
|
match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v.clone(), _ => Default::default() }
|
|
}
|
|
<% end %>
|
|
}
|
|
|
|
/** Parser value constructor(s) and accessor(s). */
|
|
<% @grammar.ptypes.each do |name, typestring| %>
|
|
<% suffix = name == "default" ? "" : "_#{name}" %>
|
|
pub fn <%= @grammar.prefix %>value<%= suffix %>(v: <%= rust_ptype(typestring) %>) -> <%= @grammar.prefix %>value_t { <%= @grammar.prefix %>value_t::v_<%= name %>(v) }
|
|
pub fn <%= @grammar.prefix %>value_get<%= suffix %>(pvalue: &<%= @grammar.prefix %>value_t) -> <%= rust_ptype(typestring) %> { pvalue.get_v_<%= name %>() }
|
|
<% end %>
|
|
<% end %>
|
|
<% if @grammar.tree %>
|
|
/** Tree node ID type (index into the context node arena). ID 0 is null. */
|
|
pub type <%= @grammar.prefix %>node_id_t = u32;
|
|
|
|
/**
|
|
* Tree node record.
|
|
*
|
|
* All tree nodes are stored contiguously in the context node arena. Child
|
|
* links are stored in a shared children array: a node's children
|
|
* occupy children[child_offset .. child_offset + n_fields]. Token payload
|
|
* fields (token, pvalue, and any user fields) are only meaningful when
|
|
* is_token is true.
|
|
*/
|
|
#[derive(Clone, Default)]
|
|
pub struct <%= @grammar.prefix %>node_data_t {
|
|
pub position: <%= @grammar.prefix %>position_t,
|
|
pub end_position: <%= @grammar.prefix %>position_t,
|
|
pub child_offset: <%= @grammar.prefix %>node_id_t,
|
|
pub n_fields: u16,
|
|
pub is_token: bool,
|
|
pub token: <%= @grammar.prefix %>token_t,
|
|
pub pvalue: <%= @grammar.prefix %>value_t,
|
|
<% unless @grammar.token_user_fields.to_s.strip.empty? %>
|
|
<%= @grammar.token_user_fields %>
|
|
<% end %>
|
|
}
|
|
|
|
/** Tree node handle types. */
|
|
|
|
/** Token tree node handle. */
|
|
#[derive(Clone, Copy)]
|
|
pub struct <%= h_type("Token") %><'a> { context: &'a <%= @grammar.prefix %>context_t, id: <%= @grammar.prefix %>node_id_t }
|
|
|
|
impl<'a> <%= h_type("Token") %><'a> {
|
|
/** Return whether this handle refers to a valid (non-null) node. */
|
|
pub fn valid(&self) -> bool { self.id != 0 }
|
|
/** Return the node ID (for identity comparison). */
|
|
pub fn node_id(&self) -> <%= @grammar.prefix %>node_id_t { self.id }
|
|
/** Access the underlying node record (token, pvalue, and user fields). */
|
|
pub fn data(&self) -> &'a <%= @grammar.prefix %>node_data_t { &self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize] }
|
|
/** Text position of the first code point spanned by this node. */
|
|
pub fn position(&self) -> <%= @grammar.prefix %>position_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].position }
|
|
/** Text position of the last code point spanned by this node. */
|
|
pub fn end_position(&self) -> <%= @grammar.prefix %>position_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].end_position }
|
|
/** Number of child fields in this node. */
|
|
pub fn n_fields(&self) -> u16 { if self.id != 0 { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].n_fields } else { 0 } }
|
|
/** Token ID for this token node. */
|
|
pub fn token(&self) -> <%= @grammar.prefix %>token_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].token }
|
|
/** Parser value associated with this token node. */
|
|
pub fn pvalue(&self) -> <%= @grammar.prefix %>value_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].pvalue.clone() }
|
|
}
|
|
<% tree_node_rule_sets.each do |rule_set| %>
|
|
|
|
/** <%= rule_set.name %> tree node handle. */
|
|
#[derive(Clone, Copy)]
|
|
pub struct <%= h_type(rule_set.name) %><'a> { context: &'a <%= @grammar.prefix %>context_t, id: <%= @grammar.prefix %>node_id_t }
|
|
|
|
impl<'a> <%= h_type(rule_set.name) %><'a> {
|
|
/** Return whether this handle refers to a valid (non-null) node. */
|
|
pub fn valid(&self) -> bool { self.id != 0 }
|
|
/** Return the node ID (for identity comparison). */
|
|
pub fn node_id(&self) -> <%= @grammar.prefix %>node_id_t { self.id }
|
|
/** Access the underlying node record. */
|
|
pub fn data(&self) -> &'a <%= @grammar.prefix %>node_data_t { &self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize] }
|
|
/** Text position of the first code point spanned by this node. */
|
|
pub fn position(&self) -> <%= @grammar.prefix %>position_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].position }
|
|
/** Text position of the last code point spanned by this node. */
|
|
pub fn end_position(&self) -> <%= @grammar.prefix %>position_t { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].end_position }
|
|
/** Number of child fields in this node. */
|
|
pub fn n_fields(&self) -> u16 { if self.id != 0 { self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].n_fields } else { 0 } }
|
|
<% each_tree_field(rule_set) do |rt, field_name, child_type, slot| %>
|
|
|
|
/** Access the <%= field_name %> child node. */
|
|
pub fn <%= rust_ident(field_name) %>(&self) -> <%= child_type %><'a> {
|
|
if self.id == 0 {
|
|
return <%= child_type %> { context: self.context, id: 0 };
|
|
}
|
|
<%= child_type %> { context: self.context, id: self.context.<%= @grammar.prefix %>tree_children[self.context.<%= @grammar.prefix %>tree_nodes[self.id as usize].child_offset as usize + <%= slot %>] }
|
|
}
|
|
<% end %>
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/** Lexed token information. */
|
|
#[derive(Clone, Default)]
|
|
pub struct <%= @grammar.prefix %>token_info_t {
|
|
/** Text position of first code point in token. */
|
|
pub position: <%= @grammar.prefix %>position_t,
|
|
/** Text position of last code point in token. */
|
|
pub end_position: <%= @grammar.prefix %>position_t,
|
|
/** Number of input bytes used by the token. */
|
|
pub length: usize,
|
|
/** Token that was lexed. */
|
|
pub token: <%= @grammar.prefix %>token_t,
|
|
/** Parser value associated with the token. */
|
|
pub pvalue: <%= @grammar.prefix %>value_t,
|
|
}
|
|
|
|
/**
|
|
* Lexer and parser context.
|
|
*
|
|
* The user must allocate an instance of this structure and pass it to any
|
|
* public API function.
|
|
*/
|
|
#[derive(Default)]
|
|
pub struct <%= @grammar.prefix %>context_t {
|
|
/* Lexer context data. */
|
|
|
|
/** Input text. */
|
|
input: Vec<u8>,
|
|
/** Input text index (byte offset). */
|
|
input_index: usize,
|
|
/** Input text position (row/column). */
|
|
text_position: <%= @grammar.prefix %>position_t,
|
|
/** Current lexer mode. */
|
|
mode: usize,
|
|
|
|
/* Parser context data. */
|
|
|
|
/** Parse result value. */
|
|
<% if @grammar.tree %>
|
|
parse_result: <%= @grammar.prefix %>node_id_t,
|
|
/** Tree node arena. Node ID 0 is reserved as the null node. */
|
|
<%= @grammar.prefix %>tree_nodes: Vec<<%= @grammar.prefix %>node_data_t>,
|
|
/** Shared tree child links (CSR layout). */
|
|
<%= @grammar.prefix %>tree_children: Vec<<%= @grammar.prefix %>node_id_t>,
|
|
<% else %>
|
|
parse_result: <%= @grammar.prefix %>value_t,
|
|
<% end %>
|
|
|
|
/** Unexpected token received. */
|
|
token: <%= @grammar.prefix %>token_t,
|
|
/** User terminate code. */
|
|
pub user_terminate_code: usize,
|
|
|
|
<%= @grammar.context_user_fields %>
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Public data
|
|
*************************************************************************/
|
|
|
|
/** Token names. */
|
|
pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>] = [
|
|
<% @grammar.tokens.each do |token| %>
|
|
"<%= token.name %>",
|
|
<% end %>
|
|
];
|
|
|
|
/**************************************************************************
|
|
* Private types
|
|
*************************************************************************/
|
|
|
|
<% if @grammar.prefix.upcase != "P_" %>
|
|
/* Result codes. */
|
|
const P_SUCCESS: usize = 0;
|
|
const P_DECODE_ERROR: usize = 1;
|
|
const P_UNEXPECTED_INPUT: usize = 2;
|
|
const P_UNEXPECTED_TOKEN: usize = 3;
|
|
const P_DROP: usize = 4;
|
|
const P_EOF: usize = 5;
|
|
const P_USER_TERMINATED: usize = 6;
|
|
<% end %>
|
|
|
|
/* An invalid ID value. */
|
|
const INVALID_ID: usize = usize::MAX;
|
|
|
|
/**************************************************************************
|
|
* State initialization
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Allocate and initialize lexer/parser context structure.
|
|
*
|
|
* Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
|
|
*
|
|
* @param input
|
|
* Text input.
|
|
*
|
|
* @return Context structure for lexer/parser.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>context_t {
|
|
let mut context = <%= @grammar.prefix %>context_t::default();
|
|
|
|
/* Lexer initialization. */
|
|
context.input = input.to_vec();
|
|
context.text_position.row = 1;
|
|
context.text_position.col = 1;
|
|
context.mode = <%= @lexer.mode_id("default") %>;
|
|
<% if @grammar.tree %>
|
|
|
|
/* Reserve node ID 0 as the null tree node. */
|
|
context.<%= @grammar.prefix %>tree_nodes.push(<%= @grammar.prefix %>node_data_t::default());
|
|
<% end %>
|
|
|
|
context
|
|
}
|
|
|
|
/**
|
|
* Deinitialize and deallocate lexer/parser context structure.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>context_delete(mut context: <%= @grammar.prefix %>context_t) {
|
|
<% if @grammar.tree && @grammar.free_token_node != "" %>
|
|
for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() {
|
|
if context.<%= @grammar.prefix %>tree_nodes[i].is_token {
|
|
let token_node_id = i;
|
|
<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id]") %>
|
|
}
|
|
}
|
|
<% end %>
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Decoder
|
|
*************************************************************************/
|
|
|
|
/**
|
|
* Decode a UTF-8 code point.
|
|
*
|
|
* @param input
|
|
* Text input to decode.
|
|
* @param out_code_point
|
|
* The decoded code point is stored here if the return value is P_SUCCESS.
|
|
* @param out_code_point_length
|
|
* The number of bytes the code point used is stored here if the return value
|
|
* is P_SUCCESS.
|
|
*
|
|
* @retval P_SUCCESS on a successful code point decode
|
|
* @retval P_DECODE_ERROR when an encoding error is observed
|
|
* @retval P_EOF when the end of the text input is reached
|
|
*/
|
|
pub fn <%= @grammar.prefix %>decode_code_point(input: &[u8],
|
|
out_code_point: &mut <%= @grammar.prefix %>code_point_t, out_code_point_length: &mut u8) -> usize {
|
|
if input.len() == 0 {
|
|
return P_EOF;
|
|
}
|
|
let c = input[0];
|
|
let mut code_point: <%= @grammar.prefix %>code_point_t;
|
|
let code_point_length: u8;
|
|
if (c & 0x80u8) == 0u8 {
|
|
code_point = c as <%= @grammar.prefix %>code_point_t;
|
|
code_point_length = 1;
|
|
} else {
|
|
let following_bytes: usize;
|
|
if (c & 0xE0u8) == 0xC0u8 {
|
|
code_point = (c & 0x1Fu8) as <%= @grammar.prefix %>code_point_t;
|
|
following_bytes = 1;
|
|
} else if (c & 0xF0u8) == 0xE0u8 {
|
|
code_point = (c & 0x0Fu8) as <%= @grammar.prefix %>code_point_t;
|
|
following_bytes = 2;
|
|
} else if (c & 0xF8u8) == 0xF0u8 {
|
|
code_point = (c & 0x07u8) as <%= @grammar.prefix %>code_point_t;
|
|
following_bytes = 3;
|
|
} else if (c & 0xFCu8) == 0xF8u8 {
|
|
code_point = (c & 0x03u8) as <%= @grammar.prefix %>code_point_t;
|
|
following_bytes = 4;
|
|
} else if (c & 0xFEu8) == 0xFCu8 {
|
|
code_point = (c & 0x01u8) as <%= @grammar.prefix %>code_point_t;
|
|
following_bytes = 5;
|
|
} else {
|
|
return P_DECODE_ERROR;
|
|
}
|
|
if input.len() <= following_bytes {
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point_length = (following_bytes + 1) as u8;
|
|
for i in 0..following_bytes {
|
|
let b = input[i + 1];
|
|
if (b & 0xC0u8) != 0x80u8 {
|
|
return P_DECODE_ERROR;
|
|
}
|
|
code_point = (code_point << 6) | ((b & 0x3Fu8) as <%= @grammar.prefix %>code_point_t);
|
|
}
|
|
}
|
|
*out_code_point = code_point;
|
|
*out_code_point_length = code_point_length;
|
|
P_SUCCESS
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Lexer
|
|
*************************************************************************/
|
|
|
|
type lexer_state_id_t = <%= get_type_for(@lexer.state_table.size) %>;
|
|
|
|
/** Invalid lexer state ID. */
|
|
const INVALID_LEXER_STATE_ID: lexer_state_id_t = <%= @lexer.state_table.size %>;
|
|
|
|
/** Invalid lexer user code ID. */
|
|
<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
|
|
const INVALID_USER_CODE_ID: <%= get_type_for(user_code_id_count) %> = <%= user_code_id_count %>;
|
|
|
|
/**
|
|
* Lexer transition table entry.
|
|
*
|
|
* An incoming code point matching the range for a transition entry will cause
|
|
* the lexer to progress to the destination state.
|
|
*/
|
|
#[derive(Clone, Copy)]
|
|
struct lexer_transition_t {
|
|
/** First code point in the range for this transition. */
|
|
first: <%= @grammar.prefix %>code_point_t,
|
|
/** Last code point in the range for this transition. */
|
|
last: <%= @grammar.prefix %>code_point_t,
|
|
/** Destination lexer state ID for this transition. */
|
|
destination_state: lexer_state_id_t,
|
|
}
|
|
|
|
/** Lexer state table entry. */
|
|
#[derive(Clone, Copy)]
|
|
struct lexer_state_t {
|
|
/** Index to the transition table for this state. */
|
|
transition_table_index: <%= get_type_for(@lexer.transition_table.size - 1) %>,
|
|
/** Number of transition table entries for this state. */
|
|
n_transitions: <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %>,
|
|
/** Lexer token formed at this state. */
|
|
token: <%= @grammar.prefix %>token_t,
|
|
/** Lexer user code ID to execute at this state. */
|
|
code_id: <%= get_type_for(user_code_id_count) %>,
|
|
/** Whether this state matches a lexer pattern. */
|
|
accepts: bool,
|
|
}
|
|
|
|
/** Lexer mode table entry. */
|
|
#[derive(Clone, Copy)]
|
|
struct lexer_mode_t {
|
|
/** Offset in the state table to be used for this mode. */
|
|
state_table_offset: u32,
|
|
}
|
|
|
|
/**
|
|
* Lexer match info structure.
|
|
*
|
|
* This structure holds output values from the lexer upon a successful pattern
|
|
* match.
|
|
*/
|
|
#[derive(Clone, Copy, Default)]
|
|
struct lexer_match_info_t {
|
|
/** Number of bytes of input text used to match. */
|
|
length: usize,
|
|
/** Input text position delta to end of token. */
|
|
end_delta_position: <%= @grammar.prefix %>position_t,
|
|
/** Input text position delta to next code point after token end. */
|
|
delta_position: <%= @grammar.prefix %>position_t,
|
|
/** Accepting lexer state from the match (state ID, or INVALID). */
|
|
accepting_state: lexer_state_id_t,
|
|
}
|
|
|
|
/** Lexer transition table. */
|
|
static lexer_transition_table: [lexer_transition_t; <%= @lexer.transition_table.size %>] = [
|
|
<% @lexer.transition_table.each do |transition_table_entry| %>
|
|
lexer_transition_t { first: <%= transition_table_entry[:first] %>, last: <%= transition_table_entry[:last] %>, destination_state: <%= transition_table_entry[:destination] %> },
|
|
<% end %>
|
|
];
|
|
|
|
/** Lexer state table. */
|
|
static lexer_state_table: [lexer_state_t; <%= @lexer.state_table.size %>] = [
|
|
<% @lexer.state_table.each do |state_table_entry| %>
|
|
lexer_state_t { transition_table_index: <%= state_table_entry[:transition_table_index] %>, n_transitions: <%= state_table_entry[:n_transitions] %>, token: <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, code_id: <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, accepts: <%= state_table_entry[:accepts] %> },
|
|
<% end %>
|
|
];
|
|
|
|
/** Lexer mode table. */
|
|
static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
|
|
<% @lexer.mode_table.each do |mode_table_entry| %>
|
|
lexer_mode_t { state_table_offset: <%= mode_table_entry[:state_table_offset] %> },
|
|
<% end %>
|
|
];
|
|
|
|
/**
|
|
* Execute user code associated with a lexer pattern.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param code_id
|
|
* The ID of the user code block to execute.
|
|
* @param match_text
|
|
* Matched text for this pattern.
|
|
* @param match_length
|
|
* Matched text length.
|
|
* @param out_token_info
|
|
* Lexer token info in progress.
|
|
*
|
|
* @return Token to accept, or invalid token if the user code does
|
|
* not explicitly return a token.
|
|
*/
|
|
fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
|
|
code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8],
|
|
match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
|
|
match code_id {
|
|
<% @grammar.patterns.each do |pattern| %>
|
|
<% if pattern.code_id %>
|
|
<%= pattern.code_id %> => {
|
|
<% unless @grammar.tree %>
|
|
out_token_info.pvalue = <%= @grammar.prefix %>value_t::v_<%= pattern.ptypename %>(Default::default());
|
|
<% end %>
|
|
<%= expand_code(pattern.code, false, nil, pattern) %>
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
_ => {}
|
|
}
|
|
|
|
INVALID_TOKEN_ID
|
|
}
|
|
|
|
/**
|
|
* Check if there is a transition from the current lexer state to another
|
|
* based on the given input code point.
|
|
*
|
|
* @param current_state
|
|
* Current lexer state.
|
|
* @param code_point
|
|
* Input code point.
|
|
*
|
|
* @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
|
|
*/
|
|
fn check_lexer_transition(current_state: u32, code_point: u32) -> lexer_state_id_t {
|
|
let transition_table_index = lexer_state_table[current_state as usize].transition_table_index as u32;
|
|
for i in 0..(lexer_state_table[current_state as usize].n_transitions as u32) {
|
|
let t = &lexer_transition_table[(transition_table_index + i) as usize];
|
|
if (t.first <= code_point) && (code_point <= t.last) {
|
|
return t.destination_state;
|
|
}
|
|
}
|
|
INVALID_LEXER_STATE_ID
|
|
}
|
|
|
|
/**
|
|
* Find the longest lexer pattern match at the current position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_match_info
|
|
* The longest match information is stored here if the return value is
|
|
* P_SUCCESS or P_DECODE_ERROR.
|
|
* @param[out] out_unexpected_input_length
|
|
* The unexpected input length is stored here if the return value is
|
|
* P_UNEXPECTED_INPUT.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_EOF
|
|
* The end of the text input was reached.
|
|
*/
|
|
fn find_longest_match(context: &<%= @grammar.prefix %>context_t,
|
|
out_match_info: &mut lexer_match_info_t, out_unexpected_input_length: &mut usize) -> usize {
|
|
let mut longest_match = lexer_match_info_t::default();
|
|
longest_match.accepting_state = INVALID_LEXER_STATE_ID;
|
|
let mut attempt_match = lexer_match_info_t::default();
|
|
attempt_match.accepting_state = INVALID_LEXER_STATE_ID;
|
|
*out_match_info = longest_match;
|
|
let mut current_state: u32 = lexer_mode_table[context.mode].state_table_offset;
|
|
loop {
|
|
let input_index = context.input_index + attempt_match.length;
|
|
let input = &context.input[input_index..];
|
|
let mut code_point: <%= @grammar.prefix %>code_point_t = 0;
|
|
let mut code_point_length: u8 = 0;
|
|
let result = <%= @grammar.prefix %>decode_code_point(input, &mut code_point, &mut code_point_length);
|
|
match result {
|
|
P_SUCCESS => {
|
|
let transition_state = check_lexer_transition(current_state, code_point);
|
|
if transition_state != INVALID_LEXER_STATE_ID {
|
|
attempt_match.length += code_point_length as usize;
|
|
attempt_match.end_delta_position = attempt_match.delta_position;
|
|
if code_point == '\n' as u32 {
|
|
attempt_match.delta_position.row += 1;
|
|
attempt_match.delta_position.col = 1;
|
|
} else {
|
|
attempt_match.delta_position.col += 1;
|
|
}
|
|
current_state = transition_state as u32;
|
|
if lexer_state_table[current_state as usize].accepts {
|
|
attempt_match.accepting_state = current_state as lexer_state_id_t;
|
|
longest_match = attempt_match;
|
|
}
|
|
} else if longest_match.length > 0 {
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
} else {
|
|
*out_unexpected_input_length = attempt_match.length + code_point_length as usize;
|
|
return P_UNEXPECTED_INPUT;
|
|
}
|
|
}
|
|
P_EOF => {
|
|
/* We hit EOF. */
|
|
if longest_match.length > 0 {
|
|
*out_match_info = longest_match;
|
|
return P_SUCCESS;
|
|
} else if attempt_match.length != 0 {
|
|
/* There is a partial match - error! */
|
|
*out_unexpected_input_length = attempt_match.length;
|
|
return P_UNEXPECTED_INPUT;
|
|
} else {
|
|
return P_EOF;
|
|
}
|
|
}
|
|
P_DECODE_ERROR => {
|
|
/* If we see a decode error, we may be partially in the middle of
|
|
* matching a pattern, so return the attempted match info so that
|
|
* the input text position can be updated. */
|
|
*out_match_info = attempt_match;
|
|
return result;
|
|
}
|
|
_ => {
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Attempt to lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_DROP
|
|
* A drop pattern was matched so the lexer should continue.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
|
|
let mut token_info = <%= @grammar.prefix %>token_info_t::default();
|
|
token_info.position = context.text_position;
|
|
token_info.token = INVALID_TOKEN_ID;
|
|
let mut match_info = lexer_match_info_t::default();
|
|
let mut unexpected_input_length: usize = 0;
|
|
let result = find_longest_match(context, &mut match_info, &mut unexpected_input_length);
|
|
match result {
|
|
P_SUCCESS => {
|
|
let mut token_to_accept = lexer_state_table[match_info.accepting_state as usize].token;
|
|
/* Calculate the token length and start/end positions before invoking
|
|
* the lexer user code so that the user code can access them. The
|
|
* context input text position tracking is not updated until after the
|
|
* user code has run so that it is left unchanged if the user code
|
|
* requests to terminate the lexer. */
|
|
token_info.length = match_info.length;
|
|
if match_info.end_delta_position.row != 0 {
|
|
token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
|
|
token_info.end_position.col = match_info.end_delta_position.col;
|
|
} else {
|
|
token_info.end_position.row = token_info.position.row;
|
|
token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
|
|
}
|
|
if lexer_state_table[match_info.accepting_state as usize].code_id != INVALID_USER_CODE_ID {
|
|
let match_start = context.input_index;
|
|
let match_slice = context.input[match_start..(match_start + match_info.length)].to_vec();
|
|
let user_code_token = lexer_user_code(context,
|
|
lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, match_info.length, &mut token_info);
|
|
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
|
|
* that the user code is requesting to terminate the lexer. */
|
|
if user_code_token == TERMINATE_TOKEN_ID {
|
|
return P_USER_TERMINATED;
|
|
}
|
|
/* An invalid token returned from lexer_user_code() means that the
|
|
* user code did not explicitly return a token. So only override
|
|
* the token to return if the user code does explicitly return a
|
|
* token. */
|
|
if user_code_token != INVALID_TOKEN_ID {
|
|
token_to_accept = user_code_token;
|
|
}
|
|
}
|
|
|
|
/* Update the input position tracking. */
|
|
context.input_index += match_info.length;
|
|
context.text_position.row += match_info.delta_position.row;
|
|
if match_info.delta_position.row != 0 {
|
|
context.text_position.col = match_info.delta_position.col;
|
|
} else {
|
|
context.text_position.col += match_info.delta_position.col;
|
|
}
|
|
|
|
if token_to_accept == INVALID_TOKEN_ID {
|
|
return P_DROP;
|
|
}
|
|
token_info.token = token_to_accept;
|
|
*out_token_info = token_info;
|
|
P_SUCCESS
|
|
}
|
|
P_EOF => {
|
|
token_info.token = TOKEN___EOF;
|
|
token_info.end_position = token_info.position;
|
|
*out_token_info = token_info;
|
|
P_SUCCESS
|
|
}
|
|
P_DECODE_ERROR => {
|
|
/* Update the input position tracking. */
|
|
context.input_index += match_info.length;
|
|
context.text_position.row += match_info.delta_position.row;
|
|
if match_info.delta_position.row != 0 {
|
|
context.text_position.col = match_info.delta_position.col;
|
|
} else {
|
|
context.text_position.col += match_info.delta_position.col;
|
|
}
|
|
result
|
|
}
|
|
_ => {
|
|
result
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Lex the next token in the input stream.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param[out] out_token_info
|
|
* The lexed token information is stored here if the return value is
|
|
* P_SUCCESS.
|
|
*
|
|
* @reval P_SUCCESS
|
|
* A token was successfully lexed.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
* @retval P_USER_TERMINATED
|
|
* User code has requested to terminate the lexer.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize {
|
|
loop {
|
|
let result = attempt_lex_token(context, out_token_info);
|
|
if result != P_DROP {
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**************************************************************************
|
|
* Parser
|
|
*************************************************************************/
|
|
|
|
/** Reduce ID type. */
|
|
type reduce_id_t = <%= get_type_for(@parser.reduce_table.size) %>;
|
|
|
|
/**
|
|
* A symbol ID can hold either a token ID or a rule set ID.
|
|
*
|
|
* Token IDs and rule set IDs share the same namespace, with rule set IDs
|
|
* beginning after token IDs end.
|
|
*/
|
|
type symbol_id_t = <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>;
|
|
|
|
/** Parser state ID type. */
|
|
type parser_state_id_t = <%= get_type_for(@parser.state_table.size) %>;
|
|
|
|
/** Parser rule ID type. */
|
|
type rule_id_t = <%= get_type_for(@grammar.rules.size) %>;
|
|
|
|
/** Parser shift ID type. */
|
|
type shift_id_t = <%= get_type_for(@parser.shift_table.size) %>;
|
|
|
|
/** Shift table entry. */
|
|
#[derive(Clone, Copy)]
|
|
struct shift_t {
|
|
/** Token or rule set ID. */
|
|
symbol_id: symbol_id_t,
|
|
/** Parser state to shift to. */
|
|
state_id: parser_state_id_t,
|
|
}
|
|
|
|
/** Reduce table entry. */
|
|
#[derive(Clone, Copy)]
|
|
struct reduce_t {
|
|
/** Lookahead token. */
|
|
token: <%= @grammar.prefix %>token_t,
|
|
|
|
/**
|
|
* Rule ID.
|
|
*
|
|
* This is used to execute the parser user code block associated with a
|
|
* grammar rule.
|
|
*/
|
|
rule: rule_id_t,
|
|
|
|
/**
|
|
* Rule set ID.
|
|
*
|
|
* This is used as the new top symbol ID of the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
rule_set: symbol_id_t,
|
|
|
|
/**
|
|
* Number of states leading to this reduce action.
|
|
*
|
|
* This is the number of entries popped from the parse stack after this
|
|
* reduce action.
|
|
*/
|
|
n_states: parser_state_id_t,
|
|
<% if @grammar.tree %>
|
|
|
|
/**
|
|
* Map of rule components to rule set child fields (None for a flat map).
|
|
*/
|
|
rule_set_node_field_index_map: Option<&'static [u16]>,
|
|
|
|
/**
|
|
* Number of rule set tree node fields.
|
|
*/
|
|
rule_set_node_field_array_size: u16,
|
|
|
|
/**
|
|
* Whether this rule was a generated optional rule that matched the
|
|
* optional target. In this case, propagate the matched target node up
|
|
* instead of making a new node for this rule.
|
|
*/
|
|
propagate_optional_target: bool,
|
|
<% end %>
|
|
}
|
|
|
|
/** Parser state entry. */
|
|
#[derive(Clone, Copy)]
|
|
struct parser_state_t {
|
|
/** First shift table entry for this parser state. */
|
|
shift_table_index: shift_id_t,
|
|
|
|
/** Number of shift table entries for this parser state. */
|
|
n_shift_entries: shift_id_t,
|
|
|
|
/** First reduce table entry for this parser state. */
|
|
reduce_table_index: reduce_id_t,
|
|
|
|
/** Number of reduce table entries for this parser state. */
|
|
n_reduce_entries: reduce_id_t,
|
|
}
|
|
|
|
/**
|
|
* Structure to hold a state ID and value pair.
|
|
*
|
|
* A stack of these structures makes up the parse stack.
|
|
*/
|
|
#[derive(Clone, Default)]
|
|
struct state_value_t {
|
|
/** Parser state ID. */
|
|
state_id: usize,
|
|
|
|
<% if @grammar.tree %>
|
|
/** Tree node ID. */
|
|
node_id: <%= @grammar.prefix %>node_id_t,
|
|
<% else %>
|
|
position: <%= @grammar.prefix %>position_t,
|
|
end_position: <%= @grammar.prefix %>position_t,
|
|
/** Parser value from this state. */
|
|
pvalue: <%= @grammar.prefix %>value_t,
|
|
<% end %>
|
|
}
|
|
|
|
/** Parser shift table. */
|
|
static parser_shift_table: [shift_t; <%= @parser.shift_table.size %>] = [
|
|
<% @parser.shift_table.each do |shift| %>
|
|
shift_t { symbol_id: <%= shift[:symbol].id %>, state_id: <%= shift[:state_id] %> },
|
|
<% end %>
|
|
];
|
|
|
|
<% if @grammar.tree %>
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% unless rule.flat_rule_set_node_field_index_map? %>
|
|
static r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map: [u16; <%= rule.rule_set_node_field_index_map.size %>] = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
|
|
<% end %>
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/** Parser reduce table. */
|
|
static parser_reduce_table: [reduce_t; <%= @parser.reduce_table.size %>] = [
|
|
<% @parser.reduce_table.each do |reduce| %>
|
|
reduce_t {
|
|
token: <%= reduce[:token_id] %>, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
|
|
rule: <%= reduce[:rule_id] %>, /* Rule ID */
|
|
rule_set: <%= reduce[:rule_set_id] %>, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
|
|
n_states: <%= reduce[:n_states] %>, /* Number of states */
|
|
<% if @grammar.tree %>
|
|
<% if reduce[:rule].flat_rule_set_node_field_index_map? %>
|
|
rule_set_node_field_index_map: None,
|
|
<% else %>
|
|
rule_set_node_field_index_map: Some(&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map),
|
|
<% end %>
|
|
rule_set_node_field_array_size: <%= reduce[:rule].rule_set.tree_fields.size %>,
|
|
propagate_optional_target: <%= reduce[:propagate_optional_target] %>,
|
|
<% end %>
|
|
},
|
|
<% end %>
|
|
];
|
|
|
|
/** Parser state table. */
|
|
static parser_state_table: [parser_state_t; <%= @parser.state_table.size %>] = [
|
|
<% @parser.state_table.each do |state| %>
|
|
parser_state_t { shift_table_index: <%= state[:shift_index] %>, n_shift_entries: <%= state[:n_shifts] %>, reduce_table_index: <%= state[:reduce_index] %>, n_reduce_entries: <%= state[:n_reduces] %> },
|
|
<% end %>
|
|
];
|
|
|
|
<% if @grammar.tree %>
|
|
/* Tree arena helpers. */
|
|
|
|
/** Allocate a new (zeroed) tree node in the context arena. */
|
|
fn tree_new_node(context: &mut <%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>node_id_t {
|
|
let id = context.<%= @grammar.prefix %>tree_nodes.len() as <%= @grammar.prefix %>node_id_t;
|
|
context.<%= @grammar.prefix %>tree_nodes.push(<%= @grammar.prefix %>node_data_t::default());
|
|
id
|
|
}
|
|
|
|
/** Reserve n contiguous (zeroed) child slots in the shared children array. */
|
|
fn tree_reserve_children(context: &mut <%= @grammar.prefix %>context_t, n: usize) -> <%= @grammar.prefix %>node_id_t {
|
|
let offset = context.<%= @grammar.prefix %>tree_children.len() as <%= @grammar.prefix %>node_id_t;
|
|
let new_len = context.<%= @grammar.prefix %>tree_children.len() + n;
|
|
context.<%= @grammar.prefix %>tree_children.resize(new_len, 0);
|
|
offset
|
|
}
|
|
<% end %>
|
|
|
|
<% unless @grammar.tree %>
|
|
/**
|
|
* Get the rule position (start or end) for the currently matched rule.
|
|
*/
|
|
fn get_rule_position(statevalues: &[state_value_t], i: usize, n_states: usize, get_end: bool) -> <%= @grammar.prefix %>position_t {
|
|
let len = statevalues.len();
|
|
if n_states > 0 {
|
|
if i == 0 {
|
|
if get_end {
|
|
for j in 0..n_states {
|
|
let sv = &statevalues[len - 1 - j];
|
|
if sv.end_position.valid() {
|
|
return sv.end_position;
|
|
}
|
|
}
|
|
} else {
|
|
for j in 0..n_states {
|
|
let sv = &statevalues[len - n_states + j];
|
|
if sv.position.valid() {
|
|
return sv.position;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
if get_end {
|
|
return statevalues[len - 1 - n_states + i].end_position;
|
|
} else {
|
|
return statevalues[len - 1 - n_states + i].position;
|
|
}
|
|
}
|
|
}
|
|
INVALID_POSITION
|
|
}
|
|
<% end %>
|
|
|
|
<% if !@grammar.tree || @grammar.parser_user_code_used? %>
|
|
/**
|
|
* Execute user code associated with a parser rule.
|
|
*
|
|
* @param rule The ID of the rule.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* Continue parsing.
|
|
* @retval P_USER_TERMINATED
|
|
* User requested to terminate parsing.
|
|
*/
|
|
fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.tree ? "_node_id: #{@grammar.prefix}node_id_t" : "_pvalue: &mut #{@grammar.prefix}value_t" %>, rule: u32, statevalues: &[state_value_t], n_states: usize) -> usize {
|
|
match rule {
|
|
<% @grammar.rules.each do |rule| %>
|
|
<% if rule.code %>
|
|
<%= rule.id %> => {
|
|
<% unless @grammar.tree %>
|
|
*_pvalue = <%= @grammar.prefix %>value_t::v_<%= rule.ptypename %>(Default::default());
|
|
<% end %>
|
|
<%= expand_code(rule.code, true, rule, nil) %>
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
_ => {}
|
|
}
|
|
|
|
P_SUCCESS
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Check if the parser should shift to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param symbol_id
|
|
* Incoming token/rule set ID.
|
|
*
|
|
* @return State to shift to, or INVALID_ID if none.
|
|
*/
|
|
fn check_shift(state_id: usize, symbol_id: usize) -> usize {
|
|
let start = parser_state_table[state_id].shift_table_index as usize;
|
|
let end = start + parser_state_table[state_id].n_shift_entries as usize;
|
|
for i in start..end {
|
|
if parser_shift_table[i].symbol_id as usize == symbol_id {
|
|
return parser_shift_table[i].state_id as usize;
|
|
}
|
|
}
|
|
INVALID_ID
|
|
}
|
|
|
|
/**
|
|
* Check if the parser should reduce to a new state.
|
|
*
|
|
* @param state_id
|
|
* Parser state ID.
|
|
* @param token
|
|
* Incoming token.
|
|
*
|
|
* @return Reduce table index to reduce with, or INVALID_ID if none.
|
|
*/
|
|
fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize {
|
|
let start = parser_state_table[state_id].reduce_table_index as usize;
|
|
let end = start + parser_state_table[state_id].n_reduce_entries as usize;
|
|
for i in start..end {
|
|
if (parser_reduce_table[i].token == token) || (parser_reduce_table[i].token == INVALID_TOKEN_ID) {
|
|
return i;
|
|
}
|
|
}
|
|
INVALID_ID
|
|
}
|
|
|
|
/**
|
|
* Run the parser.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param start_state_id
|
|
* ID of the state in which to start.
|
|
* @param start_rule_set_id
|
|
* Rule set ID for the requested start rule. Only used when
|
|
* @p follow_tokens is non-empty, to gate follow-token shift success.
|
|
* @param follow_tokens
|
|
* Optional slice of caller-provided follow tokens (tokens expected to
|
|
* appear immediately after the start rule in some outer context). Used to
|
|
* drive the "parse inner" retry logic. May be null/empty for a standard
|
|
* parse.
|
|
*
|
|
* @retval P_SUCCESS
|
|
* The parser successfully matched the input text. The parse result value
|
|
* can be accessed with <%= @grammar.prefix %>result().
|
|
* @retval P_UNEXPECTED_TOKEN
|
|
* An unexpected token was encountered that does not match any grammar rule.
|
|
* The function p_token(&context) can be used to get the unexpected token.
|
|
* @reval P_DECODE_ERROR
|
|
* The decoder encountered invalid text encoding.
|
|
* @reval P_UNEXPECTED_INPUT
|
|
* Input text does not match any lexer pattern.
|
|
*/
|
|
fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usize,
|
|
start_rule_set_id: usize, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize {
|
|
let mut token_info = <%= @grammar.prefix %>token_info_t::default();
|
|
let mut token: <%= @grammar.prefix %>token_t = INVALID_TOKEN_ID;
|
|
let mut statevalues: Vec<state_value_t> = Vec::new();
|
|
let mut reduced_rule_set: usize = INVALID_ID;
|
|
let mut last_shifted_rule_set_id: usize = INVALID_ID;
|
|
<% if @grammar.tree %>
|
|
let mut reduced_parser_node: <%= @grammar.prefix %>node_id_t = 0;
|
|
<% else %>
|
|
let mut reduced_position: <%= @grammar.prefix %>position_t = INVALID_POSITION;
|
|
let mut reduced_end_position: <%= @grammar.prefix %>position_t = INVALID_POSITION;
|
|
let mut reduced_parser_value: <%= @grammar.prefix %>value_t = Default::default();
|
|
<% end %>
|
|
statevalues.push(state_value_t::default());
|
|
let sv_len = statevalues.len();
|
|
statevalues[sv_len - 1].state_id = start_state_id;
|
|
loop {
|
|
if token == INVALID_TOKEN_ID {
|
|
let lexer_result = <%= lex_fn %>(context, &mut token_info);
|
|
if lexer_result != P_SUCCESS {
|
|
return lexer_result;
|
|
}
|
|
token = token_info.token;
|
|
}
|
|
/* For a "parse inner" operation, determine once per iteration whether
|
|
* the current token is a member of the caller-provided follow token
|
|
* set. Used by both the shift-side and reduce-side retries below. */
|
|
let mut token_is_follow = false;
|
|
for &ft in follow_tokens {
|
|
if token == ft {
|
|
token_is_follow = true;
|
|
break;
|
|
}
|
|
}
|
|
let mut shift_state: usize = INVALID_ID;
|
|
if reduced_rule_set != INVALID_ID {
|
|
shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, reduced_rule_set);
|
|
}
|
|
if shift_state == INVALID_ID {
|
|
shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, token as usize);
|
|
if (shift_state != INVALID_ID) && (token == TOKEN___EOF) {
|
|
/* Successful parse. */
|
|
<% if @grammar.tree %>
|
|
context.parse_result = statevalues[statevalues.len() - 1].node_id;
|
|
<% else %>
|
|
context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
|
|
<% end %>
|
|
return P_SUCCESS;
|
|
}
|
|
if (shift_state == INVALID_ID) && token_is_follow {
|
|
/* For a "parse inner" operation, if the incoming token is one
|
|
* of the caller's follow tokens, retry the shift as
|
|
* TOKEN___EOF. Only consider the parse complete if the reduced
|
|
* start rule is the only thing on the parse stack (i.e. the
|
|
* initial state plus a single shifted start rule set entry). */
|
|
let retry_shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF as usize);
|
|
if (retry_shift_state != INVALID_ID) &&
|
|
(statevalues.len() == 2) &&
|
|
(last_shifted_rule_set_id == start_rule_set_id) {
|
|
/* Successful parse via follow token. Rewind the input
|
|
* position so that the follow token is not consumed from
|
|
* the input stream and remains available for a subsequent
|
|
* call to <%= @grammar.prefix %>lex() or a
|
|
* <%= @grammar.prefix %>parse*() function. */
|
|
context.input_index -= token_info.length;
|
|
context.text_position = token_info.position;
|
|
<% if @grammar.tree %>
|
|
context.parse_result = statevalues[statevalues.len() - 1].node_id;
|
|
<% else %>
|
|
context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone();
|
|
<% end %>
|
|
return P_SUCCESS;
|
|
}
|
|
}
|
|
}
|
|
if shift_state != INVALID_ID {
|
|
/* We have something to shift. Track the last shifted rule set ID
|
|
* (INVALID_ID if we just shifted a token) so the follow-token
|
|
* shift retry can gate success on the reduced start rule being the
|
|
* only thing on top of the initial state. */
|
|
last_shifted_rule_set_id = reduced_rule_set;
|
|
statevalues.push(state_value_t::default());
|
|
let new_index = statevalues.len() - 1;
|
|
statevalues[new_index].state_id = shift_state;
|
|
if reduced_rule_set == INVALID_ID {
|
|
/* We shifted a token, mark it consumed. */
|
|
<% if @grammar.tree %>
|
|
let token_node_id = tree_new_node(context);
|
|
{
|
|
let token_tree_node = &mut context.<%= @grammar.prefix %>tree_nodes[token_node_id as usize];
|
|
token_tree_node.position = token_info.position;
|
|
token_tree_node.end_position = token_info.end_position;
|
|
token_tree_node.n_fields = 0;
|
|
token_tree_node.is_token = true;
|
|
token_tree_node.token = token;
|
|
token_tree_node.pvalue = token_info.pvalue.clone();
|
|
}
|
|
<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub(/\btoken_tree_node\b/, "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %>
|
|
statevalues[new_index].node_id = token_node_id;
|
|
<% else %>
|
|
statevalues[new_index].position = token_info.position;
|
|
statevalues[new_index].end_position = token_info.end_position;
|
|
statevalues[new_index].pvalue = token_info.pvalue.clone();
|
|
<% end %>
|
|
token = INVALID_TOKEN_ID;
|
|
} else {
|
|
/* We shifted a RuleSet. */
|
|
<% if @grammar.tree %>
|
|
statevalues[new_index].node_id = reduced_parser_node;
|
|
<% else %>
|
|
statevalues[new_index].pvalue = reduced_parser_value.clone();
|
|
statevalues[new_index].position = reduced_position;
|
|
statevalues[new_index].end_position = reduced_end_position;
|
|
reduced_parser_value = Default::default();
|
|
<% end %>
|
|
reduced_rule_set = INVALID_ID;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
let mut reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, token);
|
|
if (reduce_index == INVALID_ID) && token_is_follow {
|
|
/* For a "parse inner" operation, if the incoming token is one of
|
|
* the caller's follow tokens, retry the reduce lookup as
|
|
* TOKEN___EOF. Whatever reduce_index results (if any) is used
|
|
* regardless of which rule set it reduces to; this allows chains
|
|
* of reductions leading up to the start rule. */
|
|
reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF);
|
|
}
|
|
if reduce_index != INVALID_ID {
|
|
/* We have something to reduce. */
|
|
let n_states = parser_reduce_table[reduce_index].n_states as usize;
|
|
<% if @grammar.tree %>
|
|
if parser_reduce_table[reduce_index].propagate_optional_target {
|
|
reduced_parser_node = statevalues[statevalues.len() - 1].node_id;
|
|
} else if n_states > 0 {
|
|
let n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
|
|
let child_offset = tree_reserve_children(context, n_fields as usize);
|
|
match parser_reduce_table[reduce_index].rule_set_node_field_index_map {
|
|
None => {
|
|
for i in 0..n_states {
|
|
context.<%= @grammar.prefix %>tree_children[child_offset as usize + i] = statevalues[statevalues.len() - n_states + i].node_id;
|
|
}
|
|
}
|
|
Some(map) => {
|
|
for i in 0..n_states {
|
|
context.<%= @grammar.prefix %>tree_children[child_offset as usize + map[i] as usize] = statevalues[statevalues.len() - n_states + i].node_id;
|
|
}
|
|
}
|
|
}
|
|
let node_id = tree_new_node(context);
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].position = INVALID_POSITION;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].end_position = INVALID_POSITION;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].child_offset = child_offset;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].n_fields = n_fields;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].is_token = false;
|
|
let mut position_found = false;
|
|
for i in 0..(n_fields as usize) {
|
|
let child_id = context.<%= @grammar.prefix %>tree_children[child_offset as usize + i];
|
|
if (child_id != 0) && context.<%= @grammar.prefix %>tree_nodes[child_id as usize].position.valid() {
|
|
if !position_found {
|
|
let p = context.<%= @grammar.prefix %>tree_nodes[child_id as usize].position;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].position = p;
|
|
position_found = true;
|
|
}
|
|
let ep = context.<%= @grammar.prefix %>tree_nodes[child_id as usize].end_position;
|
|
context.<%= @grammar.prefix %>tree_nodes[node_id as usize].end_position = ep;
|
|
}
|
|
}
|
|
reduced_parser_node = node_id;
|
|
} else {
|
|
reduced_parser_node = 0;
|
|
}
|
|
<% if @grammar.parser_user_code_used? %>
|
|
if parser_user_code(context, reduced_parser_node, parser_reduce_table[reduce_index].rule as u32, &statevalues, n_states) == P_USER_TERMINATED {
|
|
return P_USER_TERMINATED;
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
let mut reduced_parser_value2: <%= @grammar.prefix %>value_t = Default::default();
|
|
if parser_user_code(context, &mut reduced_parser_value2, parser_reduce_table[reduce_index].rule as u32, &statevalues, n_states) == P_USER_TERMINATED {
|
|
return P_USER_TERMINATED;
|
|
}
|
|
reduced_parser_value = reduced_parser_value2;
|
|
if n_states > 0 {
|
|
reduced_position = get_rule_position(&statevalues, 0, n_states, false);
|
|
reduced_end_position = get_rule_position(&statevalues, 0, n_states, true);
|
|
} else {
|
|
reduced_position = INVALID_POSITION;
|
|
reduced_end_position = INVALID_POSITION;
|
|
}
|
|
<% end %>
|
|
reduced_rule_set = parser_reduce_table[reduce_index].rule_set as usize;
|
|
let new_len = statevalues.len() - n_states;
|
|
statevalues.truncate(new_len);
|
|
continue;
|
|
}
|
|
|
|
/* A token was successfully lexed, so the input text position was
|
|
* advanced. However, this is an unexpected token, so we want to reset
|
|
* the context text position to point to the token rather than the text
|
|
* after it, so that if the caller wants to report the error position,
|
|
* it will point to the correct position of the unexpected token. */
|
|
context.text_position = token_info.position;
|
|
context.token = token;
|
|
return P_UNEXPECTED_TOKEN;
|
|
}
|
|
}
|
|
|
|
pub fn <%= @grammar.prefix %>parse(context: &mut <%= @grammar.prefix %>context_t) -> usize {
|
|
parse_from(context, 0, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>, &[])
|
|
}
|
|
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
pub fn <%= @grammar.prefix %>parse_<%= start_rule %>(context: &mut <%= @grammar.prefix %>context_t) -> usize {
|
|
parse_from(context, <%= i %>, <%= @parser.rule_sets[start_rule].id %>, &[])
|
|
}
|
|
|
|
pub fn <%= @grammar.prefix %>parse_inner_<%= start_rule %>(context: &mut <%= @grammar.prefix %>context_t, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize {
|
|
parse_from(context, <%= i %>, <%= @parser.rule_sets[start_rule].id %>, follow_tokens)
|
|
}
|
|
<% end %>
|
|
|
|
/**
|
|
* Get the parse result value.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Parse result value.
|
|
*/
|
|
<% if @grammar.tree %>
|
|
pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(@grammar.start_rules[0]) %><'_> {
|
|
<%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result") %>
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(start_rule) %><'_> {
|
|
<%= tree_handle(h_type(start_rule), "context.parse_result") %>
|
|
}
|
|
<% end %>
|
|
<% else %>
|
|
pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= rust_ptype(start_rule_type[1]) %> {
|
|
context.parse_result.get_v_<%= start_rule_type[0] %>()
|
|
}
|
|
<% @grammar.start_rules.each_with_index do |start_rule, i| %>
|
|
pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= rust_ptype(start_rule_type(i)[1]) %> {
|
|
context.parse_result.get_v_<%= start_rule_type(i)[0] %>()
|
|
}
|
|
<% end %>
|
|
<% end %>
|
|
|
|
/**
|
|
* Get the current text input position.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current text position.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>position(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>position_t {
|
|
context.text_position
|
|
}
|
|
|
|
/**
|
|
* Set the current text input position.
|
|
*
|
|
* This can be used to set the initial text position to something other than
|
|
* (1, 1) for a nested parse operation so that error positions reported by
|
|
* subsequent lexer/parser calls are relative to a larger enclosing document.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param position
|
|
* Text position to set.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>set_position(context: &mut <%= @grammar.prefix %>context_t, position: <%= @grammar.prefix %>position_t) {
|
|
context.text_position = position;
|
|
}
|
|
|
|
/**
|
|
* Get the current input text byte offset.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return Current input text byte offset (measured from the start of the
|
|
* input text passed to <%= @grammar.prefix %>context_new()).
|
|
*/
|
|
pub fn <%= @grammar.prefix %>input_index(context: &<%= @grammar.prefix %>context_t) -> usize {
|
|
context.input_index
|
|
}
|
|
|
|
/**
|
|
* Set the current input text byte offset.
|
|
*
|
|
* This moves the lexer's read cursor to the given byte offset (measured from
|
|
* the start of the input text passed to <%= @grammar.prefix %>context_new()).
|
|
* It can be used together with <%= @grammar.prefix %>set_position() to rewind
|
|
* the input part-way through a parse in order to re-read an earlier section of
|
|
* the input. The byte offset is not validated; the caller is responsible for
|
|
* providing an offset within the bounds of the input text. A value previously
|
|
* returned by <%= @grammar.prefix %>input_index() is a suitable argument.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
* @param input_index
|
|
* Input text byte offset to set.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>set_input_index(context: &mut <%= @grammar.prefix %>context_t, input_index: usize) {
|
|
context.input_index = input_index;
|
|
}
|
|
|
|
/**
|
|
* Get the user terminate code.
|
|
*
|
|
* @param context
|
|
* Lexer/parser context structure.
|
|
*
|
|
* @return User terminate code.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>user_terminate_code(context: &<%= @grammar.prefix %>context_t) -> usize {
|
|
context.user_terminate_code
|
|
}
|
|
|
|
/**
|
|
* Get the parse token.
|
|
*
|
|
* @return Parse token.
|
|
*/
|
|
pub fn <%= @grammar.prefix %>token(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>token_t {
|
|
context.token
|
|
}
|