From 89f1f848573d5ee8d6854a19c6f9279a5220cfb8 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 11 Aug 2026 22:46:04 -0400 Subject: [PATCH] Add Rust target --- CHANGELOG.md | 4 + README.md | 2 +- assets/parser.rs.erb | 1052 +++++++++++++++++ doc/user_guide.md | 2 +- lib/propane/generator.rb | 151 ++- spec/json_parser.rust.propane | 176 +++ spec/macros.rust.propane | 80 ++ spec/parse_inner_nested.rust.propane | 41 + spec/parse_inner_nested_tree.rust.propane | 44 + spec/propane_spec.rb | 464 +++++++- spec/rewind.rust.propane | 67 ++ spec/test_basic_math_grammar.rs | 16 + spec/test_custom_lex_fn.rs | 8 + spec/test_drop_code_block.rs | 7 + spec/test_error_positions.rs | 35 + spec/test_field_aliases.rs | 7 + spec/test_input_index.rs | 28 + spec/test_lexer.rs | 49 + spec/test_lexer_match_text.rs | 8 + spec/test_lexer_modes.rs | 13 + spec/test_lexer_multiple_modes.rs | 13 + spec/test_lexer_positions.rs | 38 + spec/test_lexer_result_value.rs | 13 + spec/test_lexer_unknown_character.rs | 12 + spec/test_macros.rs | 9 + spec/test_match_backslashes.rs | 7 + spec/test_multiple_parsers.rs | 12 + ...test_named_optional_rule_component_tree.rs | 43 + spec/test_optional_rule_component.rs | 9 + spec/test_optional_rule_component_tree.rs | 40 + spec/test_parse_inner.rs | 30 + spec/test_parse_inner_nested.rs | 17 + spec/test_parse_inner_nested_tree.rs | 36 + spec/test_parse_inner_recursive.rs | 31 + spec/test_parse_inner_shared.rs | 47 + spec/test_parse_inner_tree.rs | 36 + spec/test_parser_identical_rules_lookahead.rs | 9 + spec/test_parser_rule_from_multiple_states.rs | 18 + spec/test_parser_rule_user_code.rs | 7 + spec/test_parser_user_code_tree.rs | 24 + spec/test_parsing_json.rs | 31 + spec/test_parsing_lists.rs | 11 + spec/test_pattern.rs | 13 + spec/test_positions.rs | 19 + spec/test_return_token_from_pattern.rs | 7 + spec/test_rewind.rs | 13 + spec/test_set_position.rs | 66 ++ spec/test_start_rule.rs | 2 + spec/test_start_rule_tree.rs | 12 + spec/test_starting_rules.rs | 18 + spec/test_starting_rules_tree.rs | 34 + spec/test_token_user_fields.rs | 17 + spec/test_tree.rs | 46 + spec/test_tree_field_aliases.rs | 13 + spec/test_tree_invalid_positions.rs | 89 ++ spec/test_tree_node_memory_remains.rs | 85 ++ spec/test_tree_ps.rs | 46 + spec/test_tree_token_positions.rs | 27 + spec/test_user_code.rs | 13 + spec/test_user_context_fields.rs | 9 + spec/test_user_terminate.rs | 12 + spec/test_user_terminate_lexer.rs | 12 + spec/test_value_accessors.rs | 20 + spec/tree_node_memory_remains.rust.propane | 148 +++ 64 files changed, 3441 insertions(+), 27 deletions(-) create mode 100644 assets/parser.rs.erb create mode 100644 spec/json_parser.rust.propane create mode 100644 spec/macros.rust.propane create mode 100644 spec/parse_inner_nested.rust.propane create mode 100644 spec/parse_inner_nested_tree.rust.propane create mode 100644 spec/rewind.rust.propane create mode 100644 spec/test_basic_math_grammar.rs create mode 100644 spec/test_custom_lex_fn.rs create mode 100644 spec/test_drop_code_block.rs create mode 100644 spec/test_error_positions.rs create mode 100644 spec/test_field_aliases.rs create mode 100644 spec/test_input_index.rs create mode 100644 spec/test_lexer.rs create mode 100644 spec/test_lexer_match_text.rs create mode 100644 spec/test_lexer_modes.rs create mode 100644 spec/test_lexer_multiple_modes.rs create mode 100644 spec/test_lexer_positions.rs create mode 100644 spec/test_lexer_result_value.rs create mode 100644 spec/test_lexer_unknown_character.rs create mode 100644 spec/test_macros.rs create mode 100644 spec/test_match_backslashes.rs create mode 100644 spec/test_multiple_parsers.rs create mode 100644 spec/test_named_optional_rule_component_tree.rs create mode 100644 spec/test_optional_rule_component.rs create mode 100644 spec/test_optional_rule_component_tree.rs create mode 100644 spec/test_parse_inner.rs create mode 100644 spec/test_parse_inner_nested.rs create mode 100644 spec/test_parse_inner_nested_tree.rs create mode 100644 spec/test_parse_inner_recursive.rs create mode 100644 spec/test_parse_inner_shared.rs create mode 100644 spec/test_parse_inner_tree.rs create mode 100644 spec/test_parser_identical_rules_lookahead.rs create mode 100644 spec/test_parser_rule_from_multiple_states.rs create mode 100644 spec/test_parser_rule_user_code.rs create mode 100644 spec/test_parser_user_code_tree.rs create mode 100644 spec/test_parsing_json.rs create mode 100644 spec/test_parsing_lists.rs create mode 100644 spec/test_pattern.rs create mode 100644 spec/test_positions.rs create mode 100644 spec/test_return_token_from_pattern.rs create mode 100644 spec/test_rewind.rs create mode 100644 spec/test_set_position.rs create mode 100644 spec/test_start_rule.rs create mode 100644 spec/test_start_rule_tree.rs create mode 100644 spec/test_starting_rules.rs create mode 100644 spec/test_starting_rules_tree.rs create mode 100644 spec/test_token_user_fields.rs create mode 100644 spec/test_tree.rs create mode 100644 spec/test_tree_field_aliases.rs create mode 100644 spec/test_tree_invalid_positions.rs create mode 100644 spec/test_tree_node_memory_remains.rs create mode 100644 spec/test_tree_ps.rs create mode 100644 spec/test_tree_token_positions.rs create mode 100644 spec/test_user_code.rs create mode 100644 spec/test_user_context_fields.rs create mode 100644 spec/test_user_terminate.rs create mode 100644 spec/test_user_terminate_lexer.rs create mode 100644 spec/test_value_accessors.rs create mode 100644 spec/tree_node_memory_remains.rust.propane diff --git a/CHANGELOG.md b/CHANGELOG.md index d702b3b..2153489 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## v5.0.0 +### New Features + +- Add Rust target language output. + ### API Changes - Tree generation mode now stores all tree nodes in a compact arena owned by diff --git a/README.md b/README.md index 790c511..3c83e7f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which: * generates a built-in lexer to tokenize input * supports UTF-8 lexer inputs * generates a table-driven shift/reduce parser to parse input in linear time - * targets C, C++, or D language outputs + * targets C, C++, D, or Rust language outputs * optionally supports automatic full parse tree generation * supports starting parsing from multiple start rules * tracks input text start and end positions for all matched tokens/rules diff --git a/assets/parser.rs.erb b/assets/parser.rs.erb new file mode 100644 index 0000000..1c2e8ec --- /dev/null +++ b/assets/parser.rs.erb @@ -0,0 +1,1052 @@ +/* + * This file is generated by Propane. + */ + +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(non_upper_case_globals)] +#![allow(dead_code)] +#![allow(unused_variables)] +#![allow(unused_mut)] +#![allow(unused_parens)] +#![allow(unused_assignments)] +#![allow(unreachable_patterns)] + +/************************************************************************** + * User code blocks + *************************************************************************/ + +<% unless @grammar.code_blocks.fetch("header", "").empty? %> +<%= @grammar.code_blocks.fetch("header", "") %> +<% end %> +<%= @grammar.code_blocks.fetch("", "") %> + +/************************************************************************** + * Public types + *************************************************************************/ + +/* Result codes. */ +pub const P_SUCCESS: usize = 0; +pub const P_DECODE_ERROR: usize = 1; +pub const P_UNEXPECTED_INPUT: usize = 2; +pub const P_UNEXPECTED_TOKEN: usize = 3; +pub const P_DROP: usize = 4; +pub const P_EOF: usize = 5; +pub const P_USER_TERMINATED: usize = 6; + +/** Token type. */ +pub type <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>; + +/** Token IDs. */ +<% @grammar.tokens.each_with_index do |token, index| %> +pub const TOKEN_<%= token.code_name %>: <%= @grammar.prefix %>token_t = <%= index %>; +<% unless token.id == index %> +<% raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %> +<% end %> +<% end %> +pub const INVALID_TOKEN_ID: <%= @grammar.prefix %>token_t = <%= @grammar.invalid_token_id %>; +pub const TERMINATE_TOKEN_ID: <%= @grammar.prefix %>token_t = <%= @grammar.terminate_token_id %>; + +/** Code point type. */ +pub type <%= @grammar.prefix %>code_point_t = u32; + +/** + * A structure to keep track of input position. + * + * This is useful for reporting errors, etc... + */ +#[derive(Clone, Copy, Default, PartialEq)] +pub struct <%= @grammar.prefix %>position_t { + /** Input text row (1-based). */ + pub row: u32, + /** Input text column (1-based). */ + pub col: u32, +} + +impl <%= @grammar.prefix %>position_t { + /** Return whether the position is valid. */ + pub fn valid(&self) -> bool { + self.row != 0 + } +} + +/** An invalid position value. */ +const INVALID_POSITION: <%= @grammar.prefix %>position_t = <%= @grammar.prefix %>position_t { row: 0, col: 0 }; + +<% if @grammar.tree %> +/** Parser values type. */ +pub type <%= @grammar.prefix %>value_t = <%= rust_ptype(@grammar.ptype) %>; +<% else %> +/** Parser values type(s). */ +#[derive(Clone, Default)] +pub enum <%= @grammar.prefix %>value_t { + #[default] + __None, +<% @grammar.ptypes.each do |name, typestring| %> + v_<%= name %>(<%= rust_ptype(typestring) %>), +<% end %> +} + +impl <%= @grammar.prefix %>value_t { +<% @grammar.ptypes.each do |name, typestring| %> + fn v_<%= name %>_mut(&mut self) -> &mut <%= rust_ptype(typestring) %> { + match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v, _ => unreachable!() } + } + fn get_v_<%= name %>(&self) -> <%= rust_ptype(typestring) %> { + match self { <%= @grammar.prefix %>value_t::v_<%= name %>(v) => v.clone(), _ => Default::default() } + } +<% end %> +} + +/** Parser value constructor(s) and accessor(s). */ +<% @grammar.ptypes.each do |name, typestring| %> +<% suffix = name == "default" ? "" : "_#{name}" %> +pub fn <%= @grammar.prefix %>value<%= suffix %>(v: <%= rust_ptype(typestring) %>) -> <%= @grammar.prefix %>value_t { <%= @grammar.prefix %>value_t::v_<%= name %>(v) } +pub fn <%= @grammar.prefix %>value_get<%= suffix %>(pvalue: &<%= @grammar.prefix %>value_t) -> <%= rust_ptype(typestring) %> { pvalue.get_v_<%= name %>() } +<% end %> +<% end %> +<% if @grammar.tree %> +/** Tree node ID type (index into the context node arena). ID 0 is null. */ +pub type <%= @grammar.prefix %>node_id_t = u32; + +<%= rust_tree_types %> +<% end %> + +/** Lexed token information. */ +#[derive(Clone, Default)] +pub struct <%= @grammar.prefix %>token_info_t { + /** Text position of first code point in token. */ + pub position: <%= @grammar.prefix %>position_t, + /** Text position of last code point in token. */ + pub end_position: <%= @grammar.prefix %>position_t, + /** Number of input bytes used by the token. */ + pub length: usize, + /** Token that was lexed. */ + pub token: <%= @grammar.prefix %>token_t, + /** Parser value associated with the token. */ + pub pvalue: <%= @grammar.prefix %>value_t, +} + +/** + * Lexer and parser context. + * + * The user must allocate an instance of this structure and pass it to any + * public API function. + */ +#[derive(Default)] +pub struct <%= @grammar.prefix %>context_t { + /* Lexer context data. */ + + /** Input text. */ + input: Vec, + /** Input text length. */ + input_length: usize, + /** Input text index (byte offset). */ + input_index: usize, + /** Input text position (row/column). */ + text_position: <%= @grammar.prefix %>position_t, + /** Current lexer mode. */ + mode: usize, + + /* Parser context data. */ + + /** Parse result value. */ +<% if @grammar.tree %> + parse_result: <%= @grammar.prefix %>node_id_t, + /** Tree node arena. Node ID 0 is reserved as the null node. */ + <%= @grammar.prefix %>tree_nodes: Vec<<%= @grammar.prefix %>node_data_t>, + /** Shared tree child links (CSR layout). */ + <%= @grammar.prefix %>tree_children: Vec<<%= @grammar.prefix %>node_id_t>, +<% else %> + parse_result: <%= @grammar.prefix %>value_t, +<% end %> + + /** Unexpected token received. */ + token: <%= @grammar.prefix %>token_t, + /** User terminate code. */ + pub user_terminate_code: usize, + +<%= @grammar.context_user_fields %> +} + +/************************************************************************** + * Public data + *************************************************************************/ + +/** Token names. */ +pub const <%= @grammar.prefix %>token_names: [&str; <%= @grammar.tokens.size %>] = [ +<% @grammar.tokens.each do |token| %> + "<%= token.name %>", +<% end %> +]; + +/************************************************************************** + * State initialization + *************************************************************************/ + +/** + * Allocate and initialize lexer/parser context structure. + * + * @param input + * Text input. + * + * @return Context structure for lexer/parser. + */ +pub fn <%= @grammar.prefix %>context_new(input: &[u8]) -> <%= @grammar.prefix %>context_t { + let mut context = <%= @grammar.prefix %>context_t::default(); + + /* Lexer initialization. */ + context.input = input.to_vec(); + context.input_length = input.len(); + context.text_position.row = 1; + context.text_position.col = 1; + context.mode = <%= @lexer.mode_id("default") %>; +<% if @grammar.tree %> + + /* Reserve node ID 0 as the null tree node. */ + context.<%= @grammar.prefix %>tree_nodes.push(<%= @grammar.prefix %>node_data_t::default()); +<% end %> + + context +} + +/** + * Deinitialize and deallocate lexer/parser context structure. + * + * @param context + * Lexer/parser context structure. + */ +pub fn <%= @grammar.prefix %>context_delete(mut context: <%= @grammar.prefix %>context_t) { +<% if @grammar.tree && @grammar.free_token_node != "" %> + for i in 0..context.<%= @grammar.prefix %>tree_nodes.len() { + if context.<%= @grammar.prefix %>tree_nodes[i].is_token { + let token_node_id = i; +<%= expand_code(@grammar.free_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id]") %> + } + } +<% end %> + drop(context); +} + +/************************************************************************** + * Decoder + *************************************************************************/ + +/** + * Decode a UTF-8 code point. + * + * @param input + * Text input to decode. + * @param out_code_point + * The decoded code point is stored here if the return value is P_SUCCESS. + * @param out_code_point_length + * The number of bytes the code point used is stored here if the return value + * is P_SUCCESS. + * + * @retval P_SUCCESS on a successful code point decode + * @retval P_DECODE_ERROR when an encoding error is observed + * @retval P_EOF when the end of the text input is reached + */ +pub fn <%= @grammar.prefix %>decode_code_point(input: &[u8], + out_code_point: &mut <%= @grammar.prefix %>code_point_t, out_code_point_length: &mut u8) -> usize { + if input.len() == 0 { + return P_EOF; + } + let c = input[0]; + let mut code_point: <%= @grammar.prefix %>code_point_t; + let code_point_length: u8; + if (c & 0x80u8) == 0u8 { + code_point = c as <%= @grammar.prefix %>code_point_t; + code_point_length = 1; + } else { + let following_bytes: usize; + if (c & 0xE0u8) == 0xC0u8 { + code_point = (c & 0x1Fu8) as <%= @grammar.prefix %>code_point_t; + following_bytes = 1; + } else if (c & 0xF0u8) == 0xE0u8 { + code_point = (c & 0x0Fu8) as <%= @grammar.prefix %>code_point_t; + following_bytes = 2; + } else if (c & 0xF8u8) == 0xF0u8 { + code_point = (c & 0x07u8) as <%= @grammar.prefix %>code_point_t; + following_bytes = 3; + } else if (c & 0xFCu8) == 0xF8u8 { + code_point = (c & 0x03u8) as <%= @grammar.prefix %>code_point_t; + following_bytes = 4; + } else if (c & 0xFEu8) == 0xFCu8 { + code_point = (c & 0x01u8) as <%= @grammar.prefix %>code_point_t; + following_bytes = 5; + } else { + return P_DECODE_ERROR; + } + if input.len() <= following_bytes { + return P_DECODE_ERROR; + } + code_point_length = (following_bytes + 1) as u8; + for i in 0..following_bytes { + let b = input[i + 1]; + if (b & 0xC0u8) != 0x80u8 { + return P_DECODE_ERROR; + } + code_point = (code_point << 6) | ((b & 0x3Fu8) as <%= @grammar.prefix %>code_point_t); + } + } + *out_code_point = code_point; + *out_code_point_length = code_point_length; + P_SUCCESS +} + +/************************************************************************** + * Lexer + *************************************************************************/ + +/** Invalid lexer state ID. */ +const INVALID_LEXER_STATE_ID: <%= get_type_for(@lexer.state_table.size) %> = <%= @lexer.state_table.size %>; + +/** Invalid lexer user code ID. */ +<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %> +const INVALID_USER_CODE_ID: <%= get_type_for(user_code_id_count) %> = <%= user_code_id_count %>; + +/** Lexer transition table entry. */ +#[derive(Clone, Copy)] +struct lexer_transition_t { + /** First code point in the range for this transition. */ + first: <%= @grammar.prefix %>code_point_t, + /** Last code point in the range for this transition. */ + last: <%= @grammar.prefix %>code_point_t, + /** Destination lexer state ID for this transition. */ + destination_state: <%= get_type_for(@lexer.state_table.size) %>, +} + +/** Lexer state table entry. */ +#[derive(Clone, Copy)] +struct lexer_state_t { + /** Index to the transition table for this state. */ + transition_table_index: <%= get_type_for(@lexer.transition_table.size - 1) %>, + /** Number of transition table entries for this state. */ + n_transitions: <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %>, + /** Lexer token formed at this state. */ + token: <%= @grammar.prefix %>token_t, + /** Lexer user code ID to execute at this state. */ + code_id: <%= get_type_for(user_code_id_count) %>, + /** Whether this state matches a lexer pattern. */ + accepts: bool, +} + +/** Lexer mode table entry. */ +#[derive(Clone, Copy)] +struct lexer_mode_t { + /** Offset in the state table to be used for this mode. */ + state_table_offset: u32, +} + +/** + * Lexer match info structure. + * + * This structure holds output values from the lexer upon a successful pattern + * match. + */ +#[derive(Clone, Copy, Default)] +struct lexer_match_info_t { + /** Number of bytes of input text used to match. */ + length: usize, + /** Input text position delta to end of token. */ + end_delta_position: <%= @grammar.prefix %>position_t, + /** Input text position delta to next code point after token end. */ + delta_position: <%= @grammar.prefix %>position_t, + /** Accepting lexer state from the match (state ID, or INVALID). */ + accepting_state: <%= get_type_for(@lexer.state_table.size) %>, +} + +/** Lexer transition table. */ +static lexer_transition_table: [lexer_transition_t; <%= @lexer.transition_table.size %>] = [ +<% @lexer.transition_table.each do |transition_table_entry| %> + lexer_transition_t { first: <%= transition_table_entry[:first] %>, last: <%= transition_table_entry[:last] %>, destination_state: <%= transition_table_entry[:destination] %> }, +<% end %> +]; + +/** Lexer state table. */ +static lexer_state_table: [lexer_state_t; <%= @lexer.state_table.size %>] = [ +<% @lexer.state_table.each do |state_table_entry| %> + lexer_state_t { transition_table_index: <%= state_table_entry[:transition_table_index] %>, n_transitions: <%= state_table_entry[:n_transitions] %>, token: <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, code_id: <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, accepts: <%= state_table_entry[:accepts] %> }, +<% end %> +]; + +/** Lexer mode table. */ +static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [ +<% @lexer.mode_table.each do |mode_table_entry| %> + lexer_mode_t { state_table_offset: <%= mode_table_entry[:state_table_offset] %> }, +<% end %> +]; + +/** + * Execute user code associated with a lexer pattern. + * + * @return Token to accept, or invalid token if the user code does + * not explicitly return a token. + */ +fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t, + code_id: <%= get_type_for(user_code_id_count) %>, match_: &[u8], + match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t { + match code_id { +<% @grammar.patterns.each do |pattern| %> +<% if pattern.code_id %> + <%= pattern.code_id %> => { +<% unless @grammar.tree %> + out_token_info.pvalue = <%= @grammar.prefix %>value_t::v_<%= pattern.ptypename %>(Default::default()); +<% end %> +<%= expand_code(pattern.code, false, nil, pattern) %> + } +<% end %> +<% end %> + _ => {} + } + + INVALID_TOKEN_ID +} + +/** + * Check if there is a transition from the current lexer state to another + * based on the given input code point. + * + * @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none. + */ +fn check_lexer_transition(current_state: u32, code_point: u32) -> <%= get_type_for(@lexer.state_table.size) %> { + let transition_table_index = lexer_state_table[current_state as usize].transition_table_index as u32; + for i in 0..(lexer_state_table[current_state as usize].n_transitions as u32) { + let t = &lexer_transition_table[(transition_table_index + i) as usize]; + if (t.first <= code_point) && (code_point <= t.last) { + return t.destination_state; + } + } + INVALID_LEXER_STATE_ID +} + +/** + * Find the longest lexer pattern match at the current position. + */ +fn find_longest_match(context: &<%= @grammar.prefix %>context_t, + out_match_info: &mut lexer_match_info_t, out_unexpected_input_length: &mut usize) -> usize { + let mut longest_match = lexer_match_info_t::default(); + longest_match.accepting_state = INVALID_LEXER_STATE_ID; + let mut attempt_match = lexer_match_info_t::default(); + attempt_match.accepting_state = INVALID_LEXER_STATE_ID; + *out_match_info = longest_match; + let mut current_state: u32 = lexer_mode_table[context.mode].state_table_offset; + loop { + let input_index = context.input_index + attempt_match.length; + let input = &context.input[input_index..]; + let mut code_point: <%= @grammar.prefix %>code_point_t = 0; + let mut code_point_length: u8 = 0; + let result = <%= @grammar.prefix %>decode_code_point(input, &mut code_point, &mut code_point_length); + match result { + P_SUCCESS => { + let transition_state = check_lexer_transition(current_state, code_point); + if transition_state != INVALID_LEXER_STATE_ID { + attempt_match.length += code_point_length as usize; + attempt_match.end_delta_position = attempt_match.delta_position; + if code_point == '\n' as u32 { + attempt_match.delta_position.row += 1; + attempt_match.delta_position.col = 1; + } else { + attempt_match.delta_position.col += 1; + } + current_state = transition_state as u32; + if lexer_state_table[current_state as usize].accepts { + attempt_match.accepting_state = current_state as <%= get_type_for(@lexer.state_table.size) %>; + longest_match = attempt_match; + } + } else if longest_match.length > 0 { + *out_match_info = longest_match; + return P_SUCCESS; + } else { + *out_unexpected_input_length = attempt_match.length + code_point_length as usize; + return P_UNEXPECTED_INPUT; + } + } + P_EOF => { + /* We hit EOF. */ + if longest_match.length > 0 { + *out_match_info = longest_match; + return P_SUCCESS; + } else if attempt_match.length != 0 { + *out_unexpected_input_length = attempt_match.length; + return P_UNEXPECTED_INPUT; + } else { + return P_EOF; + } + } + P_DECODE_ERROR => { + *out_match_info = attempt_match; + return result; + } + _ => { + return result; + } + } + } +} + +/** + * Attempt to lex the next token in the input stream. + */ +fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize { + let mut token_info = <%= @grammar.prefix %>token_info_t::default(); + token_info.position = context.text_position; + token_info.token = INVALID_TOKEN_ID; + let mut match_info = lexer_match_info_t::default(); + let mut unexpected_input_length: usize = 0; + let result = find_longest_match(context, &mut match_info, &mut unexpected_input_length); + match result { + P_SUCCESS => { + let mut token_to_accept = lexer_state_table[match_info.accepting_state as usize].token; + token_info.length = match_info.length; + if match_info.end_delta_position.row != 0 { + token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row; + token_info.end_position.col = match_info.end_delta_position.col; + } else { + token_info.end_position.row = token_info.position.row; + token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col; + } + if lexer_state_table[match_info.accepting_state as usize].code_id != INVALID_USER_CODE_ID { + let match_start = context.input_index; + let match_slice = context.input[match_start..(match_start + match_info.length)].to_vec(); + let user_code_token = lexer_user_code(context, + lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, match_info.length, &mut token_info); + if user_code_token == TERMINATE_TOKEN_ID { + return P_USER_TERMINATED; + } + if user_code_token != INVALID_TOKEN_ID { + token_to_accept = user_code_token; + } + } + + /* Update the input position tracking. */ + context.input_index += match_info.length; + context.text_position.row += match_info.delta_position.row; + if match_info.delta_position.row != 0 { + context.text_position.col = match_info.delta_position.col; + } else { + context.text_position.col += match_info.delta_position.col; + } + + if token_to_accept == INVALID_TOKEN_ID { + return P_DROP; + } + token_info.token = token_to_accept; + *out_token_info = token_info; + P_SUCCESS + } + P_EOF => { + token_info.token = TOKEN___EOF; + token_info.end_position = token_info.position; + *out_token_info = token_info; + P_SUCCESS + } + P_DECODE_ERROR => { + /* Update the input position tracking. */ + context.input_index += match_info.length; + context.text_position.row += match_info.delta_position.row; + if match_info.delta_position.row != 0 { + context.text_position.col = match_info.delta_position.col; + } else { + context.text_position.col += match_info.delta_position.col; + } + result + } + _ => { + result + } + } +} + +/** + * Lex the next token in the input stream. + */ +pub fn <%= @grammar.prefix %>lex(context: &mut <%= @grammar.prefix %>context_t, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> usize { + loop { + let result = attempt_lex_token(context, out_token_info); + if result != P_DROP { + return result; + } + } +} + +/************************************************************************** + * Parser + *************************************************************************/ + +/* An invalid ID value. */ +const INVALID_ID: usize = usize::MAX; + +/** Shift table entry. */ +#[derive(Clone, Copy)] +struct shift_t { + /** Token or rule set ID. */ + symbol_id: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>, + /** Parser state to shift to. */ + state_id: <%= get_type_for(@parser.state_table.size) %>, +} + +/** Reduce table entry. */ +#[derive(Clone, Copy)] +struct reduce_t { + /** Lookahead token. */ + token: <%= @grammar.prefix %>token_t, + /** Rule ID. */ + rule: <%= get_type_for(@grammar.rules.size) %>, + /** Rule set ID. */ + rule_set: <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %>, + /** Number of states leading to this reduce action. */ + n_states: <%= get_type_for(@parser.state_table.size) %>, +<% if @grammar.tree %> + /** Map of rule components to rule set child fields (None for a flat map). */ + rule_set_node_field_index_map: Option<&'static [u16]>, + /** Number of rule set tree node fields. */ + rule_set_node_field_array_size: u16, + /** Whether this rule propagates a matched optional target node. */ + propagate_optional_target: bool, +<% end %> +} + +/** Parser state entry. */ +#[derive(Clone, Copy)] +struct parser_state_t { + /** First shift table entry for this parser state. */ + shift_table_index: <%= get_type_for(@parser.shift_table.size) %>, + /** Number of shift table entries for this parser state. */ + n_shift_entries: <%= get_type_for(@parser.shift_table.size) %>, + /** First reduce table entry for this parser state. */ + reduce_table_index: <%= get_type_for(@parser.reduce_table.size) %>, + /** Number of reduce table entries for this parser state. */ + n_reduce_entries: <%= get_type_for(@parser.reduce_table.size) %>, +} + +/** + * Structure to hold a state ID and value pair. + * + * A stack of these structures makes up the parse stack. + */ +#[derive(Clone, Default)] +struct state_value_t { + /** Parser state ID. */ + state_id: usize, +<% if @grammar.tree %> + /** Tree node ID. */ + node_id: <%= @grammar.prefix %>node_id_t, +<% else %> + position: <%= @grammar.prefix %>position_t, + end_position: <%= @grammar.prefix %>position_t, + /** Parser value from this state. */ + pvalue: <%= @grammar.prefix %>value_t, +<% end %> +} + +/** Parser shift table. */ +static parser_shift_table: [shift_t; <%= @parser.shift_table.size %>] = [ +<% @parser.shift_table.each do |shift| %> + shift_t { symbol_id: <%= shift[:symbol].id %>, state_id: <%= shift[:state_id] %> }, +<% end %> +]; + +<% if @grammar.tree %> +<% @grammar.rules.each do |rule| %> +<% unless rule.flat_rule_set_node_field_index_map? %> +static r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map: [u16; <%= rule.rule_set_node_field_index_map.size %>] = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>]; +<% end %> +<% end %> +<% end %> + +/** Parser reduce table. */ +static parser_reduce_table: [reduce_t; <%= @parser.reduce_table.size %>] = [ +<% @parser.reduce_table.each do |reduce| %> + reduce_t { + token: <%= reduce[:token_id] %>, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */ + rule: <%= reduce[:rule_id] %>, /* Rule ID */ + rule_set: <%= reduce[:rule_set_id] %>, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */ + n_states: <%= reduce[:n_states] %>, /* Number of states */ +<% if @grammar.tree %> +<% if reduce[:rule].flat_rule_set_node_field_index_map? %> + rule_set_node_field_index_map: None, +<% else %> + rule_set_node_field_index_map: Some(&r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map), +<% end %> + rule_set_node_field_array_size: <%= reduce[:rule].rule_set.tree_fields.size %>, + propagate_optional_target: <%= reduce[:propagate_optional_target] %>, +<% end %> + }, +<% end %> +]; + +/** Parser state table. */ +static parser_state_table: [parser_state_t; <%= @parser.state_table.size %>] = [ +<% @parser.state_table.each do |state| %> + parser_state_t { shift_table_index: <%= state[:shift_index] %>, n_shift_entries: <%= state[:n_shifts] %>, reduce_table_index: <%= state[:reduce_index] %>, n_reduce_entries: <%= state[:n_reduces] %> }, +<% end %> +]; + +<% if @grammar.tree %> +/* Tree arena helpers. */ + +/** Allocate a new (zeroed) tree node in the context arena. */ +fn tree_new_node(context: &mut <%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>node_id_t { + let id = context.<%= @grammar.prefix %>tree_nodes.len() as <%= @grammar.prefix %>node_id_t; + context.<%= @grammar.prefix %>tree_nodes.push(<%= @grammar.prefix %>node_data_t::default()); + id +} + +/** Reserve n contiguous (zeroed) child slots in the shared children array. */ +fn tree_reserve_children(context: &mut <%= @grammar.prefix %>context_t, n: usize) -> <%= @grammar.prefix %>node_id_t { + let offset = context.<%= @grammar.prefix %>tree_children.len() as <%= @grammar.prefix %>node_id_t; + let new_len = context.<%= @grammar.prefix %>tree_children.len() + n; + context.<%= @grammar.prefix %>tree_children.resize(new_len, 0); + offset +} +<% end %> + +<% unless @grammar.tree %> +/** + * Get the rule position (start or end) for the currently matched rule. + */ +fn get_rule_position(statevalues: &[state_value_t], i: usize, n_states: usize, get_end: bool) -> <%= @grammar.prefix %>position_t { + let len = statevalues.len(); + if n_states > 0 { + if i == 0 { + if get_end { + for j in 0..n_states { + let sv = &statevalues[len - 1 - j]; + if sv.end_position.valid() { + return sv.end_position; + } + } + } else { + for j in 0..n_states { + let sv = &statevalues[len - n_states + j]; + if sv.position.valid() { + return sv.position; + } + } + } + } else { + if get_end { + return statevalues[len - 1 - n_states + i].end_position; + } else { + return statevalues[len - 1 - n_states + i].position; + } + } + } + INVALID_POSITION +} +<% end %> + +<% if !@grammar.tree || @grammar.parser_user_code_used? %> +/** + * Execute user code associated with a parser rule. + */ +fn parser_user_code(context: &mut <%= @grammar.prefix %>context_t, <%= @grammar.tree ? "_node_id: #{@grammar.prefix}node_id_t" : "_pvalue: &mut #{@grammar.prefix}value_t" %>, rule: u32, statevalues: &[state_value_t], n_states: usize) -> usize { + match rule { +<% @grammar.rules.each do |rule| %> +<% if rule.code %> + <%= rule.id %> => { +<% unless @grammar.tree %> + *_pvalue = <%= @grammar.prefix %>value_t::v_<%= rule.ptypename %>(Default::default()); +<% end %> +<%= expand_code(rule.code, true, rule, nil) %> + } +<% end %> +<% end %> + _ => {} + } + + P_SUCCESS +} +<% end %> + +/** + * Check if the parser should shift to a new state. + * + * @return State to shift to, or INVALID_ID if none. + */ +fn check_shift(state_id: usize, symbol_id: usize) -> usize { + let start = parser_state_table[state_id].shift_table_index as usize; + let end = start + parser_state_table[state_id].n_shift_entries as usize; + for i in start..end { + if parser_shift_table[i].symbol_id as usize == symbol_id { + return parser_shift_table[i].state_id as usize; + } + } + INVALID_ID +} + +/** + * Check if the parser should reduce to a new state. + * + * @return Reduce table index to reduce with, or INVALID_ID if none. + */ +fn check_reduce(state_id: usize, token: <%= @grammar.prefix %>token_t) -> usize { + let start = parser_state_table[state_id].reduce_table_index as usize; + let end = start + parser_state_table[state_id].n_reduce_entries as usize; + for i in start..end { + if (parser_reduce_table[i].token == token) || (parser_reduce_table[i].token == INVALID_TOKEN_ID) { + return i; + } + } + INVALID_ID +} + +/** + * Run the parser. + */ +fn parse_from(context: &mut <%= @grammar.prefix %>context_t, start_state_id: usize, + start_rule_set_id: usize, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize { + let mut token_info = <%= @grammar.prefix %>token_info_t::default(); + let mut token: <%= @grammar.prefix %>token_t = INVALID_TOKEN_ID; + let mut statevalues: Vec = Vec::new(); + let mut reduced_rule_set: usize = INVALID_ID; + let mut last_shifted_rule_set_id: usize = INVALID_ID; +<% if @grammar.tree %> + let mut reduced_parser_node: <%= @grammar.prefix %>node_id_t = 0; +<% else %> + let mut reduced_position: <%= @grammar.prefix %>position_t = INVALID_POSITION; + let mut reduced_end_position: <%= @grammar.prefix %>position_t = INVALID_POSITION; + let mut reduced_parser_value: <%= @grammar.prefix %>value_t = Default::default(); +<% end %> + statevalues.push(state_value_t::default()); + let sv_len = statevalues.len(); + statevalues[sv_len - 1].state_id = start_state_id; + let result; + loop { + if token == INVALID_TOKEN_ID { + let lexer_result = <%= lex_fn %>(context, &mut token_info); + if lexer_result != P_SUCCESS { + result = lexer_result; + break; + } + token = token_info.token; + } + let mut token_is_follow = false; + for &ft in follow_tokens { + if token == ft { + token_is_follow = true; + break; + } + } + let mut shift_state: usize = INVALID_ID; + if reduced_rule_set != INVALID_ID { + shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, reduced_rule_set); + } + if shift_state == INVALID_ID { + shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, token as usize); + if (shift_state != INVALID_ID) && (token == TOKEN___EOF) { + /* Successful parse. */ +<% if @grammar.tree %> + context.parse_result = statevalues[statevalues.len() - 1].node_id; +<% else %> + context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone(); +<% end %> + result = P_SUCCESS; + break; + } + if (shift_state == INVALID_ID) && token_is_follow { + let retry_shift_state = check_shift(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF as usize); + if (retry_shift_state != INVALID_ID) && (statevalues.len() == 2) && (last_shifted_rule_set_id == start_rule_set_id) { + context.input_index -= token_info.length; + context.text_position = token_info.position; +<% if @grammar.tree %> + context.parse_result = statevalues[statevalues.len() - 1].node_id; +<% else %> + context.parse_result = statevalues[statevalues.len() - 1].pvalue.clone(); +<% end %> + result = P_SUCCESS; + break; + } + } + } + if shift_state != INVALID_ID { + last_shifted_rule_set_id = reduced_rule_set; + statevalues.push(state_value_t::default()); + let new_index = statevalues.len() - 1; + statevalues[new_index].state_id = shift_state; + if reduced_rule_set == INVALID_ID { + /* We shifted a token, mark it consumed. */ +<% if @grammar.tree %> + let token_node_id = tree_new_node(context); + { + let token_tree_node = &mut context.<%= @grammar.prefix %>tree_nodes[token_node_id as usize]; + token_tree_node.position = token_info.position; + token_tree_node.end_position = token_info.end_position; + token_tree_node.n_fields = 0; + token_tree_node.is_token = true; + token_tree_node.token = token; + token_tree_node.pvalue = token_info.pvalue.clone(); + } +<%= expand_code(@grammar.on_token_node, false, nil, nil).gsub("token_tree_node", "context.#{@grammar.prefix}tree_nodes[token_node_id as usize]") %> + statevalues[new_index].node_id = token_node_id; +<% else %> + statevalues[new_index].position = token_info.position; + statevalues[new_index].end_position = token_info.end_position; + statevalues[new_index].pvalue = token_info.pvalue.clone(); +<% end %> + token = INVALID_TOKEN_ID; + } else { + /* We shifted a RuleSet. */ +<% if @grammar.tree %> + statevalues[new_index].node_id = reduced_parser_node; +<% else %> + statevalues[new_index].pvalue = reduced_parser_value.clone(); + statevalues[new_index].position = reduced_position; + statevalues[new_index].end_position = reduced_end_position; + reduced_parser_value = Default::default(); +<% end %> + reduced_rule_set = INVALID_ID; + } + continue; + } + + let mut reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, token); + if (reduce_index == INVALID_ID) && token_is_follow { + reduce_index = check_reduce(statevalues[statevalues.len() - 1].state_id, TOKEN___EOF); + } + if reduce_index != INVALID_ID { + /* We have something to reduce. */ + let n_states = parser_reduce_table[reduce_index].n_states as usize; +<% if @grammar.tree %> + if parser_reduce_table[reduce_index].propagate_optional_target { + reduced_parser_node = statevalues[statevalues.len() - 1].node_id; + } else if n_states > 0 { + let n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size; + let child_offset = tree_reserve_children(context, n_fields as usize); + match parser_reduce_table[reduce_index].rule_set_node_field_index_map { + None => { + for i in 0..n_states { + context.<%= @grammar.prefix %>tree_children[child_offset as usize + i] = statevalues[statevalues.len() - n_states + i].node_id; + } + } + Some(map) => { + for i in 0..n_states { + context.<%= @grammar.prefix %>tree_children[child_offset as usize + map[i] as usize] = statevalues[statevalues.len() - n_states + i].node_id; + } + } + } + let node_id = tree_new_node(context); + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].position = INVALID_POSITION; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].end_position = INVALID_POSITION; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].child_offset = child_offset; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].n_fields = n_fields; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].is_token = false; + let mut position_found = false; + for i in 0..(n_fields as usize) { + let child_id = context.<%= @grammar.prefix %>tree_children[child_offset as usize + i]; + if (child_id != 0) && context.<%= @grammar.prefix %>tree_nodes[child_id as usize].position.valid() { + if !position_found { + let p = context.<%= @grammar.prefix %>tree_nodes[child_id as usize].position; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].position = p; + position_found = true; + } + let ep = context.<%= @grammar.prefix %>tree_nodes[child_id as usize].end_position; + context.<%= @grammar.prefix %>tree_nodes[node_id as usize].end_position = ep; + } + } + reduced_parser_node = node_id; + } else { + reduced_parser_node = 0; + } +<% if @grammar.parser_user_code_used? %> + if parser_user_code(context, reduced_parser_node, parser_reduce_table[reduce_index].rule as u32, &statevalues, n_states) == P_USER_TERMINATED { + return P_USER_TERMINATED; + } +<% end %> +<% else %> + let mut reduced_parser_value2: <%= @grammar.prefix %>value_t = Default::default(); + if parser_user_code(context, &mut reduced_parser_value2, parser_reduce_table[reduce_index].rule as u32, &statevalues, n_states) == P_USER_TERMINATED { + return P_USER_TERMINATED; + } + reduced_parser_value = reduced_parser_value2; + if n_states > 0 { + reduced_position = get_rule_position(&statevalues, 0, n_states, false); + reduced_end_position = get_rule_position(&statevalues, 0, n_states, true); + } else { + reduced_position = INVALID_POSITION; + reduced_end_position = INVALID_POSITION; + } +<% end %> + reduced_rule_set = parser_reduce_table[reduce_index].rule_set as usize; + let new_len = statevalues.len() - n_states; + statevalues.truncate(new_len); + continue; + } + + /* Unexpected token. Reset the context text position to the token. */ + context.text_position = token_info.position; + context.token = token; + result = P_UNEXPECTED_TOKEN; + break; + } + result +} + +pub fn <%= @grammar.prefix %>parse(context: &mut <%= @grammar.prefix %>context_t) -> usize { + parse_from(context, 0, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>, &[]) +} + +<% @grammar.start_rules.each_with_index do |start_rule, i| %> +pub fn <%= @grammar.prefix %>parse_<%= start_rule %>(context: &mut <%= @grammar.prefix %>context_t) -> usize { + parse_from(context, <%= i %>, <%= @parser.rule_sets[start_rule].id %>, &[]) +} + +pub fn <%= @grammar.prefix %>parse_inner_<%= start_rule %>(context: &mut <%= @grammar.prefix %>context_t, follow_tokens: &[<%= @grammar.prefix %>token_t]) -> usize { + parse_from(context, <%= i %>, <%= @parser.rule_sets[start_rule].id %>, follow_tokens) +} +<% end %> + +/** + * Get the parse result value. + */ +<% if @grammar.tree %> +pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(@grammar.start_rules[0]) %><'_> { + <%= tree_handle(h_type(@grammar.start_rules[0]), "context.parse_result") %> +} +<% @grammar.start_rules.each_with_index do |start_rule, i| %> +pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= h_type(start_rule) %><'_> { + <%= tree_handle(h_type(start_rule), "context.parse_result") %> +} +<% end %> +<% else %> +pub fn <%= @grammar.prefix %>result(context: &<%= @grammar.prefix %>context_t) -> <%= rust_ptype(start_rule_type[1]) %> { + context.parse_result.get_v_<%= start_rule_type[0] %>() +} +<% @grammar.start_rules.each_with_index do |start_rule, i| %> +pub fn <%= @grammar.prefix %>result_<%= start_rule %>(context: &<%= @grammar.prefix %>context_t) -> <%= rust_ptype(start_rule_type(i)[1]) %> { + context.parse_result.get_v_<%= start_rule_type(i)[0] %>() +} +<% end %> +<% end %> + +/** Get the current text input position. */ +pub fn <%= @grammar.prefix %>position(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>position_t { + context.text_position +} + +/** Set the current text input position. */ +pub fn <%= @grammar.prefix %>set_position(context: &mut <%= @grammar.prefix %>context_t, position: <%= @grammar.prefix %>position_t) { + context.text_position = position; +} + +/** Get the current input text byte offset. */ +pub fn <%= @grammar.prefix %>input_index(context: &<%= @grammar.prefix %>context_t) -> usize { + context.input_index +} + +/** Set the current input text byte offset. */ +pub fn <%= @grammar.prefix %>set_input_index(context: &mut <%= @grammar.prefix %>context_t, input_index: usize) { + context.input_index = input_index; +} + +/** Get the user terminate code. */ +pub fn <%= @grammar.prefix %>user_terminate_code(context: &<%= @grammar.prefix %>context_t) -> usize { + context.user_terminate_code +} + +/** Get the parse token. */ +pub fn <%= @grammar.prefix %>token(context: &<%= @grammar.prefix %>context_t) -> <%= @grammar.prefix %>token_t { + context.token +} diff --git a/doc/user_guide.md b/doc/user_guide.md index fb2ca9c..973f0ed 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -13,7 +13,7 @@ Propane is a LALR Parser Generator (LPG) which: * generates a built-in lexer to tokenize input * supports UTF-8 lexer inputs * generates a table-driven shift/reduce parser to parse input in linear time - * targets C, C++, or D language outputs + * targets C, C++, D, or Rust language outputs * optionally supports automatic full parse tree generation * supports starting parsing from multiple start rules * tracks input text start and end positions for all matched tokens/rules diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 684dfaf..3d8a218 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -18,6 +18,8 @@ class Propane elsif output_file =~ %r{\.(cc|cpp|cxx)$} @cpp = true "c" + elsif output_file.end_with?(".rs") + "rust" else raise Error.new("Could not determine target language from output file name (#{output_file})") end @@ -31,7 +33,8 @@ class Propane extensions += %w[h] end extensions.each do |extension| - template = Assets.get("parser.#{extension || @language}.erb") + template_language = @language == "rust" ? "rs" : @language + template = Assets.get("parser.#{extension || template_language}.erb") if extension output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}") else @@ -39,7 +42,12 @@ class Propane end erb = ERB.new(template, trim_mode: "<>") result = erb.result(binding.clone).lines.each_with_index.map do |line, i| - if line == "#linereset\n" + if @language == "rust" + # Rust has no #line directive support, so strip the directives that + # the grammar embeds in user code blocks. + line = line.sub(/^#line \d+ "[^"]*"/, "") + line == "#linereset\n" ? "" : line + elsif line == "#linereset\n" %[#line #{i + 2} "#{output_file}"\n] else line @@ -275,6 +283,8 @@ class Propane "context->user_terminate_code = (#{user_terminate_code}); return #{retval};" when "d" "context.user_terminate_code = (#{user_terminate_code}); return #{retval};" + when "rust" + "context.user_terminate_code = (#{user_terminate_code}); return #{retval};" end end code = code.gsub(/\$\{context\.(\w+)\}/) do |match| @@ -284,6 +294,8 @@ class Propane "context->#{fieldname}" when "d" "context.#{fieldname}" + when "rust" + "context.#{fieldname}" end end code = code.gsub(/\$\{token\.(\w+)\}/) do |match| @@ -293,6 +305,8 @@ class Propane "token_tree_node->#{fieldname}" when "d" "token_tree_node.#{fieldname}" + when "rust" + "token_tree_node.#{fieldname}" end end if parser @@ -304,6 +318,8 @@ class Propane tree_handle(typename, "_node_id") when "d" tree_handle(typename, "_node_id") + when "rust" + tree_handle(typename, "_node_id") end else case @language @@ -311,6 +327,8 @@ class Propane "_pvalue->v_#{rule.ptypename}" when "d" "_pvalue.v_#{rule.ptypename}" + when "rust" + "(*_pvalue.v_#{rule.ptypename}_mut())" end end end @@ -345,6 +363,8 @@ class Propane "out_token_info->pvalue" when "d" "out_token_info.pvalue" + when "rust" + "out_token_info.pvalue" end else case @language @@ -352,6 +372,8 @@ class Propane "out_token_info->pvalue.v_#{pattern.ptypename}" when "d" "out_token_info.pvalue.v_#{pattern.ptypename}" + when "rust" + "(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())" end end end @@ -361,6 +383,8 @@ class Propane "out_token_info->position" when "d" "out_token_info.position" + when "rust" + "out_token_info.position" end end code = code.gsub(/\$\{end_position\}/) do |match| @@ -369,6 +393,8 @@ class Propane "out_token_info->end_position" when "d" "out_token_info.end_position" + when "rust" + "out_token_info.end_position" end end code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match| @@ -382,6 +408,8 @@ class Propane "context->mode = #{mode_id}u" when "d" "context.mode = #{mode_id}u" + when "rust" + "context.mode = #{mode_id}" end end end @@ -416,6 +444,8 @@ class Propane tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id") when "d" tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id") + when "rust" + tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id") end else case @language @@ -423,6 +453,8 @@ class Propane "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}" when "d" "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}" + when "rust" + "statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()" end end end @@ -446,6 +478,8 @@ class Propane "(#{typename}{context, #{id_expr}})" elsif @language == "c" "((#{typename}){context, #{id_expr}})" + elsif @language == "rust" + "(#{typename} { context, id: #{id_expr} })" else "#{typename}(context, #{id_expr})" end @@ -687,6 +721,113 @@ class Propane out.join("\n") end + # Rust keywords that must be escaped as raw identifiers when used as a + # generated identifier (e.g. a field alias named `type`). + RUST_KEYWORDS = %w[ + as break const continue dyn else enum extern false fn for if impl in let + loop match mod move mut pub ref return static struct trait true type + unsafe use where while async await abstract become box do final macro + override priv typeof unsized virtual yield try gen + ] + + # Escape a name as a Rust raw identifier if it is a reserved keyword. + # + # @param name [String] + # Identifier name. + # + # @return [String] + # Name, escaped as a raw identifier if necessary. + def rust_ident(name) + RUST_KEYWORDS.include?(name) ? "r##{name}" : name + end + + # Map a ptype type string to a valid Rust type. + # + # The default ptype is a C "void *"; for Rust with no declared ptype we use + # the unit type instead. + # + # @param typestring [String] + # ptype type string. + # + # @return [String] + # Rust type string. + def rust_ptype(typestring) + typestring == "void *" ? "()" : typestring + end + + # Generate the Rust tree node record and handle types. + # + # Mirrors the C tree node record plus the C++ handle structs: each rule set + # and the Token node get a handle type ({context, id}) with accessor methods. + # + # @return [String] + # Rust tree node type definitions. + def rust_tree_types + p = @grammar.prefix + out = [] + out << "/** Tree node record. */" + out << "#[derive(Clone, Default)]" + out << "pub struct #{p}node_data_t {" + out << " pub position: #{p}position_t," + out << " pub end_position: #{p}position_t," + out << " pub child_offset: #{p}node_id_t," + out << " pub n_fields: u16," + out << " pub is_token: bool," + out << " pub token: #{p}token_t," + out << " pub pvalue: #{p}value_t," + unless @grammar.token_user_fields.to_s.strip.empty? + out << @grammar.token_user_fields + end + out << "}" + out << "" + out << "/** Tree node handle types. */" + tree_handle_types.each do |t| + out << "#[derive(Clone, Copy)]" + out << "pub struct #{t}<'a> { context: &'a #{p}context_t, id: #{p}node_id_t }" + end + out << "" + # Common accessors for every handle type. + tree_handle_types.each do |t| + out << "impl<'a> #{t}<'a> {" + out << " /** Return whether this handle refers to a valid (non-null) node. */" + out << " pub fn valid(&self) -> bool { self.id != 0 }" + out << " /** Return the node ID (for identity comparison). */" + out << " pub fn node_id(&self) -> #{p}node_id_t { self.id }" + out << " /** Access the underlying node record. */" + out << " pub fn data(&self) -> &'a #{p}node_data_t { &self.context.#{p}tree_nodes[self.id as usize] }" + out << " /** Text position of the first code point spanned by this node. */" + out << " pub fn position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].position }" + out << " /** Text position of the last code point spanned by this node. */" + out << " pub fn end_position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].end_position }" + out << " /** Number of child fields in this node. */" + out << " pub fn n_fields(&self) -> u16 { if self.id != 0 { self.context.#{p}tree_nodes[self.id as usize].n_fields } else { 0 } }" + if t == h_type("Token") + out << " /** Token ID for this token node. */" + out << " pub fn token(&self) -> #{p}token_t { self.context.#{p}tree_nodes[self.id as usize].token }" + out << " /** Parser value associated with this token node. */" + out << " pub fn pvalue(&self) -> #{p}value_t { self.context.#{p}tree_nodes[self.id as usize].pvalue.clone() }" + end + out << "}" + end + out << "" + # Navigation accessors for rule set handles. + tree_node_rule_sets.each do |rule_set| + rtype = h_type(rule_set.name) + out << "impl<'a> #{rtype}<'a> {" + each_tree_field(rule_set) do |rt, field_name, child_type, slot| + out << " /** Access the #{field_name} child node. */" + out << " pub fn #{rust_ident(field_name)}(&self) -> #{child_type}<'a> {" + out << " if self.id == 0 {" + out << " return #{child_type} { context: self.context, id: 0 };" + out << " }" + out << " #{child_type} { context: self.context, id: self.context.#{p}tree_children[self.context.#{p}tree_nodes[self.id as usize].child_offset as usize + #{slot}] }" + out << " }" + end + out << "}" + end + out.join("\n") + end + # Get the lex function to use. # # @return [String] @@ -720,6 +861,8 @@ class Propane "uint8_t" when "d" "ubyte" + when "rust" + "u8" end elsif max <= 0xFFFF case @language @@ -727,11 +870,15 @@ class Propane "uint16_t" when "d" "ushort" + when "rust" + "u16" end else case @language when "c" "uint32_t" + when "rust" + "u32" else "uint" end diff --git a/spec/json_parser.rust.propane b/spec/json_parser.rust.propane new file mode 100644 index 0000000..fcabc2f --- /dev/null +++ b/spec/json_parser.rust.propane @@ -0,0 +1,176 @@ +<
), + Array(Vec), + Number(f64), + StringVal(String), + True, + False, +} + +impl JSONValue { + pub fn id(&self) -> usize { + match self { + JSONValue::Object(_) => JSON_OBJECT, + JSONValue::Array(_) => JSON_ARRAY, + JSONValue::Number(_) => JSON_NUMBER, + JSONValue::StringVal(_) => JSON_STRING, + JSONValue::True => JSON_TRUE, + JSONValue::False => JSON_FALSE, + JSONValue::Null => JSON_NULL, + } + } + pub fn number(&self) -> f64 { + if let JSONValue::Number(n) = self { *n } else { 0.0 } + } + pub fn string(&self) -> &str { + if let JSONValue::StringVal(s) = self { s.as_str() } else { "" } + } + pub fn object_len(&self) -> usize { + if let JSONValue::Object(e) = self { e.len() } else { 0 } + } + pub fn array_len(&self) -> usize { + if let JSONValue::Array(e) = self { e.len() } else { 0 } + } +} +>> + +context_user_fields << + pub string_value: String, +>> + +ptype JSONValue; + +drop /\s+/; +token lbrace /\{/; +token rbrace /\}/; +token lbracket /\[/; +token rbracket /\]/; +token comma /,/; +token colon /:/; +token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ << + let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap(); + $$ = JSONValue::Number(n); +>> +token true << + $$ = JSONValue::True; +>> +token false << + $$ = JSONValue::False; +>> +token null << + $$ = JSONValue::Null; +>> +/"/ << + $mode(string); + ${context.string_value} = String::new(); +>> +string: token string /"/ << + $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value})); + $mode(default); +>> +string: /\\"/ << + ${context.string_value}.push('"'); +>> +string: /\\\\/ << + ${context.string_value}.push('\\'); +>> +string: /\\\// << + ${context.string_value}.push('/'); +>> +string: /\\b/ << + ${context.string_value}.push('\u{0008}'); +>> +string: /\\f/ << + ${context.string_value}.push('\u{000C}'); +>> +string: /\\n/ << + ${context.string_value}.push('\n'); +>> +string: /\\r/ << + ${context.string_value}.push('\r'); +>> +string: /\\t/ << + ${context.string_value}.push('\t'); +>> +string: /\\u[0-9a-fA-F]{4}/ << + /* Not actually going to encode the code point for this example... */ + let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect(); + ${context.string_value}.push_str(&s); +>> +string: /[^\\]/ << + ${context.string_value}.push(match_[0] as char); +>> +Start -> Value << + $$ = $1; +>> +Value -> string << + $$ = $1; +>> +Value -> number << + $$ = $1; +>> +Value -> Object << + $$ = $1; +>> +Value -> Array << + $$ = $1; +>> +Value -> true << + $$ = $1; +>> +Value -> false << + $$ = $1; +>> +Value -> null << + $$ = $1; +>> +Object -> lbrace rbrace << + $$ = JSONValue::Object(Vec::new()); +>> +Object -> lbrace KeyValues rbrace << + $$ = $2; +>> +KeyValues -> KeyValue << + $$ = $1; +>> +KeyValues -> KeyValues comma KeyValue << + let mut obj = $1; + if let JSONValue::Object(kve) = $3 { + if let JSONValue::Object(entries) = &mut obj { + entries.extend(kve); + } + } + $$ = obj; +>> +KeyValue -> string colon Value << + let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() }; + $$ = JSONValue::Object(vec![(name, $3)]); +>> +Array -> lbracket rbracket << + $$ = JSONValue::Array(Vec::new()); +>> +Array -> lbracket Values rbracket << + $$ = $2; +>> +Values -> Value << + $$ = $1; +>> +Values -> Values comma Value << + let mut arr = $1; + if let JSONValue::Array(elems) = &mut arr { + elems.push($3); + } + $$ = arr; +>> diff --git a/spec/macros.rust.propane b/spec/macros.rust.propane new file mode 100644 index 0000000..8515d73 --- /dev/null +++ b/spec/macros.rust.propane @@ -0,0 +1,80 @@ +<< +fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize { + loop { + if context.expanding { + let ei = context.expand_i; + context.expand_i += 1; + if context.expand_i >= context.token_infos.len() { + context.expanding = false; + } + *out_token_info = context.token_infos[ei].clone(); + return P_SUCCESS; + } + + let lex_result = p_lex(context, out_token_info); + if lex_result != P_SUCCESS { + return lex_result; + } + + if out_token_info.token == TOKEN_macro { + context.defining = true; + } else if out_token_info.token == TOKEN_macroname { + if !context.defining { + context.expanding = true; + context.expand_i = 0; + continue; + } + } else if out_token_info.token == TOKEN_lbrace { + if context.defining { + /* Capture the macro body tokens (up to the closing '}'). */ + let mut infos: Vec = Vec::new(); + loop { + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(context, &mut ti)); + if ti.token == TOKEN_rbrace { + break; + } + infos.push(ti); + } + context.token_infos = infos; + context.defining = false; + } + } else { + context.defining = false; + } + return lex_result; + } +} +>> + +context_user_fields << + pub defining: bool, + pub expanding: bool, + pub expand_i: usize, + pub token_infos: Vec, + pub nums: Vec, +>> + +ptype i64; + +lex_fn mylexfn; + +drop /\s+/; +token lbrace /\{/; +token rbrace /\}/; +token plus /\+/; +token macro; +token macroname /@[a-zA-Z_]\w*/; +token num /\d+/ << + let mut v: i64 = 0; + for c in match_ { v = v * 10 + (*c - b'0') as i64; } + $$ = v; +>> + +Start -> Statements; +Statements -> ; +Statements -> Statement Statements; +Statement -> Add; +Statement -> MacroStart; +Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >> +MacroStart -> macro macroname lbrace; diff --git a/spec/parse_inner_nested.rust.propane b/spec/parse_inner_nested.rust.propane new file mode 100644 index 0000000..27763fb --- /dev/null +++ b/spec/parse_inner_nested.rust.propane @@ -0,0 +1,41 @@ +<< +fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize { + let result = p_lex(context, out_token_info); + if result != P_SUCCESS { + return result; + } + if out_token_info.token == TOKEN_lparen { + /* Reentrant nested parse of the parenthesized sub-expression. */ + let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]); + if inner_result != P_SUCCESS { + return inner_result; + } + let value = p_result_Start(context); + /* p_parse_inner rewound the input so ')' was not consumed; consume it. */ + let mut rparen_info = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info)); + assert_eq!(TOKEN_rparen, rparen_info.token); + out_token_info.token = TOKEN_num; + out_token_info.pvalue = p_value(value); + } + P_SUCCESS +} +>> + +ptype i64; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/ << + let mut v: i64 = 0; + for c in match_ { v = v * 10 + (*c - b'0') as i64; } + $$ = v; +>> + +Start -> Expr << $$ = $1; >> +Expr -> num << $$ = $1; >> +Expr -> Expr plus num << $$ = $1 + $3; >> diff --git a/spec/parse_inner_nested_tree.rust.propane b/spec/parse_inner_nested_tree.rust.propane new file mode 100644 index 0000000..04d1c60 --- /dev/null +++ b/spec/parse_inner_nested_tree.rust.propane @@ -0,0 +1,44 @@ +<< +fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize { + let result = p_lex(context, out_token_info); + if result != P_SUCCESS { + return result; + } + if out_token_info.token == TOKEN_lparen { + let start_position = out_token_info.position; + let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]); + if inner_result != P_SUCCESS { + return inner_result; + } + /* Read the inner subtree's span before re-borrowing context to lex. */ + let inner = p_result_Start(context); + assert!(inner.valid()); + let inner_start_col = inner.position().col; + let inner_end_col = inner.end_position().col; + let mut rparen_info = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info)); + assert_eq!(TOKEN_rparen, rparen_info.token); + assert_eq!(start_position.col + 1, inner_start_col); + assert_eq!(rparen_info.position.col - 1, inner_end_col); + /* Synthesize a num token spanning the whole "( ... )" group. */ + out_token_info.token = TOKEN_num; + out_token_info.position = start_position; + out_token_info.end_position = rparen_info.end_position; + } + P_SUCCESS +} +>> + +tree; + +lex_fn mylexfn; + +drop /\s+/; +token lparen /\(/; +token rparen /\)/; +token plus /\+/; +token num /\d+/; + +Start -> Expr; +Expr -> num; +Expr -> Expr plus num; diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 51499ee..2d0f208 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -53,7 +53,7 @@ EOF if options[:args] command += options[:args] else - command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log] + command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{lang_ext(options[:language])} --log spec/run/testparser#{options[:name]}.log] end command += (options[:extra_args] || []) if (options[:capture]) @@ -65,9 +65,16 @@ EOF end end + # Map a spec language name to the generated parser source file extension. + def lang_ext(language) + language == "rust" ? "rs" : language + end + def compile(test_files, options = {}) test_files = Array(test_files).map do |test_file| - if !File.exist?(test_file) && test_file.end_with?(".cpp") + if test_file.end_with?(".rust") + test_file.sub(%r{\.rust$}, ".rs") + elsif !File.exist?(test_file) && test_file.end_with?(".cpp") test_file.sub(%r{\.cpp$}, ".c") else test_file @@ -75,7 +82,7 @@ EOF end options[:parsers] ||= [""] parsers = options[:parsers].map do |name| - "spec/run/testparser#{name}.#{options[:language]}" + "spec/run/testparser#{name}.#{lang_ext(options[:language])}" end case options[:language] when "c" @@ -84,6 +91,21 @@ EOF command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"] when "d" command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"] + when "rust" + # Compile each generated parser to an rlib, then compile the test crate + # against them. Safe Rust needs no valgrind, but it is run anyway. + externs = [] + options[:parsers].each do |name| + crate = "testparser#{name}" + rlib = "spec/run/lib#{crate}.rlib" + rustc = [*%w[rustc --edition 2021 --crate-type=rlib -A warnings], + "--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"] + expect(system(*rustc)).to be_truthy + externs += ["--extern", "#{crate}=#{rlib}"] + end + # rustc takes a single crate root; any additional Rust test files are + # expected to be pulled in as modules or provided by the parser crate. + command = [*%w[rustc --edition 2021 -A warnings -o spec/run/testparser], *externs, test_files.first] end result = system(*command) expect(result).to be_truthy @@ -285,7 +307,7 @@ EOF expect(results.status).to_not eq 0 end - %w[d c cpp].each do |language| + %w[d c cpp rust].each do |language| context "#{language.upcase} language" do @@ -335,6 +357,16 @@ token int /\\d+/ << $$ = v; >> Start -> int << $$ = $1; >> +EOF + when "rust" + write_grammar <> +Start -> int << $$ = $1; >> EOF end run_propane(language: language) @@ -429,6 +461,31 @@ E3 -> E4 << $$ = $1; >> E3 -> E3 power E4 << $$ = pow($1, $3); >> E4 -> integer << $$ = $1; >> E4 -> lparen E1 rparen << $$ = $2; >> +EOF + when "rust" + write_grammar <> +token lparen /\\(/; +token rparen /\\)/; +drop /\\s+/; + +Start -> E1 << $$ = $1; >> +E1 -> E2 << $$ = $1; >> +E1 -> E1 plus E2 << $$ = $1 + $3; >> +E2 -> E3 << $$ = $1; >> +E2 -> E2 times E3 << $$ = $1 * $3; >> +E3 -> E4 << $$ = $1; >> +E3 -> E3 power E4 << $$ = $1.pow($3 as u32); >> +E4 -> integer << $$ = $1; >> +E4 -> lparen E1 rparen << $$ = $2; >> EOF end run_propane(language: language) @@ -505,6 +562,16 @@ token def; Start -> Abcs def; Abcs -> ; Abcs -> abc Abcs; +EOF + when "rust" + write_grammar <> +token def; +Start -> Abcs def; +Abcs -> ; +Abcs -> abc Abcs; EOF end run_propane(language: language) @@ -541,6 +608,12 @@ import std.stdio; token abc; /def/ << writeln("def!"); >> Start -> abc; +EOF + when "rust" + write_grammar <> +Start -> abc; EOF end run_propane(language: language) @@ -583,6 +656,16 @@ token abc; return $token(abc); >> Start -> abc; +EOF + when "rust" + write_grammar <> +/ghi/ << + println!("ghi!"); + return $token(abc); +>> +Start -> abc; EOF end run_propane(language: language) @@ -641,6 +724,25 @@ string: /"/ << return $token(string); >> Start -> abc string def; +EOF + when "rust" + write_grammar <> +string: /[^"]+/ << + println!("captured string"); +>> +string: /"/ << + $mode(default); + return $token(string); +>> +Start -> abc string def; EOF end run_propane(language: language) @@ -699,6 +801,24 @@ default, identonly: drop /\\s+/; Start -> abc dot ident << writeln("ident: ", $3); >> +EOF + when "rust" + write_grammar <> +token dot /\\./ << + $mode(identonly); +>> +default, identonly: drop /\\s+/; +Start -> abc dot ident << + println!("ident: {}", $3 as char); +>> EOF end run_propane(language: language) @@ -736,6 +856,14 @@ token b; Start -> A B << writeln("Start!"); >> A -> a << writeln("A!"); >> B -> b << writeln("B!"); >> +EOF + when "rust" + write_grammar < A B << println!("Start!"); >> +A -> a << println!("A!"); >> +B -> b << println!("B!"); >> EOF end run_propane(language: language) @@ -843,6 +971,37 @@ Start -> A:ay B:bee C << A -> a; B -> b; C -> << ${context.c_is_null} = ($$.valid) ? 0 : 1; >> +EOF + when "rust" + write_grammar <> +ptype i64; +token a << $$ = 11; >> +token b << $$ = 22; >> +Start -> A:ay B:bee C << + ${context.start_n_fields} = $$.n_fields() as i64; + ${context.start_a_value} = $$.pA().pToken1().pvalue(); + ${context.a_value} = $1.pToken1().pvalue(); + ${context.b_value} = $2.pToken1().pvalue(); + ${context.b_token} = $2.pToken1().token(); + ${context.c_field_is_null} = if $$.pC().valid() { 0 } else { 1 }; + ${context.alias_a_value} = ${ay}.pToken1().pvalue(); + ${context.alias_b_value} = ${bee}.pToken1().pvalue(); +>> +A -> a; +B -> b; +C -> << ${context.c_is_null} = if $$.valid() { 0 } else { 1 }; >> EOF end run_propane(language: language) @@ -853,12 +1012,15 @@ EOF end it "parses lists" do + ptype = language == "d" ? "uint" : (language == "rust" ? "u32" : "uint32_t") + zero = language == "rust" ? "0" : "0u" + one = language == "rust" ? "1" : "1u" write_grammar < As << $$ = $1; >> -As -> << $$ = 0u; >> -As -> As a << $$ = $1 + 1u; >> +As -> << $$ = #{zero}; >> +As -> As a << $$ = $1 + #{one}; >> EOF run_propane(language: language) compile("spec/test_parsing_lists.#{language}", language: language) @@ -912,6 +1074,13 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ << writeln("Matched token is ", match); >> Start -> id; +EOF + when "rust" + write_grammar <> +Start -> id; EOF end run_propane(language: language) @@ -945,6 +1114,16 @@ token word /[a-z]+/ << Start -> word << $$ = $1; >> +EOF + when "rust" + write_grammar <> +Start -> word << + $$ = $1; +>> EOF end run_propane(language: language) @@ -1097,6 +1276,25 @@ Start -> Words; Words -> ; Words -> word Words; Words -> stop Words; +EOF + when "rust" + write_grammar <> +drop /\\s+/; +token word /[a-z]+/ << + ${context.last_start} = ${position}; + ${context.last_end} = ${end_position}; +>> +token stop /!/ << + $terminate(42); +>> +Start -> Words; +Words -> ; +Words -> word Words; +Words -> stop Words; EOF end run_propane(language: language) @@ -1159,6 +1357,21 @@ tokenid t; return $token(t); >> Start -> t; +EOF + when "rust" + write_grammar <> +/\\b/ << println!("B"); >> +/\\t/ << println!("T"); >> +/\\n/ << println!("N"); >> +/\\v/ << println!("V"); >> +/\\f/ << println!("F"); >> +/\\r/ << println!("R"); >> +/t/ << + return $token(t); +>> +Start -> t; EOF end run_propane(language: language) @@ -1195,7 +1408,7 @@ EOF write_grammar <> token b << $$ = 22; >> @@ -1239,7 +1452,7 @@ tree; tree_prefix P ; tree_suffix S; -ptype int; +ptype #{language == "rust" ? "i64" : "int"}; token a << $$ = 11; >> token b << $$ = 22; >> @@ -1320,6 +1533,24 @@ Start -> a? b R? << >> R -> c d << $$ = "cd"; >> R (string) -> d c << $$ = "dc"; >> +EOF + elsif language == "rust" + write_grammar <> +token b << $$ = 2; >> +token c << $$ = 3; >> +token d << $$ = 4; >> +Start -> a? b R? << + println!("a: {}", $1); + println!("b: {}", $2); + println!("R: {}", $3); +>> +R -> c d << $$ = "cd".to_string(); >> +R (string) -> d c << $$ = "dc".to_string(); >> EOF else write_grammar < >> +token a; +token b; +token c; +token d; +Start -> a? b R?; +R -> c d; +R -> d c; +EOF + end + if language == "rust" + write_grammar < >> +token a; +token b; +token c; +token d; +Start -> a?:a b R?:r; +R -> c d; +R -> d c; +EOF + end + if language == "rust" + write_grammar <> +drop /\\s+/; +Start -> id:first id:second << + println!("first is {}", ${first}); + println!("second is {}", ${second}); +>> +EOF + elsif language == "d" write_grammar <> +drop /\\s+/; +Start -> id; +Start -> Foo; +Start -> id:first id:second << + println!("first is {}", ${first}); + println!("second is {}", ${second}); +>> +Foo -> ; +EOF + elsif language == "d" write_grammar <> token b << $$ = 2; >> token c << $$ = 3; >> @@ -1660,7 +1943,7 @@ EOF it "supports parse_inner APIs that treat provided tokens as follow tokens" do write_grammar <> token b << $$ = 2; >> Start -> Y << $$ = $1; >> @@ -1675,7 +1958,7 @@ EOF it "parse_inner APIs block success when the outer rule is unfinished" do write_grammar <> token b << $$ = 2; >> token c << $$ = 3; >> @@ -1691,7 +1974,7 @@ EOF it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do write_grammar <> token b << $$ = 2; >> token c << $$ = 3; >> @@ -1795,7 +2078,16 @@ EOF end it "executes code blocks associated with drop statements" do - if language == "d" + if language == "rust" + write_grammar <> +token a; +Start -> a; +EOF + elsif language == "d" write_grammar <> +drop /\\s+/; +drop /#(.*)\\n/ << + ${context.comments} += std::str::from_utf8(match_).unwrap(); +>> +token a << + ${context.acount} += 1; +>> +Start -> As; +As -> ; +As -> a As; +EOF + elsif language == "d" write_grammar <> +token_user_fields << + pub comments: String, +>> +on_token_node << + ${token.comments} = std::mem::take(&mut ${context.comments}); +>> +tree; +drop /\\s+/; +drop /#(.*)\\n/ << + ${context.comments} += std::str::from_utf8(match_).unwrap(); +>> +token id /\\w+/; +Start -> IDs; +IDs -> ; +IDs -> id:id IDs; +EOF + elsif language == "d" write_grammar < usize { + let mut result = P_SUCCESS; + if context.count > 0 { + out_token_info.token = TOKEN_a; + out_token_info.pvalue = p_value(context.count); + context.count -= 1; + } else { + result = p_lex(context, out_token_info); + if out_token_info.token == TOKEN_c { + context.count = 3; + } + } + result +} +>> + +context_user_fields << + pub count: usize, +>> +ptype usize; +lex_fn mylexfn; + +token a << $$ = 7; >> +token b << $$ = 8; >> +token c << $$ = 9; >> +Start -> << $$ = 0; >> +Start -> Start ID << $$ = ($1 << 4) | $2; >> +ID -> a << $$ = $1; >> +ID -> b << $$ = $1; >> +ID -> c << $$ = $1; >> +EOF + elsif language == "d" write_grammar <> +token flt (float) /f/ << $$ = 1.5; >> +token str (string) /s/ << $$ = "hello".to_string(); >> +Start -> num flt str; +EOF + elsif language == "d" write_grammar < num; +Start -> ident Num << + println!("ident start: {}, {}", ${1.position}.row, ${1.position}.col); + println!("ident end: {}, {}", ${1.end_position}.row, ${1.end_position}.col); + println!("Num start: {}, {}", ${2.position}.row, ${2.position}.col); + println!("Num end: {}, {}", ${2.end_position}.row, ${2.end_position}.col); + println!("Start start: {}, {}", ${$.position}.row, ${$.position}.col); + println!("Start end: {}, {}", ${$.end_position}.row, ${$.end_position}.col); +>> +R -> Empty tok2 << + println!("Empty start: {}, {}", ${1.position}.row, ${1.position}.col); + println!("Empty end: {}, {}", ${1.end_position}.row, ${1.end_position}.col); + println!("tok2 start: {}, {}", ${2.position}.row, ${2.position}.col); + println!("tok2 end: {}, {}", ${2.end_position}.row, ${2.end_position}.col); + println!("R start: {}, {}", ${$.position}.row, ${$.position}.col); + println!("R end: {}, {}", ${$.end_position}.row, ${$.end_position}.col); +>> +R -> tok1 Empty << + println!("tok1 start: {}, {}", ${1.position}.row, ${1.position}.col); + println!("tok1 end: {}, {}", ${1.end_position}.row, ${1.end_position}.col); + println!("Empty start: {}, {}", ${2.position}.row, ${2.position}.col); + println!("Empty end: {}, {}", ${2.end_position}.row, ${2.end_position}.col); + println!("R2 start: {}, {}", ${$.position}.row, ${$.position}.col); + println!("R2 end: {}, {}", ${$.end_position}.row, ${$.end_position}.col); +>> +Empty -> ; +Start -> R << + println!("StartR start: {}, {}", ${$.position}.row, ${$.position}.col); + println!("StartR end: {}, {}", ${$.end_position}.row, ${$.end_position}.col); +>> +Start -> Empty << + println!("StartEmpty start: {}, {}", ${$.position}.row, ${$.position}.col); + println!("StartEmpty end: {}, {}", ${$.end_position}.row, ${$.end_position}.col); +>> +EOF + elsif language == "d" write_grammar < usize { + loop { + let result = p_lex(context, out_token_info); + if result != P_SUCCESS { + return result; + } + if out_token_info.token == TOKEN_repeat { + let mut count_info = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(context, &mut count_info)); + assert_eq!(TOKEN_num, count_info.token); + let mut brace_info = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info)); + assert_eq!(TOKEN_lbrace, brace_info.token); + context.remaining = p_value_get(&count_info.pvalue); + context.body_index = p_input_index(context); + context.body_position = p_position(context); + continue; + } + if out_token_info.token == TOKEN_rbrace { + if context.remaining > 1 { + context.remaining -= 1; + let bi = context.body_index; + let bp = context.body_position; + p_set_input_index(context, bi); + p_set_position(context, bp); + continue; + } + context.remaining = 0; + continue; + } + if out_token_info.token == TOKEN_num { + context.num_cols.push(out_token_info.position.col); + } + return result; + } +} +>> + +context_user_fields << + pub nums: Vec, + pub num_cols: Vec, + pub remaining: i64, + pub body_index: usize, + pub body_position: p_position_t, +>> + +ptype i64; + +lex_fn mylexfn; + +drop /\s+/; +token repeat /repeat/; +token lbrace /\{/; +token rbrace /\}/; +token plus /\+/; +token num /\d+/ << + let mut v: i64 = 0; + for c in match_ { v = v * 10 + (*c - b'0') as i64; } + $$ = v; +>> + +Start -> Statements; +Statements -> ; +Statements -> Statement Statements; +Statement -> Add; +Add -> num plus num << ${context.nums}.push($1 + $3); >> diff --git a/spec/test_basic_math_grammar.rs b/spec/test_basic_math_grammar.rs new file mode 100644 index 0000000..15ef591 --- /dev/null +++ b/spec/test_basic_math_grammar.rs @@ -0,0 +1,16 @@ +use testparser::*; + +fn main() { + let cases: [(&[u8], u64); 4] = [ + (b"1 + 2 * 3 + 4", 11), + (b"1 * 2 ** 4 * 3", 48), + (b"(1 + 2) * 3 + 4", 13), + (b"(2 * 2) ** 3 + 4 + 5", 73), + ]; + for (input, expected) in cases { + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert_eq!(expected, p_result(&context)); + p_context_delete(context); + } +} diff --git a/spec/test_custom_lex_fn.rs b/spec/test_custom_lex_fn.rs new file mode 100644 index 0000000..f8339b1 --- /dev/null +++ b/spec/test_custom_lex_fn.rs @@ -0,0 +1,8 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"cbacba"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(0x932187932187, p_result(&c)); + p_context_delete(c); +} diff --git a/spec/test_drop_code_block.rs b/spec/test_drop_code_block.rs new file mode 100644 index 0000000..5c20b5e --- /dev/null +++ b/spec/test_drop_code_block.rs @@ -0,0 +1,7 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b" # comment 1\n# comment 2\na\n"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); +} diff --git a/spec/test_error_positions.rs b/spec/test_error_positions.rs new file mode 100644 index 0000000..e045870 --- /dev/null +++ b/spec/test_error_positions.rs @@ -0,0 +1,35 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"a 42"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"a\n123\na a"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c)); + assert_eq!(3, p_position(&c).row); + assert_eq!(4, p_position(&c).col); + assert_eq!(TOKEN_a, p_token(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"12"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c)); + assert_eq!(1, p_position(&c).row); + assert_eq!(1, p_position(&c).col); + assert_eq!(TOKEN_num, p_token(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"a 12\n\nab"); + assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c)); + assert_eq!(3, p_position(&c).row); + assert_eq!(2, p_position(&c).col); + p_context_delete(c); + + let mut c = p_context_new(b"a 12\n\na\n\n77\na \xAA"); + assert_eq!(P_DECODE_ERROR, p_parse(&mut c)); + assert_eq!(6, p_position(&c).row); + assert_eq!(5, p_position(&c).col); + assert_eq!("a", p_token_names[TOKEN_a as usize]); + assert_eq!("num", p_token_names[TOKEN_num as usize]); + p_context_delete(c); +} diff --git a/spec/test_field_aliases.rs b/spec/test_field_aliases.rs new file mode 100644 index 0000000..421c387 --- /dev/null +++ b/spec/test_field_aliases.rs @@ -0,0 +1,7 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"foo1\nbar2"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); +} diff --git a/spec/test_input_index.rs b/spec/test_input_index.rs new file mode 100644 index 0000000..2b00a5e --- /dev/null +++ b/spec/test_input_index.rs @@ -0,0 +1,28 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"ab"); + assert_eq!(0, p_input_index(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"a b"); + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + assert_eq!(1, p_input_index(&c)); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_b, ti.token); + assert_eq!(3, p_input_index(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + assert_eq!(2, p_input_index(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"abb"); + let follow = [TOKEN_b]; + assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow)); + assert_eq!(2, p_input_index(&c)); + p_context_delete(c); +} diff --git a/spec/test_lexer.rs b/spec/test_lexer.rs new file mode 100644 index 0000000..741236b --- /dev/null +++ b/spec/test_lexer.rs @@ -0,0 +1,49 @@ +use testparser::*; + +fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) { + assert_eq!(row, ti.position.row); + assert_eq!(col, ti.position.col); + assert_eq!(erow, ti.end_position.row); + assert_eq!(ecol, ti.end_position.col); + assert_eq!(len, ti.length); + assert_eq!(token, ti.token); +} + +fn main() { + let mut cp: p_code_point_t = 0; + let mut cpl: u8 = 0; + + assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl)); + assert_eq!('5' as u32, cp); + assert_eq!(1, cpl); + + assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl)); + + assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl)); + assert_eq!(0xA9, cp); + assert_eq!(2, cpl); + + assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl)); + assert_eq!(0x1F9E1, cp); + assert_eq!(4, cpl); + + assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl)); + assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl)); + assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl)); + + let mut context = p_context_new(b"5 + 4 * \n677 + 567"); + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF); + p_context_delete(context); + + let mut context = p_context_new(b""); + assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF); + p_context_delete(context); +} diff --git a/spec/test_lexer_match_text.rs b/spec/test_lexer_match_text.rs new file mode 100644 index 0000000..6ee055b --- /dev/null +++ b/spec/test_lexer_match_text.rs @@ -0,0 +1,8 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"identifier_123"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass1"); + p_context_delete(context); +} diff --git a/spec/test_lexer_modes.rs b/spec/test_lexer_modes.rs new file mode 100644 index 0000000..29d9072 --- /dev/null +++ b/spec/test_lexer_modes.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"abc \"a string\" def"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass1"); + p_context_delete(context); + + let mut context = p_context_new(b"abc \"abc def\" def"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass2"); + p_context_delete(context); +} diff --git a/spec/test_lexer_multiple_modes.rs b/spec/test_lexer_multiple_modes.rs new file mode 100644 index 0000000..ee80c97 --- /dev/null +++ b/spec/test_lexer_multiple_modes.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"abc.def"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass1"); + p_context_delete(context); + + let mut context = p_context_new(b"abc . abc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass2"); + p_context_delete(context); +} diff --git a/spec/test_lexer_positions.rs b/spec/test_lexer_positions.rs new file mode 100644 index 0000000..c90979b --- /dev/null +++ b/spec/test_lexer_positions.rs @@ -0,0 +1,38 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"abc\n defg hi\n!"); + let mut ti = p_token_info_t::default(); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_word, ti.token); + assert_eq!(1, c.last_start.row); + assert_eq!(1, c.last_start.col); + assert_eq!(1, c.last_end.row); + assert_eq!(3, c.last_end.col); + assert_eq!(c.last_start.row, ti.position.row); + assert_eq!(c.last_start.col, ti.position.col); + assert_eq!(c.last_end.row, ti.end_position.row); + assert_eq!(c.last_end.col, ti.end_position.col); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_word, ti.token); + assert_eq!(2, c.last_start.row); + assert_eq!(3, c.last_start.col); + assert_eq!(2, c.last_end.row); + assert_eq!(6, c.last_end.col); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_word, ti.token); + assert_eq!(2, c.last_start.row); + assert_eq!(8, c.last_start.col); + assert_eq!(2, c.last_end.row); + assert_eq!(9, c.last_end.col); + + assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti)); + assert_eq!(42, p_user_terminate_code(&c)); + assert_eq!(3, p_position(&c).row); + assert_eq!(1, p_position(&c).col); + + p_context_delete(c); +} diff --git a/spec/test_lexer_result_value.rs b/spec/test_lexer_result_value.rs new file mode 100644 index 0000000..da7ecd3 --- /dev/null +++ b/spec/test_lexer_result_value.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"x"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert_eq!(1, p_result(&context)); + p_context_delete(context); + + let mut context = p_context_new(b"fabulous"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert_eq!(8, p_result(&context)); + p_context_delete(context); +} diff --git a/spec/test_lexer_unknown_character.rs b/spec/test_lexer_unknown_character.rs new file mode 100644 index 0000000..7015dc4 --- /dev/null +++ b/spec/test_lexer_unknown_character.rs @@ -0,0 +1,12 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"x"); + assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context)); + p_context_delete(context); + + let mut context = p_context_new(b"123"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert_eq!(123, p_result(&context)); + p_context_delete(context); +} diff --git a/spec/test_macros.rs b/spec/test_macros.rs new file mode 100644 index 0000000..89ab7f4 --- /dev/null +++ b/spec/test_macros.rs @@ -0,0 +1,9 @@ +use testparser::*; + +fn main() { + let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n"; + let mut c = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(vec![166, 223, 88, 223], c.nums); + p_context_delete(c); +} diff --git a/spec/test_match_backslashes.rs b/spec/test_match_backslashes.rs new file mode 100644 index 0000000..68f5d8b --- /dev/null +++ b/spec/test_match_backslashes.rs @@ -0,0 +1,7 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); +} diff --git a/spec/test_multiple_parsers.rs b/spec/test_multiple_parsers.rs new file mode 100644 index 0000000..b589f90 --- /dev/null +++ b/spec/test_multiple_parsers.rs @@ -0,0 +1,12 @@ +use testparsermyp1 as m1; +use testparsermyp2 as m2; + +fn main() { + let mut context1 = m1::myp1_context_new(b"a\n1"); + assert_eq!(m1::P_SUCCESS, m1::myp1_parse(&mut context1)); + m1::myp1_context_delete(context1); + + let mut context2 = m2::myp2_context_new(b"bcb"); + assert_eq!(m2::P_SUCCESS, m2::myp2_parse(&mut context2)); + m2::myp2_context_delete(context2); +} diff --git a/spec/test_named_optional_rule_component_tree.rs b/spec/test_named_optional_rule_component_tree.rs new file mode 100644 index 0000000..f55332f --- /dev/null +++ b/spec/test_named_optional_rule_component_tree.rs @@ -0,0 +1,43 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"b"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(!start.a().valid()); + assert!(start.pToken2().valid()); + assert_eq!(TOKEN_b, start.pToken2().token()); + assert!(!start.pR3().valid()); + assert!(!start.pR().valid()); + assert!(!start.r().valid()); + } + p_context_delete(context); + + let mut context = p_context_new(b"abcd"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.a().valid()); + assert_eq!(TOKEN_a, start.pToken1().token()); + assert!(start.pToken2().valid()); + assert!(start.pR3().valid()); + assert!(start.pR().valid()); + assert!(start.r().valid()); + assert_eq!(start.pR().node_id(), start.pR3().node_id()); + assert_eq!(start.pR().node_id(), start.r().node_id()); + assert_eq!(TOKEN_c, start.pR().pToken1().token()); + } + p_context_delete(context); + + let mut context = p_context_new(b"bdc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(!start.a().valid()); + assert!(start.pToken2().valid()); + assert!(start.r().valid()); + assert_eq!(TOKEN_d, start.pR().pToken1().token()); + } + p_context_delete(context); +} diff --git a/spec/test_optional_rule_component.rs b/spec/test_optional_rule_component.rs new file mode 100644 index 0000000..a5080ee --- /dev/null +++ b/spec/test_optional_rule_component.rs @@ -0,0 +1,9 @@ +use testparser::*; + +fn main() { + for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] { + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); + } +} diff --git a/spec/test_optional_rule_component_tree.rs b/spec/test_optional_rule_component_tree.rs new file mode 100644 index 0000000..a1b18e7 --- /dev/null +++ b/spec/test_optional_rule_component_tree.rs @@ -0,0 +1,40 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"b"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(!start.pToken1().valid()); + assert!(start.pToken2().valid()); + assert_eq!(TOKEN_b, start.pToken2().token()); + assert!(!start.pR3().valid()); + assert!(!start.pR().valid()); + } + p_context_delete(context); + + let mut context = p_context_new(b"abcd"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.pToken1().valid()); + assert_eq!(TOKEN_a, start.pToken1().token()); + assert!(start.pToken2().valid()); + assert!(start.pR3().valid()); + assert!(start.pR().valid()); + assert_eq!(start.pR().node_id(), start.pR3().node_id()); + assert_eq!(TOKEN_c, start.pR().pToken1().token()); + } + p_context_delete(context); + + let mut context = p_context_new(b"bdc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(!start.pToken1().valid()); + assert!(start.pToken2().valid()); + assert!(start.pR().valid()); + assert_eq!(TOKEN_d, start.pR().pToken1().token()); + } + p_context_delete(context); +} diff --git a/spec/test_parse_inner.rs b/spec/test_parse_inner.rs new file mode 100644 index 0000000..d86f6fe --- /dev/null +++ b/spec/test_parse_inner.rs @@ -0,0 +1,30 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"a"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + assert_eq!(1, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b])); + assert_eq!(1, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[])); + p_context_delete(c); + + let mut c = p_context_new(b"a"); + assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b])); + assert_eq!(1, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF])); + p_context_delete(c); +} diff --git a/spec/test_parse_inner_nested.rs b/spec/test_parse_inner_nested.rs new file mode 100644 index 0000000..a3ae912 --- /dev/null +++ b/spec/test_parse_inner_nested.rs @@ -0,0 +1,17 @@ +use testparser::*; + +fn eval(input: &[u8]) -> i64 { + let mut c = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + let v = p_result(&c); + p_context_delete(c); + v +} + +fn main() { + assert_eq!(5, eval(b"2 + 3")); + assert_eq!(3, eval(b"(1 + 2)")); + assert_eq!(14, eval(b"2 + (3 + 4) + 5")); + assert_eq!(37, eval(b"2 + (10 + (20 + 5))")); + assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))")); +} diff --git a/spec/test_parse_inner_nested_tree.rs b/spec/test_parse_inner_nested_tree.rs new file mode 100644 index 0000000..872c9d7 --- /dev/null +++ b/spec/test_parse_inner_nested_tree.rs @@ -0,0 +1,36 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"(3 + 4) + (5 + 6)"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + { + let tree = p_result(&c); + assert!(tree.valid()); + + /* Start -> Expr, where the top Expr is "Expr plus num". */ + let top = tree.pExpr(); + assert!(top.valid()); + assert!(top.pExpr().valid()); + assert!(top.pToken2().valid()); + assert!(top.pToken3().valid()); + + /* The '+' joining the two groups is at column 9. */ + assert_eq!(1, top.pToken2().position().row); + assert_eq!(9, top.pToken2().position().col); + + /* Right operand: synthesized num for "(5 + 6)", columns 11..17. */ + assert_eq!(11, top.pToken3().position().col); + assert_eq!(17, top.pToken3().end_position().col); + + /* Left operand: synthesized num for "(3 + 4)", columns 1..7. */ + let left = top.pExpr(); + assert!(left.pToken1().valid()); + assert_eq!(1, left.pToken1().position().col); + assert_eq!(7, left.pToken1().end_position().col); + + /* The whole tree spans columns 1..17. */ + assert_eq!(1, tree.position().col); + assert_eq!(17, tree.end_position().col); + } + p_context_delete(c); +} diff --git a/spec/test_parse_inner_recursive.rs b/spec/test_parse_inner_recursive.rs new file mode 100644 index 0000000..d7f8e4f --- /dev/null +++ b/spec/test_parse_inner_recursive.rs @@ -0,0 +1,31 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"c"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + assert_eq!(3, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"acb"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + assert_eq!(3, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"ac"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"ac"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF])); + p_context_delete(c); + + let mut c = p_context_new(b"acb"); + assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b])); + assert_eq!(3, p_result_Start(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"c"); + assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b])); + assert_eq!(3, p_result_Start(&c)); + p_context_delete(c); +} diff --git a/spec/test_parse_inner_shared.rs b/spec/test_parse_inner_shared.rs new file mode 100644 index 0000000..8b01d30 --- /dev/null +++ b/spec/test_parse_inner_shared.rs @@ -0,0 +1,47 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"aba"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"abb"); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse_R1(&mut c)); + assert_eq!(11, p_result_R1(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"abb"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"abb"); + assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b])); + assert_eq!(11, p_result_R1(&c)); + let pos = p_position(&c); + assert_eq!(1, pos.row); + assert_eq!(3, pos.col); + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_b, ti.token); + assert_eq!(3, ti.position.col); + p_context_delete(c); + + let mut c = p_context_new(b"aba"); + assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a])); + assert_eq!(11, p_result_R1(&c)); + let pos = p_position(&c); + assert_eq!(3, pos.col); + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + p_context_delete(c); + + let mut c = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[])); + assert_eq!(11, p_result_R1(&c)); + p_context_delete(c); +} diff --git a/spec/test_parse_inner_tree.rs b/spec/test_parse_inner_tree.rs new file mode 100644 index 0000000..7d8062f --- /dev/null +++ b/spec/test_parse_inner_tree.rs @@ -0,0 +1,36 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse_R1(&mut c)); + { + let tree = p_result_R1(&c); + assert!(tree.valid()); + assert!(tree.pToken1().valid()); + assert_eq!(TOKEN_a, tree.pToken1().token()); + assert!(tree.pToken2().valid()); + assert_eq!(TOKEN_b, tree.pToken2().token()); + } + p_context_delete(c); + + let mut c = p_context_new(b"abb"); + assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b])); + { + let tree = p_result_R1(&c); + assert!(tree.valid()); + assert_eq!(TOKEN_a, tree.pToken1().token()); + assert_eq!(1, tree.pToken1().position().row); + assert_eq!(1, tree.pToken1().position().col); + assert_eq!(TOKEN_b, tree.pToken2().token()); + assert_eq!(2, tree.pToken2().position().col); + assert_eq!(1, tree.position().col); + assert_eq!(2, tree.end_position().col); + } + let pos = p_position(&c); + assert_eq!(3, pos.col); + let mut ti = p_token_info_t::default(); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_b, ti.token); + assert_eq!(3, ti.position.col); + p_context_delete(c); +} diff --git a/spec/test_parser_identical_rules_lookahead.rs b/spec/test_parser_identical_rules_lookahead.rs new file mode 100644 index 0000000..40d82ca --- /dev/null +++ b/spec/test_parser_identical_rules_lookahead.rs @@ -0,0 +1,9 @@ +use testparser::*; + +fn main() { + for input in [&b"aba"[..], &b"abb"[..]] { + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); + } +} diff --git a/spec/test_parser_rule_from_multiple_states.rs b/spec/test_parser_rule_from_multiple_states.rs new file mode 100644 index 0000000..6de5a7b --- /dev/null +++ b/spec/test_parser_rule_from_multiple_states.rs @@ -0,0 +1,18 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"a"); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context)); + assert_eq!(1, p_position(&context).row); + assert_eq!(2, p_position(&context).col); + assert_eq!(TOKEN___EOF, p_token(&context)); + p_context_delete(context); + + let mut context = p_context_new(b"a b"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); + + let mut context = p_context_new(b"bb"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); +} diff --git a/spec/test_parser_rule_user_code.rs b/spec/test_parser_rule_user_code.rs new file mode 100644 index 0000000..9973303 --- /dev/null +++ b/spec/test_parser_rule_user_code.rs @@ -0,0 +1,7 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); +} diff --git a/spec/test_parser_user_code_tree.rs b/spec/test_parser_user_code_tree.rs new file mode 100644 index 0000000..ca1ea38 --- /dev/null +++ b/spec/test_parser_user_code_tree.rs @@ -0,0 +1,24 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"ab"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + + assert_eq!(3, context.start_n_fields); + assert_eq!(11, context.start_a_value); + assert_eq!(11, context.a_value); + assert_eq!(22, context.b_value); + assert_eq!(TOKEN_b, context.b_token); + assert_eq!(1, context.c_is_null); + assert_eq!(1, context.c_field_is_null); + assert_eq!(11, context.alias_a_value); + assert_eq!(22, context.alias_b_value); + + { + let start = p_result(&context); + assert!(start.pA().valid()); + assert!(start.pB().valid()); + assert!(!start.pC().valid()); + } + p_context_delete(context); +} diff --git a/spec/test_parsing_json.rs b/spec/test_parsing_json.rs new file mode 100644 index 0000000..3c3ff89 --- /dev/null +++ b/spec/test_parsing_json.rs @@ -0,0 +1,31 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"{}"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(JSON_OBJECT, p_result(&c).id()); + p_context_delete(c); + + let mut c = p_context_new(b"[]"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(JSON_ARRAY, p_result(&c).id()); + p_context_delete(c); + + let mut c = p_context_new(b"-45.6"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(JSON_NUMBER, p_result(&c).id()); + assert_eq!(-45.6, p_result(&c).number()); + p_context_delete(c); + + let mut c = p_context_new(b"{\"hi\":true}"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(JSON_OBJECT, p_result(&c).id()); + p_context_delete(c); + + let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + let v = p_result(&c); + assert_eq!(JSON_ARRAY, v.id()); + assert_eq!(5, v.array_len()); + p_context_delete(c); +} diff --git a/spec/test_parsing_lists.rs b/spec/test_parsing_lists.rs new file mode 100644 index 0000000..06bf1f6 --- /dev/null +++ b/spec/test_parsing_lists.rs @@ -0,0 +1,11 @@ +use testparser::*; + +fn main() { + let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)]; + for (input, expected) in cases { + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert_eq!(expected, p_result(&context)); + p_context_delete(context); + } +} diff --git a/spec/test_pattern.rs b/spec/test_pattern.rs new file mode 100644 index 0000000..74d5e38 --- /dev/null +++ b/spec/test_pattern.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"abcdef"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass1"); + p_context_delete(context); + + let mut context = p_context_new(b"defabcdef"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass2"); + p_context_delete(context); +} diff --git a/spec/test_positions.rs b/spec/test_positions.rs new file mode 100644 index 0000000..9a6efa2 --- /dev/null +++ b/spec/test_positions.rs @@ -0,0 +1,19 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b" Hello\n\n 4200\n"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); + + println!(); + + let mut c = p_context_new(b"\n tok2"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); + + println!(); + + let mut c = p_context_new(b" tok1"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); +} diff --git a/spec/test_return_token_from_pattern.rs b/spec/test_return_token_from_pattern.rs new file mode 100644 index 0000000..f9df3a2 --- /dev/null +++ b/spec/test_return_token_from_pattern.rs @@ -0,0 +1,7 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"defghidef"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + p_context_delete(context); +} diff --git a/spec/test_rewind.rs b/spec/test_rewind.rs new file mode 100644 index 0000000..0027e48 --- /dev/null +++ b/spec/test_rewind.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three + * times (recording 30 each time), followed by "5 + 5" (recording 10). */ + let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + + assert_eq!(vec![30, 30, 30, 10], c.nums); + assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols); + + p_context_delete(c); +} diff --git a/spec/test_set_position.rs b/spec/test_set_position.rs new file mode 100644 index 0000000..6696e15 --- /dev/null +++ b/spec/test_set_position.rs @@ -0,0 +1,66 @@ +use testparser::*; + +fn main() { + let mut ti = p_token_info_t::default(); + + /* Baseline: default (1, 1). */ + let mut c = p_context_new(b"ab"); + let pos = p_position(&c); + assert_eq!(1, pos.row); + assert_eq!(1, pos.col); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + assert_eq!(1, ti.position.row); + assert_eq!(1, ti.position.col); + p_context_delete(c); + + /* p_set_position overrides the initial position. */ + let mut c = p_context_new(b"ab"); + p_set_position(&mut c, p_position_t { row: 5, col: 20 }); + let pos = p_position(&c); + assert_eq!(5, pos.row); + assert_eq!(20, pos.col); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + assert_eq!(5, ti.position.row); + assert_eq!(20, ti.position.col); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_b, ti.token); + assert_eq!(5, ti.position.row); + assert_eq!(21, ti.position.col); + p_context_delete(c); + + /* Set position before a full parse. */ + let mut c = p_context_new(b"ab"); + p_set_position(&mut c, p_position_t { row: 3, col: 7 }); + assert_eq!(P_SUCCESS, p_parse_Start(&mut c)); + p_context_delete(c); + + /* Set position before a failing parse: error position is relative. */ + let mut c = p_context_new(b"aa"); + p_set_position(&mut c, p_position_t { row: 10, col: 2 }); + assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c)); + let ep = p_position(&c); + assert_eq!(10, ep.row); + assert_eq!(3, ep.col); + p_context_delete(c); + + /* p_set_input_index rewinds the byte cursor to re-read a section. */ + let mut c = p_context_new(b"ab"); + let start_index = p_input_index(&c); + let start_position = p_position(&c); + assert_eq!(0, start_index); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_b, ti.token); + assert_eq!(2, p_input_index(&c)); + p_set_input_index(&mut c, start_index); + p_set_position(&mut c, start_position); + assert_eq!(0, p_input_index(&c)); + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_a, ti.token); + assert_eq!(1, ti.position.row); + assert_eq!(1, ti.position.col); + p_context_delete(c); +} diff --git a/spec/test_start_rule.rs b/spec/test_start_rule.rs new file mode 100644 index 0000000..f79c691 --- /dev/null +++ b/spec/test_start_rule.rs @@ -0,0 +1,2 @@ +fn main() { +} diff --git a/spec/test_start_rule_tree.rs b/spec/test_start_rule_tree.rs new file mode 100644 index 0000000..73a4529 --- /dev/null +++ b/spec/test_start_rule_tree.rs @@ -0,0 +1,12 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"hi"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let top = p_result(&context); + assert!(top.pToken().valid()); + assert_eq!(TOKEN_hi, top.pToken().token()); + } + p_context_delete(context); +} diff --git a/spec/test_starting_rules.rs b/spec/test_starting_rules.rs new file mode 100644 index 0000000..f1d6913 --- /dev/null +++ b/spec/test_starting_rules.rs @@ -0,0 +1,18 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"bbbb"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + assert_eq!(8, p_result(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"bbbb"); + assert_eq!(P_SUCCESS, p_parse_Bs(&mut c)); + assert_eq!(8, p_result_Bs(&c)); + p_context_delete(c); + + let mut c = p_context_new(b"c"); + assert_eq!(P_SUCCESS, p_parse_R(&mut c)); + assert_eq!(3, p_result_R(&c)); + p_context_delete(c); +} diff --git a/spec/test_starting_rules_tree.rs b/spec/test_starting_rules_tree.rs new file mode 100644 index 0000000..c61d350 --- /dev/null +++ b/spec/test_starting_rules_tree.rs @@ -0,0 +1,34 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"bbbb"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + { + let start = p_result(&c); + assert!(start.bs().valid()); + assert!(start.bs().b().valid()); + assert!(start.bs().bs().b().valid()); + assert!(start.bs().bs().bs().b().valid()); + assert!(start.bs().bs().bs().bs().b().valid()); + } + p_context_delete(c); + + let mut c = p_context_new(b"bbbb"); + assert_eq!(P_SUCCESS, p_parse_Bs(&mut c)); + { + let bs = p_result_Bs(&c); + assert!(bs.b().valid()); + assert!(bs.bs().b().valid()); + assert!(bs.bs().bs().b().valid()); + assert!(bs.bs().bs().bs().b().valid()); + } + p_context_delete(c); + + let mut c = p_context_new(b"c"); + assert_eq!(P_SUCCESS, p_parse_R(&mut c)); + { + let r = p_result_R(&c); + assert!(r.c().valid()); + } + p_context_delete(c); +} diff --git a/spec/test_token_user_fields.rs b/spec/test_token_user_fields.rs new file mode 100644 index 0000000..3f2757c --- /dev/null +++ b/spec/test_token_user_fields.rs @@ -0,0 +1,17 @@ +use testparser::*; + +fn main() { + let input = b"# c1\n# c2\n\nfirst\n\n \n \n # s1\n # s2\nsecond\n"; + let mut c = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + { + let start = p_result(&c); + assert!(start.pIDs().valid()); + assert!(start.pIDs().id().valid()); + assert_eq!("# c1\n# c2\n", start.pIDs().id().data().comments.as_str()); + assert!(start.pIDs().pIDs().valid()); + assert!(start.pIDs().pIDs().id().valid()); + assert_eq!("# s1\n# s2\n", start.pIDs().pIDs().id().data().comments.as_str()); + } + p_context_delete(c); +} diff --git a/spec/test_tree.rs b/spec/test_tree.rs new file mode 100644 index 0000000..534d65c --- /dev/null +++ b/spec/test_tree.rs @@ -0,0 +1,46 @@ +use testparser::*; + +fn main() { + let input = b"a, ((b)), b"; + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.pItems1().valid()); + assert!(start.pItems().valid()); + let items = start.pItems(); + assert!(items.pItem().valid()); + assert!(items.pItem().pToken1().valid()); + assert_eq!(TOKEN_a, items.pItem().pToken1().token()); + assert_eq!(11, items.pItem().pToken1().pvalue()); + let itemsmore = items.pItemsMore(); + assert!(itemsmore.pItem().pItem().pItem().pToken1().valid()); + assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token()); + assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue()); + assert!(itemsmore.pItemsMore().valid()); + let itemsmore = itemsmore.pItemsMore(); + assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token()); + assert!(!itemsmore.pItemsMore().valid()); + } + p_context_delete(context); + + /* Empty input yields a Start node with no Items child. */ + let mut context = p_context_new(b""); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert!(!p_result(&context).pItems().valid()); + p_context_delete(context); + + /* Dual rule alternative field positions. */ + let mut context = p_context_new(b"2 1"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.pItems().pItem().pDual().pTwo1().valid()); + assert!(start.pItems().pItem().pDual().pOne2().valid()); + assert!(!start.pItems().pItem().pDual().pTwo2().valid()); + assert!(!start.pItems().pItem().pDual().pOne1().valid()); + } + p_context_delete(context); + + println!("ok"); +} diff --git a/spec/test_tree_field_aliases.rs b/spec/test_tree_field_aliases.rs new file mode 100644 index 0000000..3320f11 --- /dev/null +++ b/spec/test_tree_field_aliases.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"\na\nb\nc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert_eq!(TOKEN_a, start.first().pToken().token()); + assert_eq!(TOKEN_b, start.second().pToken().token()); + assert_eq!(TOKEN_c, start.third().pToken().token()); + } + p_context_delete(context); +} diff --git a/spec/test_tree_invalid_positions.rs b/spec/test_tree_invalid_positions.rs new file mode 100644 index 0000000..76714ed --- /dev/null +++ b/spec/test_tree_invalid_positions.rs @@ -0,0 +1,89 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"\na\n bb ccc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + let t1 = start.pT1(); + let k1 = t1.pToken(); + let a1 = t1.pA(); + assert_eq!(2, k1.position().row); + assert_eq!(1, k1.position().col); + assert_eq!(2, k1.end_position().row); + assert_eq!(1, k1.end_position().col); + assert!(a1.position().valid()); + assert_eq!(3, a1.position().row); + assert_eq!(3, a1.position().col); + assert_eq!(3, a1.end_position().row); + assert_eq!(8, a1.end_position().col); + assert_eq!(2, t1.position().row); + assert_eq!(1, t1.position().col); + assert_eq!(3, t1.end_position().row); + assert_eq!(8, t1.end_position().col); + assert_eq!(2, start.position().row); + assert_eq!(1, start.position().col); + assert_eq!(3, start.end_position().row); + assert_eq!(8, start.end_position().col); + } + p_context_delete(context); + + let mut context = p_context_new(b"a\nbb"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + let t1 = start.pT1(); + let k1 = t1.pToken(); + let a1 = t1.pA(); + assert_eq!(1, k1.position().row); + assert_eq!(1, k1.position().col); + assert!(a1.position().valid()); + assert_eq!(2, a1.position().row); + assert_eq!(1, a1.position().col); + assert_eq!(2, a1.end_position().row); + assert_eq!(2, a1.end_position().col); + assert_eq!(1, t1.position().row); + assert_eq!(2, t1.end_position().row); + assert_eq!(2, t1.end_position().col); + assert_eq!(1, start.position().row); + assert_eq!(2, start.end_position().row); + assert_eq!(2, start.end_position().col); + } + p_context_delete(context); + + let mut context = p_context_new(b"a\nc\nc"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + let t1 = start.pT1(); + let a1 = t1.pA(); + assert!(a1.position().valid()); + assert_eq!(2, a1.position().row); + assert_eq!(1, a1.position().col); + assert_eq!(3, a1.end_position().row); + assert_eq!(1, a1.end_position().col); + assert_eq!(1, t1.position().row); + assert_eq!(3, t1.end_position().row); + assert_eq!(1, t1.end_position().col); + assert_eq!(3, start.end_position().row); + assert_eq!(1, start.end_position().col); + } + p_context_delete(context); + + let mut context = p_context_new(b"a"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + let t1 = start.pT1(); + let a1 = t1.pA(); + assert!(!a1.position().valid()); + assert_eq!(1, t1.position().row); + assert_eq!(1, t1.position().col); + assert_eq!(1, t1.end_position().row); + assert_eq!(1, t1.end_position().col); + assert_eq!(1, start.position().row); + assert_eq!(1, start.end_position().row); + assert_eq!(1, start.end_position().col); + } + p_context_delete(context); +} diff --git a/spec/test_tree_node_memory_remains.rs b/spec/test_tree_node_memory_remains.rs new file mode 100644 index 0000000..b66f2ce --- /dev/null +++ b/spec/test_tree_node_memory_remains.rs @@ -0,0 +1,85 @@ +use testparser::*; + +fn main() { + let entries: [(&str, &str, p_token_t); 51] = [ + ("byte_val", "byte", TOKEN_byte), + ("short_val", "short", TOKEN_short), + ("int_val", "int", TOKEN_int), + ("long_val", "long", TOKEN_long), + ("ssize_t_val", "ssize_t", TOKEN_ssize_t), + ("byte_to_short", "short", TOKEN_short), + ("byte_to_int", "int", TOKEN_int), + ("byte_to_long", "long", TOKEN_long), + ("byte_to_ssize_t", "ssize_t", TOKEN_ssize_t), + ("short_to_byte", "byte", TOKEN_byte), + ("short_to_int", "int", TOKEN_int), + ("short_to_long", "long", TOKEN_long), + ("short_to_ssize_t", "ssize_t", TOKEN_ssize_t), + ("int_to_byte", "byte", TOKEN_byte), + ("int_to_short", "short", TOKEN_short), + ("int_to_long", "long", TOKEN_long), + ("int_to_ssize_t", "ssize_t", TOKEN_ssize_t), + ("long_to_byte", "byte", TOKEN_byte), + ("long_to_short", "short", TOKEN_short), + ("long_to_int", "int", TOKEN_int), + ("long_to_ssize_t", "ssize_t", TOKEN_ssize_t), + ("ssize_t_to_byte", "byte", TOKEN_byte), + ("ssize_t_to_short", "short", TOKEN_short), + ("ssize_t_to_int", "int", TOKEN_int), + ("ssize_t_to_long", "long", TOKEN_long), + ("ubyte_val", "ubyte", TOKEN_ubyte), + ("ushort_val", "ushort", TOKEN_ushort), + ("uint_val", "uint", TOKEN_uint), + ("ulong_val", "ulong", TOKEN_ulong), + ("size_t_val", "size_t", TOKEN_size_t), + ("ubyte_to_ushort", "ushort", TOKEN_ushort), + ("ubyte_to_uint", "uint", TOKEN_uint), + ("ubyte_to_ulong", "ulong", TOKEN_ulong), + ("ubyte_to_size_t", "size_t", TOKEN_size_t), + ("ushort_to_ubyte", "ubyte", TOKEN_ubyte), + ("ushort_to_uint", "uint", TOKEN_uint), + ("ushort_to_ulong", "ulong", TOKEN_ulong), + ("ushort_to_size_t", "size_t", TOKEN_size_t), + ("uint_to_ubyte", "ubyte", TOKEN_ubyte), + ("uint_to_ushort", "ushort", TOKEN_ushort), + ("uint_to_ulong", "ulong", TOKEN_ulong), + ("uint_to_size_t", "size_t", TOKEN_size_t), + ("ulong_to_ubyte", "ubyte", TOKEN_ubyte), + ("ulong_to_ushort", "ushort", TOKEN_ushort), + ("ulong_to_uint", "uint", TOKEN_uint), + ("ulong_to_size_t", "size_t", TOKEN_size_t), + ("size_t_to_ubyte", "ubyte", TOKEN_ubyte), + ("size_t_to_ushort", "ushort", TOKEN_ushort), + ("size_t_to_int", "int", TOKEN_int), + ("size_t_to_ulong", "ulong", TOKEN_ulong), + ("main", "int", TOKEN_int), + ]; + + let mut input = String::new(); + for (name, ret, _) in entries.iter() { + input.push_str(&format!("def {}() -> {} {{\nreturn 0x42;\n}}\n", name, ret)); + } + + let mut c = p_context_new(input.as_bytes()); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + { + let pmod = p_result(&c); + let mut pfds = Vec::new(); + let mut pmis = pmod.pModuleItems(); + while pmis.valid() { + let pmi = pmis.pModuleItem(); + assert!(pmi.valid()); + let pfd = pmi.pFunctionDefinition(); + if pfd.valid() { + pfds.insert(0, pfd); + } + pmis = pmis.pModuleItems(); + } + assert_eq!(51, pfds.len()); + for i in 0..pfds.len() { + assert_eq!(entries[i].0, pfds[i].name().data().pvalue.s.as_str()); + assert_eq!(entries[i].2, pfds[i].returntype().pType().pTypeBase().pToken1().token()); + } + } + p_context_delete(c); +} diff --git a/spec/test_tree_ps.rs b/spec/test_tree_ps.rs new file mode 100644 index 0000000..534d65c --- /dev/null +++ b/spec/test_tree_ps.rs @@ -0,0 +1,46 @@ +use testparser::*; + +fn main() { + let input = b"a, ((b)), b"; + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.pItems1().valid()); + assert!(start.pItems().valid()); + let items = start.pItems(); + assert!(items.pItem().valid()); + assert!(items.pItem().pToken1().valid()); + assert_eq!(TOKEN_a, items.pItem().pToken1().token()); + assert_eq!(11, items.pItem().pToken1().pvalue()); + let itemsmore = items.pItemsMore(); + assert!(itemsmore.pItem().pItem().pItem().pToken1().valid()); + assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token()); + assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue()); + assert!(itemsmore.pItemsMore().valid()); + let itemsmore = itemsmore.pItemsMore(); + assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token()); + assert!(!itemsmore.pItemsMore().valid()); + } + p_context_delete(context); + + /* Empty input yields a Start node with no Items child. */ + let mut context = p_context_new(b""); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + assert!(!p_result(&context).pItems().valid()); + p_context_delete(context); + + /* Dual rule alternative field positions. */ + let mut context = p_context_new(b"2 1"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + assert!(start.pItems().pItem().pDual().pTwo1().valid()); + assert!(start.pItems().pItem().pDual().pOne2().valid()); + assert!(!start.pItems().pItem().pDual().pTwo2().valid()); + assert!(!start.pItems().pItem().pDual().pOne1().valid()); + } + p_context_delete(context); + + println!("ok"); +} diff --git a/spec/test_tree_token_positions.rs b/spec/test_tree_token_positions.rs new file mode 100644 index 0000000..a68ce9e --- /dev/null +++ b/spec/test_tree_token_positions.rs @@ -0,0 +1,27 @@ +use testparser::*; + +fn main() { + let input = b"abbccc"; + let mut context = p_context_new(input); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + { + let start = p_result(&context); + let t1 = start.pT1(); + let t2 = start.pT2(); + let t3 = start.pT3(); + assert_eq!(1, t1.position().col); + assert_eq!(1, t1.end_position().col); + assert_eq!(2, t2.position().col); + assert_eq!(3, t2.end_position().col); + assert_eq!(4, t3.position().col); + assert_eq!(6, t3.end_position().col); + /* Token node position within T1. */ + assert_eq!(1, t1.pToken().position().col); + /* Overall start node spans the whole input. */ + assert_eq!(1, start.position().col); + assert_eq!(6, start.end_position().col); + assert_eq!(3, start.n_fields()); + } + p_context_delete(context); + println!("ok"); +} diff --git a/spec/test_user_code.rs b/spec/test_user_code.rs new file mode 100644 index 0000000..6ecbd39 --- /dev/null +++ b/spec/test_user_code.rs @@ -0,0 +1,13 @@ +use testparser::*; + +fn main() { + let mut context = p_context_new(b"abcdef"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass1"); + p_context_delete(context); + + let mut context = p_context_new(b"abcabcdef"); + assert_eq!(P_SUCCESS, p_parse(&mut context)); + println!("pass2"); + p_context_delete(context); +} diff --git a/spec/test_user_context_fields.rs b/spec/test_user_context_fields.rs new file mode 100644 index 0000000..e6e34fd --- /dev/null +++ b/spec/test_user_context_fields.rs @@ -0,0 +1,9 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"aaa\n\n\na\n # comment 1\na a aa\n\naa\n# comment 2\na\n"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + eprint!("comments: {}", c.comments); + eprintln!("acount: {}", c.acount); + p_context_delete(c); +} diff --git a/spec/test_user_terminate.rs b/spec/test_user_terminate.rs new file mode 100644 index 0000000..03d9baf --- /dev/null +++ b/spec/test_user_terminate.rs @@ -0,0 +1,12 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"aacc"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"abc"); + assert_eq!(P_USER_TERMINATED, p_parse(&mut c)); + assert_eq!(4200, p_user_terminate_code(&c)); + p_context_delete(c); +} diff --git a/spec/test_user_terminate_lexer.rs b/spec/test_user_terminate_lexer.rs new file mode 100644 index 0000000..c5951f0 --- /dev/null +++ b/spec/test_user_terminate_lexer.rs @@ -0,0 +1,12 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"a"); + assert_eq!(P_SUCCESS, p_parse(&mut c)); + p_context_delete(c); + + let mut c = p_context_new(b"b"); + assert_eq!(P_USER_TERMINATED, p_parse(&mut c)); + assert_eq!(8675309, p_user_terminate_code(&c)); + p_context_delete(c); +} diff --git a/spec/test_value_accessors.rs b/spec/test_value_accessors.rs new file mode 100644 index 0000000..ac0a042 --- /dev/null +++ b/spec/test_value_accessors.rs @@ -0,0 +1,20 @@ +use testparser::*; + +fn main() { + let mut c = p_context_new(b"42 f s"); + let mut ti = p_token_info_t::default(); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_num, ti.token); + assert_eq!(42, p_value_get(&ti.pvalue)); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_flt, ti.token); + assert_eq!(1.5, p_value_get_float(&ti.pvalue)); + + assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti)); + assert_eq!(TOKEN_str, ti.token); + assert_eq!("hello", p_value_get_string(&ti.pvalue)); + + p_context_delete(c); +} diff --git a/spec/tree_node_memory_remains.rust.propane b/spec/tree_node_memory_remains.rust.propane new file mode 100644 index 0000000..f9cb138 --- /dev/null +++ b/spec/tree_node_memory_remains.rust.propane @@ -0,0 +1,148 @@ +tree; +tree_prefix P; + +<
> + +ptype TokenVal; + +# Keywords. +token byte; +token def; +token int; +token long; +token module; +token return; +token short; +token size_t; +token ssize_t; +token ubyte; +token uint; +token ulong; +token ushort; + +# Symbols. +token arrow /->/; +token comma /,/; +token lbrace /\{/; +token lparen /\(/; +token rbrace /\}/; +token rparen /\)/; +token semicolon /;/; + +# Integer literals. +token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ << + $$.i64_ = 64; +>> + +# Identifier. +token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ << + $$.s = std::str::from_utf8(match_).unwrap().to_string(); + $mode(default); + return $token(ident); +>> + +# Comments. +drop /#.*/; + +# Whitespace. +drop /[ \r\n]*/; + +start Module; + +# Assignment operators - right associative +Expression -> Expression_Or:exp0; + +# Logical OR operator - left associative +Expression_Or -> Expression_And:exp0; + +# Logical AND operator - left associative +Expression_And -> Expression_Comp:exp0; + +# Equality operators - left associative +Expression_Comp -> Expression_Relational:exp0; + +# Relational operators - left associative +Expression_Relational -> Expression_REMatch:exp0; + +# Regular expression - left associative +Expression_REMatch -> Expression_BinOr:exp0; + +# Binary OR operator - left associative +Expression_BinOr -> Expression_Xor:exp0; + +# Binary XOR operator - left associative +Expression_Xor -> Expression_BinAnd:exp0; + +# Binary AND operator - left associative +Expression_BinAnd -> Expression_BitShift:exp0; + +# Bit shift operators - left associative +Expression_BitShift -> Expression_Plus:exp0; + +# Add/subtract operators - left associative +Expression_Plus -> Expression_Mul:exp0; + +# Multiplication/divide/modulus operators - left associative +Expression_Mul -> Expression_Range:exp0; + +# Range construction operators - left associative +Expression_Range -> Expression_UnaryPrefix:exp0; + +# Unary prefix operators +Expression_UnaryPrefix -> Expression_Dot:exp0; + +# Postfix operators +Expression_Dot -> Expression_Ident:exp0; +Expression_Dot -> Expression_Dot:exp1 lparen rparen; + +# Literals, identifiers, and parenthesized expressions +Expression_Ident -> Literal; +Expression_Ident -> ident; + +FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace; + +FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more; +FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more; + +FunctionReturnType -> arrow Type; + +Literal -> LiteralInteger; +LiteralInteger -> hex_int_l; + +Module -> ModuleStatement? ModuleItems; + +ModuleItem -> FunctionDefinition; + +ModuleItems -> ; +ModuleItems -> ModuleItems ModuleItem; + +ModulePath -> ident; + +ModuleStatement -> module ModulePath semicolon; + +ReturnStatement -> return Expression?:exp0 semicolon; + +Statements -> ; +Statements -> Statements Statement; +Statement -> Expression semicolon; +Statement -> ReturnStatement; + +Type -> TypeBase; + +TypeBase -> byte; +TypeBase -> ubyte; +TypeBase -> short; +TypeBase -> ushort; +TypeBase -> int; +TypeBase -> uint; +TypeBase -> long; +TypeBase -> ulong; +TypeBase -> size_t; +TypeBase -> ssize_t;