josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Add Rust target

Browse Source
This commit is contained in:
Josh Holtrop 2026-08-11 22:46:04 -04:00
parent 5fc712c6ee
commit 89f1f84857
64 changed files with 3441 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -1,5 +1,9 @@
## v5.0.0
### New Features
- Add Rust target language output.
### API Changes
- Tree generation mode now stores all tree nodes in a compact arena owned by

2
README.md
View File

@ -6,7 +6,7 @@ Propane is a LALR Parser Generator (LPG) which:
* generates a built-in lexer to tokenize input
* supports UTF-8 lexer inputs
* generates a table-driven shift/reduce parser to parse input in linear time
* targets C, C++, or D language outputs
* targets C, C++, D, or Rust language outputs
* optionally supports automatic full parse tree generation
* supports starting parsing from multiple start rules
* tracks input text start and end positions for all matched tokens/rules

1052
assets/parser.rs.erb Normal file
View File

File diff suppressed because it is too large Load Diff

2
doc/user_guide.md
View File

@ -13,7 +13,7 @@ Propane is a LALR Parser Generator (LPG) which:
* generates a built-in lexer to tokenize input
* supports UTF-8 lexer inputs
* generates a table-driven shift/reduce parser to parse input in linear time
* targets C, C++, or D language outputs
* targets C, C++, D, or Rust language outputs
* optionally supports automatic full parse tree generation
* supports starting parsing from multiple start rules
* tracks input text start and end positions for all matched tokens/rules

151
lib/propane/generator.rb
View File

@ -18,6 +18,8 @@ class Propane
elsif output_file =~ %r{\.(cc|cpp|cxx)$}
@cpp = true
"c"
elsif output_file.end_with?(".rs")
"rust"
else
raise Error.new("Could not determine target language from output file name (#{output_file})")
end
@ -31,7 +33,8 @@ class Propane
extensions += %w[h]
end
extensions.each do |extension|
template = Assets.get("parser.#{extension || @language}.erb")
template_language = @language == "rust" ? "rs" : @language
template = Assets.get("parser.#{extension || template_language}.erb")
if extension
output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
else
@ -39,7 +42,12 @@ class Propane
end
erb = ERB.new(template, trim_mode: "<>")
result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
if line == "#linereset\n"
if @language == "rust"
# Rust has no #line directive support, so strip the directives that
# the grammar embeds in user code blocks.
line = line.sub(/^#line \d+ "[^"]*"/, "")
line == "#linereset\n" ? "" : line
elsif line == "#linereset\n"
%[#line #{i + 2} "#{output_file}"\n]
else
line
@ -275,6 +283,8 @@ class Propane
"context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
when "d"
"context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
when "rust"
"context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
end
end
code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
@ -284,6 +294,8 @@ class Propane
"context->#{fieldname}"
when "d"
"context.#{fieldname}"
when "rust"
"context.#{fieldname}"
end
end
code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
@ -293,6 +305,8 @@ class Propane
"token_tree_node->#{fieldname}"
when "d"
"token_tree_node.#{fieldname}"
when "rust"
"token_tree_node.#{fieldname}"
end
end
if parser
@ -304,6 +318,8 @@ class Propane
tree_handle(typename, "_node_id")
when "d"
tree_handle(typename, "_node_id")
when "rust"
tree_handle(typename, "_node_id")
end
else
case @language
@ -311,6 +327,8 @@ class Propane
"_pvalue->v_#{rule.ptypename}"
when "d"
"_pvalue.v_#{rule.ptypename}"
when "rust"
"(*_pvalue.v_#{rule.ptypename}_mut())"
end
end
end
@ -345,6 +363,8 @@ class Propane
"out_token_info->pvalue"
when "d"
"out_token_info.pvalue"
when "rust"
"out_token_info.pvalue"
end
else
case @language
@ -352,6 +372,8 @@ class Propane
"out_token_info->pvalue.v_#{pattern.ptypename}"
when "d"
"out_token_info.pvalue.v_#{pattern.ptypename}"
when "rust"
"(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())"
end
end
end
@ -361,6 +383,8 @@ class Propane
"out_token_info->position"
when "d"
"out_token_info.position"
when "rust"
"out_token_info.position"
end
end
code = code.gsub(/\$\{end_position\}/) do |match|
@ -369,6 +393,8 @@ class Propane
"out_token_info->end_position"
when "d"
"out_token_info.end_position"
when "rust"
"out_token_info.end_position"
end
end
code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
@ -382,6 +408,8 @@ class Propane
"context->mode = #{mode_id}u"
when "d"
"context.mode = #{mode_id}u"
when "rust"
"context.mode = #{mode_id}"
end
end
end
@ -416,6 +444,8 @@ class Propane
tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
when "d"
tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
when "rust"
tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id")
end
else
case @language
@ -423,6 +453,8 @@ class Propane
"state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
when "d"
"statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
when "rust"
"statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()"
end
end
end
@ -446,6 +478,8 @@ class Propane
"(#{typename}{context, #{id_expr}})"
elsif @language == "c"
"((#{typename}){context, #{id_expr}})"
elsif @language == "rust"
"(#{typename} { context, id: #{id_expr} })"
else
"#{typename}(context, #{id_expr})"
end
@ -687,6 +721,113 @@ class Propane
out.join("\n")
end
# Rust keywords that must be escaped as raw identifiers when used as a
# generated identifier (e.g. a field alias named `type`).
RUST_KEYWORDS = %w[
as break const continue dyn else enum extern false fn for if impl in let
loop match mod move mut pub ref return static struct trait true type
unsafe use where while async await abstract become box do final macro
override priv typeof unsized virtual yield try gen
]
# Escape a name as a Rust raw identifier if it is a reserved keyword.
#
# @param name [String]
# Identifier name.
#
# @return [String]
# Name, escaped as a raw identifier if necessary.
def rust_ident(name)
RUST_KEYWORDS.include?(name) ? "r##{name}" : name
end
# Map a ptype type string to a valid Rust type.
#
# The default ptype is a C "void *"; for Rust with no declared ptype we use
# the unit type instead.
#
# @param typestring [String]
# ptype type string.
#
# @return [String]
# Rust type string.
def rust_ptype(typestring)
typestring == "void *" ? "()" : typestring
end
# Generate the Rust tree node record and handle types.
#
# Mirrors the C tree node record plus the C++ handle structs: each rule set
# and the Token node get a handle type ({context, id}) with accessor methods.
#
# @return [String]
# Rust tree node type definitions.
def rust_tree_types
p = @grammar.prefix
out = []
out << "/** Tree node record. */"
out << "#[derive(Clone, Default)]"
out << "pub struct #{p}node_data_t {"
out << " pub position: #{p}position_t,"
out << " pub end_position: #{p}position_t,"
out << " pub child_offset: #{p}node_id_t,"
out << " pub n_fields: u16,"
out << " pub is_token: bool,"
out << " pub token: #{p}token_t,"
out << " pub pvalue: #{p}value_t,"
unless @grammar.token_user_fields.to_s.strip.empty?
out << @grammar.token_user_fields
end
out << "}"
out << ""
out << "/** Tree node handle types. */"
tree_handle_types.each do |t|
out << "#[derive(Clone, Copy)]"
out << "pub struct #{t}<'a> { context: &'a #{p}context_t, id: #{p}node_id_t }"
end
out << ""
# Common accessors for every handle type.
tree_handle_types.each do |t|
out << "impl<'a> #{t}<'a> {"
out << " /** Return whether this handle refers to a valid (non-null) node. */"
out << " pub fn valid(&self) -> bool { self.id != 0 }"
out << " /** Return the node ID (for identity comparison). */"
out << " pub fn node_id(&self) -> #{p}node_id_t { self.id }"
out << " /** Access the underlying node record. */"
out << " pub fn data(&self) -> &'a #{p}node_data_t { &self.context.#{p}tree_nodes[self.id as usize] }"
out << " /** Text position of the first code point spanned by this node. */"
out << " pub fn position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].position }"
out << " /** Text position of the last code point spanned by this node. */"
out << " pub fn end_position(&self) -> #{p}position_t { self.context.#{p}tree_nodes[self.id as usize].end_position }"
out << " /** Number of child fields in this node. */"
out << " pub fn n_fields(&self) -> u16 { if self.id != 0 { self.context.#{p}tree_nodes[self.id as usize].n_fields } else { 0 } }"
if t == h_type("Token")
out << " /** Token ID for this token node. */"
out << " pub fn token(&self) -> #{p}token_t { self.context.#{p}tree_nodes[self.id as usize].token }"
out << " /** Parser value associated with this token node. */"
out << " pub fn pvalue(&self) -> #{p}value_t { self.context.#{p}tree_nodes[self.id as usize].pvalue.clone() }"
end
out << "}"
end
out << ""
# Navigation accessors for rule set handles.
tree_node_rule_sets.each do |rule_set|
rtype = h_type(rule_set.name)
out << "impl<'a> #{rtype}<'a> {"
each_tree_field(rule_set) do |rt, field_name, child_type, slot|
out << " /** Access the #{field_name} child node. */"
out << " pub fn #{rust_ident(field_name)}(&self) -> #{child_type}<'a> {"
out << " if self.id == 0 {"
out << " return #{child_type} { context: self.context, id: 0 };"
out << " }"
out << " #{child_type} { context: self.context, id: self.context.#{p}tree_children[self.context.#{p}tree_nodes[self.id as usize].child_offset as usize + #{slot}] }"
out << " }"
end
out << "}"
end
out.join("\n")
end
# Get the lex function to use.
#
# @return [String]
@ -720,6 +861,8 @@ class Propane
"uint8_t"
when "d"
"ubyte"
when "rust"
"u8"
end
elsif max <= 0xFFFF
case @language
@ -727,11 +870,15 @@ class Propane
"uint16_t"
when "d"
"ushort"
when "rust"
"u16"
end
else
case @language
when "c"
"uint32_t"
when "rust"
"u32"
else
"uint"
end

176
spec/json_parser.rust.propane Normal file
View File

@ -0,0 +1,176 @@
<<header
pub const JSON_OBJECT: usize = 0;
pub const JSON_ARRAY: usize = 1;
pub const JSON_NUMBER: usize = 2;
pub const JSON_STRING: usize = 3;
pub const JSON_TRUE: usize = 4;
pub const JSON_FALSE: usize = 5;
pub const JSON_NULL: usize = 6;
#[derive(Clone, Default)]
pub enum JSONValue {
#[default]
Null,
Object(Vec<(String, JSONValue)>),
Array(Vec<JSONValue>),
Number(f64),
StringVal(String),
True,
False,
}
impl JSONValue {
pub fn id(&self) -> usize {
match self {
JSONValue::Object(_) => JSON_OBJECT,
JSONValue::Array(_) => JSON_ARRAY,
JSONValue::Number(_) => JSON_NUMBER,
JSONValue::StringVal(_) => JSON_STRING,
JSONValue::True => JSON_TRUE,
JSONValue::False => JSON_FALSE,
JSONValue::Null => JSON_NULL,
}
}
pub fn number(&self) -> f64 {
if let JSONValue::Number(n) = self { *n } else { 0.0 }
}
pub fn string(&self) -> &str {
if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
}
pub fn object_len(&self) -> usize {
if let JSONValue::Object(e) = self { e.len() } else { 0 }
}
pub fn array_len(&self) -> usize {
if let JSONValue::Array(e) = self { e.len() } else { 0 }
}
}
>>
context_user_fields <<
pub string_value: String,
>>
ptype JSONValue;
drop /\s+/;
token lbrace /\{/;
token rbrace /\}/;
token lbracket /\[/;
token rbracket /\]/;
token comma /,/;
token colon /:/;
token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
$$ = JSONValue::Number(n);
>>
token true <<
$$ = JSONValue::True;
>>
token false <<
$$ = JSONValue::False;
>>
token null <<
$$ = JSONValue::Null;
>>
/"/ <<
$mode(string);
${context.string_value} = String::new();
>>
string: token string /"/ <<
$$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
$mode(default);
>>
string: /\\"/ <<
${context.string_value}.push('"');
>>
string: /\\\\/ <<
${context.string_value}.push('\\');
>>
string: /\\\// <<
${context.string_value}.push('/');
>>
string: /\\b/ <<
${context.string_value}.push('\u{0008}');
>>
string: /\\f/ <<
${context.string_value}.push('\u{000C}');
>>
string: /\\n/ <<
${context.string_value}.push('\n');
>>
string: /\\r/ <<
${context.string_value}.push('\r');
>>
string: /\\t/ <<
${context.string_value}.push('\t');
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
${context.string_value}.push_str(&s);
>>
string: /[^\\]/ <<
${context.string_value}.push(match_[0] as char);
>>
Start -> Value <<
$$ = $1;
>>
Value -> string <<
$$ = $1;
>>
Value -> number <<
$$ = $1;
>>
Value -> Object <<
$$ = $1;
>>
Value -> Array <<
$$ = $1;
>>
Value -> true <<
$$ = $1;
>>
Value -> false <<
$$ = $1;
>>
Value -> null <<
$$ = $1;
>>
Object -> lbrace rbrace <<
$$ = JSONValue::Object(Vec::new());
>>
Object -> lbrace KeyValues rbrace <<
$$ = $2;
>>
KeyValues -> KeyValue <<
$$ = $1;
>>
KeyValues -> KeyValues comma KeyValue <<
let mut obj = $1;
if let JSONValue::Object(kve) = $3 {
if let JSONValue::Object(entries) = &mut obj {
entries.extend(kve);
}
}
$$ = obj;
>>
KeyValue -> string colon Value <<
let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
$$ = JSONValue::Object(vec![(name, $3)]);
>>
Array -> lbracket rbracket <<
$$ = JSONValue::Array(Vec::new());
>>
Array -> lbracket Values rbracket <<
$$ = $2;
>>
Values -> Value <<
$$ = $1;
>>
Values -> Values comma Value <<
let mut arr = $1;
if let JSONValue::Array(elems) = &mut arr {
elems.push($3);
}
$$ = arr;
>>

80
spec/macros.rust.propane Normal file
View File

@ -0,0 +1,80 @@
<<
fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
loop {
if context.expanding {
let ei = context.expand_i;
context.expand_i += 1;
if context.expand_i >= context.token_infos.len() {
context.expanding = false;
}
*out_token_info = context.token_infos[ei].clone();
return P_SUCCESS;
}
let lex_result = p_lex(context, out_token_info);
if lex_result != P_SUCCESS {
return lex_result;
}
if out_token_info.token == TOKEN_macro {
context.defining = true;
} else if out_token_info.token == TOKEN_macroname {
if !context.defining {
context.expanding = true;
context.expand_i = 0;
continue;
}
} else if out_token_info.token == TOKEN_lbrace {
if context.defining {
/* Capture the macro body tokens (up to the closing '}'). */
let mut infos: Vec<p_token_info_t> = Vec::new();
loop {
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
if ti.token == TOKEN_rbrace {
break;
}
infos.push(ti);
}
context.token_infos = infos;
context.defining = false;
}
} else {
context.defining = false;
}
return lex_result;
}
}
>>
context_user_fields <<
pub defining: bool,
pub expanding: bool,
pub expand_i: usize,
pub token_infos: Vec<p_token_info_t>,
pub nums: Vec<i64>,
>>
ptype i64;
lex_fn mylexfn;
drop /\s+/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> Statements;
Statements -> ;
Statements -> Statement Statements;
Statement -> Add;
Statement -> MacroStart;
Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
MacroStart -> macro macroname lbrace;

41
spec/parse_inner_nested.rust.propane Normal file
View File

@ -0,0 +1,41 @@
<<
fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
let result = p_lex(context, out_token_info);
if result != P_SUCCESS {
return result;
}
if out_token_info.token == TOKEN_lparen {
/* Reentrant nested parse of the parenthesized sub-expression. */
let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
if inner_result != P_SUCCESS {
return inner_result;
}
let value = p_result_Start(context);
/* p_parse_inner rewound the input so ')' was not consumed; consume it. */
let mut rparen_info = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
assert_eq!(TOKEN_rparen, rparen_info.token);
out_token_info.token = TOKEN_num;
out_token_info.pvalue = p_value(value);
}
P_SUCCESS
}
>>
ptype i64;
lex_fn mylexfn;
drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> Expr << $$ = $1; >>
Expr -> num << $$ = $1; >>
Expr -> Expr plus num << $$ = $1 + $3; >>

44
spec/parse_inner_nested_tree.rust.propane Normal file
View File

@ -0,0 +1,44 @@
<<
fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
let result = p_lex(context, out_token_info);
if result != P_SUCCESS {
return result;
}
if out_token_info.token == TOKEN_lparen {
let start_position = out_token_info.position;
let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
if inner_result != P_SUCCESS {
return inner_result;
}
/* Read the inner subtree's span before re-borrowing context to lex. */
let inner = p_result_Start(context);
assert!(inner.valid());
let inner_start_col = inner.position().col;
let inner_end_col = inner.end_position().col;
let mut rparen_info = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
assert_eq!(TOKEN_rparen, rparen_info.token);
assert_eq!(start_position.col + 1, inner_start_col);
assert_eq!(rparen_info.position.col - 1, inner_end_col);
/* Synthesize a num token spanning the whole "( ... )" group. */
out_token_info.token = TOKEN_num;
out_token_info.position = start_position;
out_token_info.end_position = rparen_info.end_position;
}
P_SUCCESS
}
>>
tree;
lex_fn mylexfn;
drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/;
Start -> Expr;
Expr -> num;
Expr -> Expr plus num;

464
spec/propane_spec.rb
View File

@ -53,7 +53,7 @@ EOF
if options[:args]
command += options[:args]
else
command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{lang_ext(options[:language])} --log spec/run/testparser#{options[:name]}.log]
end
command += (options[:extra_args] || [])
if (options[:capture])
@ -65,9 +65,16 @@ EOF
end
end
# Map a spec language name to the generated parser source file extension.
def lang_ext(language)
language == "rust" ? "rs" : language
end
def compile(test_files, options = {})
test_files = Array(test_files).map do |test_file|
if !File.exist?(test_file) && test_file.end_with?(".cpp")
if test_file.end_with?(".rust")
test_file.sub(%r{\.rust$}, ".rs")
elsif !File.exist?(test_file) && test_file.end_with?(".cpp")
test_file.sub(%r{\.cpp$}, ".c")
else
test_file
@ -75,7 +82,7 @@ EOF
end
options[:parsers] ||= [""]
parsers = options[:parsers].map do |name|
"spec/run/testparser#{name}.#{options[:language]}"
"spec/run/testparser#{name}.#{lang_ext(options[:language])}"
end
case options[:language]
when "c"
@ -84,6 +91,21 @@ EOF
command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
when "d"
command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
when "rust"
# Compile each generated parser to an rlib, then compile the test crate
# against them. Safe Rust needs no valgrind, but it is run anyway.
externs = []
options[:parsers].each do |name|
crate = "testparser#{name}"
rlib = "spec/run/lib#{crate}.rlib"
rustc = [*%w[rustc --edition 2021 --crate-type=rlib -A warnings],
"--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"]
expect(system(*rustc)).to be_truthy
externs += ["--extern", "#{crate}=#{rlib}"]
end
# rustc takes a single crate root; any additional Rust test files are
# expected to be pulled in as modules or provided by the parser crate.
command = [*%w[rustc --edition 2021 -A warnings -o spec/run/testparser], *externs, test_files.first]
end
result = system(*command)
expect(result).to be_truthy
@ -285,7 +307,7 @@ EOF
expect(results.status).to_not eq 0
end
%w[d c cpp].each do |language|
%w[d c cpp rust].each do |language|
context "#{language.upcase} language" do
@ -335,6 +357,16 @@ token int /\\d+/ <<
$$ = v;
>>
Start -> int << $$ = $1; >>
EOF
when "rust"
write_grammar <<EOF
ptype i64;
token int /\\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> int << $$ = $1; >>
EOF
end
run_propane(language: language)
@ -429,6 +461,31 @@ E3 -> E4 << $$ = $1; >>
E3 -> E3 power E4 << $$ = pow($1, $3); >>
E4 -> integer << $$ = $1; >>
E4 -> lparen E1 rparen << $$ = $2; >>
EOF
when "rust"
write_grammar <<EOF
ptype u64;
token plus /\\+/;
token times /\\*/;
token power /\\*\\*/;
token integer /\\d+/ <<
let mut v: u64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as u64; }
$$ = v;
>>
token lparen /\\(/;
token rparen /\\)/;
drop /\\s+/;
Start -> E1 << $$ = $1; >>
E1 -> E2 << $$ = $1; >>
E1 -> E1 plus E2 << $$ = $1 + $3; >>
E2 -> E3 << $$ = $1; >>
E2 -> E2 times E3 << $$ = $1 * $3; >>
E3 -> E4 << $$ = $1; >>
E3 -> E3 power E4 << $$ = $1.pow($3 as u32); >>
E4 -> integer << $$ = $1; >>
E4 -> lparen E1 rparen << $$ = $2; >>
EOF
end
run_propane(language: language)
@ -505,6 +562,16 @@ token def;
Start -> Abcs def;
Abcs -> ;
Abcs -> abc Abcs;
EOF
when "rust"
write_grammar <<EOF
token abc <<
println!("abc!");
>>
token def;
Start -> Abcs def;
Abcs -> ;
Abcs -> abc Abcs;
EOF
end
run_propane(language: language)
@ -541,6 +608,12 @@ import std.stdio;
token abc;
/def/ << writeln("def!"); >>
Start -> abc;
EOF
when "rust"
write_grammar <<EOF
token abc;
/def/ << println!("def!"); >>
Start -> abc;
EOF
end
run_propane(language: language)
@ -583,6 +656,16 @@ token abc;
return $token(abc);
>>
Start -> abc;
EOF
when "rust"
write_grammar <<EOF
token abc;
/def/ << println!("def!"); >>
/ghi/ <<
println!("ghi!");
return $token(abc);
>>
Start -> abc;
EOF
end
run_propane(language: language)
@ -641,6 +724,25 @@ string: /"/ <<
return $token(string);
>>
Start -> abc string def;
EOF
when "rust"
write_grammar <<EOF
token abc;
token def;
tokenid string;
drop /\\s+/;
/"/ <<
println!("begin string mode");
$mode(string);
>>
string: /[^"]+/ <<
println!("captured string");
>>
string: /"/ <<
$mode(default);
return $token(string);
>>
Start -> abc string def;
EOF
end
run_propane(language: language)
@ -699,6 +801,24 @@ default, identonly: drop /\\s+/;
Start -> abc dot ident <<
writeln("ident: ", $3);
>>
EOF
when "rust"
write_grammar <<EOF
ptype u8;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match_[0];
$mode(default);
return $token(ident);
>>
token dot /\\./ <<
$mode(identonly);
>>
default, identonly: drop /\\s+/;
Start -> abc dot ident <<
println!("ident: {}", $3 as char);
>>
EOF
end
run_propane(language: language)
@ -736,6 +856,14 @@ token b;
Start -> A B << writeln("Start!"); >>
A -> a << writeln("A!"); >>
B -> b << writeln("B!"); >>
EOF
when "rust"
write_grammar <<EOF
token a;
token b;
Start -> A B << println!("Start!"); >>
A -> a << println!("A!"); >>
B -> b << println!("B!"); >>
EOF
end
run_propane(language: language)
@ -843,6 +971,37 @@ Start -> A:ay B:bee C <<
A -> a;
B -> b;
C -> << ${context.c_is_null} = ($$.valid) ? 0 : 1; >>
EOF
when "rust"
write_grammar <<EOF
tree;
context_user_fields <<
pub start_n_fields: i64,
pub start_a_value: i64,
pub a_value: i64,
pub b_value: i64,
pub b_token: p_token_t,
pub c_is_null: i64,
pub c_field_is_null: i64,
pub alias_a_value: i64,
pub alias_b_value: i64,
>>
ptype i64;
token a << $$ = 11; >>
token b << $$ = 22; >>
Start -> A:ay B:bee C <<
${context.start_n_fields} = $$.n_fields() as i64;
${context.start_a_value} = $$.pA().pToken1().pvalue();
${context.a_value} = $1.pToken1().pvalue();
${context.b_value} = $2.pToken1().pvalue();
${context.b_token} = $2.pToken1().token();
${context.c_field_is_null} = if $$.pC().valid() { 0 } else { 1 };
${context.alias_a_value} = ${ay}.pToken1().pvalue();
${context.alias_b_value} = ${bee}.pToken1().pvalue();
>>
A -> a;
B -> b;
C -> << ${context.c_is_null} = if $$.valid() { 0 } else { 1 }; >>
EOF
end
run_propane(language: language)
@ -853,12 +1012,15 @@ EOF
end
it "parses lists" do
ptype = language == "d" ? "uint" : (language == "rust" ? "u32" : "uint32_t")
zero = language == "rust" ? "0" : "0u"
one = language == "rust" ? "1" : "1u"
write_grammar <<EOF
ptype #{language == "d" ? "uint" : "uint32_t"};
ptype #{ptype};
token a;
Start -> As << $$ = $1; >>
As -> << $$ = 0u; >>
As -> As a << $$ = $1 + 1u; >>
As -> << $$ = #{zero}; >>
As -> As a << $$ = $1 + #{one}; >>
EOF
run_propane(language: language)
compile("spec/test_parsing_lists.#{language}", language: language)
@ -912,6 +1074,13 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
writeln("Matched token is ", match);
>>
Start -> id;
EOF
when "rust"
write_grammar <<EOF
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
>>
Start -> id;
EOF
end
run_propane(language: language)
@ -945,6 +1114,16 @@ token word /[a-z]+/ <<
Start -> word <<
$$ = $1;
>>
EOF
when "rust"
write_grammar <<EOF
ptype u64;
token word /[a-z]+/ <<
$$ = match_length as u64;
>>
Start -> word <<
$$ = $1;
>>
EOF
end
run_propane(language: language)
@ -1097,6 +1276,25 @@ Start -> Words;
Words -> ;
Words -> word Words;
Words -> stop Words;
EOF
when "rust"
write_grammar <<EOF
context_user_fields <<
pub last_start: p_position_t,
pub last_end: p_position_t,
>>
drop /\\s+/;
token word /[a-z]+/ <<
${context.last_start} = ${position};
${context.last_end} = ${end_position};
>>
token stop /!/ <<
$terminate(42);
>>
Start -> Words;
Words -> ;
Words -> word Words;
Words -> stop Words;
EOF
end
run_propane(language: language)
@ -1159,6 +1357,21 @@ tokenid t;
return $token(t);
>>
Start -> t;
EOF
when "rust"
write_grammar <<EOF
tokenid t;
/\\a/ << println!("A"); >>
/\\b/ << println!("B"); >>
/\\t/ << println!("T"); >>
/\\n/ << println!("N"); >>
/\\v/ << println!("V"); >>
/\\f/ << println!("F"); >>
/\\r/ << println!("R"); >>
/t/ <<
return $token(t);
>>
Start -> t;
EOF
end
run_propane(language: language)
@ -1195,7 +1408,7 @@ EOF
write_grammar <<EOF
tree;
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 11; >>
token b << $$ = 22; >>
@ -1239,7 +1452,7 @@ tree;
tree_prefix P ;
tree_suffix S;
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 11; >>
token b << $$ = 22; >>
@ -1320,6 +1533,24 @@ Start -> a? b R? <<
>>
R -> c d << $$ = "cd"; >>
R (string) -> d c << $$ = "dc"; >>
EOF
elsif language == "rust"
write_grammar <<EOF
ptype i64;
ptype float = f64;
ptype string = String;
token a (float) << $$ = 1.5; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
token d << $$ = 4; >>
Start -> a? b R? <<
println!("a: {}", $1);
println!("b: {}", $2);
println!("R: {}", $3);
>>
R -> c d << $$ = "cd".to_string(); >>
R (string) -> d c << $$ = "dc".to_string(); >>
EOF
else
write_grammar <<EOF
@ -1350,7 +1581,7 @@ EOF
expect(results.stderr).to eq ""
expect(results.status).to eq 0
verify_lines(results.stdout, [
"a: 0#{language == "d" ? "" : ".0"}",
"a: 0#{["d", "rust"].include?(language) ? "" : ".0"}",
"b: 2",
"R: ",
"a: 1.5",
@ -1387,6 +1618,18 @@ tree;
#include <stdio.h>
>>
token a;
token b;
token c;
token d;
Start -> a? b R?;
R -> c d;
R -> d c;
EOF
end
if language == "rust"
write_grammar <<EOF
tree;
token a;
token b;
token c;
@ -1398,6 +1641,7 @@ EOF
end
run_propane(language: language)
compile("spec/test_optional_rule_component_tree.#{language}", language: language)
# (rust grammar written above overrides the C/D form for the Rust run)
results = run_test(language: language)
expect(results.stderr).to eq ""
expect(results.status).to eq 0
@ -1428,6 +1672,18 @@ tree;
#include <stdio.h>
>>
token a;
token b;
token c;
token d;
Start -> a?:a b R?:r;
R -> c d;
R -> d c;
EOF
end
if language == "rust"
write_grammar <<EOF
tree;
token a;
token b;
token c;
@ -1528,7 +1784,19 @@ EOF
end
it "allows specifying field aliases when tree mode is not enabled" do
if language == "d"
if language == "rust"
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
>>
drop /\\s+/;
Start -> id:first id:second <<
println!("first is {}", ${first});
println!("second is {}", ${second});
>>
EOF
elsif language == "d"
write_grammar <<EOF
<<
import std.stdio;
@ -1574,7 +1842,22 @@ EOF
end
it "aliases the correct field when multiple rules are in a rule set when tree mode is not enabled" do
if language == "d"
if language == "rust"
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
>>
drop /\\s+/;
Start -> id;
Start -> Foo;
Start -> id:first id:second <<
println!("first is {}", ${first});
println!("second is {}", ${second});
>>
Foo -> ;
EOF
elsif language == "d"
write_grammar <<EOF
<<
import std.stdio;
@ -1640,7 +1923,7 @@ EOF
it "allows multiple starting rules" do
write_grammar <<EOF
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
@ -1660,7 +1943,7 @@ EOF
it "supports parse_inner APIs that treat provided tokens as follow tokens" do
write_grammar <<EOF
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 1; >>
token b << $$ = 2; >>
Start -> Y << $$ = $1; >>
@ -1675,7 +1958,7 @@ EOF
it "parse_inner APIs block success when the outer rule is unfinished" do
write_grammar <<EOF
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
@ -1691,7 +1974,7 @@ EOF
it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
write_grammar <<EOF
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a;
token b;
start Start;
@ -1750,7 +2033,7 @@ EOF
it "allows multiple starting rules in tree mode" do
write_grammar <<EOF
tree;
ptype int;
ptype #{language == "rust" ? "i64" : "int"};
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
@ -1795,7 +2078,16 @@ EOF
end
it "executes code blocks associated with drop statements" do
if language == "d"
if language == "rust"
write_grammar <<EOF
drop /\\s+/;
drop /#(.*)\\n/ <<
eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
>>
token a;
Start -> a;
EOF
elsif language == "d"
write_grammar <<EOF
<<
import std.stdio;
@ -1829,7 +2121,24 @@ EOF
end
it "allows user-defined context fields" do
if language == "d"
if language == "rust"
write_grammar <<EOF
context_user_fields <<
pub comments: String,
pub acount: u32,
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
>>
token a <<
${context.acount} += 1;
>>
Start -> As;
As -> ;
As -> a As;
EOF
elsif language == "d"
write_grammar <<EOF
context_user_fields <<
string comments;
@ -1889,7 +2198,28 @@ EOF
end
it "allows custom token user fields" do
if language == "d"
if language == "rust"
write_grammar <<EOF
context_user_fields <<
pub comments: String,
>>
token_user_fields <<
pub comments: String,
>>
on_token_node <<
${token.comments} = std::mem::take(&mut ${context.comments});
>>
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
>>
token id /\\w+/;
Start -> IDs;
IDs -> ;
IDs -> id:id IDs;
EOF
elsif language == "d"
write_grammar <<EOF
context_user_fields <<
string comments;
@ -1986,7 +2316,41 @@ EOF
end
it "allows a custom lex function" do
if language == "d"
if language == "rust"
write_grammar <<EOF
<<
fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
let mut result = P_SUCCESS;
if context.count > 0 {
out_token_info.token = TOKEN_a;
out_token_info.pvalue = p_value(context.count);
context.count -= 1;
} else {
result = p_lex(context, out_token_info);
if out_token_info.token == TOKEN_c {
context.count = 3;
}
}
result
}
>>
context_user_fields <<
pub count: usize,
>>
ptype usize;
lex_fn mylexfn;
token a << $$ = 7; >>
token b << $$ = 8; >>
token c << $$ = 9; >>
Start -> << $$ = 0; >>
Start -> Start ID << $$ = ($1 << 4) | $2; >>
ID -> a << $$ = $1; >>
ID -> b << $$ = $1; >>
ID -> c << $$ = $1; >>
EOF
elsif language == "d"
write_grammar <<EOF
<<
private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
@ -2068,7 +2432,19 @@ EOF
end
it "provides accessor functions to extract user values from a parser value" do
if language == "d"
if language == "rust"
write_grammar <<EOF
ptype i64;
ptype float = f64;
ptype string = String;
drop /\\s+/;
token num /\\d+/ << $$ = 42; >>
token flt (float) /f/ << $$ = 1.5; >>
token str (string) /s/ << $$ = "hello".to_string(); >>
Start -> num flt str;
EOF
elsif language == "d"
write_grammar <<EOF
ptype int;
ptype float = float;
@ -2101,7 +2477,49 @@ EOF
end
it "allows accessing rule and component text positions" do
if language == "d"
if language == "rust"
write_grammar <<EOF
drop /\\s+/;
token tok1;
token tok2;
token ident /[a-zA-Z_]\\w*/;
token num /\\d+/;
Num -> num;
Start -> ident Num <<
println!("ident start: {}, {}", ${1.position}.row, ${1.position}.col);
println!("ident end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
println!("Num start: {}, {}", ${2.position}.row, ${2.position}.col);
println!("Num end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
println!("Start start: {}, {}", ${$.position}.row, ${$.position}.col);
println!("Start end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
>>
R -> Empty tok2 <<
println!("Empty start: {}, {}", ${1.position}.row, ${1.position}.col);
println!("Empty end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
println!("tok2 start: {}, {}", ${2.position}.row, ${2.position}.col);
println!("tok2 end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
println!("R start: {}, {}", ${$.position}.row, ${$.position}.col);
println!("R end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
>>
R -> tok1 Empty <<
println!("tok1 start: {}, {}", ${1.position}.row, ${1.position}.col);
println!("tok1 end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
println!("Empty start: {}, {}", ${2.position}.row, ${2.position}.col);
println!("Empty end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
println!("R2 start: {}, {}", ${$.position}.row, ${$.position}.col);
println!("R2 end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
>>
Empty -> ;
Start -> R <<
println!("StartR start: {}, {}", ${$.position}.row, ${$.position}.col);
println!("StartR end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
>>
Start -> Empty <<
println!("StartEmpty start: {}, {}", ${$.position}.row, ${$.position}.col);
println!("StartEmpty end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
>>
EOF
elsif language == "d"
write_grammar <<EOF
<<
import std.stdio;

67
spec/rewind.rust.propane Normal file
View File

@ -0,0 +1,67 @@
<<
fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
loop {
let result = p_lex(context, out_token_info);
if result != P_SUCCESS {
return result;
}
if out_token_info.token == TOKEN_repeat {
let mut count_info = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
assert_eq!(TOKEN_num, count_info.token);
let mut brace_info = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
assert_eq!(TOKEN_lbrace, brace_info.token);
context.remaining = p_value_get(&count_info.pvalue);
context.body_index = p_input_index(context);
context.body_position = p_position(context);
continue;
}
if out_token_info.token == TOKEN_rbrace {
if context.remaining > 1 {
context.remaining -= 1;
let bi = context.body_index;
let bp = context.body_position;
p_set_input_index(context, bi);
p_set_position(context, bp);
continue;
}
context.remaining = 0;
continue;
}
if out_token_info.token == TOKEN_num {
context.num_cols.push(out_token_info.position.col);
}
return result;
}
}
>>
context_user_fields <<
pub nums: Vec<i64>,
pub num_cols: Vec<u32>,
pub remaining: i64,
pub body_index: usize,
pub body_position: p_position_t,
>>
ptype i64;
lex_fn mylexfn;
drop /\s+/;
token repeat /repeat/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> Statements;
Statements -> ;
Statements -> Statement Statements;
Statement -> Add;
Add -> num plus num << ${context.nums}.push($1 + $3); >>

16
spec/test_basic_math_grammar.rs Normal file
View File

@ -0,0 +1,16 @@
use testparser::*;
fn main() {
let cases: [(&[u8], u64); 4] = [
(b"1 + 2 * 3 + 4", 11),
(b"1 * 2 ** 4 * 3", 48),
(b"(1 + 2) * 3 + 4", 13),
(b"(2 * 2) ** 3 + 4 + 5", 73),
];
for (input, expected) in cases {
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(expected, p_result(&context));
p_context_delete(context);
}
}

8
spec/test_custom_lex_fn.rs Normal file
View File

@ -0,0 +1,8 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"cbacba");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(0x932187932187, p_result(&c));
p_context_delete(c);
}

7
spec/test_drop_code_block.rs Normal file
View File

@ -0,0 +1,7 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b" # comment 1\n# comment 2\na\n");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
}

35
spec/test_error_positions.rs Normal file
View File

@ -0,0 +1,35 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"a 42");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"a\n123\na a");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
assert_eq!(3, p_position(&c).row);
assert_eq!(4, p_position(&c).col);
assert_eq!(TOKEN_a, p_token(&c));
p_context_delete(c);
let mut c = p_context_new(b"12");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
assert_eq!(1, p_position(&c).row);
assert_eq!(1, p_position(&c).col);
assert_eq!(TOKEN_num, p_token(&c));
p_context_delete(c);
let mut c = p_context_new(b"a 12\n\nab");
assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
assert_eq!(3, p_position(&c).row);
assert_eq!(2, p_position(&c).col);
p_context_delete(c);
let mut c = p_context_new(b"a 12\n\na\n\n77\na \xAA");
assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
assert_eq!(6, p_position(&c).row);
assert_eq!(5, p_position(&c).col);
assert_eq!("a", p_token_names[TOKEN_a as usize]);
assert_eq!("num", p_token_names[TOKEN_num as usize]);
p_context_delete(c);
}

7
spec/test_field_aliases.rs Normal file
View File

@ -0,0 +1,7 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"foo1\nbar2");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
}

28
spec/test_input_index.rs Normal file
View File

@ -0,0 +1,28 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"ab");
assert_eq!(0, p_input_index(&c));
p_context_delete(c);
let mut c = p_context_new(b"a b");
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
assert_eq!(1, p_input_index(&c));
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_b, ti.token);
assert_eq!(3, p_input_index(&c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
assert_eq!(2, p_input_index(&c));
p_context_delete(c);
let mut c = p_context_new(b"abb");
let follow = [TOKEN_b];
assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
assert_eq!(2, p_input_index(&c));
p_context_delete(c);
}

49
spec/test_lexer.rs Normal file
View File

@ -0,0 +1,49 @@
use testparser::*;
fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
assert_eq!(row, ti.position.row);
assert_eq!(col, ti.position.col);
assert_eq!(erow, ti.end_position.row);
assert_eq!(ecol, ti.end_position.col);
assert_eq!(len, ti.length);
assert_eq!(token, ti.token);
}
fn main() {
let mut cp: p_code_point_t = 0;
let mut cpl: u8 = 0;
assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
assert_eq!('5' as u32, cp);
assert_eq!(1, cpl);
assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
assert_eq!(0xA9, cp);
assert_eq!(2, cpl);
assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
assert_eq!(0x1F9E1, cp);
assert_eq!(4, cpl);
assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
let mut context = p_context_new(b"5 + 4 * \n677 + 567");
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
p_context_delete(context);
let mut context = p_context_new(b"");
assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
p_context_delete(context);
}

8
spec/test_lexer_match_text.rs Normal file
View File

@ -0,0 +1,8 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"identifier_123");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass1");
p_context_delete(context);
}

13
spec/test_lexer_modes.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"abc \"a string\" def");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass1");
p_context_delete(context);
let mut context = p_context_new(b"abc \"abc def\" def");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass2");
p_context_delete(context);
}

13
spec/test_lexer_multiple_modes.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"abc.def");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass1");
p_context_delete(context);
let mut context = p_context_new(b"abc . abc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass2");
p_context_delete(context);
}

38
spec/test_lexer_positions.rs Normal file
View File

@ -0,0 +1,38 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"abc\n defg hi\n!");
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_word, ti.token);
assert_eq!(1, c.last_start.row);
assert_eq!(1, c.last_start.col);
assert_eq!(1, c.last_end.row);
assert_eq!(3, c.last_end.col);
assert_eq!(c.last_start.row, ti.position.row);
assert_eq!(c.last_start.col, ti.position.col);
assert_eq!(c.last_end.row, ti.end_position.row);
assert_eq!(c.last_end.col, ti.end_position.col);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_word, ti.token);
assert_eq!(2, c.last_start.row);
assert_eq!(3, c.last_start.col);
assert_eq!(2, c.last_end.row);
assert_eq!(6, c.last_end.col);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_word, ti.token);
assert_eq!(2, c.last_start.row);
assert_eq!(8, c.last_start.col);
assert_eq!(2, c.last_end.row);
assert_eq!(9, c.last_end.col);
assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
assert_eq!(42, p_user_terminate_code(&c));
assert_eq!(3, p_position(&c).row);
assert_eq!(1, p_position(&c).col);
p_context_delete(c);
}

13
spec/test_lexer_result_value.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"x");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(1, p_result(&context));
p_context_delete(context);
let mut context = p_context_new(b"fabulous");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(8, p_result(&context));
p_context_delete(context);
}

12
spec/test_lexer_unknown_character.rs Normal file
View File

@ -0,0 +1,12 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"x");
assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
p_context_delete(context);
let mut context = p_context_new(b"123");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(123, p_result(&context));
p_context_delete(context);
}

9
spec/test_macros.rs Normal file
View File

@ -0,0 +1,9 @@
use testparser::*;
fn main() {
let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n";
let mut c = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(vec![166, 223, 88, 223], c.nums);
p_context_delete(c);
}

7
spec/test_match_backslashes.rs Normal file
View File

@ -0,0 +1,7 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
}

12
spec/test_multiple_parsers.rs Normal file
View File

@ -0,0 +1,12 @@
use testparsermyp1 as m1;
use testparsermyp2 as m2;
fn main() {
let mut context1 = m1::myp1_context_new(b"a\n1");
assert_eq!(m1::P_SUCCESS, m1::myp1_parse(&mut context1));
m1::myp1_context_delete(context1);
let mut context2 = m2::myp2_context_new(b"bcb");
assert_eq!(m2::P_SUCCESS, m2::myp2_parse(&mut context2));
m2::myp2_context_delete(context2);
}

43
spec/test_named_optional_rule_component_tree.rs Normal file
View File

@ -0,0 +1,43 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"b");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(!start.a().valid());
assert!(start.pToken2().valid());
assert_eq!(TOKEN_b, start.pToken2().token());
assert!(!start.pR3().valid());
assert!(!start.pR().valid());
assert!(!start.r().valid());
}
p_context_delete(context);
let mut context = p_context_new(b"abcd");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.a().valid());
assert_eq!(TOKEN_a, start.pToken1().token());
assert!(start.pToken2().valid());
assert!(start.pR3().valid());
assert!(start.pR().valid());
assert!(start.r().valid());
assert_eq!(start.pR().node_id(), start.pR3().node_id());
assert_eq!(start.pR().node_id(), start.r().node_id());
assert_eq!(TOKEN_c, start.pR().pToken1().token());
}
p_context_delete(context);
let mut context = p_context_new(b"bdc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(!start.a().valid());
assert!(start.pToken2().valid());
assert!(start.r().valid());
assert_eq!(TOKEN_d, start.pR().pToken1().token());
}
p_context_delete(context);
}

9
spec/test_optional_rule_component.rs Normal file
View File

@ -0,0 +1,9 @@
use testparser::*;
fn main() {
for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] {
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
}
}

40
spec/test_optional_rule_component_tree.rs Normal file
View File

@ -0,0 +1,40 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"b");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(!start.pToken1().valid());
assert!(start.pToken2().valid());
assert_eq!(TOKEN_b, start.pToken2().token());
assert!(!start.pR3().valid());
assert!(!start.pR().valid());
}
p_context_delete(context);
let mut context = p_context_new(b"abcd");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.pToken1().valid());
assert_eq!(TOKEN_a, start.pToken1().token());
assert!(start.pToken2().valid());
assert!(start.pR3().valid());
assert!(start.pR().valid());
assert_eq!(start.pR().node_id(), start.pR3().node_id());
assert_eq!(TOKEN_c, start.pR().pToken1().token());
}
p_context_delete(context);
let mut context = p_context_new(b"bdc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(!start.pToken1().valid());
assert!(start.pToken2().valid());
assert!(start.pR().valid());
assert_eq!(TOKEN_d, start.pR().pToken1().token());
}
p_context_delete(context);
}

30
spec/test_parse_inner.rs Normal file
View File

@ -0,0 +1,30 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"a");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
assert_eq!(1, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
assert_eq!(1, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[]));
p_context_delete(c);
let mut c = p_context_new(b"a");
assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
assert_eq!(1, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF]));
p_context_delete(c);
}

17
spec/test_parse_inner_nested.rs Normal file
View File

@ -0,0 +1,17 @@
use testparser::*;
fn eval(input: &[u8]) -> i64 {
let mut c = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut c));
let v = p_result(&c);
p_context_delete(c);
v
}
fn main() {
assert_eq!(5, eval(b"2 + 3"));
assert_eq!(3, eval(b"(1 + 2)"));
assert_eq!(14, eval(b"2 + (3 + 4) + 5"));
assert_eq!(37, eval(b"2 + (10 + (20 + 5))"));
assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))"));
}

36
spec/test_parse_inner_nested_tree.rs Normal file
View File

@ -0,0 +1,36 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"(3 + 4) + (5 + 6)");
assert_eq!(P_SUCCESS, p_parse(&mut c));
{
let tree = p_result(&c);
assert!(tree.valid());
/* Start -> Expr, where the top Expr is "Expr plus num". */
let top = tree.pExpr();
assert!(top.valid());
assert!(top.pExpr().valid());
assert!(top.pToken2().valid());
assert!(top.pToken3().valid());
/* The '+' joining the two groups is at column 9. */
assert_eq!(1, top.pToken2().position().row);
assert_eq!(9, top.pToken2().position().col);
/* Right operand: synthesized num for "(5 + 6)", columns 11..17. */
assert_eq!(11, top.pToken3().position().col);
assert_eq!(17, top.pToken3().end_position().col);
/* Left operand: synthesized num for "(3 + 4)", columns 1..7. */
let left = top.pExpr();
assert!(left.pToken1().valid());
assert_eq!(1, left.pToken1().position().col);
assert_eq!(7, left.pToken1().end_position().col);
/* The whole tree spans columns 1..17. */
assert_eq!(1, tree.position().col);
assert_eq!(17, tree.end_position().col);
}
p_context_delete(c);
}

31
spec/test_parse_inner_recursive.rs Normal file
View File

@ -0,0 +1,31 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"c");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
assert_eq!(3, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"acb");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
assert_eq!(3, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"ac");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"ac");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF]));
p_context_delete(c);
let mut c = p_context_new(b"acb");
assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
assert_eq!(3, p_result_Start(&c));
p_context_delete(c);
let mut c = p_context_new(b"c");
assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
assert_eq!(3, p_result_Start(&c));
p_context_delete(c);
}

47
spec/test_parse_inner_shared.rs Normal file
View File

@ -0,0 +1,47 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"aba");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"abb");
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
assert_eq!(11, p_result_R1(&c));
p_context_delete(c);
let mut c = p_context_new(b"abb");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"abb");
assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
assert_eq!(11, p_result_R1(&c));
let pos = p_position(&c);
assert_eq!(1, pos.row);
assert_eq!(3, pos.col);
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_b, ti.token);
assert_eq!(3, ti.position.col);
p_context_delete(c);
let mut c = p_context_new(b"aba");
assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a]));
assert_eq!(11, p_result_R1(&c));
let pos = p_position(&c);
assert_eq!(3, pos.col);
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
p_context_delete(c);
let mut c = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[]));
assert_eq!(11, p_result_R1(&c));
p_context_delete(c);
}

36
spec/test_parse_inner_tree.rs Normal file
View File

@ -0,0 +1,36 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
{
let tree = p_result_R1(&c);
assert!(tree.valid());
assert!(tree.pToken1().valid());
assert_eq!(TOKEN_a, tree.pToken1().token());
assert!(tree.pToken2().valid());
assert_eq!(TOKEN_b, tree.pToken2().token());
}
p_context_delete(c);
let mut c = p_context_new(b"abb");
assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
{
let tree = p_result_R1(&c);
assert!(tree.valid());
assert_eq!(TOKEN_a, tree.pToken1().token());
assert_eq!(1, tree.pToken1().position().row);
assert_eq!(1, tree.pToken1().position().col);
assert_eq!(TOKEN_b, tree.pToken2().token());
assert_eq!(2, tree.pToken2().position().col);
assert_eq!(1, tree.position().col);
assert_eq!(2, tree.end_position().col);
}
let pos = p_position(&c);
assert_eq!(3, pos.col);
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_b, ti.token);
assert_eq!(3, ti.position.col);
p_context_delete(c);
}

9
spec/test_parser_identical_rules_lookahead.rs Normal file
View File

@ -0,0 +1,9 @@
use testparser::*;
fn main() {
for input in [&b"aba"[..], &b"abb"[..]] {
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
}
}

18
spec/test_parser_rule_from_multiple_states.rs Normal file
View File

@ -0,0 +1,18 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"a");
assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context));
assert_eq!(1, p_position(&context).row);
assert_eq!(2, p_position(&context).col);
assert_eq!(TOKEN___EOF, p_token(&context));
p_context_delete(context);
let mut context = p_context_new(b"a b");
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
let mut context = p_context_new(b"bb");
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
}

7
spec/test_parser_rule_user_code.rs Normal file
View File

@ -0,0 +1,7 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
}

24
spec/test_parser_user_code_tree.rs Normal file
View File

@ -0,0 +1,24 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"ab");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(3, context.start_n_fields);
assert_eq!(11, context.start_a_value);
assert_eq!(11, context.a_value);
assert_eq!(22, context.b_value);
assert_eq!(TOKEN_b, context.b_token);
assert_eq!(1, context.c_is_null);
assert_eq!(1, context.c_field_is_null);
assert_eq!(11, context.alias_a_value);
assert_eq!(22, context.alias_b_value);
{
let start = p_result(&context);
assert!(start.pA().valid());
assert!(start.pB().valid());
assert!(!start.pC().valid());
}
p_context_delete(context);
}

31
spec/test_parsing_json.rs Normal file
View File

@ -0,0 +1,31 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"{}");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(JSON_OBJECT, p_result(&c).id());
p_context_delete(c);
let mut c = p_context_new(b"[]");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(JSON_ARRAY, p_result(&c).id());
p_context_delete(c);
let mut c = p_context_new(b"-45.6");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(JSON_NUMBER, p_result(&c).id());
assert_eq!(-45.6, p_result(&c).number());
p_context_delete(c);
let mut c = p_context_new(b"{\"hi\":true}");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(JSON_OBJECT, p_result(&c).id());
p_context_delete(c);
let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]");
assert_eq!(P_SUCCESS, p_parse(&mut c));
let v = p_result(&c);
assert_eq!(JSON_ARRAY, v.id());
assert_eq!(5, v.array_len());
p_context_delete(c);
}

11
spec/test_parsing_lists.rs Normal file
View File

@ -0,0 +1,11 @@
use testparser::*;
fn main() {
let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)];
for (input, expected) in cases {
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert_eq!(expected, p_result(&context));
p_context_delete(context);
}
}

13
spec/test_pattern.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"abcdef");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass1");
p_context_delete(context);
let mut context = p_context_new(b"defabcdef");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass2");
p_context_delete(context);
}

19
spec/test_positions.rs Normal file
View File

@ -0,0 +1,19 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b" Hello\n\n 4200\n");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
println!();
let mut c = p_context_new(b"\n tok2");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
println!();
let mut c = p_context_new(b" tok1");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
}

7
spec/test_return_token_from_pattern.rs Normal file
View File

@ -0,0 +1,7 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"defghidef");
assert_eq!(P_SUCCESS, p_parse(&mut context));
p_context_delete(context);
}

13
spec/test_rewind.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
/* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
* times (recording 30 each time), followed by "5 + 5" (recording 10). */
let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(vec![30, 30, 30, 10], c.nums);
assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols);
p_context_delete(c);
}

66
spec/test_set_position.rs Normal file
View File

@ -0,0 +1,66 @@
use testparser::*;
fn main() {
let mut ti = p_token_info_t::default();
/* Baseline: default (1, 1). */
let mut c = p_context_new(b"ab");
let pos = p_position(&c);
assert_eq!(1, pos.row);
assert_eq!(1, pos.col);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
assert_eq!(1, ti.position.row);
assert_eq!(1, ti.position.col);
p_context_delete(c);
/* p_set_position overrides the initial position. */
let mut c = p_context_new(b"ab");
p_set_position(&mut c, p_position_t { row: 5, col: 20 });
let pos = p_position(&c);
assert_eq!(5, pos.row);
assert_eq!(20, pos.col);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
assert_eq!(5, ti.position.row);
assert_eq!(20, ti.position.col);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_b, ti.token);
assert_eq!(5, ti.position.row);
assert_eq!(21, ti.position.col);
p_context_delete(c);
/* Set position before a full parse. */
let mut c = p_context_new(b"ab");
p_set_position(&mut c, p_position_t { row: 3, col: 7 });
assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
p_context_delete(c);
/* Set position before a failing parse: error position is relative. */
let mut c = p_context_new(b"aa");
p_set_position(&mut c, p_position_t { row: 10, col: 2 });
assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
let ep = p_position(&c);
assert_eq!(10, ep.row);
assert_eq!(3, ep.col);
p_context_delete(c);
/* p_set_input_index rewinds the byte cursor to re-read a section. */
let mut c = p_context_new(b"ab");
let start_index = p_input_index(&c);
let start_position = p_position(&c);
assert_eq!(0, start_index);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_b, ti.token);
assert_eq!(2, p_input_index(&c));
p_set_input_index(&mut c, start_index);
p_set_position(&mut c, start_position);
assert_eq!(0, p_input_index(&c));
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_a, ti.token);
assert_eq!(1, ti.position.row);
assert_eq!(1, ti.position.col);
p_context_delete(c);
}

2
spec/test_start_rule.rs Normal file
View File

@ -0,0 +1,2 @@
fn main() {
}

12
spec/test_start_rule_tree.rs Normal file
View File

@ -0,0 +1,12 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"hi");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let top = p_result(&context);
assert!(top.pToken().valid());
assert_eq!(TOKEN_hi, top.pToken().token());
}
p_context_delete(context);
}

18
spec/test_starting_rules.rs Normal file
View File

@ -0,0 +1,18 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"bbbb");
assert_eq!(P_SUCCESS, p_parse(&mut c));
assert_eq!(8, p_result(&c));
p_context_delete(c);
let mut c = p_context_new(b"bbbb");
assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
assert_eq!(8, p_result_Bs(&c));
p_context_delete(c);
let mut c = p_context_new(b"c");
assert_eq!(P_SUCCESS, p_parse_R(&mut c));
assert_eq!(3, p_result_R(&c));
p_context_delete(c);
}

34
spec/test_starting_rules_tree.rs Normal file
View File

@ -0,0 +1,34 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"bbbb");
assert_eq!(P_SUCCESS, p_parse(&mut c));
{
let start = p_result(&c);
assert!(start.bs().valid());
assert!(start.bs().b().valid());
assert!(start.bs().bs().b().valid());
assert!(start.bs().bs().bs().b().valid());
assert!(start.bs().bs().bs().bs().b().valid());
}
p_context_delete(c);
let mut c = p_context_new(b"bbbb");
assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
{
let bs = p_result_Bs(&c);
assert!(bs.b().valid());
assert!(bs.bs().b().valid());
assert!(bs.bs().bs().b().valid());
assert!(bs.bs().bs().bs().b().valid());
}
p_context_delete(c);
let mut c = p_context_new(b"c");
assert_eq!(P_SUCCESS, p_parse_R(&mut c));
{
let r = p_result_R(&c);
assert!(r.c().valid());
}
p_context_delete(c);
}

17
spec/test_token_user_fields.rs Normal file
View File

@ -0,0 +1,17 @@
use testparser::*;
fn main() {
let input = b"# c1\n# c2\n\nfirst\n\n \n \n # s1\n # s2\nsecond\n";
let mut c = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut c));
{
let start = p_result(&c);
assert!(start.pIDs().valid());
assert!(start.pIDs().id().valid());
assert_eq!("# c1\n# c2\n", start.pIDs().id().data().comments.as_str());
assert!(start.pIDs().pIDs().valid());
assert!(start.pIDs().pIDs().id().valid());
assert_eq!("# s1\n# s2\n", start.pIDs().pIDs().id().data().comments.as_str());
}
p_context_delete(c);
}

46
spec/test_tree.rs Normal file
View File

@ -0,0 +1,46 @@
use testparser::*;
fn main() {
let input = b"a, ((b)), b";
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.pItems1().valid());
assert!(start.pItems().valid());
let items = start.pItems();
assert!(items.pItem().valid());
assert!(items.pItem().pToken1().valid());
assert_eq!(TOKEN_a, items.pItem().pToken1().token());
assert_eq!(11, items.pItem().pToken1().pvalue());
let itemsmore = items.pItemsMore();
assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
assert!(itemsmore.pItemsMore().valid());
let itemsmore = itemsmore.pItemsMore();
assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
assert!(!itemsmore.pItemsMore().valid());
}
p_context_delete(context);
/* Empty input yields a Start node with no Items child. */
let mut context = p_context_new(b"");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert!(!p_result(&context).pItems().valid());
p_context_delete(context);
/* Dual rule alternative field positions. */
let mut context = p_context_new(b"2 1");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.pItems().pItem().pDual().pTwo1().valid());
assert!(start.pItems().pItem().pDual().pOne2().valid());
assert!(!start.pItems().pItem().pDual().pTwo2().valid());
assert!(!start.pItems().pItem().pDual().pOne1().valid());
}
p_context_delete(context);
println!("ok");
}

13
spec/test_tree_field_aliases.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"\na\nb\nc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert_eq!(TOKEN_a, start.first().pToken().token());
assert_eq!(TOKEN_b, start.second().pToken().token());
assert_eq!(TOKEN_c, start.third().pToken().token());
}
p_context_delete(context);
}

89
spec/test_tree_invalid_positions.rs Normal file
View File

@ -0,0 +1,89 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"\na\n bb ccc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
let t1 = start.pT1();
let k1 = t1.pToken();
let a1 = t1.pA();
assert_eq!(2, k1.position().row);
assert_eq!(1, k1.position().col);
assert_eq!(2, k1.end_position().row);
assert_eq!(1, k1.end_position().col);
assert!(a1.position().valid());
assert_eq!(3, a1.position().row);
assert_eq!(3, a1.position().col);
assert_eq!(3, a1.end_position().row);
assert_eq!(8, a1.end_position().col);
assert_eq!(2, t1.position().row);
assert_eq!(1, t1.position().col);
assert_eq!(3, t1.end_position().row);
assert_eq!(8, t1.end_position().col);
assert_eq!(2, start.position().row);
assert_eq!(1, start.position().col);
assert_eq!(3, start.end_position().row);
assert_eq!(8, start.end_position().col);
}
p_context_delete(context);
let mut context = p_context_new(b"a\nbb");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
let t1 = start.pT1();
let k1 = t1.pToken();
let a1 = t1.pA();
assert_eq!(1, k1.position().row);
assert_eq!(1, k1.position().col);
assert!(a1.position().valid());
assert_eq!(2, a1.position().row);
assert_eq!(1, a1.position().col);
assert_eq!(2, a1.end_position().row);
assert_eq!(2, a1.end_position().col);
assert_eq!(1, t1.position().row);
assert_eq!(2, t1.end_position().row);
assert_eq!(2, t1.end_position().col);
assert_eq!(1, start.position().row);
assert_eq!(2, start.end_position().row);
assert_eq!(2, start.end_position().col);
}
p_context_delete(context);
let mut context = p_context_new(b"a\nc\nc");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
let t1 = start.pT1();
let a1 = t1.pA();
assert!(a1.position().valid());
assert_eq!(2, a1.position().row);
assert_eq!(1, a1.position().col);
assert_eq!(3, a1.end_position().row);
assert_eq!(1, a1.end_position().col);
assert_eq!(1, t1.position().row);
assert_eq!(3, t1.end_position().row);
assert_eq!(1, t1.end_position().col);
assert_eq!(3, start.end_position().row);
assert_eq!(1, start.end_position().col);
}
p_context_delete(context);
let mut context = p_context_new(b"a");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
let t1 = start.pT1();
let a1 = t1.pA();
assert!(!a1.position().valid());
assert_eq!(1, t1.position().row);
assert_eq!(1, t1.position().col);
assert_eq!(1, t1.end_position().row);
assert_eq!(1, t1.end_position().col);
assert_eq!(1, start.position().row);
assert_eq!(1, start.end_position().row);
assert_eq!(1, start.end_position().col);
}
p_context_delete(context);
}

85
spec/test_tree_node_memory_remains.rs Normal file
View File

@ -0,0 +1,85 @@
use testparser::*;
fn main() {
let entries: [(&str, &str, p_token_t); 51] = [
("byte_val", "byte", TOKEN_byte),
("short_val", "short", TOKEN_short),
("int_val", "int", TOKEN_int),
("long_val", "long", TOKEN_long),
("ssize_t_val", "ssize_t", TOKEN_ssize_t),
("byte_to_short", "short", TOKEN_short),
("byte_to_int", "int", TOKEN_int),
("byte_to_long", "long", TOKEN_long),
("byte_to_ssize_t", "ssize_t", TOKEN_ssize_t),
("short_to_byte", "byte", TOKEN_byte),
("short_to_int", "int", TOKEN_int),
("short_to_long", "long", TOKEN_long),
("short_to_ssize_t", "ssize_t", TOKEN_ssize_t),
("int_to_byte", "byte", TOKEN_byte),
("int_to_short", "short", TOKEN_short),
("int_to_long", "long", TOKEN_long),
("int_to_ssize_t", "ssize_t", TOKEN_ssize_t),
("long_to_byte", "byte", TOKEN_byte),
("long_to_short", "short", TOKEN_short),
("long_to_int", "int", TOKEN_int),
("long_to_ssize_t", "ssize_t", TOKEN_ssize_t),
("ssize_t_to_byte", "byte", TOKEN_byte),
("ssize_t_to_short", "short", TOKEN_short),
("ssize_t_to_int", "int", TOKEN_int),
("ssize_t_to_long", "long", TOKEN_long),
("ubyte_val", "ubyte", TOKEN_ubyte),
("ushort_val", "ushort", TOKEN_ushort),
("uint_val", "uint", TOKEN_uint),
("ulong_val", "ulong", TOKEN_ulong),
("size_t_val", "size_t", TOKEN_size_t),
("ubyte_to_ushort", "ushort", TOKEN_ushort),
("ubyte_to_uint", "uint", TOKEN_uint),
("ubyte_to_ulong", "ulong", TOKEN_ulong),
("ubyte_to_size_t", "size_t", TOKEN_size_t),
("ushort_to_ubyte", "ubyte", TOKEN_ubyte),
("ushort_to_uint", "uint", TOKEN_uint),
("ushort_to_ulong", "ulong", TOKEN_ulong),
("ushort_to_size_t", "size_t", TOKEN_size_t),
("uint_to_ubyte", "ubyte", TOKEN_ubyte),
("uint_to_ushort", "ushort", TOKEN_ushort),
("uint_to_ulong", "ulong", TOKEN_ulong),
("uint_to_size_t", "size_t", TOKEN_size_t),
("ulong_to_ubyte", "ubyte", TOKEN_ubyte),
("ulong_to_ushort", "ushort", TOKEN_ushort),
("ulong_to_uint", "uint", TOKEN_uint),
("ulong_to_size_t", "size_t", TOKEN_size_t),
("size_t_to_ubyte", "ubyte", TOKEN_ubyte),
("size_t_to_ushort", "ushort", TOKEN_ushort),
("size_t_to_int", "int", TOKEN_int),
("size_t_to_ulong", "ulong", TOKEN_ulong),
("main", "int", TOKEN_int),
];
let mut input = String::new();
for (name, ret, _) in entries.iter() {
input.push_str(&format!("def {}() -> {} {{\nreturn 0x42;\n}}\n", name, ret));
}
let mut c = p_context_new(input.as_bytes());
assert_eq!(P_SUCCESS, p_parse(&mut c));
{
let pmod = p_result(&c);
let mut pfds = Vec::new();
let mut pmis = pmod.pModuleItems();
while pmis.valid() {
let pmi = pmis.pModuleItem();
assert!(pmi.valid());
let pfd = pmi.pFunctionDefinition();
if pfd.valid() {
pfds.insert(0, pfd);
}
pmis = pmis.pModuleItems();
}
assert_eq!(51, pfds.len());
for i in 0..pfds.len() {
assert_eq!(entries[i].0, pfds[i].name().data().pvalue.s.as_str());
assert_eq!(entries[i].2, pfds[i].returntype().pType().pTypeBase().pToken1().token());
}
}
p_context_delete(c);
}

46
spec/test_tree_ps.rs Normal file
View File

@ -0,0 +1,46 @@
use testparser::*;
fn main() {
let input = b"a, ((b)), b";
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.pItems1().valid());
assert!(start.pItems().valid());
let items = start.pItems();
assert!(items.pItem().valid());
assert!(items.pItem().pToken1().valid());
assert_eq!(TOKEN_a, items.pItem().pToken1().token());
assert_eq!(11, items.pItem().pToken1().pvalue());
let itemsmore = items.pItemsMore();
assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
assert!(itemsmore.pItemsMore().valid());
let itemsmore = itemsmore.pItemsMore();
assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
assert!(!itemsmore.pItemsMore().valid());
}
p_context_delete(context);
/* Empty input yields a Start node with no Items child. */
let mut context = p_context_new(b"");
assert_eq!(P_SUCCESS, p_parse(&mut context));
assert!(!p_result(&context).pItems().valid());
p_context_delete(context);
/* Dual rule alternative field positions. */
let mut context = p_context_new(b"2 1");
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
assert!(start.pItems().pItem().pDual().pTwo1().valid());
assert!(start.pItems().pItem().pDual().pOne2().valid());
assert!(!start.pItems().pItem().pDual().pTwo2().valid());
assert!(!start.pItems().pItem().pDual().pOne1().valid());
}
p_context_delete(context);
println!("ok");
}

27
spec/test_tree_token_positions.rs Normal file
View File

@ -0,0 +1,27 @@
use testparser::*;
fn main() {
let input = b"abbccc";
let mut context = p_context_new(input);
assert_eq!(P_SUCCESS, p_parse(&mut context));
{
let start = p_result(&context);
let t1 = start.pT1();
let t2 = start.pT2();
let t3 = start.pT3();
assert_eq!(1, t1.position().col);
assert_eq!(1, t1.end_position().col);
assert_eq!(2, t2.position().col);
assert_eq!(3, t2.end_position().col);
assert_eq!(4, t3.position().col);
assert_eq!(6, t3.end_position().col);
/* Token node position within T1. */
assert_eq!(1, t1.pToken().position().col);
/* Overall start node spans the whole input. */
assert_eq!(1, start.position().col);
assert_eq!(6, start.end_position().col);
assert_eq!(3, start.n_fields());
}
p_context_delete(context);
println!("ok");
}

13
spec/test_user_code.rs Normal file
View File

@ -0,0 +1,13 @@
use testparser::*;
fn main() {
let mut context = p_context_new(b"abcdef");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass1");
p_context_delete(context);
let mut context = p_context_new(b"abcabcdef");
assert_eq!(P_SUCCESS, p_parse(&mut context));
println!("pass2");
p_context_delete(context);
}

9
spec/test_user_context_fields.rs Normal file
View File

@ -0,0 +1,9 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"aaa\n\n\na\n # comment 1\na a aa\n\naa\n# comment 2\na\n");
assert_eq!(P_SUCCESS, p_parse(&mut c));
eprint!("comments: {}", c.comments);
eprintln!("acount: {}", c.acount);
p_context_delete(c);
}

12
spec/test_user_terminate.rs Normal file
View File

@ -0,0 +1,12 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"aacc");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"abc");
assert_eq!(P_USER_TERMINATED, p_parse(&mut c));
assert_eq!(4200, p_user_terminate_code(&c));
p_context_delete(c);
}

12
spec/test_user_terminate_lexer.rs Normal file
View File

@ -0,0 +1,12 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"a");
assert_eq!(P_SUCCESS, p_parse(&mut c));
p_context_delete(c);
let mut c = p_context_new(b"b");
assert_eq!(P_USER_TERMINATED, p_parse(&mut c));
assert_eq!(8675309, p_user_terminate_code(&c));
p_context_delete(c);
}

20
spec/test_value_accessors.rs Normal file
View File

@ -0,0 +1,20 @@
use testparser::*;
fn main() {
let mut c = p_context_new(b"42 f s");
let mut ti = p_token_info_t::default();
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_num, ti.token);
assert_eq!(42, p_value_get(&ti.pvalue));
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_flt, ti.token);
assert_eq!(1.5, p_value_get_float(&ti.pvalue));
assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
assert_eq!(TOKEN_str, ti.token);
assert_eq!("hello", p_value_get_string(&ti.pvalue));
p_context_delete(c);
}

148
spec/tree_node_memory_remains.rust.propane Normal file
View File

@ -0,0 +1,148 @@
tree;
tree_prefix P;
<<header
#[derive(Clone, Default)]
pub struct TokenVal {
pub i64_: u64,
pub s: String,
pub dou: f64,
}
>>
ptype TokenVal;
# Keywords.
token byte;
token def;
token int;
token long;
token module;
token return;
token short;
token size_t;
token ssize_t;
token ubyte;
token uint;
token ulong;
token ushort;
# Symbols.
token arrow /->/;
token comma /,/;
token lbrace /\{/;
token lparen /\(/;
token rbrace /\}/;
token rparen /\)/;
token semicolon /;/;
# Integer literals.
token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
$$.i64_ = 64;
>>
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
$$.s = std::str::from_utf8(match_).unwrap().to_string();
$mode(default);
return $token(ident);
>>
# Comments.
drop /#.*/;
# Whitespace.
drop /[ \r\n]*/;
start Module;
# Assignment operators - right associative
Expression -> Expression_Or:exp0;
# Logical OR operator - left associative
Expression_Or -> Expression_And:exp0;
# Logical AND operator - left associative
Expression_And -> Expression_Comp:exp0;
# Equality operators - left associative
Expression_Comp -> Expression_Relational:exp0;
# Relational operators - left associative
Expression_Relational -> Expression_REMatch:exp0;
# Regular expression - left associative
Expression_REMatch -> Expression_BinOr:exp0;
# Binary OR operator - left associative
Expression_BinOr -> Expression_Xor:exp0;
# Binary XOR operator - left associative
Expression_Xor -> Expression_BinAnd:exp0;
# Binary AND operator - left associative
Expression_BinAnd -> Expression_BitShift:exp0;
# Bit shift operators - left associative
Expression_BitShift -> Expression_Plus:exp0;
# Add/subtract operators - left associative
Expression_Plus -> Expression_Mul:exp0;
# Multiplication/divide/modulus operators - left associative
Expression_Mul -> Expression_Range:exp0;
# Range construction operators - left associative
Expression_Range -> Expression_UnaryPrefix:exp0;
# Unary prefix operators
Expression_UnaryPrefix -> Expression_Dot:exp0;
# Postfix operators
Expression_Dot -> Expression_Ident:exp0;
Expression_Dot -> Expression_Dot:exp1 lparen rparen;
# Literals, identifiers, and parenthesized expressions
Expression_Ident -> Literal;
Expression_Ident -> ident;
FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
FunctionReturnType -> arrow Type;
Literal -> LiteralInteger;
LiteralInteger -> hex_int_l;
Module -> ModuleStatement? ModuleItems;
ModuleItem -> FunctionDefinition;
ModuleItems -> ;
ModuleItems -> ModuleItems ModuleItem;
ModulePath -> ident;
ModuleStatement -> module ModulePath semicolon;
ReturnStatement -> return Expression?:exp0 semicolon;
Statements -> ;
Statements -> Statements Statement;
Statement -> Expression semicolon;
Statement -> ReturnStatement;
Type -> TypeBase;
TypeBase -> byte;
TypeBase -> ubyte;
TypeBase -> short;
TypeBase -> ushort;
TypeBase -> int;
TypeBase -> uint;
TypeBase -> long;
TypeBase -> ulong;
TypeBase -> size_t;
TypeBase -> ssize_t;