Add Rust target

spec/json_parser.rust.propane

@@ -0,0 +1,176 @@
+<<header
+pub const JSON_OBJECT: usize = 0;
+pub const JSON_ARRAY: usize = 1;
+pub const JSON_NUMBER: usize = 2;
+pub const JSON_STRING: usize = 3;
+pub const JSON_TRUE: usize = 4;
+pub const JSON_FALSE: usize = 5;
+pub const JSON_NULL: usize = 6;
+
+#[derive(Clone, Default)]
+pub enum JSONValue {
+    #[default]
+    Null,
+    Object(Vec<(String, JSONValue)>),
+    Array(Vec<JSONValue>),
+    Number(f64),
+    StringVal(String),
+    True,
+    False,
+}
+
+impl JSONValue {
+    pub fn id(&self) -> usize {
+        match self {
+            JSONValue::Object(_) => JSON_OBJECT,
+            JSONValue::Array(_) => JSON_ARRAY,
+            JSONValue::Number(_) => JSON_NUMBER,
+            JSONValue::StringVal(_) => JSON_STRING,
+            JSONValue::True => JSON_TRUE,
+            JSONValue::False => JSON_FALSE,
+            JSONValue::Null => JSON_NULL,
+        }
+    }
+    pub fn number(&self) -> f64 {
+        if let JSONValue::Number(n) = self { *n } else { 0.0 }
+    }
+    pub fn string(&self) -> &str {
+        if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
+    }
+    pub fn object_len(&self) -> usize {
+        if let JSONValue::Object(e) = self { e.len() } else { 0 }
+    }
+    pub fn array_len(&self) -> usize {
+        if let JSONValue::Array(e) = self { e.len() } else { 0 }
+    }
+}
+>>
+
+context_user_fields <<
+    pub string_value: String,
+>>
+
+ptype JSONValue;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
+    $$ = JSONValue::Number(n);
+>>
+token true <<
+  $$ = JSONValue::True;
+>>
+token false <<
+  $$ = JSONValue::False;
+>>
+token null <<
+  $$ = JSONValue::Null;
+>>
+/"/ <<
+  $mode(string);
+  ${context.string_value} = String::new();
+>>
+string: token string /"/ <<
+  $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
+  $mode(default);
+>>
+string: /\\"/ <<
+  ${context.string_value}.push('"');
+>>
+string: /\\\\/ <<
+  ${context.string_value}.push('\\');
+>>
+string: /\\\// <<
+  ${context.string_value}.push('/');
+>>
+string: /\\b/ <<
+  ${context.string_value}.push('\u{0008}');
+>>
+string: /\\f/ <<
+  ${context.string_value}.push('\u{000C}');
+>>
+string: /\\n/ <<
+  ${context.string_value}.push('\n');
+>>
+string: /\\r/ <<
+  ${context.string_value}.push('\r');
+>>
+string: /\\t/ <<
+  ${context.string_value}.push('\t');
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
+  ${context.string_value}.push_str(&s);
+>>
+string: /[^\\]/ <<
+  ${context.string_value}.push(match_[0] as char);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONValue::Object(Vec::new());
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  let mut obj = $1;
+  if let JSONValue::Object(kve) = $3 {
+      if let JSONValue::Object(entries) = &mut obj {
+          entries.extend(kve);
+      }
+  }
+  $$ = obj;
+>>
+KeyValue -> string colon Value <<
+  let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
+  $$ = JSONValue::Object(vec![(name, $3)]);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONValue::Array(Vec::new());
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  let mut arr = $1;
+  if let JSONValue::Array(elems) = &mut arr {
+      elems.push($3);
+  }
+  $$ = arr;
+>>

spec/macros.rust.propane

@@ -0,0 +1,80 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        if context.expanding {
+            let ei = context.expand_i;
+            context.expand_i += 1;
+            if context.expand_i >= context.token_infos.len() {
+                context.expanding = false;
+            }
+            *out_token_info = context.token_infos[ei].clone();
+            return P_SUCCESS;
+        }
+
+        let lex_result = p_lex(context, out_token_info);
+        if lex_result != P_SUCCESS {
+            return lex_result;
+        }
+
+        if out_token_info.token == TOKEN_macro {
+            context.defining = true;
+        } else if out_token_info.token == TOKEN_macroname {
+            if !context.defining {
+                context.expanding = true;
+                context.expand_i = 0;
+                continue;
+            }
+        } else if out_token_info.token == TOKEN_lbrace {
+            if context.defining {
+                /* Capture the macro body tokens (up to the closing '}'). */
+                let mut infos: Vec<p_token_info_t> = Vec::new();
+                loop {
+                    let mut ti = p_token_info_t::default();
+                    assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
+                    if ti.token == TOKEN_rbrace {
+                        break;
+                    }
+                    infos.push(ti);
+                }
+                context.token_infos = infos;
+                context.defining = false;
+            }
+        } else {
+            context.defining = false;
+        }
+        return lex_result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub defining: bool,
+    pub expanding: bool,
+    pub expand_i: usize,
+    pub token_infos: Vec<p_token_info_t>,
+    pub nums: Vec<i64>,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token macro;
+token macroname /@[a-zA-Z_]\w*/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Statement -> MacroStart;
+Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
+MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.rust.propane

@@ -0,0 +1,41 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        /* Reentrant nested parse of the parenthesized sub-expression. */
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        let value = p_result_Start(context);
+        /* p_parse_inner rewound the input so ')' was not consumed; consume it. */
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        out_token_info.token = TOKEN_num;
+        out_token_info.pvalue = p_value(value);
+    }
+    P_SUCCESS
+}
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Expr << $$ = $1; >>
+Expr -> num << $$ = $1; >>
+Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.rust.propane

@@ -0,0 +1,44 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let result = p_lex(context, out_token_info);
+    if result != P_SUCCESS {
+        return result;
+    }
+    if out_token_info.token == TOKEN_lparen {
+        let start_position = out_token_info.position;
+        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
+        if inner_result != P_SUCCESS {
+            return inner_result;
+        }
+        /* Read the inner subtree's span before re-borrowing context to lex. */
+        let inner = p_result_Start(context);
+        assert!(inner.valid());
+        let inner_start_col = inner.position().col;
+        let inner_end_col = inner.end_position().col;
+        let mut rparen_info = p_token_info_t::default();
+        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
+        assert_eq!(TOKEN_rparen, rparen_info.token);
+        assert_eq!(start_position.col + 1, inner_start_col);
+        assert_eq!(rparen_info.position.col - 1, inner_end_col);
+        /* Synthesize a num token spanning the whole "( ... )" group. */
+        out_token_info.token = TOKEN_num;
+        out_token_info.position = start_position;
+        out_token_info.end_position = rparen_info.end_position;
+    }
+    P_SUCCESS
+}
+>>
+
+tree;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token lparen /\(/;
+token rparen /\)/;
+token plus /\+/;
+token num /\d+/;
+
+Start -> Expr;
+Expr -> num;
+Expr -> Expr plus num;

spec/propane_spec.rb

@@ -53,7 +53,7 @@ EOF
     if options[:args]
       command += options[:args]
     else
-      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{lang_ext(options[:language])} --log spec/run/testparser#{options[:name]}.log]
     end
     command += (options[:extra_args] || [])
     if (options[:capture])
@@ -65,9 +65,16 @@ EOF
     end
   end
 
+  # Map a spec language name to the generated parser source file extension.
+  def lang_ext(language)
+    language == "rust" ? "rs" : language
+  end
+
   def compile(test_files, options = {})
     test_files = Array(test_files).map do |test_file|
-      if !File.exist?(test_file) && test_file.end_with?(".cpp")
+      if test_file.end_with?(".rust")
+        test_file.sub(%r{\.rust$}, ".rs")
+      elsif !File.exist?(test_file) && test_file.end_with?(".cpp")
         test_file.sub(%r{\.cpp$}, ".c")
       else
         test_file
@@ -75,7 +82,7 @@ EOF
     end
     options[:parsers] ||= [""]
     parsers = options[:parsers].map do |name|
-      "spec/run/testparser#{name}.#{options[:language]}"
+      "spec/run/testparser#{name}.#{lang_ext(options[:language])}"
     end
     case options[:language]
     when "c"
@@ -84,6 +91,21 @@ EOF
       command = [*%w[g++ -g -x c++ -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm"]
     when "d"
       command = [*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d"]
+    when "rust"
+      # Compile each generated parser to an rlib, then compile the test crate
+      # against them. Safe Rust needs no valgrind, but it is run anyway.
+      externs = []
+      options[:parsers].each do |name|
+        crate = "testparser#{name}"
+        rlib = "spec/run/lib#{crate}.rlib"
+        rustc = [*%w[rustc --edition 2021 --crate-type=rlib -A warnings],
+                 "--crate-name", crate, "-o", rlib, "spec/run/testparser#{name}.rs"]
+        expect(system(*rustc)).to be_truthy
+        externs += ["--extern", "#{crate}=#{rlib}"]
+      end
+      # rustc takes a single crate root; any additional Rust test files are
+      # expected to be pulled in as modules or provided by the parser crate.
+      command = [*%w[rustc --edition 2021 -A warnings -o spec/run/testparser], *externs, test_files.first]
     end
     result = system(*command)
     expect(result).to be_truthy
@@ -285,7 +307,7 @@ EOF
     expect(results.status).to_not eq 0
   end
 
-  %w[d c cpp].each do |language|
+  %w[d c cpp rust].each do |language|
 
     context "#{language.upcase} language" do
 
@@ -335,6 +357,16 @@ token int /\\d+/ <<
   $$ = v;
 >>
 Start -> int << $$ = $1; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype i64;
+token int /\\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+Start -> int << $$ = $1; >>
 EOF
         end
         run_propane(language: language)
@@ -429,6 +461,31 @@ E3 -> E4 << $$ = $1; >>
 E3 -> E3 power E4 << $$ = pow($1, $3); >>
 E4 -> integer << $$ = $1; >>
 E4 -> lparen E1 rparen << $$ = $2; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u64;
+token plus /\\+/;
+token times /\\*/;
+token power /\\*\\*/;
+token integer /\\d+/ <<
+  let mut v: u64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as u64; }
+  $$ = v;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = $1.pow($3 as u32); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
         end
         run_propane(language: language)
@@ -505,6 +562,16 @@ token def;
 Start -> Abcs def;
 Abcs -> ;
 Abcs -> abc Abcs;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc <<
+  println!("abc!");
+>>
+token def;
+Start -> Abcs def;
+Abcs -> ;
+Abcs -> abc Abcs;
 EOF
         end
         run_propane(language: language)
@@ -541,6 +608,12 @@ import std.stdio;
 token abc;
 /def/ << writeln("def!"); >>
 Start -> abc;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+/def/ << println!("def!"); >>
+Start -> abc;
 EOF
         end
         run_propane(language: language)
@@ -583,6 +656,16 @@ token abc;
   return $token(abc);
 >>
 Start -> abc;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+/def/ << println!("def!"); >>
+/ghi/ <<
+  println!("ghi!");
+  return $token(abc);
+>>
+Start -> abc;
 EOF
         end
         run_propane(language: language)
@@ -641,6 +724,25 @@ string: /"/ <<
   return $token(string);
 >>
 Start -> abc string def;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token abc;
+token def;
+tokenid string;
+drop /\\s+/;
+/"/ <<
+  println!("begin string mode");
+  $mode(string);
+>>
+string: /[^"]+/ <<
+  println!("captured string");
+>>
+string: /"/ <<
+  $mode(default);
+  return $token(string);
+>>
+Start -> abc string def;
 EOF
         end
         run_propane(language: language)
@@ -699,6 +801,24 @@ default, identonly: drop /\\s+/;
 Start -> abc dot ident <<
   writeln("ident: ", $3);
 >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u8;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match_[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  println!("ident: {}", $3 as char);
+>>
 EOF
         end
         run_propane(language: language)
@@ -736,6 +856,14 @@ token b;
 Start -> A B << writeln("Start!"); >>
 A -> a << writeln("A!"); >>
 B -> b << writeln("B!"); >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+token a;
+token b;
+Start -> A B << println!("Start!"); >>
+A -> a << println!("A!"); >>
+B -> b << println!("B!"); >>
 EOF
         end
         run_propane(language: language)
@@ -843,6 +971,37 @@ Start -> A:ay B:bee C <<
 A -> a;
 B -> b;
 C -> << ${context.c_is_null} = ($$.valid) ? 0 : 1; >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+tree;
+context_user_fields <<
+    pub start_n_fields: i64,
+    pub start_a_value: i64,
+    pub a_value: i64,
+    pub b_value: i64,
+    pub b_token: p_token_t,
+    pub c_is_null: i64,
+    pub c_field_is_null: i64,
+    pub alias_a_value: i64,
+    pub alias_b_value: i64,
+>>
+ptype i64;
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+Start -> A:ay B:bee C <<
+  ${context.start_n_fields} = $$.n_fields() as i64;
+  ${context.start_a_value} = $$.pA().pToken1().pvalue();
+  ${context.a_value} = $1.pToken1().pvalue();
+  ${context.b_value} = $2.pToken1().pvalue();
+  ${context.b_token} = $2.pToken1().token();
+  ${context.c_field_is_null} = if $$.pC().valid() { 0 } else { 1 };
+  ${context.alias_a_value} = ${ay}.pToken1().pvalue();
+  ${context.alias_b_value} = ${bee}.pToken1().pvalue();
+>>
+A -> a;
+B -> b;
+C -> << ${context.c_is_null} = if $$.valid() { 0 } else { 1 }; >>
 EOF
         end
         run_propane(language: language)
@@ -853,12 +1012,15 @@ EOF
       end
 
       it "parses lists" do
+        ptype = language == "d" ? "uint" : (language == "rust" ? "u32" : "uint32_t")
+        zero = language == "rust" ? "0" : "0u"
+        one = language == "rust" ? "1" : "1u"
         write_grammar <<EOF
-ptype #{language == "d" ? "uint" : "uint32_t"};
+ptype #{ptype};
 token a;
 Start -> As << $$ = $1; >>
-As -> << $$ = 0u; >>
-As -> As a << $$ = $1 + 1u; >>
+As -> << $$ = #{zero}; >>
+As -> As a << $$ = $1 + #{one}; >>
 EOF
         run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
@@ -912,6 +1074,13 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
   writeln("Matched token is ", match);
 >>
 Start -> id;
+EOF
+        when "rust"
+          write_grammar <<EOF
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
+>>
+Start -> id;
 EOF
         end
         run_propane(language: language)
@@ -945,6 +1114,16 @@ token word /[a-z]+/ <<
 Start -> word <<
   $$ = $1;
 >>
+EOF
+        when "rust"
+          write_grammar <<EOF
+ptype u64;
+token word /[a-z]+/ <<
+  $$ = match_length as u64;
+>>
+Start -> word <<
+  $$ = $1;
+>>
 EOF
         end
         run_propane(language: language)
@@ -1097,6 +1276,25 @@ Start -> Words;
 Words -> ;
 Words -> word Words;
 Words -> stop Words;
+EOF
+        when "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub last_start: p_position_t,
+    pub last_end: p_position_t,
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
 EOF
         end
         run_propane(language: language)
@@ -1159,6 +1357,21 @@ tokenid t;
   return $token(t);
 >>
 Start -> t;
+EOF
+        when "rust"
+          write_grammar <<EOF
+tokenid t;
+/\\a/ << println!("A"); >>
+/\\b/ << println!("B"); >>
+/\\t/ << println!("T"); >>
+/\\n/ << println!("N"); >>
+/\\v/ << println!("V"); >>
+/\\f/ << println!("F"); >>
+/\\r/ << println!("R"); >>
+/t/ <<
+  return $token(t);
+>>
+Start -> t;
 EOF
         end
         run_propane(language: language)
@@ -1195,7 +1408,7 @@ EOF
         write_grammar <<EOF
 tree;
 
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 
 token a << $$ = 11; >>
 token b << $$ = 22; >>
@@ -1239,7 +1452,7 @@ tree;
 tree_prefix P ;
 tree_suffix  S;
 
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 
 token a << $$ = 11; >>
 token b << $$ = 22; >>
@@ -1320,6 +1533,24 @@ Start -> a? b R? <<
 >>
 R -> c d << $$ = "cd"; >>
 R (string) -> d c << $$ = "dc"; >>
+EOF
+        elsif language == "rust"
+          write_grammar <<EOF
+ptype i64;
+ptype float = f64;
+ptype string = String;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  println!("a: {}", $1);
+  println!("b: {}", $2);
+  println!("R: {}", $3);
+>>
+R -> c d << $$ = "cd".to_string(); >>
+R (string) -> d c << $$ = "dc".to_string(); >>
 EOF
         else
           write_grammar <<EOF
@@ -1350,7 +1581,7 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
-          "a: 0#{language == "d" ? "" : ".0"}",
+          "a: 0#{["d", "rust"].include?(language) ? "" : ".0"}",
           "b: 2",
           "R: ",
           "a: 1.5",
@@ -1387,6 +1618,18 @@ tree;
 #include <stdio.h>
 >>
 
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        end
+        if language == "rust"
+          write_grammar <<EOF
+tree;
 token a;
 token b;
 token c;
@@ -1398,6 +1641,7 @@ EOF
         end
         run_propane(language: language)
         compile("spec/test_optional_rule_component_tree.#{language}", language: language)
+        # (rust grammar written above overrides the C/D form for the Rust run)
         results = run_test(language: language)
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
@@ -1428,6 +1672,18 @@ tree;
 #include <stdio.h>
 >>
 
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        end
+        if language == "rust"
+          write_grammar <<EOF
+tree;
 token a;
 token b;
 token c;
@@ -1528,7 +1784,19 @@ EOF
       end
 
       it "allows specifying field aliases when tree mode is not enabled" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype String;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = std::str::from_utf8(match_).unwrap().to_string();
+>>
+drop /\\s+/;
+Start -> id:first id:second <<
+  println!("first is {}", ${first});
+  println!("second is {}", ${second});
+>>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -1574,7 +1842,22 @@ EOF
       end
 
       it "aliases the correct field when multiple rules are in a rule set when tree mode is not enabled" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype String;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = std::str::from_utf8(match_).unwrap().to_string();
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  println!("first is {}", ${first});
+  println!("second is {}", ${second});
+>>
+Foo -> ;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -1640,7 +1923,7 @@ EOF
 
       it "allows multiple starting rules" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1660,7 +1943,7 @@ EOF
 
       it "supports parse_inner APIs that treat provided tokens as follow tokens" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 Start -> Y << $$ = $1; >>
@@ -1675,7 +1958,7 @@ EOF
 
       it "parse_inner APIs block success when the outer rule is unfinished" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1691,7 +1974,7 @@ EOF
 
       it "parse_inner APIs work when the reduce state uses lookahead disambiguation" do
     write_grammar <<EOF
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a;
 token b;
 start Start;
@@ -1750,7 +2033,7 @@ EOF
       it "allows multiple starting rules in tree mode" do
     write_grammar <<EOF
 tree;
-ptype int;
+ptype #{language == "rust" ? "i64" : "int"};
 token a << $$ = 1; >>
 token b << $$ = 2; >>
 token c << $$ = 3; >>
@@ -1795,7 +2078,16 @@ EOF
       end
 
       it "executes code blocks associated with drop statements" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
+>>
+token a;
+Start -> a;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;
@@ -1829,7 +2121,24 @@ EOF
       end
 
       it "allows user-defined context fields" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub comments: String,
+    pub acount: u32,
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::str::from_utf8(match_).unwrap();
+>>
+token a <<
+    ${context.acount} += 1;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 context_user_fields <<
     string comments;
@@ -1889,7 +2198,28 @@ EOF
       end
 
       it "allows custom token user fields" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+context_user_fields <<
+    pub comments: String,
+>>
+token_user_fields <<
+    pub comments: String,
+>>
+on_token_node <<
+    ${token.comments} = std::mem::take(&mut ${context.comments});
+>>
+tree;
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} += std::str::from_utf8(match_).unwrap();
+>>
+token id /\\w+/;
+Start -> IDs;
+IDs -> ;
+IDs -> id:id IDs;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 context_user_fields <<
     string comments;
@@ -1986,7 +2316,41 @@ EOF
       end
 
       it "allows a custom lex function" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    let mut result = P_SUCCESS;
+    if context.count > 0 {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(context.count);
+        context.count -= 1;
+    } else {
+        result = p_lex(context, out_token_info);
+        if out_token_info.token == TOKEN_c {
+            context.count = 3;
+        }
+    }
+    result
+}
+>>
+
+context_user_fields <<
+    pub count: usize,
+>>
+ptype usize;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
@@ -2068,7 +2432,19 @@ EOF
       end
 
       it "provides accessor functions to extract user values from a parser value" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+ptype i64;
+ptype float = f64;
+ptype string = String;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello".to_string(); >>
+Start -> num flt str;
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 ptype int;
 ptype float = float;
@@ -2101,7 +2477,49 @@ EOF
       end
 
       it "allows accessing rule and component text positions" do
-        if language == "d"
+        if language == "rust"
+          write_grammar <<EOF
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    println!("ident start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("ident end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("Num start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("Num end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("Start start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("Start end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    println!("Empty start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("Empty end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("tok2 start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("tok2 end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("R start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("R end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    println!("tok1 start: {}, {}", ${1.position}.row, ${1.position}.col);
+    println!("tok1 end: {}, {}", ${1.end_position}.row, ${1.end_position}.col);
+    println!("Empty start: {}, {}", ${2.position}.row, ${2.position}.col);
+    println!("Empty end: {}, {}", ${2.end_position}.row, ${2.end_position}.col);
+    println!("R2 start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("R2 end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    println!("StartR start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("StartR end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    println!("StartEmpty start: {}, {}", ${$.position}.row, ${$.position}.col);
+    println!("StartEmpty end: {}, {}", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        elsif language == "d"
           write_grammar <<EOF
 <<
 import std.stdio;

spec/rewind.rust.propane

@@ -0,0 +1,67 @@
+<<
+fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
+    loop {
+        let result = p_lex(context, out_token_info);
+        if result != P_SUCCESS {
+            return result;
+        }
+        if out_token_info.token == TOKEN_repeat {
+            let mut count_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
+            assert_eq!(TOKEN_num, count_info.token);
+            let mut brace_info = p_token_info_t::default();
+            assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
+            assert_eq!(TOKEN_lbrace, brace_info.token);
+            context.remaining = p_value_get(&count_info.pvalue);
+            context.body_index = p_input_index(context);
+            context.body_position = p_position(context);
+            continue;
+        }
+        if out_token_info.token == TOKEN_rbrace {
+            if context.remaining > 1 {
+                context.remaining -= 1;
+                let bi = context.body_index;
+                let bp = context.body_position;
+                p_set_input_index(context, bi);
+                p_set_position(context, bp);
+                continue;
+            }
+            context.remaining = 0;
+            continue;
+        }
+        if out_token_info.token == TOKEN_num {
+            context.num_cols.push(out_token_info.position.col);
+        }
+        return result;
+    }
+}
+>>
+
+context_user_fields <<
+    pub nums: Vec<i64>,
+    pub num_cols: Vec<u32>,
+    pub remaining: i64,
+    pub body_index: usize,
+    pub body_position: p_position_t,
+>>
+
+ptype i64;
+
+lex_fn mylexfn;
+
+drop /\s+/;
+token repeat /repeat/;
+token lbrace /\{/;
+token rbrace /\}/;
+token plus /\+/;
+token num /\d+/ <<
+  let mut v: i64 = 0;
+  for c in match_ { v = v * 10 + (*c - b'0') as i64; }
+  $$ = v;
+>>
+
+Start -> Statements;
+Statements -> ;
+Statements -> Statement Statements;
+Statement -> Add;
+Add -> num plus num << ${context.nums}.push($1 + $3); >>

spec/test_basic_math_grammar.rs

@@ -0,0 +1,16 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u64); 4] = [
+        (b"1 + 2 * 3 + 4", 11),
+        (b"1 * 2 ** 4 * 3", 48),
+        (b"(1 + 2) * 3 + 4", 13),
+        (b"(2 * 2) ** 3 + 4 + 5", 73),
+    ];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_custom_lex_fn.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"cbacba");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(0x932187932187, p_result(&c));
+    p_context_delete(c);
+}

spec/test_drop_code_block.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    # comment 1\n#    comment 2\na\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_error_positions.rs

@@ -0,0 +1,35 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a 42");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a\n123\na  a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(4, p_position(&c).col);
+    assert_eq!(TOKEN_a, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"12");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
+    assert_eq!(1, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+    assert_eq!(TOKEN_num, p_token(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\nab");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(2, p_position(&c).col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a 12\n\na\n\n77\na   \xAA");
+    assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
+    assert_eq!(6, p_position(&c).row);
+    assert_eq!(5, p_position(&c).col);
+    assert_eq!("a", p_token_names[TOKEN_a as usize]);
+    assert_eq!("num", p_token_names[TOKEN_num as usize]);
+    p_context_delete(c);
+}

spec/test_field_aliases.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"foo1\nbar2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_input_index.rs

@@ -0,0 +1,28 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"ab");
+    assert_eq!(0, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a b");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    let follow = [TOKEN_b];
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
+    assert_eq!(2, p_input_index(&c));
+    p_context_delete(c);
+}

spec/test_lexer.rs

@@ -0,0 +1,49 @@
+use testparser::*;
+
+fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
+    assert_eq!(row, ti.position.row);
+    assert_eq!(col, ti.position.col);
+    assert_eq!(erow, ti.end_position.row);
+    assert_eq!(ecol, ti.end_position.col);
+    assert_eq!(len, ti.length);
+    assert_eq!(token, ti.token);
+}
+
+fn main() {
+    let mut cp: p_code_point_t = 0;
+    let mut cpl: u8 = 0;
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
+    assert_eq!('5' as u32, cp);
+    assert_eq!(1, cpl);
+
+    assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
+    assert_eq!(0xA9, cp);
+    assert_eq!(2, cpl);
+
+    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
+    assert_eq!(0x1F9E1, cp);
+    assert_eq!(4, cpl);
+
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
+    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
+
+    let mut context = p_context_new(b"5 + 4 * \n677 + 567");
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
+    p_context_delete(context);
+}

spec/test_lexer_match_text.rs

@@ -0,0 +1,8 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"identifier_123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+}

spec/test_lexer_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc \"a string\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc \"abc def\" def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_multiple_modes.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abc.def");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abc .  abc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_lexer_positions.rs

@@ -0,0 +1,38 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"abc\n  defg hi\n!");
+    let mut ti = p_token_info_t::default();
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(1, c.last_start.row);
+    assert_eq!(1, c.last_start.col);
+    assert_eq!(1, c.last_end.row);
+    assert_eq!(3, c.last_end.col);
+    assert_eq!(c.last_start.row, ti.position.row);
+    assert_eq!(c.last_start.col, ti.position.col);
+    assert_eq!(c.last_end.row, ti.end_position.row);
+    assert_eq!(c.last_end.col, ti.end_position.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(3, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(6, c.last_end.col);
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_word, ti.token);
+    assert_eq!(2, c.last_start.row);
+    assert_eq!(8, c.last_start.col);
+    assert_eq!(2, c.last_end.row);
+    assert_eq!(9, c.last_end.col);
+
+    assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
+    assert_eq!(42, p_user_terminate_code(&c));
+    assert_eq!(3, p_position(&c).row);
+    assert_eq!(1, p_position(&c).col);
+
+    p_context_delete(c);
+}

spec/test_lexer_result_value.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(1, p_result(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"fabulous");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(8, p_result(&context));
+    p_context_delete(context);
+}

spec/test_lexer_unknown_character.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"x");
+    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"123");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert_eq!(123, p_result(&context));
+    p_context_delete(context);
+}

spec/test_macros.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    let input = b"macro @m { 23 + 200 }\n66 + 100\n@m\n33 + 55\n@m\n";
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(vec![166, 223, 88, 223], c.nums);
+    p_context_delete(c);
+}

spec/test_match_backslashes.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"\x07\x08\t\n\x0b\x0c\rt");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_multiple_parsers.rs

@@ -0,0 +1,12 @@
+use testparsermyp1 as m1;
+use testparsermyp2 as m2;
+
+fn main() {
+    let mut context1 = m1::myp1_context_new(b"a\n1");
+    assert_eq!(m1::P_SUCCESS, m1::myp1_parse(&mut context1));
+    m1::myp1_context_delete(context1);
+
+    let mut context2 = m2::myp2_context_new(b"bcb");
+    assert_eq!(m2::P_SUCCESS, m2::myp2_parse(&mut context2));
+    m2::myp2_context_delete(context2);
+}

spec/test_named_optional_rule_component_tree.rs

@@ -0,0 +1,43 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+        assert!(!start.r().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.a().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert!(start.r().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(start.pR().node_id(), start.r().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.a().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.r().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_optional_rule_component.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"b"[..], &b"abcd"[..], &b"abdc"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_optional_rule_component_tree.rs

@@ -0,0 +1,40 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert_eq!(TOKEN_b, start.pToken2().token());
+        assert!(!start.pR3().valid());
+        assert!(!start.pR().valid());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcd");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pToken1().valid());
+        assert_eq!(TOKEN_a, start.pToken1().token());
+        assert!(start.pToken2().valid());
+        assert!(start.pR3().valid());
+        assert!(start.pR().valid());
+        assert_eq!(start.pR().node_id(), start.pR3().node_id());
+        assert_eq!(TOKEN_c, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bdc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(!start.pToken1().valid());
+        assert!(start.pToken2().valid());
+        assert!(start.pR().valid());
+        assert_eq!(TOKEN_d, start.pR().pToken1().token());
+    }
+    p_context_delete(context);
+}

spec/test_parse_inner.rs

@@ -0,0 +1,30 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(1, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN___EOF]));
+    p_context_delete(c);
+}

spec/test_parse_inner_nested.rs

@@ -0,0 +1,17 @@
+use testparser::*;
+
+fn eval(input: &[u8]) -> i64 {
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    p_context_delete(c);
+    v
+}
+
+fn main() {
+    assert_eq!(5, eval(b"2 + 3"));
+    assert_eq!(3, eval(b"(1 + 2)"));
+    assert_eq!(14, eval(b"2 + (3 + 4) + 5"));
+    assert_eq!(37, eval(b"2 + (10 + (20 + 5))"));
+    assert_eq!(15, eval(b"(1 + 2) + (3 + (4 + 5))"));
+}

spec/test_parse_inner_nested_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"(3 + 4) + (5 + 6)");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let tree = p_result(&c);
+        assert!(tree.valid());
+
+        /* Start -> Expr, where the top Expr is "Expr plus num". */
+        let top = tree.pExpr();
+        assert!(top.valid());
+        assert!(top.pExpr().valid());
+        assert!(top.pToken2().valid());
+        assert!(top.pToken3().valid());
+
+        /* The '+' joining the two groups is at column 9. */
+        assert_eq!(1, top.pToken2().position().row);
+        assert_eq!(9, top.pToken2().position().col);
+
+        /* Right operand: synthesized num for "(5 + 6)", columns 11..17. */
+        assert_eq!(11, top.pToken3().position().col);
+        assert_eq!(17, top.pToken3().end_position().col);
+
+        /* Left operand: synthesized num for "(3 + 4)", columns 1..7. */
+        let left = top.pExpr();
+        assert!(left.pToken1().valid());
+        assert_eq!(1, left.pToken1().position().col);
+        assert_eq!(7, left.pToken1().end_position().col);
+
+        /* The whole tree spans columns 1..17. */
+        assert_eq!(1, tree.position().col);
+        assert_eq!(17, tree.end_position().col);
+    }
+    p_context_delete(c);
+}

spec/test_parse_inner_recursive.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ac");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_inner_Start(&mut c, &[TOKEN_b, TOKEN___EOF]));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"acb");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &[TOKEN_b]));
+    assert_eq!(3, p_result_Start(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_shared.rs

@@ -0,0 +1,47 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_R1(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"aba");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_a]));
+    assert_eq!(11, p_result_R1(&c));
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[]));
+    assert_eq!(11, p_result_R1(&c));
+    p_context_delete(c);
+}

spec/test_parse_inner_tree.rs

@@ -0,0 +1,36 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse_R1(&mut c));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert!(tree.pToken1().valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert!(tree.pToken2().valid());
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abb");
+    assert_eq!(P_SUCCESS, p_parse_inner_R1(&mut c, &[TOKEN_b]));
+    {
+        let tree = p_result_R1(&c);
+        assert!(tree.valid());
+        assert_eq!(TOKEN_a, tree.pToken1().token());
+        assert_eq!(1, tree.pToken1().position().row);
+        assert_eq!(1, tree.pToken1().position().col);
+        assert_eq!(TOKEN_b, tree.pToken2().token());
+        assert_eq!(2, tree.pToken2().position().col);
+        assert_eq!(1, tree.position().col);
+        assert_eq!(2, tree.end_position().col);
+    }
+    let pos = p_position(&c);
+    assert_eq!(3, pos.col);
+    let mut ti = p_token_info_t::default();
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(3, ti.position.col);
+    p_context_delete(c);
+}

spec/test_parser_identical_rules_lookahead.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    for input in [&b"aba"[..], &b"abb"[..]] {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        p_context_delete(context);
+    }
+}

spec/test_parser_rule_from_multiple_states.rs

@@ -0,0 +1,18 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"a");
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut context));
+    assert_eq!(1, p_position(&context).row);
+    assert_eq!(2, p_position(&context).col);
+    assert_eq!(TOKEN___EOF, p_token(&context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a b");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"bb");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_rule_user_code.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_parser_user_code_tree.rs

@@ -0,0 +1,24 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"ab");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+
+    assert_eq!(3, context.start_n_fields);
+    assert_eq!(11, context.start_a_value);
+    assert_eq!(11, context.a_value);
+    assert_eq!(22, context.b_value);
+    assert_eq!(TOKEN_b, context.b_token);
+    assert_eq!(1, context.c_is_null);
+    assert_eq!(1, context.c_field_is_null);
+    assert_eq!(11, context.alias_a_value);
+    assert_eq!(22, context.alias_b_value);
+
+    {
+        let start = p_result(&context);
+        assert!(start.pA().valid());
+        assert!(start.pB().valid());
+        assert!(!start.pC().valid());
+    }
+    p_context_delete(context);
+}

spec/test_parsing_json.rs

@@ -0,0 +1,31 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"{}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_ARRAY, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"-45.6");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_NUMBER, p_result(&c).id());
+    assert_eq!(-45.6, p_result(&c).number());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"{\"hi\":true}");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(JSON_OBJECT, p_result(&c).id());
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"[1, 2, \"three\", [4, 5], {\"six\": 6}]");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    let v = p_result(&c);
+    assert_eq!(JSON_ARRAY, v.id());
+    assert_eq!(5, v.array_len());
+    p_context_delete(c);
+}

spec/test_parsing_lists.rs

@@ -0,0 +1,11 @@
+use testparser::*;
+
+fn main() {
+    let cases: [(&[u8], u32); 3] = [(b"a", 1), (b"", 0), (b"aaaaaaaaaaaaaaaa", 16)];
+    for (input, expected) in cases {
+        let mut context = p_context_new(input);
+        assert_eq!(P_SUCCESS, p_parse(&mut context));
+        assert_eq!(expected, p_result(&context));
+        p_context_delete(context);
+    }
+}

spec/test_pattern.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"defabcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_positions.rs

@@ -0,0 +1,19 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"    Hello\n\n        4200\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"\n tok2");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    println!();
+
+    let mut c = p_context_new(b"  tok1");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+}

spec/test_return_token_from_pattern.rs

@@ -0,0 +1,7 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"defghidef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    p_context_delete(context);
+}

spec/test_rewind.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    /* "repeat 3 { 10 + 20 } 5 + 5": the body "10 + 20" is expanded three
+     * times (recording 30 each time), followed by "5 + 5" (recording 10). */
+    let mut c = p_context_new(b"repeat 3 { 10 + 20 } 5 + 5");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+
+    assert_eq!(vec![30, 30, 30, 10], c.nums);
+    assert_eq!(vec![12, 17, 12, 17, 12, 17, 22, 26], c.num_cols);
+
+    p_context_delete(c);
+}

spec/test_set_position.rs

@@ -0,0 +1,66 @@
+use testparser::*;
+
+fn main() {
+    let mut ti = p_token_info_t::default();
+
+    /* Baseline: default (1, 1). */
+    let mut c = p_context_new(b"ab");
+    let pos = p_position(&c);
+    assert_eq!(1, pos.row);
+    assert_eq!(1, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+
+    /* p_set_position overrides the initial position. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 5, col: 20 });
+    let pos = p_position(&c);
+    assert_eq!(5, pos.row);
+    assert_eq!(20, pos.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(20, ti.position.col);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(5, ti.position.row);
+    assert_eq!(21, ti.position.col);
+    p_context_delete(c);
+
+    /* Set position before a full parse. */
+    let mut c = p_context_new(b"ab");
+    p_set_position(&mut c, p_position_t { row: 3, col: 7 });
+    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
+    p_context_delete(c);
+
+    /* Set position before a failing parse: error position is relative. */
+    let mut c = p_context_new(b"aa");
+    p_set_position(&mut c, p_position_t { row: 10, col: 2 });
+    assert_eq!(P_UNEXPECTED_TOKEN, p_parse_Start(&mut c));
+    let ep = p_position(&c);
+    assert_eq!(10, ep.row);
+    assert_eq!(3, ep.col);
+    p_context_delete(c);
+
+    /* p_set_input_index rewinds the byte cursor to re-read a section. */
+    let mut c = p_context_new(b"ab");
+    let start_index = p_input_index(&c);
+    let start_position = p_position(&c);
+    assert_eq!(0, start_index);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_b, ti.token);
+    assert_eq!(2, p_input_index(&c));
+    p_set_input_index(&mut c, start_index);
+    p_set_position(&mut c, start_position);
+    assert_eq!(0, p_input_index(&c));
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_a, ti.token);
+    assert_eq!(1, ti.position.row);
+    assert_eq!(1, ti.position.col);
+    p_context_delete(c);
+}

spec/test_start_rule.rs

@@ -0,0 +1,2 @@
+fn main() {
+}

spec/test_start_rule_tree.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"hi");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let top = p_result(&context);
+        assert!(top.pToken().valid());
+        assert_eq!(TOKEN_hi, top.pToken().token());
+    }
+    p_context_delete(context);
+}

spec/test_starting_rules.rs

@@ -0,0 +1,18 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    assert_eq!(8, p_result(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
+    assert_eq!(8, p_result_Bs(&c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_R(&mut c));
+    assert_eq!(3, p_result_R(&c));
+    p_context_delete(c);
+}

spec/test_starting_rules_tree.rs

@@ -0,0 +1,34 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let start = p_result(&c);
+        assert!(start.bs().valid());
+        assert!(start.bs().b().valid());
+        assert!(start.bs().bs().b().valid());
+        assert!(start.bs().bs().bs().b().valid());
+        assert!(start.bs().bs().bs().bs().b().valid());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"bbbb");
+    assert_eq!(P_SUCCESS, p_parse_Bs(&mut c));
+    {
+        let bs = p_result_Bs(&c);
+        assert!(bs.b().valid());
+        assert!(bs.bs().b().valid());
+        assert!(bs.bs().bs().b().valid());
+        assert!(bs.bs().bs().bs().b().valid());
+    }
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"c");
+    assert_eq!(P_SUCCESS, p_parse_R(&mut c));
+    {
+        let r = p_result_R(&c);
+        assert!(r.c().valid());
+    }
+    p_context_delete(c);
+}

spec/test_token_user_fields.rs

@@ -0,0 +1,17 @@
+use testparser::*;
+
+fn main() {
+    let input = b"# c1\n#  c2\n\nfirst\n\n   \n  \n  # s1\n   #   s2\nsecond\n";
+    let mut c = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let start = p_result(&c);
+        assert!(start.pIDs().valid());
+        assert!(start.pIDs().id().valid());
+        assert_eq!("# c1\n#  c2\n", start.pIDs().id().data().comments.as_str());
+        assert!(start.pIDs().pIDs().valid());
+        assert!(start.pIDs().pIDs().id().valid());
+        assert_eq!("# s1\n#   s2\n", start.pIDs().pIDs().id().data().comments.as_str());
+    }
+    p_context_delete(c);
+}

spec/test_tree.rs

@@ -0,0 +1,46 @@
+use testparser::*;
+
+fn main() {
+    let input = b"a, ((b)), b";
+    let mut context = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems1().valid());
+        assert!(start.pItems().valid());
+        let items = start.pItems();
+        assert!(items.pItem().valid());
+        assert!(items.pItem().pToken1().valid());
+        assert_eq!(TOKEN_a, items.pItem().pToken1().token());
+        assert_eq!(11, items.pItem().pToken1().pvalue());
+        let itemsmore = items.pItemsMore();
+        assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
+        assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
+        assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
+        assert!(itemsmore.pItemsMore().valid());
+        let itemsmore = itemsmore.pItemsMore();
+        assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
+        assert!(!itemsmore.pItemsMore().valid());
+    }
+    p_context_delete(context);
+
+    /* Empty input yields a Start node with no Items child. */
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert!(!p_result(&context).pItems().valid());
+    p_context_delete(context);
+
+    /* Dual rule alternative field positions. */
+    let mut context = p_context_new(b"2 1");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems().pItem().pDual().pTwo1().valid());
+        assert!(start.pItems().pItem().pDual().pOne2().valid());
+        assert!(!start.pItems().pItem().pDual().pTwo2().valid());
+        assert!(!start.pItems().pItem().pDual().pOne1().valid());
+    }
+    p_context_delete(context);
+
+    println!("ok");
+}

spec/test_tree_field_aliases.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"\na\nb\nc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert_eq!(TOKEN_a, start.first().pToken().token());
+        assert_eq!(TOKEN_b, start.second().pToken().token());
+        assert_eq!(TOKEN_c, start.third().pToken().token());
+    }
+    p_context_delete(context);
+}

spec/test_tree_invalid_positions.rs

@@ -0,0 +1,89 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"\na\n  bb ccc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let k1 = t1.pToken();
+        let a1 = t1.pA();
+        assert_eq!(2, k1.position().row);
+        assert_eq!(1, k1.position().col);
+        assert_eq!(2, k1.end_position().row);
+        assert_eq!(1, k1.end_position().col);
+        assert!(a1.position().valid());
+        assert_eq!(3, a1.position().row);
+        assert_eq!(3, a1.position().col);
+        assert_eq!(3, a1.end_position().row);
+        assert_eq!(8, a1.end_position().col);
+        assert_eq!(2, t1.position().row);
+        assert_eq!(1, t1.position().col);
+        assert_eq!(3, t1.end_position().row);
+        assert_eq!(8, t1.end_position().col);
+        assert_eq!(2, start.position().row);
+        assert_eq!(1, start.position().col);
+        assert_eq!(3, start.end_position().row);
+        assert_eq!(8, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a\nbb");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let k1 = t1.pToken();
+        let a1 = t1.pA();
+        assert_eq!(1, k1.position().row);
+        assert_eq!(1, k1.position().col);
+        assert!(a1.position().valid());
+        assert_eq!(2, a1.position().row);
+        assert_eq!(1, a1.position().col);
+        assert_eq!(2, a1.end_position().row);
+        assert_eq!(2, a1.end_position().col);
+        assert_eq!(1, t1.position().row);
+        assert_eq!(2, t1.end_position().row);
+        assert_eq!(2, t1.end_position().col);
+        assert_eq!(1, start.position().row);
+        assert_eq!(2, start.end_position().row);
+        assert_eq!(2, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a\nc\nc");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let a1 = t1.pA();
+        assert!(a1.position().valid());
+        assert_eq!(2, a1.position().row);
+        assert_eq!(1, a1.position().col);
+        assert_eq!(3, a1.end_position().row);
+        assert_eq!(1, a1.end_position().col);
+        assert_eq!(1, t1.position().row);
+        assert_eq!(3, t1.end_position().row);
+        assert_eq!(1, t1.end_position().col);
+        assert_eq!(3, start.end_position().row);
+        assert_eq!(1, start.end_position().col);
+    }
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let a1 = t1.pA();
+        assert!(!a1.position().valid());
+        assert_eq!(1, t1.position().row);
+        assert_eq!(1, t1.position().col);
+        assert_eq!(1, t1.end_position().row);
+        assert_eq!(1, t1.end_position().col);
+        assert_eq!(1, start.position().row);
+        assert_eq!(1, start.end_position().row);
+        assert_eq!(1, start.end_position().col);
+    }
+    p_context_delete(context);
+}

spec/test_tree_node_memory_remains.rs

@@ -0,0 +1,85 @@
+use testparser::*;
+
+fn main() {
+    let entries: [(&str, &str, p_token_t); 51] = [
+        ("byte_val", "byte", TOKEN_byte),
+        ("short_val", "short", TOKEN_short),
+        ("int_val", "int", TOKEN_int),
+        ("long_val", "long", TOKEN_long),
+        ("ssize_t_val", "ssize_t", TOKEN_ssize_t),
+        ("byte_to_short", "short", TOKEN_short),
+        ("byte_to_int", "int", TOKEN_int),
+        ("byte_to_long", "long", TOKEN_long),
+        ("byte_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("short_to_byte", "byte", TOKEN_byte),
+        ("short_to_int", "int", TOKEN_int),
+        ("short_to_long", "long", TOKEN_long),
+        ("short_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("int_to_byte", "byte", TOKEN_byte),
+        ("int_to_short", "short", TOKEN_short),
+        ("int_to_long", "long", TOKEN_long),
+        ("int_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("long_to_byte", "byte", TOKEN_byte),
+        ("long_to_short", "short", TOKEN_short),
+        ("long_to_int", "int", TOKEN_int),
+        ("long_to_ssize_t", "ssize_t", TOKEN_ssize_t),
+        ("ssize_t_to_byte", "byte", TOKEN_byte),
+        ("ssize_t_to_short", "short", TOKEN_short),
+        ("ssize_t_to_int", "int", TOKEN_int),
+        ("ssize_t_to_long", "long", TOKEN_long),
+        ("ubyte_val", "ubyte", TOKEN_ubyte),
+        ("ushort_val", "ushort", TOKEN_ushort),
+        ("uint_val", "uint", TOKEN_uint),
+        ("ulong_val", "ulong", TOKEN_ulong),
+        ("size_t_val", "size_t", TOKEN_size_t),
+        ("ubyte_to_ushort", "ushort", TOKEN_ushort),
+        ("ubyte_to_uint", "uint", TOKEN_uint),
+        ("ubyte_to_ulong", "ulong", TOKEN_ulong),
+        ("ubyte_to_size_t", "size_t", TOKEN_size_t),
+        ("ushort_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("ushort_to_uint", "uint", TOKEN_uint),
+        ("ushort_to_ulong", "ulong", TOKEN_ulong),
+        ("ushort_to_size_t", "size_t", TOKEN_size_t),
+        ("uint_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("uint_to_ushort", "ushort", TOKEN_ushort),
+        ("uint_to_ulong", "ulong", TOKEN_ulong),
+        ("uint_to_size_t", "size_t", TOKEN_size_t),
+        ("ulong_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("ulong_to_ushort", "ushort", TOKEN_ushort),
+        ("ulong_to_uint", "uint", TOKEN_uint),
+        ("ulong_to_size_t", "size_t", TOKEN_size_t),
+        ("size_t_to_ubyte", "ubyte", TOKEN_ubyte),
+        ("size_t_to_ushort", "ushort", TOKEN_ushort),
+        ("size_t_to_int", "int", TOKEN_int),
+        ("size_t_to_ulong", "ulong", TOKEN_ulong),
+        ("main", "int", TOKEN_int),
+    ];
+
+    let mut input = String::new();
+    for (name, ret, _) in entries.iter() {
+        input.push_str(&format!("def {}() -> {} {{\nreturn 0x42;\n}}\n", name, ret));
+    }
+
+    let mut c = p_context_new(input.as_bytes());
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    {
+        let pmod = p_result(&c);
+        let mut pfds = Vec::new();
+        let mut pmis = pmod.pModuleItems();
+        while pmis.valid() {
+            let pmi = pmis.pModuleItem();
+            assert!(pmi.valid());
+            let pfd = pmi.pFunctionDefinition();
+            if pfd.valid() {
+                pfds.insert(0, pfd);
+            }
+            pmis = pmis.pModuleItems();
+        }
+        assert_eq!(51, pfds.len());
+        for i in 0..pfds.len() {
+            assert_eq!(entries[i].0, pfds[i].name().data().pvalue.s.as_str());
+            assert_eq!(entries[i].2, pfds[i].returntype().pType().pTypeBase().pToken1().token());
+        }
+    }
+    p_context_delete(c);
+}

spec/test_tree_ps.rs

@@ -0,0 +1,46 @@
+use testparser::*;
+
+fn main() {
+    let input = b"a, ((b)), b";
+    let mut context = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems1().valid());
+        assert!(start.pItems().valid());
+        let items = start.pItems();
+        assert!(items.pItem().valid());
+        assert!(items.pItem().pToken1().valid());
+        assert_eq!(TOKEN_a, items.pItem().pToken1().token());
+        assert_eq!(11, items.pItem().pToken1().pvalue());
+        let itemsmore = items.pItemsMore();
+        assert!(itemsmore.pItem().pItem().pItem().pToken1().valid());
+        assert_eq!(TOKEN_b, itemsmore.pItem().pItem().pItem().pToken1().token());
+        assert_eq!(22, itemsmore.pItem().pItem().pItem().pToken1().pvalue());
+        assert!(itemsmore.pItemsMore().valid());
+        let itemsmore = itemsmore.pItemsMore();
+        assert_eq!(TOKEN_b, itemsmore.pItem().pToken1().token());
+        assert!(!itemsmore.pItemsMore().valid());
+    }
+    p_context_delete(context);
+
+    /* Empty input yields a Start node with no Items child. */
+    let mut context = p_context_new(b"");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    assert!(!p_result(&context).pItems().valid());
+    p_context_delete(context);
+
+    /* Dual rule alternative field positions. */
+    let mut context = p_context_new(b"2 1");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        assert!(start.pItems().pItem().pDual().pTwo1().valid());
+        assert!(start.pItems().pItem().pDual().pOne2().valid());
+        assert!(!start.pItems().pItem().pDual().pTwo2().valid());
+        assert!(!start.pItems().pItem().pDual().pOne1().valid());
+    }
+    p_context_delete(context);
+
+    println!("ok");
+}

spec/test_tree_token_positions.rs

@@ -0,0 +1,27 @@
+use testparser::*;
+
+fn main() {
+    let input = b"abbccc";
+    let mut context = p_context_new(input);
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    {
+        let start = p_result(&context);
+        let t1 = start.pT1();
+        let t2 = start.pT2();
+        let t3 = start.pT3();
+        assert_eq!(1, t1.position().col);
+        assert_eq!(1, t1.end_position().col);
+        assert_eq!(2, t2.position().col);
+        assert_eq!(3, t2.end_position().col);
+        assert_eq!(4, t3.position().col);
+        assert_eq!(6, t3.end_position().col);
+        /* Token node position within T1. */
+        assert_eq!(1, t1.pToken().position().col);
+        /* Overall start node spans the whole input. */
+        assert_eq!(1, start.position().col);
+        assert_eq!(6, start.end_position().col);
+        assert_eq!(3, start.n_fields());
+    }
+    p_context_delete(context);
+    println!("ok");
+}

spec/test_user_code.rs

@@ -0,0 +1,13 @@
+use testparser::*;
+
+fn main() {
+    let mut context = p_context_new(b"abcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass1");
+    p_context_delete(context);
+
+    let mut context = p_context_new(b"abcabcdef");
+    assert_eq!(P_SUCCESS, p_parse(&mut context));
+    println!("pass2");
+    p_context_delete(context);
+}

spec/test_user_context_fields.rs

@@ -0,0 +1,9 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    eprint!("comments: {}", c.comments);
+    eprintln!("acount: {}", c.acount);
+    p_context_delete(c);
+}

spec/test_user_terminate.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"aacc");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"abc");
+    assert_eq!(P_USER_TERMINATED, p_parse(&mut c));
+    assert_eq!(4200, p_user_terminate_code(&c));
+    p_context_delete(c);
+}

spec/test_user_terminate_lexer.rs

@@ -0,0 +1,12 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"a");
+    assert_eq!(P_SUCCESS, p_parse(&mut c));
+    p_context_delete(c);
+
+    let mut c = p_context_new(b"b");
+    assert_eq!(P_USER_TERMINATED, p_parse(&mut c));
+    assert_eq!(8675309, p_user_terminate_code(&c));
+    p_context_delete(c);
+}

spec/test_value_accessors.rs

@@ -0,0 +1,20 @@
+use testparser::*;
+
+fn main() {
+    let mut c = p_context_new(b"42 f s");
+    let mut ti = p_token_info_t::default();
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_num, ti.token);
+    assert_eq!(42, p_value_get(&ti.pvalue));
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_flt, ti.token);
+    assert_eq!(1.5, p_value_get_float(&ti.pvalue));
+
+    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
+    assert_eq!(TOKEN_str, ti.token);
+    assert_eq!("hello", p_value_get_string(&ti.pvalue));
+
+    p_context_delete(c);
+}

spec/tree_node_memory_remains.rust.propane

@@ -0,0 +1,148 @@
+tree;
+tree_prefix P;
+
+<<header
+#[derive(Clone, Default)]
+pub struct TokenVal {
+    pub i64_: u64,
+    pub s: String,
+    pub dou: f64,
+}
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.i64_ = 64;
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  $$.s = std::str::from_utf8(match_).unwrap().to_string();
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;