josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Switch to new API - close #8

Browse Source 
The new API is more C-like and will allow consistency across all future
supported language targets.
This commit is contained in:
Josh Holtrop 2023-07-12 15:46:13 -04:00
parent e0e5e87338
commit 7a1b4064c1
15 changed files with 611 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

897
assets/parser.d.erb
View File

File diff suppressed because it is too large Load Diff

2
lib/propane/generator.rb
View File

@ -203,7 +203,7 @@ class Propane
unless mode_id unless mode_id
raise Error.new("Lexer mode '#{mode_name}' not found") raise Error.new("Lexer mode '#{mode_name}' not found")
end end
"m_mode = #{mode_id}u" "context.mode = #{mode_id}u"
end end
end end
code code

37
spec/test_d_lexer.d
View File

@ -12,31 +12,31 @@ unittest
CodePoint code_point; CodePoint code_point;
ubyte code_point_length; ubyte code_point_length;
result = Decoder.decode_code_point("5", &code_point, &code_point_length); result = p_decode_code_point("5", &code_point, &code_point_length);
assert(result == P_SUCCESS); assert(result == P_SUCCESS);
assert(code_point == '5'); assert(code_point == '5');
assert(code_point_length == 1u); assert(code_point_length == 1u);
result = Decoder.decode_code_point("", &code_point, &code_point_length); result = p_decode_code_point("", &code_point, &code_point_length);
assert(result == P_EOF); assert(result == P_EOF);
result = Decoder.decode_code_point("\xC2\xA9", &code_point, &code_point_length); result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
assert(result == P_SUCCESS); assert(result == P_SUCCESS);
assert(code_point == 0xA9u); assert(code_point == 0xA9u);
assert(code_point_length == 2u); assert(code_point_length == 2u);
result = Decoder.decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length); result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
assert(result == P_SUCCESS); assert(result == P_SUCCESS);
assert(code_point == 0x1F9E1u); assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u); assert(code_point_length == 4u);
result = Decoder.decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length); result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR); assert(result == P_DECODE_ERROR);
result = Decoder.decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length); result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR); assert(result == P_DECODE_ERROR);
result = Decoder.decode_code_point("\xfe", &code_point, &code_point_length); result = p_decode_code_point("\xfe", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR); assert(result == P_DECODE_ERROR);
} }
@ -44,25 +44,26 @@ unittest
{ {
TokenInfo token_info; TokenInfo token_info;
string input = "5 + 4 * \n677 + 567"; string input = "5 + 4 * \n677 + 567";
Lexer lexer = new Lexer(input); p_context_t context;
assert(lexer.lex_token(&token_info) == P_SUCCESS); p_context_init(&context, input);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 0), 1, TOKEN_int)); assert(token_info == TokenInfo(Position(0, 0), 1, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 2), 1, TOKEN_plus)); assert(token_info == TokenInfo(Position(0, 2), 1, TOKEN_plus));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 4), 1, TOKEN_int)); assert(token_info == TokenInfo(Position(0, 4), 1, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 6), 1, TOKEN_times)); assert(token_info == TokenInfo(Position(0, 6), 1, TOKEN_times));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 0), 3, TOKEN_int)); assert(token_info == TokenInfo(Position(1, 0), 3, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 4), 1, TOKEN_plus)); assert(token_info == TokenInfo(Position(1, 4), 1, TOKEN_plus));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 6), 3, TOKEN_int)); assert(token_info == TokenInfo(Position(1, 6), 3, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 9), 0, TOKEN___EOF)); assert(token_info == TokenInfo(Position(1, 9), 0, TOKEN___EOF));
lexer = new Lexer(""); p_context_init(&context, "");
assert(lexer.lex_token(&token_info) == P_SUCCESS); assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 0), 0, TOKEN___EOF)); assert(token_info == TokenInfo(Position(0, 0), 0, TOKEN___EOF));
} }

9
spec/test_d_parser_identical_rules_lookahead.d
View File

@ -9,10 +9,11 @@ int main()
unittest unittest
{ {
string input = "aba"; string input = "aba";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
input = "abb"; input = "abb";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
} }

13
spec/test_d_parser_rule_from_multiple_states.d
View File

@ -9,14 +9,15 @@ int main()
unittest unittest
{ {
string input = "a"; string input = "a";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_UNEXPECTED_TOKEN); p_context_init(&context, input);
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
input = "a b"; input = "a b";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
input = "bb"; input = "bb";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
} }

5
spec/test_lexer_match_text.d
View File

@ -9,7 +9,8 @@ int main()
unittest unittest
{ {
string input = `identifier_123`; string input = `identifier_123`;
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1"); writeln("pass1");
} }

9
spec/test_lexer_modes.d
View File

@ -9,12 +9,13 @@ int main()
unittest unittest
{ {
string input = `abc "a string" def`; string input = `abc "a string" def`;
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1"); writeln("pass1");
input = `abc "abc def" def`; input = `abc "abc def" def`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
writeln("pass2"); writeln("pass2");
} }

13
spec/test_lexer_result_value.d
View File

@ -9,12 +9,13 @@ int main()
unittest unittest
{ {
string input = `x`; string input = `x`;
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(parser.result == 1u); assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 1u);
input = `fabulous`; input = `fabulous`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(parser.result == 8u); assert(p_result(&context) == 8u);
} }

11
spec/test_lexer_unknown_character.d
View File

@ -9,11 +9,12 @@ int main()
unittest unittest
{ {
string input = `x`; string input = `x`;
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_UNEXPECTED_INPUT); p_context_init(&context, input);
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
input = `123`; input = `123`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(parser.result == 123u); assert(p_result(&context) == 123u);
} }

5
spec/test_parser_rule_user_code.d
View File

@ -9,6 +9,7 @@ int main()
unittest unittest
{ {
string input = "ab"; string input = "ab";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
} }

49
spec/test_parsing_json.d
View File

@ -10,44 +10,45 @@ int main()
unittest unittest
{ {
string input = ``; string input = ``;
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
input = `{}`; input = `{}`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)parser.result); assert(cast(JSONObject)p_result(&context));
input = `[]`; input = `[]`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONArray)parser.result); assert(cast(JSONArray)p_result(&context));
input = `-45.6`; input = `-45.6`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONNumber)parser.result); assert(cast(JSONNumber)p_result(&context));
assert((cast(JSONNumber)parser.result).value == -45.6); assert((cast(JSONNumber)p_result(&context)).value == -45.6);
input = `2E-2`; input = `2E-2`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONNumber)parser.result); assert(cast(JSONNumber)p_result(&context));
assert((cast(JSONNumber)parser.result).value == 0.02); assert((cast(JSONNumber)p_result(&context)).value == 0.02);
input = `{"hi":true}`; input = `{"hi":true}`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)parser.result); assert(cast(JSONObject)p_result(&context));
JSONObject o = cast(JSONObject)parser.result; JSONObject o = cast(JSONObject)p_result(&context);
assert(o.value["hi"]); assert(o.value["hi"]);
assert(cast(JSONTrue)o.value["hi"]); assert(cast(JSONTrue)o.value["hi"]);
input = `{"ff": false, "nn": null}`; input = `{"ff": false, "nn": null}`;
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)parser.result); assert(cast(JSONObject)p_result(&context));
o = cast(JSONObject)parser.result; o = cast(JSONObject)p_result(&context);
assert(o.value["ff"]); assert(o.value["ff"]);
assert(cast(JSONFalse)o.value["ff"]); assert(cast(JSONFalse)o.value["ff"]);
assert(o.value["nn"]); assert(o.value["nn"]);

19
spec/test_parsing_lists.d
View File

@ -9,17 +9,18 @@ int main()
unittest unittest
{ {
string input = "a"; string input = "a";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(parser.result == 1u); assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 1u);
input = ""; input = "";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(parser.result == 0u); assert(p_result(&context) == 0u);
input = "aaaaaaaaaaaaaaaa"; input = "aaaaaaaaaaaaaaaa";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
assert(parser.result == 16u); assert(p_result(&context) == 16u);
} }

9
spec/test_pattern.d
View File

@ -9,12 +9,13 @@ int main()
unittest unittest
{ {
string input = "abcdef"; string input = "abcdef";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1"); writeln("pass1");
input = "defabcdef"; input = "defabcdef";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
writeln("pass2"); writeln("pass2");
} }

5
spec/test_return_token_from_pattern.d
View File

@ -9,6 +9,7 @@ int main()
unittest unittest
{ {
string input = "defghidef"; string input = "defghidef";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
} }

9
spec/test_user_code.d
View File

@ -9,12 +9,13 @@ int main()
unittest unittest
{ {
string input = "abcdef"; string input = "abcdef";
auto parser = new Parser(input); p_context_t context;
assert(parser.parse() == P_SUCCESS); p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1"); writeln("pass1");
input = "abcabcdef"; input = "abcabcdef";
parser = new Parser(input); p_context_init(&context, input);
assert(parser.parse() == P_SUCCESS); assert(p_parse(&context) == P_SUCCESS);
writeln("pass2"); writeln("pass2");
} }