josh/propane
1
0 Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Work on real D strings instead of ubyte pointer and length

Browse Source 
Also fix a couple UTF-8 decoder bugs!
This commit is contained in:
josh committed 2022-10-15 13:32:33 -04:00
1 parent de93d23585
commit 623c644e74
11 files changed
+43 -77
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

spec/propane_spec.rb
+1
View File
@@ -76,6 +76,7 @@ EOF
build_parser
compile("spec/test_d_lexer.d")
results = run
expect(results.stderr).to eq ""
expect(results.status).to eq 0
end
spec/test_d_lexer.d
+13 -45
View File
@@ -9,59 +9,27 @@ int main()
unittest
{
alias DCP = Testparser.Decoder.DecodedCodePoint;
string inputstring = "5+\n 66";
const(ubyte) * input = cast(const(ubyte) *)inputstring.ptr;
size_t input_length = inputstring.length;
DCP dcp;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("5");
assert(dcp == DCP('5', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
assert(dcp == DCP('+', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
assert(dcp == DCP('\n', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
assert(dcp == DCP(' ', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
assert(dcp == DCP('6', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
assert(dcp == DCP('6', 1u));
input += dcp.code_point_length;
input_length -= dcp.code_point_length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
inputstring = "\xf0\x1f\x27\x21";
input = cast(const(ubyte) *)inputstring.ptr;
input_length = inputstring.length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("\xC2\xA9");
assert(dcp == DCP(0xA9u, 2u));
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
assert(dcp == DCP(0x1F9E1, 4u));
inputstring = "\xf0\x1f\x27";
input = cast(const(ubyte) *)inputstring.ptr;
input_length = inputstring.length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
inputstring = "\xf0\x1f\x27\xFF";
input = cast(const(ubyte) *)inputstring.ptr;
input_length = inputstring.length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
inputstring = "\xfe";
input = cast(const(ubyte) *)inputstring.ptr;
input_length = inputstring.length;
dcp = Testparser.Decoder.decode_code_point(input, input_length);
dcp = Testparser.Decoder.decode_code_point("\xfe");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
}
@@ -69,7 +37,7 @@ unittest
{
alias LT = Testparser.Lexer.LexedToken;
string input = "5 + 4 * \n677 + 567";
Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
Testparser.Lexer lexer = new Testparser.Lexer(input);
assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_int));
assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_plus));
assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_int));
@@ -79,6 +47,6 @@ unittest
assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_int));
assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_0EOF));
lexer = new Testparser.Lexer(null, 0u);
lexer = new Testparser.Lexer("");
assert(lexer.lex_token() == LT(0, 0, 0, Testparser.TOKEN_0EOF));
}
spec/test_d_parser_identical_rules_lookahead.d
+2 -2
View File
@@ -9,10 +9,10 @@ int main()
unittest
{
string input = "aba";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
input = "abb";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
}
spec/test_d_parser_rule_from_multiple_states.d
+3 -3
View File
@@ -9,14 +9,14 @@ int main()
unittest
{
string input = "a";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == false);
input = "a b";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
input = "bb";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
}
spec/test_lexer_modes.d
+2 -2
View File
@@ -9,12 +9,12 @@ int main()
unittest
{
string input = `abc "a string" def`;
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass1");
input = `abc "abc def" def`;
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass2");
}
spec/test_parser_rule_user_code.d
+1 -1
View File
@@ -9,6 +9,6 @@ int main()
unittest
{
string input = "ab";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
}
spec/test_parsing_lists.d
+3 -3
View File
@@ -9,17 +9,17 @@ int main()
unittest
{
string input = "a";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
assert(parser.result == 1u);
input = "";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
assert(parser.result == 0u);
input = "aaaaaaaaaaaaaaaa";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
assert(parser.result == 16u);
}
spec/test_pattern.d
+2 -2
View File
@@ -9,12 +9,12 @@ int main()
unittest
{
string input = "abcdef";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass1");
input = "defabcdef";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass2");
}
spec/test_return_token_from_pattern.d
+1 -1
View File
@@ -9,6 +9,6 @@ int main()
unittest
{
string input = "defghidef";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
}
spec/test_user_code.d
+2 -2
View File
@@ -9,12 +9,12 @@ int main()
unittest
{
string input = "abcdef";
auto parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
auto parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass1");
input = "abcabcdef";
parser = new Testparser.Parser(cast(const(ubyte) *)input.ptr, input.length);
parser = new Testparser.Parser(input);
assert(parser.parse() == true);
writeln("pass2");
}