josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Clean up Decoder result handling

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-06 20:32:01 -05:00
parent 2e5083d823
commit b00f53050b
2 changed files with 48 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
assets/parser.d.erb
View File

@ -36,23 +36,40 @@ class <%= @classname %>
static class Decoder
{
enum : uint
struct Result
{
CODE_POINT_INVALID = 0xFFFF_FFFE,
CODE_POINT_EOF = 0xFFFF_FFFF,
enum : ubyte
{
SUCCESS,
EOF,
DECODE_ERROR,
}
struct DecodedCodePoint
{
ubyte result;
alias result this;
uint code_point;
uint code_point_length;
static Result success(uint code_point, uint code_point_length)
{
return Result(SUCCESS, code_point, code_point_length);
}
static DecodedCodePoint decode_code_point(string input)
static Result eof()
{
return Result(EOF);
}
static Result decode_error()
{
return Result(DECODE_ERROR);
}
}
static Result decode_code_point(string input)
{
if (input.length == 0u)
{
return DecodedCodePoint(CODE_POINT_EOF, 0u);
return Result.eof();
}
char c = input[0];
uint code_point;
@ -92,11 +109,11 @@ class <%= @classname %>
}
else
{
return DecodedCodePoint(CODE_POINT_INVALID, 0u);
return Result.decode_error();
}
if (input.length <= following_bytes)
{
return DecodedCodePoint(CODE_POINT_INVALID, 0u);
return Result.decode_error();
}
code_point_length = following_bytes + 1u;
for (size_t i = 0u; i < following_bytes; i++)
@ -104,12 +121,12 @@ class <%= @classname %>
char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u)
{
return DecodedCodePoint(CODE_POINT_INVALID, 0u);
return Result.decode_error();
}
code_point = (code_point << 6u) | (b & 0x3Fu);
}
}
return DecodedCodePoint(code_point, code_point_length);
return Result.success(code_point, code_point_length);
}
}
@ -238,13 +255,13 @@ class <%= @classname %>
for (;;)
{
auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match_info.length)..(m_input.length)]);
if (decoded.code_point == Decoder.CODE_POINT_INVALID)
if (decoded == Decoder.Result.DECODE_ERROR)
{
lt.token = _TOKEN_DECODE_ERROR;
return lt;
}
bool lex_continue = false;
if (decoded.code_point != Decoder.CODE_POINT_EOF)
if (decoded != Decoder.Result.EOF)
{
uint dest = transition(current_state, decoded.code_point);
if (dest != cast(uint)-1)

32
spec/test_d_lexer.d
View File

@ -8,29 +8,29 @@ int main()
unittest
{
alias DCP = Testparser.Decoder.DecodedCodePoint;
DCP dcp;
alias Result = Testparser.Decoder.Result;
Result result;
dcp = Testparser.Decoder.decode_code_point("5");
assert(dcp == DCP('5', 1u));
result = Testparser.Decoder.decode_code_point("5");
assert(result == Result.success('5', 1u));
dcp = Testparser.Decoder.decode_code_point("");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
result = Testparser.Decoder.decode_code_point("");
assert(result == Result.eof());
dcp = Testparser.Decoder.decode_code_point("\xC2\xA9");
assert(dcp == DCP(0xA9u, 2u));
result = Testparser.Decoder.decode_code_point("\xC2\xA9");
assert(result == Result.success(0xA9u, 2u));
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
assert(dcp == DCP(0x1F9E1, 4u));
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
assert(result == Result.success(0x1F9E1, 4u));
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
assert(result == Result.decode_error());
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
assert(result == Result.decode_error());
dcp = Testparser.Decoder.decode_code_point("\xfe");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
result = Testparser.Decoder.decode_code_point("\xfe");
assert(result == Result.decode_error());
}
unittest