josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Clean up Decoder result handling

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-06 20:32:01 -05:00
parent 2e5083d823
commit b00f53050b
2 changed files with 48 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
assets/parser.d.erb
View File

@ -36,23 +36,40 @@ class <%= @classname %>
static class Decoder static class Decoder
{ {
enum : uint struct Result
{
CODE_POINT_INVALID = 0xFFFF_FFFE,
CODE_POINT_EOF = 0xFFFF_FFFF,
}
struct DecodedCodePoint
{ {
enum : ubyte
{
SUCCESS,
EOF,
DECODE_ERROR,
}
ubyte result;
alias result this;
uint code_point; uint code_point;
uint code_point_length; uint code_point_length;
static Result success(uint code_point, uint code_point_length)
{
return Result(SUCCESS, code_point, code_point_length);
}
static Result eof()
{
return Result(EOF);
}
static Result decode_error()
{
return Result(DECODE_ERROR);
}
} }
static DecodedCodePoint decode_code_point(string input) static Result decode_code_point(string input)
{ {
if (input.length == 0u) if (input.length == 0u)
{ {
return DecodedCodePoint(CODE_POINT_EOF, 0u); return Result.eof();
} }
char c = input[0]; char c = input[0];
uint code_point; uint code_point;
@ -92,11 +109,11 @@ class <%= @classname %>
} }
else else
{ {
return DecodedCodePoint(CODE_POINT_INVALID, 0u); return Result.decode_error();
} }
if (input.length <= following_bytes) if (input.length <= following_bytes)
{ {
return DecodedCodePoint(CODE_POINT_INVALID, 0u); return Result.decode_error();
} }
code_point_length = following_bytes + 1u; code_point_length = following_bytes + 1u;
for (size_t i = 0u; i < following_bytes; i++) for (size_t i = 0u; i < following_bytes; i++)
@ -104,12 +121,12 @@ class <%= @classname %>
char b = input[i + 1u]; char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u) if ((b & 0xC0u) != 0x80u)
{ {
return DecodedCodePoint(CODE_POINT_INVALID, 0u); return Result.decode_error();
} }
code_point = (code_point << 6u) | (b & 0x3Fu); code_point = (code_point << 6u) | (b & 0x3Fu);
} }
} }
return DecodedCodePoint(code_point, code_point_length); return Result.success(code_point, code_point_length);
} }
} }
@ -238,13 +255,13 @@ class <%= @classname %>
for (;;) for (;;)
{ {
auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match_info.length)..(m_input.length)]); auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match_info.length)..(m_input.length)]);
if (decoded.code_point == Decoder.CODE_POINT_INVALID) if (decoded == Decoder.Result.DECODE_ERROR)
{ {
lt.token = _TOKEN_DECODE_ERROR; lt.token = _TOKEN_DECODE_ERROR;
return lt; return lt;
} }
bool lex_continue = false; bool lex_continue = false;
if (decoded.code_point != Decoder.CODE_POINT_EOF) if (decoded != Decoder.Result.EOF)
{ {
uint dest = transition(current_state, decoded.code_point); uint dest = transition(current_state, decoded.code_point);
if (dest != cast(uint)-1) if (dest != cast(uint)-1)

32
spec/test_d_lexer.d
View File

@ -8,29 +8,29 @@ int main()
unittest unittest
{ {
alias DCP = Testparser.Decoder.DecodedCodePoint; alias Result = Testparser.Decoder.Result;
DCP dcp; Result result;
dcp = Testparser.Decoder.decode_code_point("5"); result = Testparser.Decoder.decode_code_point("5");
assert(dcp == DCP('5', 1u)); assert(result == Result.success('5', 1u));
dcp = Testparser.Decoder.decode_code_point(""); result = Testparser.Decoder.decode_code_point("");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u)); assert(result == Result.eof());
dcp = Testparser.Decoder.decode_code_point("\xC2\xA9"); result = Testparser.Decoder.decode_code_point("\xC2\xA9");
assert(dcp == DCP(0xA9u, 2u)); assert(result == Result.success(0xA9u, 2u));
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
assert(dcp == DCP(0x1F9E1, 4u)); assert(result == Result.success(0x1F9E1, 4u));
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\x27"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u)); assert(result == Result.decode_error());
dcp = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u)); assert(result == Result.decode_error());
dcp = Testparser.Decoder.decode_code_point("\xfe"); result = Testparser.Decoder.decode_code_point("\xfe");
assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u)); assert(result == Result.decode_error());
} }
unittest unittest