josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Just return integer result code from Decoder.decode_code_point()

Browse Source
This commit is contained in:
Josh Holtrop 2023-07-05 16:54:52 -04:00
parent 9e33a32930
commit d56cc2deeb
2 changed files with 44 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
assets/parser.d.erb
View File

@ -71,40 +71,20 @@ class <%= @classname %>
static class Decoder static class Decoder
{ {
struct Result enum Result
{ {
enum SUCCESS,
{ EOF,
SUCCESS, DECODE_ERROR,
EOF,
DECODE_ERROR,
}
private ubyte type;
alias type this;
ubyte code_point_length;
CodePoint code_point;
static Result success(CodePoint code_point, ubyte code_point_length)
{
return Result(SUCCESS, code_point_length, code_point);
}
static Result eof()
{
return Result(EOF);
}
static Result decode_error()
{
return Result(DECODE_ERROR);
}
} }
static Result decode_code_point(string input) static Result decode_code_point(string input,
ref CodePoint out_code_point,
ref ubyte out_code_point_length)
{ {
if (input.length == 0u) if (input.length == 0u)
{ {
return Result.eof(); return Result.EOF;
} }
char c = input[0]; char c = input[0];
CodePoint code_point; CodePoint code_point;
@ -144,11 +124,11 @@ class <%= @classname %>
} }
else else
{ {
return Result.decode_error(); return Result.DECODE_ERROR;
} }
if (input.length <= following_bytes) if (input.length <= following_bytes)
{ {
return Result.decode_error(); return Result.DECODE_ERROR;
} }
code_point_length = cast(ubyte)(following_bytes + 1u); code_point_length = cast(ubyte)(following_bytes + 1u);
for (size_t i = 0u; i < following_bytes; i++) for (size_t i = 0u; i < following_bytes; i++)
@ -156,12 +136,14 @@ class <%= @classname %>
char b = input[i + 1u]; char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u) if ((b & 0xC0u) != 0x80u)
{ {
return Result.decode_error(); return Result.DECODE_ERROR;
} }
code_point = (code_point << 6u) | (b & 0x3Fu); code_point = (code_point << 6u) | (b & 0x3Fu);
} }
} }
return Result.success(code_point, code_point_length); out_code_point = code_point;
out_code_point_length = code_point_length;
return Result.SUCCESS;
} }
} }
@ -484,15 +466,17 @@ class <%= @classname %>
uint current_state = modes[m_mode].state_table_offset; uint current_state = modes[m_mode].state_table_offset;
for (;;) for (;;)
{ {
auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]); string input = m_input[(m_input_position + attempt_match.length)..(m_input.length)];
switch (decoded) CodePoint code_point;
ubyte code_point_length;
switch (Decoder.decode_code_point(input, code_point, code_point_length))
{ {
case Decoder.Result.SUCCESS: case Decoder.Result.SUCCESS:
auto transition_result = transition(current_state, decoded.code_point); auto transition_result = transition(current_state, code_point);
if (transition_result.found()) if (transition_result.found())
{ {
attempt_match.length += decoded.code_point_length; attempt_match.length += code_point_length;
if (decoded.code_point == '\n') if (code_point == '\n')
{ {
attempt_match.delta_row++; attempt_match.delta_row++;
attempt_match.delta_col = 0u; attempt_match.delta_col = 0u;
@ -514,7 +498,7 @@ class <%= @classname %>
} }
else else
{ {
return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length); return FindLongestMatchResult.unexpected_input(attempt_match.length + code_point_length);
} }
break; break;

36
spec/test_d_lexer.d
View File

@ -10,27 +10,35 @@ unittest
{ {
alias Result = Testparser.Decoder.Result; alias Result = Testparser.Decoder.Result;
Result result; Result result;
Testparser.CodePoint code_point;
ubyte code_point_length;
result = Testparser.Decoder.decode_code_point("5"); result = Testparser.Decoder.decode_code_point("5", code_point, code_point_length);
assert(result == Result.success('5', 1u)); assert(result == Result.SUCCESS);
assert(code_point == '5');
assert(code_point_length == 1u);
result = Testparser.Decoder.decode_code_point(""); result = Testparser.Decoder.decode_code_point("", code_point, code_point_length);
assert(result == Result.eof()); assert(result == Result.EOF);
result = Testparser.Decoder.decode_code_point("\xC2\xA9"); result = Testparser.Decoder.decode_code_point("\xC2\xA9", code_point, code_point_length);
assert(result == Result.success(0xA9u, 2u)); assert(result == Result.SUCCESS);
assert(code_point == 0xA9u);
assert(code_point_length == 2u);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1", code_point, code_point_length);
assert(result == Result.success(0x1F9E1, 4u)); assert(result == Result.SUCCESS);
assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27", code_point, code_point_length);
assert(result == Result.decode_error()); assert(result == Result.DECODE_ERROR);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF"); result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF", code_point, code_point_length);
assert(result == Result.decode_error()); assert(result == Result.DECODE_ERROR);
result = Testparser.Decoder.decode_code_point("\xfe"); result = Testparser.Decoder.decode_code_point("\xfe", code_point, code_point_length);
assert(result == Result.decode_error()); assert(result == Result.DECODE_ERROR);
} }
unittest unittest