josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Just return integer result code from Decoder.decode_code_point()

Browse Source
This commit is contained in:
Josh Holtrop 2023-07-05 16:54:52 -04:00
parent 9e33a32930
commit d56cc2deeb
2 changed files with 44 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
assets/parser.d.erb
View File

@ -71,40 +71,20 @@ class <%= @classname %>
static class Decoder
{
struct Result
enum Result
{
enum
{
SUCCESS,
EOF,
DECODE_ERROR,
}
private ubyte type;
alias type this;
ubyte code_point_length;
CodePoint code_point;
static Result success(CodePoint code_point, ubyte code_point_length)
{
return Result(SUCCESS, code_point_length, code_point);
}
static Result eof()
{
return Result(EOF);
}
static Result decode_error()
{
return Result(DECODE_ERROR);
}
SUCCESS,
EOF,
DECODE_ERROR,
}
static Result decode_code_point(string input)
static Result decode_code_point(string input,
ref CodePoint out_code_point,
ref ubyte out_code_point_length)
{
if (input.length == 0u)
{
return Result.eof();
return Result.EOF;
}
char c = input[0];
CodePoint code_point;
@ -144,11 +124,11 @@ class <%= @classname %>
}
else
{
return Result.decode_error();
return Result.DECODE_ERROR;
}
if (input.length <= following_bytes)
{
return Result.decode_error();
return Result.DECODE_ERROR;
}
code_point_length = cast(ubyte)(following_bytes + 1u);
for (size_t i = 0u; i < following_bytes; i++)
@ -156,12 +136,14 @@ class <%= @classname %>
char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u)
{
return Result.decode_error();
return Result.DECODE_ERROR;
}
code_point = (code_point << 6u) | (b & 0x3Fu);
}
}
return Result.success(code_point, code_point_length);
out_code_point = code_point;
out_code_point_length = code_point_length;
return Result.SUCCESS;
}
}
@ -484,15 +466,17 @@ class <%= @classname %>
uint current_state = modes[m_mode].state_table_offset;
for (;;)
{
auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]);
switch (decoded)
string input = m_input[(m_input_position + attempt_match.length)..(m_input.length)];
CodePoint code_point;
ubyte code_point_length;
switch (Decoder.decode_code_point(input, code_point, code_point_length))
{
case Decoder.Result.SUCCESS:
auto transition_result = transition(current_state, decoded.code_point);
auto transition_result = transition(current_state, code_point);
if (transition_result.found())
{
attempt_match.length += decoded.code_point_length;
if (decoded.code_point == '\n')
attempt_match.length += code_point_length;
if (code_point == '\n')
{
attempt_match.delta_row++;
attempt_match.delta_col = 0u;
@ -514,7 +498,7 @@ class <%= @classname %>
}
else
{
return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length);
return FindLongestMatchResult.unexpected_input(attempt_match.length + code_point_length);
}
break;

36
spec/test_d_lexer.d
View File

@ -10,27 +10,35 @@ unittest
{
alias Result = Testparser.Decoder.Result;
Result result;
Testparser.CodePoint code_point;
ubyte code_point_length;
result = Testparser.Decoder.decode_code_point("5");
assert(result == Result.success('5', 1u));
result = Testparser.Decoder.decode_code_point("5", code_point, code_point_length);
assert(result == Result.SUCCESS);
assert(code_point == '5');
assert(code_point_length == 1u);
result = Testparser.Decoder.decode_code_point("");
assert(result == Result.eof());
result = Testparser.Decoder.decode_code_point("", code_point, code_point_length);
assert(result == Result.EOF);
result = Testparser.Decoder.decode_code_point("\xC2\xA9");
assert(result == Result.success(0xA9u, 2u));
result = Testparser.Decoder.decode_code_point("\xC2\xA9", code_point, code_point_length);
assert(result == Result.SUCCESS);
assert(code_point == 0xA9u);
assert(code_point_length == 2u);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
assert(result == Result.success(0x1F9E1, 4u));
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1", code_point, code_point_length);
assert(result == Result.SUCCESS);
assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
assert(result == Result.decode_error());
result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27", code_point, code_point_length);
assert(result == Result.DECODE_ERROR);
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
assert(result == Result.decode_error());
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF", code_point, code_point_length);
assert(result == Result.DECODE_ERROR);
result = Testparser.Decoder.decode_code_point("\xfe");
assert(result == Result.decode_error());
result = Testparser.Decoder.decode_code_point("\xfe", code_point, code_point_length);
assert(result == Result.DECODE_ERROR);
}
unittest