josh/propane
1
0 Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Consolidate return codes

Browse Source
This commit is contained in:
josh committed 2023-07-08 09:23:31 -04:00
1 parent 0232b204c6
commit ecef933255
13 files changed
+113 -111
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

assets/parser.d.erb
+70 -67
View File
@@ -11,6 +11,18 @@ import std.stdio;
class <%= @classname %>
{
/* Result codes. */
public enum : size_t
{
P_SUCCESS,
P_DECODE_ERROR,
P_UNEXPECTED_INPUT,
P_UNEXPECTED_TOKEN,
P_TOKEN,
P_DROP,
P_EOF,
}
alias TokenID = uint;
enum : TokenID
@@ -71,20 +83,21 @@ class <%= @classname %>
static class Decoder
{
enum Result
{
SUCCESS,
EOF,
DECODE_ERROR,
}
static Result decode_code_point(string input,
/**
* Decode a UTF-8 code point.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_EOF
*/
static size_t decode_code_point(string input,
ref CodePoint out_code_point,
ref ubyte out_code_point_length)
{
if (input.length == 0u)
{
return Result.EOF;
return P_EOF;
}
char c = input[0];
CodePoint code_point;
@@ -124,11 +137,11 @@ class <%= @classname %>
}
else
{
return Result.DECODE_ERROR;
return P_DECODE_ERROR;
}
if (input.length <= following_bytes)
{
return Result.DECODE_ERROR;
return P_DECODE_ERROR;
}
code_point_length = cast(ubyte)(following_bytes + 1u);
for (size_t i = 0u; i < following_bytes; i++)
@@ -136,14 +149,14 @@ class <%= @classname %>
char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u)
{
return Result.DECODE_ERROR;
return P_DECODE_ERROR;
}
code_point = (code_point << 6u) | (b & 0x3Fu);
}
}
out_code_point = code_point;
out_code_point_length = code_point_length;
return Result.SUCCESS;
return P_SUCCESS;
}
}
@@ -230,14 +243,6 @@ class <%= @classname %>
<% end %>
];
public enum : size_t
{
P_TOKEN,
P_UNEXPECTED_INPUT,
P_DECODE_ERROR,
P_DROP,
}
public static struct TokenInfo
{
size_t row;
@@ -259,6 +264,14 @@ class <%= @classname %>
m_mode = <%= @lexer.mode_id("default") %>;
}
/**
* Lex the next token in the input stream.
*
* Returns one of:
* - P_TOKEN
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
*/
size_t lex_token(TokenInfo * out_token_info)
{
for (;;)
@@ -298,6 +311,15 @@ class <%= @classname %>
return Token.invalid();
}
/**
* Attempt to lex the next token in the input stream.
*
* Returns one of:
* - P_TOKEN
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
* - P_DROP
*/
private size_t attempt_lex_token(TokenInfo * out_token_info)
{
TokenInfo token_info;
@@ -307,9 +329,10 @@ class <%= @classname %>
*out_token_info = token_info; // TODO: remove
MatchInfo match_info;
size_t unexpected_input_length;
switch (find_longest_match(match_info, unexpected_input_length))
size_t result = find_longest_match(match_info, unexpected_input_length);
switch (result)
{
case FindLongestMatchResult.FOUND_MATCH:
case P_SUCCESS:
uint token_to_accept = match_info.accepting_state.token;
if (match_info.accepting_state.code_id.is_valid())
{
@@ -344,30 +367,16 @@ class <%= @classname %>
*out_token_info = token_info;
return P_TOKEN;
case FindLongestMatchResult.DECODE_ERROR:
return P_DECODE_ERROR;
case FindLongestMatchResult.EOF:
case P_EOF:
token_info.token = TOKEN___EOF;
*out_token_info = token_info;
return P_TOKEN;
case FindLongestMatchResult.UNEXPECTED_INPUT:
return P_UNEXPECTED_INPUT;
default:
assert(false);
return result;
}
}
enum FindLongestMatchResult
{
FOUND_MATCH,
DECODE_ERROR,
EOF,
UNEXPECTED_INPUT,
}
struct MatchInfo
{
size_t length;
@@ -376,7 +385,16 @@ class <%= @classname %>
const(State) * accepting_state;
}
private FindLongestMatchResult find_longest_match(
/**
* Find the longest lexer pattern match at the current position.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
* - P_EOF
*/
private size_t find_longest_match(
ref MatchInfo out_match_info,
ref size_t out_unexpected_input_length)
{
@@ -388,9 +406,10 @@ class <%= @classname %>
string input = m_input[(m_input_position + attempt_match.length)..(m_input.length)];
CodePoint code_point;
ubyte code_point_length;
switch (Decoder.decode_code_point(input, code_point, code_point_length))
size_t result = Decoder.decode_code_point(input, code_point, code_point_length);
switch (result)
{
case Decoder.Result.SUCCESS:
case P_SUCCESS:
auto transition_result = transition(current_state, code_point);
if (transition_result.found())
{
@@ -414,41 +433,38 @@ class <%= @classname %>
else if (longest_match.length > 0)
{
out_match_info = longest_match;
return FindLongestMatchResult.FOUND_MATCH;
return P_SUCCESS;
}
else
{
out_unexpected_input_length = attempt_match.length + code_point_length;
return FindLongestMatchResult.UNEXPECTED_INPUT;
return P_UNEXPECTED_INPUT;
}
break;
case Decoder.Result.EOF:
case P_EOF:
/* We hit EOF. */
if (longest_match.length > 0)
{
/* We have a match, so use it. */
out_match_info = longest_match;
return FindLongestMatchResult.FOUND_MATCH;
return P_SUCCESS;
}
else if (attempt_match.length != 0)
{
/* There is a partial match - error! */
out_unexpected_input_length = attempt_match.length;
return FindLongestMatchResult.UNEXPECTED_INPUT;
return P_UNEXPECTED_INPUT;
}
else
{
/* Valid EOF return. */
return FindLongestMatchResult.EOF;
return P_EOF;
}
break;
case Decoder.Result.DECODE_ERROR:
return FindLongestMatchResult.DECODE_ERROR;
default:
assert(false);
return result;
}
}
}
@@ -564,14 +580,6 @@ class <%= @classname %>
m_lexer = new Lexer(input);
}
public enum : size_t
{
P_SUCCESS,
P_DECODE_ERROR,
P_UNEXPECTED_INPUT,
P_UNEXPECTED_TOKEN,
}
size_t parse()
{
Lexer.TokenInfo token_info;
@@ -584,14 +592,9 @@ class <%= @classname %>
if (token == _TOKEN_COUNT)
{
size_t lexer_result = m_lexer.lex_token(&token_info);
switch (lexer_result)
if (lexer_result != P_TOKEN)
{
case Lexer.P_UNEXPECTED_INPUT:
return P_UNEXPECTED_INPUT;
case Lexer.P_DECODE_ERROR:
return P_DECODE_ERROR;
default:
break;
return lexer_result;
}
token = token_info.token;
}