josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Rename lexer variables to reflect trying to get longest match

Browse Source
This commit is contained in:
Josh Holtrop 2022-09-25 14:44:44 -04:00
parent 672098ad32
commit 01c9340819
1 changed files with 23 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
assets/parser.d.erb
View File

@ -198,7 +198,7 @@ class <%= @classname %>
private LexedToken attempt_lex_token() private LexedToken attempt_lex_token()
{ {
LexedToken lt = LexedToken(m_input_row, m_input_col, 0, _TOKEN_NONE); LexedToken lt = LexedToken(m_input_row, m_input_col, 0, _TOKEN_NONE);
struct LexedTokenState struct MatchInfo
{ {
size_t length; size_t length;
size_t delta_row; size_t delta_row;
@ -206,13 +206,13 @@ class <%= @classname %>
uint token; uint token;
uint code_id; uint code_id;
} }
LexedTokenState last_accepts_info; MatchInfo longest_match_info;
last_accepts_info.token = _TOKEN_NONE; longest_match_info.token = _TOKEN_NONE;
LexedTokenState attempt_info; MatchInfo attempt_match_info;
uint current_state; uint current_state;
for (;;) for (;;)
{ {
auto decoded = Decoder.decode_code_point(&m_input[m_input_position + attempt_info.length], m_input_length - m_input_position - attempt_info.length); auto decoded = Decoder.decode_code_point(&m_input[m_input_position + attempt_match_info.length], m_input_length - m_input_position - attempt_match_info.length);
if (decoded.code_point == Decoder.CODE_POINT_INVALID) if (decoded.code_point == Decoder.CODE_POINT_INVALID)
{ {
lt.token = _TOKEN_DECODE_ERROR; lt.token = _TOKEN_DECODE_ERROR;
@ -225,26 +225,26 @@ class <%= @classname %>
if (dest != cast(uint)-1) if (dest != cast(uint)-1)
{ {
lex_continue = true; lex_continue = true;
attempt_info.length += decoded.code_point_length; attempt_match_info.length += decoded.code_point_length;
if (decoded.code_point == '\n') if (decoded.code_point == '\n')
{ {
attempt_info.delta_row++; attempt_match_info.delta_row++;
attempt_info.delta_col = 0u; attempt_match_info.delta_col = 0u;
} }
else else
{ {
attempt_info.delta_col++; attempt_match_info.delta_col++;
} }
current_state = dest; current_state = dest;
if (states[current_state].accepts != _TOKEN_NONE) if (states[current_state].accepts != _TOKEN_NONE)
{ {
attempt_info.token = states[current_state].accepts; attempt_match_info.token = states[current_state].accepts;
attempt_info.code_id = states[current_state].code_id; attempt_match_info.code_id = states[current_state].code_id;
last_accepts_info = attempt_info; longest_match_info = attempt_match_info;
} }
} }
} }
else if (attempt_info.length == 0u) else if (attempt_match_info.length == 0u)
{ {
lt.token = _TOKEN_EOF; lt.token = _TOKEN_EOF;
break; break;
@ -252,10 +252,10 @@ class <%= @classname %>
if (!lex_continue) if (!lex_continue)
{ {
bool pattern_accepted = false; bool pattern_accepted = false;
uint token_to_accept = last_accepts_info.token; uint token_to_accept = longest_match_info.token;
if (last_accepts_info.code_id != 0xFFFF_FFFFu) if (longest_match_info.code_id != 0xFFFF_FFFFu)
{ {
uint user_code_token = user_code(last_accepts_info.code_id); uint user_code_token = user_code(longest_match_info.code_id);
/* A return of _TOKEN_NONE from user_code() means /* A return of _TOKEN_NONE from user_code() means
* that the user code did not explicitly return a * that the user code did not explicitly return a
* token. So only override the token to return if the * token. So only override the token to return if the
@ -269,22 +269,22 @@ class <%= @classname %>
if (pattern_accepted || (token_to_accept != _TOKEN_NONE)) if (pattern_accepted || (token_to_accept != _TOKEN_NONE))
{ {
/* Update the input position tracking. */ /* Update the input position tracking. */
m_input_position += last_accepts_info.length; m_input_position += longest_match_info.length;
m_input_row += last_accepts_info.delta_row; m_input_row += longest_match_info.delta_row;
if (last_accepts_info.delta_row != 0u) if (longest_match_info.delta_row != 0u)
{ {
m_input_col = last_accepts_info.delta_col; m_input_col = longest_match_info.delta_col;
} }
else else
{ {
m_input_col += last_accepts_info.delta_col; m_input_col += longest_match_info.delta_col;
} }
} }
if (token_to_accept != _TOKEN_NONE) if (token_to_accept != _TOKEN_NONE)
{ {
/* We have a token to accept. */ /* We have a token to accept. */
lt.token = last_accepts_info.token; lt.token = longest_match_info.token;
lt.length = last_accepts_info.length; lt.length = longest_match_info.length;
break; break;
} }
} }