josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Return a Token object from Lexer.user_code()

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-17 20:24:48 -04:00
parent ce80e00e60
commit b78827746a
2 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
assets/parser.d.erb
View File

@ -323,10 +323,10 @@ class <%= @classname %>
* @param match Matched text for this pattern. * @param match Matched text for this pattern.
* @param result Result lexer result in progress. * @param result Result lexer result in progress.
* *
* @return Token ID to accept, or _TOKEN_COUNT if the user code does * @return Token to accept, or invalid token if the user code does
* not explicitly return a token. * not explicitly return a token.
*/ */
private uint user_code(UserCodeID code_id, string match, Result * result) private Token user_code(UserCodeID code_id, string match, Result * result)
{ {
switch (code_id) switch (code_id)
{ {
@ -340,7 +340,7 @@ class <%= @classname %>
default: break; default: break;
} }
return _TOKEN_COUNT; return Token.invalid();
} }
private Result attempt_lex_token() private Result attempt_lex_token()
@ -360,14 +360,14 @@ class <%= @classname %>
uint token_to_accept = match_info.accepting_state.token; uint token_to_accept = match_info.accepting_state.token;
if (match_info.accepting_state.code_id.is_valid()) if (match_info.accepting_state.code_id.is_valid())
{ {
uint user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_info.length)], &result); Token user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_info.length)], &result);
/* A return of _TOKEN_COUNT from user_code() means /* An invalid Token from user_code() means that the user
* that the user code did not explicitly return a * code did not explicitly return a token. So only override
* token. So only override the token to return if the * the token to return if the user code does explicitly
* user code does explicitly return a token. */ * return a token. */
if (user_code_token != _TOKEN_COUNT) if (user_code_token.is_valid())
{ {
token_to_accept = user_code_token; token_to_accept = user_code_token.token;
} }
} }

2
lib/propane/generator.rb
View File

@ -183,7 +183,7 @@ class Propane
# Expanded user code block. # Expanded user code block.
def expand_code(code, parser, rule, pattern) def expand_code(code, parser, rule, pattern)
code = code.gsub(/\$token\(([$\w]+)\)/) do |match| code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
"TOKEN_#{Token.code_name($1)}" "Token(TOKEN_#{Token.code_name($1)})"
end end
if parser if parser
code = code.gsub(/\$\$/) do |match| code = code.gsub(/\$\$/) do |match|