josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Return a Token object from Lexer.user_code()

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-17 20:24:48 -04:00
parent ce80e00e60
commit b78827746a
2 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
assets/parser.d.erb
View File

@ -323,10 +323,10 @@ class <%= @classname %>
* @param match Matched text for this pattern.
* @param result Result lexer result in progress.
*
* @return Token ID to accept, or _TOKEN_COUNT if the user code does
* @return Token to accept, or invalid token if the user code does
* not explicitly return a token.
*/
private uint user_code(UserCodeID code_id, string match, Result * result)
private Token user_code(UserCodeID code_id, string match, Result * result)
{
switch (code_id)
{
@ -340,7 +340,7 @@ class <%= @classname %>
default: break;
}
return _TOKEN_COUNT;
return Token.invalid();
}
private Result attempt_lex_token()
@ -360,14 +360,14 @@ class <%= @classname %>
uint token_to_accept = match_info.accepting_state.token;
if (match_info.accepting_state.code_id.is_valid())
{
uint user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_info.length)], &result);
/* A return of _TOKEN_COUNT from user_code() means
* that the user code did not explicitly return a
* token. So only override the token to return if the
* user code does explicitly return a token. */
if (user_code_token != _TOKEN_COUNT)
Token user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_position..(m_input_position + match_info.length)], &result);
/* An invalid Token from user_code() means that the user
* code did not explicitly return a token. So only override
* the token to return if the user code does explicitly
* return a token. */
if (user_code_token.is_valid())
{
token_to_accept = user_code_token;
token_to_accept = user_code_token.token;
}
}

2
lib/propane/generator.rb
View File

@ -183,7 +183,7 @@ class Propane
# Expanded user code block.
def expand_code(code, parser, rule, pattern)
code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
"TOKEN_#{Token.code_name($1)}"
"Token(TOKEN_#{Token.code_name($1)})"
end
if parser
code = code.gsub(/\$\$/) do |match|