josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Fix returning TOKEN_EOF when lexing at EOF

Browse Source
This commit is contained in:
Josh Holtrop 2021-07-06 11:55:44 -04:00
parent ec2dcf9a72
commit d9e4f64d2e
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
assets/parser.d.erb
View File

@ -206,6 +206,11 @@ class <%= classname %>
}
}
}
else if (attempt_info.length == 0u)
{
lt.token = TOKEN_EOF;
break;
}
if (!lex_continue)
{
if (last_accepts_info.token != TOKEN_NONE)

7
spec/test_d_lexer.d
View File

@ -53,4 +53,11 @@ unittest
string input = "5 + 4 * \n677 + 567";
Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
assert(lexer.lex_token() == LT(0, 0, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 2, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(0, 4, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 6, Testparser.TOKEN_TIMES));
assert(lexer.lex_token() == LT(1, 0, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 4, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(1, 6, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 9, Testparser.TOKEN_EOF));
}