josh/propane
1
0 Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Add length field to LexedToken

Browse Source
This commit is contained in:
josh committed 2021-07-06 11:59:35 -04:00
1 parent d9e4f64d2e
commit 3158e51059
2 files changed
+11 -9
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

spec/test_d_lexer.d
+8 -8
View File
@@ -52,12 +52,12 @@ unittest
alias LT = Testparser.Lexer.LexedToken;
string input = "5 + 4 * \n677 + 567";
Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
assert(lexer.lex_token() == LT(0, 0, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 2, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(0, 4, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 6, Testparser.TOKEN_TIMES));
assert(lexer.lex_token() == LT(1, 0, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 4, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(1, 6, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 9, Testparser.TOKEN_EOF));
assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_TIMES));
assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_PLUS));
assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_INT));
assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_EOF));
}