diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 0b6c3cc..ca6ef5c 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -206,6 +206,11 @@ class <%= classname %> } } } + else if (attempt_info.length == 0u) + { + lt.token = TOKEN_EOF; + break; + } if (!lex_continue) { if (last_accepts_info.token != TOKEN_NONE) diff --git a/spec/test_d_lexer.d b/spec/test_d_lexer.d index a9d69d0..d9115da 100644 --- a/spec/test_d_lexer.d +++ b/spec/test_d_lexer.d @@ -53,4 +53,11 @@ unittest string input = "5 + 4 * \n677 + 567"; Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length); assert(lexer.lex_token() == LT(0, 0, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(0, 2, Testparser.TOKEN_PLUS)); + assert(lexer.lex_token() == LT(0, 4, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(0, 6, Testparser.TOKEN_TIMES)); + assert(lexer.lex_token() == LT(1, 0, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(1, 4, Testparser.TOKEN_PLUS)); + assert(lexer.lex_token() == LT(1, 6, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(1, 9, Testparser.TOKEN_EOF)); }