From d9e4f64d2e9102110fcd7086324e71056560e952 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Tue, 6 Jul 2021 11:55:44 -0400 Subject: [PATCH] Fix returning TOKEN_EOF when lexing at EOF --- assets/parser.d.erb | 5 +++++ spec/test_d_lexer.d | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 0b6c3cc..ca6ef5c 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -206,6 +206,11 @@ class <%= classname %> } } } + else if (attempt_info.length == 0u) + { + lt.token = TOKEN_EOF; + break; + } if (!lex_continue) { if (last_accepts_info.token != TOKEN_NONE) diff --git a/spec/test_d_lexer.d b/spec/test_d_lexer.d index a9d69d0..d9115da 100644 --- a/spec/test_d_lexer.d +++ b/spec/test_d_lexer.d @@ -53,4 +53,11 @@ unittest string input = "5 + 4 * \n677 + 567"; Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length); assert(lexer.lex_token() == LT(0, 0, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(0, 2, Testparser.TOKEN_PLUS)); + assert(lexer.lex_token() == LT(0, 4, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(0, 6, Testparser.TOKEN_TIMES)); + assert(lexer.lex_token() == LT(1, 0, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(1, 4, Testparser.TOKEN_PLUS)); + assert(lexer.lex_token() == LT(1, 6, Testparser.TOKEN_INT)); + assert(lexer.lex_token() == LT(1, 9, Testparser.TOKEN_EOF)); }