Add Lexer class

assets/parser.d.erb

@@ -69,67 +69,70 @@ class <%= classname %>
         }
     }
 
-    enum
+    class Lexer
     {
+        enum
+        {
 <% @grammar.tokens.each_with_index do |token, index| %>
 <%   if token.name %>
-        TOKEN_<%= token.c_name %> = <%= index %>,
+            TOKEN_<%= token.c_name %> = <%= index %>,
 <%   end %>
 <% end %>
-    }
+        }
 
-    static immutable string TokenNames[] = [
+        static immutable string TokenNames[] = [
 <% @grammar.tokens.each_with_index do |token, index| %>
 <%   if token.name %>
-        "<%= token.name %>",
+            "<%= token.name %>",
 <%   else %>
-        null,
+            null,
 <%   end %>
 <% end %>
-    ];
+        ];
 
-    private struct Transition
+        private struct Transition
-    {
+        {
-        uint first;
+            uint first;
-        uint last;
+            uint last;
-        uint destination;
+            uint destination;
-    }
+        }
 
-    private struct LexerState
+        private struct State
-    {
+        {
-        uint transition_table_index;
+            uint transition_table_index;
-        uint n_transitions;
+            uint n_transitions;
-        uint accepts;
+            uint accepts;
-    }
+        }
 
 <% transition_table, state_table = lexer_dfa.build_tables %>
-    private static const Transition transitions[] = [
+        private static const Transition transitions[] = [
 <% transition_table.each do |transition_table_entry| %>
-        Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
+            Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
 <% end %>
-    ];
+        ];
 
-    private static const LexerState lexer_states[] = [
+        private static const State states[] = [
 <% state_table.each do |state_table_entry| %>
-        LexerState(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:accepts] %>u),
+            State(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:accepts] %>u),
 <% end %>
-    ];
+        ];
 
-    static string[] lex(const(ubyte)[] input)
+        static string[] lex(const(ubyte)[] input)
-    {
-        string[] tokens;
-        string token = lex_token(input.ptr, input.length);
-        if (token !is null)
         {
-            tokens ~= token;
+            string[] tokens;
+            string token = lex_token(input.ptr, input.length);
+            if (token !is null)
+            {
+                tokens ~= token;
+            }
+            return tokens;
         }
-        return tokens;
-    }
 
-    private static string lex_token(const(ubyte) * input, size_t input_length)
+        private static string lex_token(const(ubyte) * input, size_t input_length)
-    {
+        {
-        size_t code_point_length;
+            size_t code_point_length;
-        uint code_point = Decoder.decode_code_point(input, input_length, &code_point_length);
+            uint code_point = Decoder.decode_code_point(input, input_length, &code_point_length);
-        return null;
+            return null;
+        }
     }
 }