josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Add Lexer class

Browse Source
This commit is contained in:
Josh Holtrop 2021-07-05 19:13:41 -04:00
parent 2f1cb47bea
commit 91d6ee25ea
1 changed files with 41 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
assets/parser.d.erb
View File

@ -69,67 +69,70 @@ class <%= classname %>
}
}
enum
class Lexer
{
enum
{
<% @grammar.tokens.each_with_index do |token, index| %>
<% if token.name %>
TOKEN_<%= token.c_name %> = <%= index %>,
TOKEN_<%= token.c_name %> = <%= index %>,
<% end %>
<% end %>
}
}
static immutable string TokenNames[] = [
static immutable string TokenNames[] = [
<% @grammar.tokens.each_with_index do |token, index| %>
<% if token.name %>
"<%= token.name %>",
"<%= token.name %>",
<% else %>
null,
null,
<% end %>
<% end %>
];
];
private struct Transition
{
uint first;
uint last;
uint destination;
}
private struct Transition
{
uint first;
uint last;
uint destination;
}
private struct LexerState
{
uint transition_table_index;
uint n_transitions;
uint accepts;
}
private struct State
{
uint transition_table_index;
uint n_transitions;
uint accepts;
}
<% transition_table, state_table = lexer_dfa.build_tables %>
private static const Transition transitions[] = [
private static const Transition transitions[] = [
<% transition_table.each do |transition_table_entry| %>
Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
<% end %>
];
];
private static const LexerState lexer_states[] = [
private static const State states[] = [
<% state_table.each do |state_table_entry| %>
LexerState(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:accepts] %>u),
State(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:accepts] %>u),
<% end %>
];
];
static string[] lex(const(ubyte)[] input)
{
string[] tokens;
string token = lex_token(input.ptr, input.length);
if (token !is null)
static string[] lex(const(ubyte)[] input)
{
tokens ~= token;
string[] tokens;
string token = lex_token(input.ptr, input.length);
if (token !is null)
{
tokens ~= token;
}
return tokens;
}
return tokens;
}
private static string lex_token(const(ubyte) * input, size_t input_length)
{
size_t code_point_length;
uint code_point = Decoder.decode_code_point(input, input_length, &code_point_length);
return null;
private static string lex_token(const(ubyte) * input, size_t input_length)
{
size_t code_point_length;
uint code_point = Decoder.decode_code_point(input, input_length, &code_point_length);
return null;
}
}
}