139 lines
3.8 KiB
Plaintext
139 lines
3.8 KiB
Plaintext
<% if @grammar.modulename %>
|
|
module <%= @grammar.modulename %>;
|
|
|
|
<% end %>
|
|
class <%= classname %>
|
|
{
|
|
class Decoder
|
|
{
|
|
enum
|
|
{
|
|
CODE_POINT_INVALID = 0xFFFFFFFE,
|
|
CODE_POINT_EOF = 0xFFFFFFFF,
|
|
}
|
|
|
|
static uint decode_code_point(const(ubyte) * input, size_t input_length, size_t * code_point_length)
|
|
{
|
|
if (input_length == 0u)
|
|
{
|
|
return CODE_POINT_EOF;
|
|
}
|
|
ubyte c = *input;
|
|
uint result;
|
|
if ((c & 0x80u) == 0u)
|
|
{
|
|
result = c;
|
|
*code_point_length = 1u;
|
|
}
|
|
else
|
|
{
|
|
ubyte following_bytes;
|
|
if ((c & 0xE0u) == 0xC0u)
|
|
{
|
|
result = c & 0x1Fu;
|
|
following_bytes = 1u;
|
|
}
|
|
else if ((c & 0xF0u) == 0xE0u)
|
|
{
|
|
result = c & 0x0Fu;
|
|
following_bytes = 2u;
|
|
}
|
|
else if ((c & 0xF8u) == 0xF0u)
|
|
{
|
|
result = c & 0x07u;
|
|
following_bytes = 3u;
|
|
}
|
|
else if ((c & 0xFCu) == 0xF8u)
|
|
{
|
|
result = c & 0x03u;
|
|
following_bytes = 4u;
|
|
}
|
|
else if ((c & 0xFEu) == 0xFCu)
|
|
{
|
|
result = c & 0x01u;
|
|
following_bytes = 5u;
|
|
}
|
|
if (input_length <= following_bytes)
|
|
{
|
|
return CODE_POINT_INVALID;
|
|
}
|
|
*code_point_length = following_bytes + 1u;
|
|
while (following_bytes-- > 0u)
|
|
{
|
|
input++;
|
|
result <<= 6u;
|
|
result |= *input & 0x3Fu;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
class Lexer
|
|
{
|
|
enum
|
|
{
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
<% if token.name %>
|
|
TOKEN_<%= token.c_name %> = <%= index %>,
|
|
<% end %>
|
|
<% end %>
|
|
}
|
|
|
|
static immutable string TokenNames[] = [
|
|
<% @grammar.tokens.each_with_index do |token, index| %>
|
|
<% if token.name %>
|
|
"<%= token.name %>",
|
|
<% else %>
|
|
null,
|
|
<% end %>
|
|
<% end %>
|
|
];
|
|
|
|
private struct Transition
|
|
{
|
|
uint first;
|
|
uint last;
|
|
uint destination;
|
|
}
|
|
|
|
private struct State
|
|
{
|
|
uint transition_table_index;
|
|
uint n_transitions;
|
|
uint accepts;
|
|
}
|
|
|
|
<% transition_table, state_table = lexer_dfa.build_tables %>
|
|
private static const Transition transitions[] = [
|
|
<% transition_table.each do |transition_table_entry| %>
|
|
Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
|
|
<% end %>
|
|
];
|
|
|
|
private static const State states[] = [
|
|
<% state_table.each do |state_table_entry| %>
|
|
State(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:accepts] %>u),
|
|
<% end %>
|
|
];
|
|
|
|
static string[] lex(const(ubyte)[] input)
|
|
{
|
|
string[] tokens;
|
|
string token = lex_token(input.ptr, input.length);
|
|
if (token !is null)
|
|
{
|
|
tokens ~= token;
|
|
}
|
|
return tokens;
|
|
}
|
|
|
|
private static string lex_token(const(ubyte) * input, size_t input_length)
|
|
{
|
|
size_t code_point_length;
|
|
uint code_point = Decoder.decode_code_point(input, input_length, &code_point_length);
|
|
return null;
|
|
}
|
|
}
|
|
}
|