josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Compare commits

base: josh:11348ca351884573fa44b444c6cd28afe678039f
Branches Tags
josh:master
josh:wip
josh:c-backend
josh:sumtype
josh:v1.5.1
josh:v1.5.0
josh:v1.4.0
josh:v1.3.0
josh:v1.2.0
josh:v1.1.0
josh:v1.0.0
...
compare: josh:d466189982a1705a4b134a023343c28332bf828b
Branches Tags
josh:master
josh:wip
josh:c-backend
josh:sumtype
josh:v1.5.1
josh:v1.5.0
josh:v1.4.0
josh:v1.3.0
josh:v1.2.0
josh:v1.1.0
josh:v1.0.0

2 Commits
11348ca351 ... d466189982

Author SHA1 Message Date
Josh Holtrop
d466189982 Convert Decoder.decode_code_point return type to a SumType 2023-06-29 19:21:30 -04:00
Josh Holtrop
9895733a05 Switch unit tests from gdc to ldc2 2023-06-29 19:16:36 -04:00
2 changed files with 74 additions and 109 deletions
Show Stats Expand all files Collapse all files

180
assets/parser.d.erb
View File

@ -2,8 +2,8 @@
module <%= @grammar.modulename %>; module <%= @grammar.modulename %>;
<% end %> <% end %>
import std.stdio; import std.stdio;
import std.sumtype;
<% @grammar.code_blocks.each do |code| %> <% @grammar.code_blocks.each do |code| %>
<%= code %> <%= code %>
@ -56,7 +56,7 @@ class <%= @classname %>
alias CodePoint = uint; alias CodePoint = uint;
static immutable string token_names[] = [ static immutable string[] token_names = [
<% @grammar.tokens.each_with_index do |token, index| %> <% @grammar.tokens.each_with_index do |token, index| %>
"<%= token.name %>", "<%= token.name %>",
<% end %> <% end %>
@ -71,54 +71,23 @@ class <%= @classname %>
static class Decoder static class Decoder
{ {
struct Result struct Success
{ {
enum Type
{
SUCCESS,
EOF,
DECODE_ERROR,
}
private Type type;
CodePoint code_point; CodePoint code_point;
uint code_point_length; uint code_point_length;
static Result success(CodePoint code_point, uint code_point_length)
{
return Result(Type.SUCCESS, code_point, code_point_length);
}
bool is_success()
{
return type == Type.SUCCESS;
}
static Result eof()
{
return Result(Type.EOF);
}
bool is_eof()
{
return type == Type.EOF;
}
static Result decode_error()
{
return Result(Type.DECODE_ERROR);
}
bool is_decode_error()
{
return type == Type.DECODE_ERROR;
}
} }
struct EOF {}
struct DecodeError {}
static Result decode_code_point(string input) static SumType!(
Success,
EOF,
DecodeError)
decode_code_point(string input)
{ {
if (input.length == 0u) if (input.length == 0u)
{ {
return Result.eof(); return cast(ReturnType!decode_code_point)EOF();
} }
char c = input[0]; char c = input[0];
CodePoint code_point; CodePoint code_point;
@ -158,11 +127,11 @@ class <%= @classname %>
} }
else else
{ {
return Result.decode_error(); return cast(ReturnType!decode_code_point)DecodeError();
} }
if (input.length <= following_bytes) if (input.length <= following_bytes)
{ {
return Result.decode_error(); return cast(ReturnType!decode_code_point)DecodeError();
} }
code_point_length = following_bytes + 1u; code_point_length = following_bytes + 1u;
for (size_t i = 0u; i < following_bytes; i++) for (size_t i = 0u; i < following_bytes; i++)
@ -170,12 +139,12 @@ class <%= @classname %>
char b = input[i + 1u]; char b = input[i + 1u];
if ((b & 0xC0u) != 0x80u) if ((b & 0xC0u) != 0x80u)
{ {
return Result.decode_error(); return cast(ReturnType!decode_code_point)DecodeError();
} }
code_point = (code_point << 6u) | (b & 0x3Fu); code_point = (code_point << 6u) | (b & 0x3Fu);
} }
} }
return Result.success(code_point, code_point_length); return cast(ReturnType!decode_code_point)Success(code_point, code_point_length);
} }
} }
@ -237,13 +206,13 @@ class <%= @classname %>
} }
<% transition_table, state_table, mode_table = @lexer.build_tables %> <% transition_table, state_table, mode_table = @lexer.build_tables %>
private static immutable Transition transitions[] = [ private static immutable Transition[] transitions = [
<% transition_table.each do |transition_table_entry| %> <% transition_table.each do |transition_table_entry| %>
Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u), Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u),
<% end %> <% end %>
]; ];
private static immutable State states[] = [ private static immutable State[] states = [
<% state_table.each do |state_table_entry| %> <% state_table.each do |state_table_entry| %>
State(<%= state_table_entry[:transition_table_index] %>u, State(<%= state_table_entry[:transition_table_index] %>u,
<%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:n_transitions] %>u,
@ -261,7 +230,7 @@ class <%= @classname %>
<% end %> <% end %>
]; ];
private static immutable Mode modes[] = [ private static immutable Mode[] modes = [
<% mode_table.each do |mode_table_entry| %> <% mode_table.each do |mode_table_entry| %>
Mode(<%= mode_table_entry[:state_table_offset] %>), Mode(<%= mode_table_entry[:state_table_offset] %>),
<% end %> <% end %>
@ -498,63 +467,58 @@ class <%= @classname %>
uint current_state = modes[m_mode].state_table_offset; uint current_state = modes[m_mode].state_table_offset;
for (;;) for (;;)
{ {
auto decoded = Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]);
if (decoded.is_decode_error())
{
return FindLongestMatchResult.decode_error();
}
bool lex_continue = false; bool lex_continue = false;
if (decoded.is_eof()) Decoder.decode_code_point(m_input[(m_input_position + attempt_match.length)..(m_input.length)]).match!(
{ (Decoder.DecodeError de) {return FindLongestMatchResult.decode_error();},
/* We hit EOF. */ (Decoder.EOF e) {
if (longest_match.length > 0) /* We hit EOF. */
{ if (longest_match.length > 0)
/* We have a match, so use it. */ {
return longest_match; /* We have a match, so use it. */
} return longest_match;
else if (attempt_match.length != 0) }
{ else if (attempt_match.length != 0)
/* There is a partial match - error! */ {
return FindLongestMatchResult.unexpected_input(attempt_match.length); /* There is a partial match - error! */
} return FindLongestMatchResult.unexpected_input(attempt_match.length);
else }
{ else
/* Valid EOF return. */ {
return FindLongestMatchResult.eof(); /* Valid EOF return. */
} return FindLongestMatchResult.eof();
} }
else },
{ (Decoder.Success decoded) {
auto transition_result = transition(current_state, decoded.code_point); auto transition_result = transition(current_state, decoded.code_point);
if (transition_result.found()) if (transition_result.found())
{ {
lex_continue = true; lex_continue = true;
attempt_match.length += decoded.code_point_length; attempt_match.length += decoded.code_point_length;
if (decoded.code_point == '\n') if (decoded.code_point == '\n')
{ {
attempt_match.delta_row++; attempt_match.delta_row++;
attempt_match.delta_col = 0u; attempt_match.delta_col = 0u;
} }
else else
{ {
attempt_match.delta_col++; attempt_match.delta_col++;
} }
current_state = transition_result.destination(); current_state = transition_result.destination();
if (states[current_state].accepts()) if (states[current_state].accepts())
{ {
attempt_match.accepting_state = &states[current_state]; attempt_match.accepting_state = &states[current_state];
longest_match = attempt_match; longest_match = attempt_match;
} }
} }
else if (longest_match.length > 0) else if (longest_match.length > 0)
{ {
return longest_match; return longest_match;
} }
else else
{ {
return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length); return FindLongestMatchResult.unexpected_input(attempt_match.length + decoded.code_point_length);
} }
} });
} }
} }
@ -642,19 +606,19 @@ class <%= @classname %>
} }
<% state_table, shift_table, reduce_table = @parser.build_tables %> <% state_table, shift_table, reduce_table = @parser.build_tables %>
private static immutable Shift shifts[] = [ private static immutable Shift[] shifts = [
<% shift_table.each do |shift| %> <% shift_table.each do |shift| %>
Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u), Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u),
<% end %> <% end %>
]; ];
private static immutable Reduce reduces[] = [ private static immutable Reduce[] reduces = [
<% reduce_table.each do |reduce| %> <% reduce_table.each do |reduce| %>
Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u), Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
<% end %> <% end %>
]; ];
private static immutable State states[] = [ private static immutable State[] states = [
<% state_table.each do |state| %> <% state_table.each do |state| %>
State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u), State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
<% end %> <% end %>

3
spec/propane_spec.rb
View File

@ -20,7 +20,7 @@ describe Propane do
end end
def compile(*test_files) def compile(*test_files)
result = system(*%w[gdc -funittest -o spec/run/testparser spec/run/testparser.d -Ispec], *test_files) result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
expect(result).to be_truthy expect(result).to be_truthy
end end
@ -28,6 +28,7 @@ describe Propane do
stdout, stderr, status = Open3.capture3("spec/run/testparser") stdout, stderr, status = Open3.capture3("spec/run/testparser")
File.binwrite("spec/run/.stderr", stderr) File.binwrite("spec/run/.stderr", stderr)
File.binwrite("spec/run/.stdout", stdout) File.binwrite("spec/run/.stdout", stdout)
stderr.sub!(/^.*modules passed unittests\n/, "")
Results.new(stdout, stderr, status) Results.new(stdout, stderr, status)
end end