josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Clean up Lexer.State a bit

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-14 19:38:42 -04:00
parent e432f62b05
commit ffccc659aa
2 changed files with 21 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
assets/parser.d.erb
View File

@ -223,8 +223,14 @@ class <%= @classname %>
{ {
uint transition_table_index; uint transition_table_index;
uint n_transitions; uint n_transitions;
uint token; Token token;
UserCodeID code_id; UserCodeID code_id;
bool drop;
bool accepts() const
{
return drop || token.is_valid() || code_id.is_valid();
}
} }
private struct Mode private struct Mode
@ -243,12 +249,17 @@ class <%= @classname %>
<% state_table.each do |state_table_entry| %> <% state_table.each do |state_table_entry| %>
State(<%= state_table_entry[:transition_table_index] %>u, State(<%= state_table_entry[:transition_table_index] %>u,
<%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:n_transitions] %>u,
<%= state_table_entry[:token] %>u, <% if state_table_entry[:token] %>
<% if state_table_entry[:code_id] %> Token(<%= state_table_entry[:token] %>u),
UserCodeID(<%= state_table_entry[:code_id] %>u)),
<% else %> <% else %>
UserCodeID.invalid()), Token.invalid(),
<% end %> <% end %>
<% if state_table_entry[:code_id] %>
UserCodeID(<%= state_table_entry[:code_id] %>u),
<% else %>
UserCodeID.invalid(),
<% end %>
<%= state_table_entry[:drop] %>),
<% end %> <% end %>
]; ];
@ -375,7 +386,7 @@ class <%= @classname %>
} }
result.token = token_to_accept; result.token = token_to_accept;
result.length = match_info.length; result.length = match_info.length;
if (result.token == _TOKEN_DROP) if (match_info.accepting_state.drop)
{ {
result.type = Result.Type.DROP; result.type = Result.Type.DROP;
} }
@ -418,8 +429,7 @@ class <%= @classname %>
attempt_match_info.delta_col++; attempt_match_info.delta_col++;
} }
current_state = dest; current_state = dest;
if ((states[current_state].token != _TOKEN_COUNT) || if (states[current_state].accepts())
(states[current_state].code_id.is_valid()))
{ {
attempt_match_info.accepting_state = &states[current_state]; attempt_match_info.accepting_state = &states[current_state];
*match_info = attempt_match_info; *match_info = attempt_match_info;

10
lib/propane/lexer.rb
View File

@ -24,15 +24,10 @@ class Propane
} }
states = mode_info[:dfa].enumerate states = mode_info[:dfa].enumerate
states.each do |state, id| states.each do |state, id|
drop = state.accepts && state.accepts.drop?
token = token =
if state.accepts.nil? if state.accepts && state.accepts.token
@grammar.tokens.size
elsif state.accepts.drop?
TOKEN_DROP
elsif state.accepts.token
state.accepts.token.id state.accepts.token.id
else
@grammar.tokens.size
end end
code_id = code_id =
if state.accepts && state.accepts.code_id if state.accepts && state.accepts.code_id
@ -41,6 +36,7 @@ class Propane
state_table << { state_table << {
transition_table_index: transition_table.size, transition_table_index: transition_table.size,
n_transitions: state.transitions.size, n_transitions: state.transitions.size,
drop: drop,
token: token, token: token,
code_id: code_id, code_id: code_id,
} }