josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Clean up Lexer.State a bit

Browse Source
This commit is contained in:
Josh Holtrop 2023-03-14 19:38:42 -04:00
parent e432f62b05
commit ffccc659aa
2 changed files with 21 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
assets/parser.d.erb
View File

@ -223,8 +223,14 @@ class <%= @classname %>
{
uint transition_table_index;
uint n_transitions;
uint token;
Token token;
UserCodeID code_id;
bool drop;
bool accepts() const
{
return drop || token.is_valid() || code_id.is_valid();
}
}
private struct Mode
@ -243,12 +249,17 @@ class <%= @classname %>
<% state_table.each do |state_table_entry| %>
State(<%= state_table_entry[:transition_table_index] %>u,
<%= state_table_entry[:n_transitions] %>u,
<%= state_table_entry[:token] %>u,
<% if state_table_entry[:code_id] %>
UserCodeID(<%= state_table_entry[:code_id] %>u)),
<% if state_table_entry[:token] %>
Token(<%= state_table_entry[:token] %>u),
<% else %>
UserCodeID.invalid()),
Token.invalid(),
<% end %>
<% if state_table_entry[:code_id] %>
UserCodeID(<%= state_table_entry[:code_id] %>u),
<% else %>
UserCodeID.invalid(),
<% end %>
<%= state_table_entry[:drop] %>),
<% end %>
];
@ -375,7 +386,7 @@ class <%= @classname %>
}
result.token = token_to_accept;
result.length = match_info.length;
if (result.token == _TOKEN_DROP)
if (match_info.accepting_state.drop)
{
result.type = Result.Type.DROP;
}
@ -418,8 +429,7 @@ class <%= @classname %>
attempt_match_info.delta_col++;
}
current_state = dest;
if ((states[current_state].token != _TOKEN_COUNT) ||
(states[current_state].code_id.is_valid()))
if (states[current_state].accepts())
{
attempt_match_info.accepting_state = &states[current_state];
*match_info = attempt_match_info;

10
lib/propane/lexer.rb
View File

@ -24,15 +24,10 @@ class Propane
}
states = mode_info[:dfa].enumerate
states.each do |state, id|
drop = state.accepts && state.accepts.drop?
token =
if state.accepts.nil?
@grammar.tokens.size
elsif state.accepts.drop?
TOKEN_DROP
elsif state.accepts.token
if state.accepts && state.accepts.token
state.accepts.token.id
else
@grammar.tokens.size
end
code_id =
if state.accepts && state.accepts.code_id
@ -41,6 +36,7 @@ class Propane
state_table << {
transition_table_index: transition_table.size,
n_transitions: state.transitions.size,
drop: drop,
token: token,
code_id: code_id,
}