josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Build Lexer and Parser tables upon construction

Browse Source
This commit is contained in:
Josh Holtrop 2023-07-08 16:52:58 -04:00
parent ffd02c16d1
commit 36ad6e0d1b
3 changed files with 72 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
assets/parser.d.erb
View File

@ -165,9 +165,8 @@ class <%= @classname %>
static class Lexer
{
<% transition_table, state_table, mode_table = @lexer.build_tables %>
alias StateID = <%= get_type_for(state_table.size) %>;
enum StateID INVALID_STATE_ID = <%= state_table.size %>u;
alias StateID = <%= get_type_for(@lexer.state_table.size) %>;
enum StateID INVALID_STATE_ID = <%= @lexer.state_table.size %>u;
private struct Transition
{
@ -207,8 +206,8 @@ class <%= @classname %>
private struct State
{
<%= get_type_for(transition_table.size - 1) %> transition_table_index;
<%= get_type_for(state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
<%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
<%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
Token token;
UserCodeID code_id;
bool accepts;
@ -220,7 +219,7 @@ class <%= @classname %>
}
private static immutable Transition[] transitions = [
<% transition_table.each do |transition_table_entry| %>
<% @lexer.transition_table.each do |transition_table_entry| %>
Transition(<%= transition_table_entry[:first] %>u,
<%= transition_table_entry[:last] %>u,
<%= transition_table_entry[:destination] %>u),
@ -228,7 +227,7 @@ class <%= @classname %>
];
private static immutable State[] states = [
<% state_table.each do |state_table_entry| %>
<% @lexer.state_table.each do |state_table_entry| %>
State(<%= state_table_entry[:transition_table_index] %>u,
<%= state_table_entry[:n_transitions] %>u,
<% if state_table_entry[:token] %>
@ -246,7 +245,7 @@ class <%= @classname %>
];
private static immutable Mode[] modes = [
<% mode_table.each do |mode_table_entry| %>
<% @lexer.mode_table.each do |mode_table_entry| %>
Mode(<%= mode_table_entry[:state_table_offset] %>),
<% end %>
];
@ -527,21 +526,20 @@ class <%= @classname %>
}
}
<% state_table, shift_table, reduce_table = @parser.build_tables %>
private static immutable Shift[] shifts = [
<% shift_table.each do |shift| %>
<% @parser.shift_table.each do |shift| %>
Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u),
<% end %>
];
private static immutable Reduce[] reduces = [
<% reduce_table.each do |reduce| %>
<% @parser.reduce_table.each do |reduce| %>
Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
<% end %>
];
private static immutable State[] states = [
<% state_table.each do |state| %>
<% @parser.state_table.each do |state| %>
State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u),
<% end %>
];

86
lib/propane/lexer.rb
View File

@ -1,48 +1,13 @@
class Propane
class Lexer
attr_reader :state_table
attr_reader :transition_table
attr_reader :mode_table
def initialize(grammar)
@grammar = grammar
end
def build_tables
@modes = @grammar.patterns.group_by do |pattern|
pattern.mode
end.transform_values do |patterns|
{dfa: DFA.new(patterns)}
end
@modes.each_with_index do |(mode_name, mode_info), index|
mode_info[:id] = index
end
transition_table = []
state_table = []
mode_table = []
@modes.each do |mode_name, mode_info|
state_table_offset = state_table.size
mode_table << {
state_table_offset: state_table_offset,
}
states = mode_info[:dfa].enumerate
states.each do |state, id|
token = state.accepts && state.accepts.token && state.accepts.token.id
code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
state_table << {
transition_table_index: transition_table.size,
n_transitions: state.transitions.size,
accepts: !!state.accepts,
token: token,
code_id: code_id,
}
state.transitions.each do |transition|
transition_table << {
first: transition.code_point_range.first,
last: transition.code_point_range.last,
destination: states[transition.destination] + state_table_offset,
}
end
end
end
[transition_table, state_table, mode_table]
build_tables!
end
# Get ID for a mode.
@ -58,5 +23,46 @@ class Propane
end
end
private
def build_tables!
@modes = @grammar.patterns.group_by do |pattern|
pattern.mode
end.transform_values do |patterns|
{dfa: DFA.new(patterns)}
end
@modes.each_with_index do |(mode_name, mode_info), index|
mode_info[:id] = index
end
@state_table = []
@transition_table = []
@mode_table = []
@modes.each do |mode_name, mode_info|
state_table_offset = @state_table.size
@mode_table << {
state_table_offset: state_table_offset,
}
states = mode_info[:dfa].enumerate
states.each do |state, id|
token = state.accepts && state.accepts.token && state.accepts.token.id
code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
@state_table << {
transition_table_index: @transition_table.size,
n_transitions: state.transitions.size,
accepts: !!state.accepts,
token: token,
code_id: code_id,
}
state.transitions.each do |transition|
@transition_table << {
first: transition.code_point_range.first,
last: transition.code_point_range.last,
destination: states[transition.destination] + state_table_offset,
}
end
end
end
end
end
end

28
lib/propane/parser.rb
View File

@ -2,6 +2,10 @@ class Propane
class Parser
attr_reader :state_table
attr_reader :shift_table
attr_reader :reduce_table
def initialize(grammar, rule_sets, log)
@grammar = grammar
@rule_sets = rule_sets
@ -33,12 +37,15 @@ class Propane
build_reduce_actions!
write_log!
build_tables!
end
def build_tables
shift_table = []
state_table = []
reduce_table = []
private
def build_tables!
@state_table = []
@shift_table = []
@reduce_table = []
@item_sets.each do |item_set|
shift_entries = item_set.following_symbols.map do |following_symbol|
state_id =
@ -65,20 +72,17 @@ class Propane
else
[]
end
state_table << {
shift_index: shift_table.size,
@state_table << {
shift_index: @shift_table.size,
n_shifts: shift_entries.size,
reduce_index: reduce_table.size,
reduce_index: @reduce_table.size,
n_reduces: reduce_entries.size,
}
shift_table += shift_entries
reduce_table += reduce_entries
@shift_table += shift_entries
@reduce_table += reduce_entries
end
[state_table, shift_table, reduce_table]
end
private
def process_item_set(item_set)
item_set.following_symbols.each do |following_symbol|
unless following_symbol.name == "$EOF"