diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 72df299..09babcc 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -165,9 +165,8 @@ class <%= @classname %> static class Lexer { -<% transition_table, state_table, mode_table = @lexer.build_tables %> - alias StateID = <%= get_type_for(state_table.size) %>; - enum StateID INVALID_STATE_ID = <%= state_table.size %>u; + alias StateID = <%= get_type_for(@lexer.state_table.size) %>; + enum StateID INVALID_STATE_ID = <%= @lexer.state_table.size %>u; private struct Transition { @@ -207,8 +206,8 @@ class <%= @classname %> private struct State { - <%= get_type_for(transition_table.size - 1) %> transition_table_index; - <%= get_type_for(state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions; + <%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index; + <%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions; Token token; UserCodeID code_id; bool accepts; @@ -220,7 +219,7 @@ class <%= @classname %> } private static immutable Transition[] transitions = [ -<% transition_table.each do |transition_table_entry| %> +<% @lexer.transition_table.each do |transition_table_entry| %> Transition(<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u), @@ -228,7 +227,7 @@ class <%= @classname %> ]; private static immutable State[] states = [ -<% state_table.each do |state_table_entry| %> +<% @lexer.state_table.each do |state_table_entry| %> State(<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <% if state_table_entry[:token] %> @@ -246,7 +245,7 @@ class <%= @classname %> ]; private static immutable Mode[] modes = [ -<% mode_table.each do |mode_table_entry| %> +<% @lexer.mode_table.each do |mode_table_entry| %> Mode(<%= mode_table_entry[:state_table_offset] %>), <% end %> ]; @@ -527,21 +526,20 @@ class <%= @classname %> } } -<% state_table, shift_table, reduce_table = @parser.build_tables %> private static immutable Shift[] shifts = [ -<% shift_table.each do |shift| %> +<% @parser.shift_table.each do |shift| %> Shift(<%= shift[:token_id] %>u, <%= shift[:state_id] %>u), <% end %> ]; private static immutable Reduce[] reduces = [ -<% reduce_table.each do |reduce| %> +<% @parser.reduce_table.each do |reduce| %> Reduce(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u), <% end %> ]; private static immutable State[] states = [ -<% state_table.each do |state| %> +<% @parser.state_table.each do |state| %> State(<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u), <% end %> ]; diff --git a/lib/propane/lexer.rb b/lib/propane/lexer.rb index cd6d6cf..3a3da8b 100644 --- a/lib/propane/lexer.rb +++ b/lib/propane/lexer.rb @@ -1,48 +1,13 @@ class Propane class Lexer + attr_reader :state_table + attr_reader :transition_table + attr_reader :mode_table + def initialize(grammar) @grammar = grammar - end - - def build_tables - @modes = @grammar.patterns.group_by do |pattern| - pattern.mode - end.transform_values do |patterns| - {dfa: DFA.new(patterns)} - end - @modes.each_with_index do |(mode_name, mode_info), index| - mode_info[:id] = index - end - transition_table = [] - state_table = [] - mode_table = [] - @modes.each do |mode_name, mode_info| - state_table_offset = state_table.size - mode_table << { - state_table_offset: state_table_offset, - } - states = mode_info[:dfa].enumerate - states.each do |state, id| - token = state.accepts && state.accepts.token && state.accepts.token.id - code_id = state.accepts && state.accepts.code_id && state.accepts.code_id - state_table << { - transition_table_index: transition_table.size, - n_transitions: state.transitions.size, - accepts: !!state.accepts, - token: token, - code_id: code_id, - } - state.transitions.each do |transition| - transition_table << { - first: transition.code_point_range.first, - last: transition.code_point_range.last, - destination: states[transition.destination] + state_table_offset, - } - end - end - end - [transition_table, state_table, mode_table] + build_tables! end # Get ID for a mode. @@ -58,5 +23,46 @@ class Propane end end + private + + def build_tables! + @modes = @grammar.patterns.group_by do |pattern| + pattern.mode + end.transform_values do |patterns| + {dfa: DFA.new(patterns)} + end + @modes.each_with_index do |(mode_name, mode_info), index| + mode_info[:id] = index + end + @state_table = [] + @transition_table = [] + @mode_table = [] + @modes.each do |mode_name, mode_info| + state_table_offset = @state_table.size + @mode_table << { + state_table_offset: state_table_offset, + } + states = mode_info[:dfa].enumerate + states.each do |state, id| + token = state.accepts && state.accepts.token && state.accepts.token.id + code_id = state.accepts && state.accepts.code_id && state.accepts.code_id + @state_table << { + transition_table_index: @transition_table.size, + n_transitions: state.transitions.size, + accepts: !!state.accepts, + token: token, + code_id: code_id, + } + state.transitions.each do |transition| + @transition_table << { + first: transition.code_point_range.first, + last: transition.code_point_range.last, + destination: states[transition.destination] + state_table_offset, + } + end + end + end + end + end end diff --git a/lib/propane/parser.rb b/lib/propane/parser.rb index d21b13a..af5f1f5 100644 --- a/lib/propane/parser.rb +++ b/lib/propane/parser.rb @@ -2,6 +2,10 @@ class Propane class Parser + attr_reader :state_table + attr_reader :shift_table + attr_reader :reduce_table + def initialize(grammar, rule_sets, log) @grammar = grammar @rule_sets = rule_sets @@ -33,12 +37,15 @@ class Propane build_reduce_actions! write_log! + build_tables! end - def build_tables - shift_table = [] - state_table = [] - reduce_table = [] + private + + def build_tables! + @state_table = [] + @shift_table = [] + @reduce_table = [] @item_sets.each do |item_set| shift_entries = item_set.following_symbols.map do |following_symbol| state_id = @@ -65,20 +72,17 @@ class Propane else [] end - state_table << { - shift_index: shift_table.size, + @state_table << { + shift_index: @shift_table.size, n_shifts: shift_entries.size, - reduce_index: reduce_table.size, + reduce_index: @reduce_table.size, n_reduces: reduce_entries.size, } - shift_table += shift_entries - reduce_table += reduce_entries + @shift_table += shift_entries + @reduce_table += reduce_entries end - [state_table, shift_table, reduce_table] end - private - def process_item_set(item_set) item_set.following_symbols.each do |following_symbol| unless following_symbol.name == "$EOF"