josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Store tokens in Hash by name

Browse Source
This commit is contained in:
Josh Holtrop 2021-09-27 21:40:12 -04:00
parent 280b749e38
commit f3ed678fe1
3 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
assets/parser.d.erb
View File

@ -6,7 +6,7 @@ class <%= classname %>
{ {
enum enum
{ {
<% @tokens.each_with_index do |token, index| %> <% @tokens.each_with_index do |(name, token), index| %>
<% if token.name %> <% if token.name %>
TOKEN_<%= token.c_name %> = <%= index %>, TOKEN_<%= token.c_name %> = <%= index %>,
<% end %> <% end %>
@ -18,7 +18,7 @@ class <%= classname %>
} }
static immutable string TokenNames[] = [ static immutable string TokenNames[] = [
<% @tokens.each_with_index do |token, index| %> <% @tokens.each_with_index do |(name, token), index| %>
<% if token.name %> <% if token.name %>
"<%= token.name %>", "<%= token.name %>",
<% else %> <% else %>

22
lib/imbecile.rb
View File

@ -35,7 +35,7 @@ class Imbecile
end end
def initialize(input) def initialize(input)
@tokens = [] @tokens = {}
@rules = {} @rules = {}
input = input.gsub("\r\n", "\n") input = input.gsub("\r\n", "\n")
while !input.empty? while !input.empty?
@ -74,10 +74,14 @@ class Imbecile
unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/ unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
raise Error.new("Invalid token name #{name}") raise Error.new("Invalid token name #{name}")
end end
@tokens << Token.new(name, pattern, @tokens.size) if @tokens[name]
raise Error.new("Duplicate token name #{name}")
else
@tokens[name] = Token.new(name, pattern, @tokens.size)
end
elsif input.slice!(/\Adrop\s+(\S+)\n/) elsif input.slice!(/\Adrop\s+(\S+)\n/)
pattern = $1 pattern = $1
@tokens << Token.new(nil, pattern, @tokens.size) @tokens[name] = Token.new(nil, pattern, @tokens.size)
elsif input.slice!(/\A(\S+)\s*:\s*\[(.*?)\] <<\n(.*?)^>>\n/m) elsif input.slice!(/\A(\S+)\s*:\s*\[(.*?)\] <<\n(.*?)^>>\n/m)
rule_name, components, code = $1, $2, $3 rule_name, components, code = $1, $2, $3
components = components.strip.split(/\s+/) components = components.strip.split(/\s+/)
@ -92,14 +96,8 @@ class Imbecile
end end
def expand_rules def expand_rules
token_names = @tokens.each_with_object({}) do |token, token_names|
if token_names.include?(token.name)
raise Error.new("Duplicate token name #{token.name}")
end
token_names[token.name] = token
end
@rules.each do |rule_name, rule| @rules.each do |rule_name, rule|
if token_names.include?(rule_name) if @tokens.include?(rule_name)
raise Error.new("Rule name collides with token name #{rule_name}") raise Error.new("Rule name collides with token name #{rule_name}")
end end
end end
@ -109,8 +107,8 @@ class Imbecile
@rules.each do |rule_name, rule| @rules.each do |rule_name, rule|
rule.patterns.each do |rule| rule.patterns.each do |rule|
rule.components.map! do |component| rule.components.map! do |component|
if token_names[component] if @tokens[component]
token_names[component] @tokens[component]
elsif @rules[component] elsif @rules[component]
@rules[component] @rules[component]
else else

2
lib/imbecile/lexer/dfa.rb
View File

@ -6,7 +6,7 @@ class Imbecile
def initialize(tokens) def initialize(tokens)
super() super()
start_nfa = Regex::NFA.new start_nfa = Regex::NFA.new
tokens.each do |token| tokens.each do |name, token|
start_nfa.start_state.add_transition(nil, token.nfa.start_state) start_nfa.start_state.add_transition(nil, token.nfa.start_state)
end end
@nfa_state_sets = {} @nfa_state_sets = {}