Rename to propane

This commit is contained in:
josh committed 2022-05-28 20:20:03 -04:00
1 parent fbd215098b
commit ddadc2008b
32 files changed
+60 -100

No files matched your search

+87
View File
@@ -0,0 +1,87 @@
class Propane
describe CodePointRange do
describe "#<=>" do
it "sorts ranges" do
arr = [
CodePointRange.new(100,102),
CodePointRange.new(65, 68),
CodePointRange.new(65, 65),
CodePointRange.new(100, 100),
CodePointRange.new(68, 70),
]
arr.sort!
expect(arr[0]).to eq CodePointRange.new(65, 65)
expect(arr[1]).to eq CodePointRange.new(65, 68)
expect(arr[2]).to eq CodePointRange.new(68, 70)
expect(arr[3]).to eq CodePointRange.new(100, 100)
expect(arr[4]).to eq CodePointRange.new(100, 102)
end
end
describe "#include?" do
it "returns whether the code point is included in the range" do
expect(CodePointRange.new(100).include?(100)).to be_truthy
expect(CodePointRange.new(100, 100).include?(99)).to be_falsey
expect(CodePointRange.new(100, 100).include?(101)).to be_falsey
expect(CodePointRange.new(100, 120).include?(99)).to be_falsey
expect(CodePointRange.new(100, 120).include?(100)).to be_truthy
expect(CodePointRange.new(100, 120).include?(110)).to be_truthy
expect(CodePointRange.new(100, 120).include?(120)).to be_truthy
expect(CodePointRange.new(100, 120).include?(121)).to be_falsey
end
it "returns whether the range is included in the range" do
expect(CodePointRange.new(100).include?(CodePointRange.new(100))).to be_truthy
expect(CodePointRange.new(100, 100).include?(CodePointRange.new(99))).to be_falsey
expect(CodePointRange.new(100, 100).include?(CodePointRange.new(99, 100))).to be_falsey
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(90, 110))).to be_falsey
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(110, 130))).to be_falsey
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(100, 120))).to be_truthy
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(100, 110))).to be_truthy
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(110, 120))).to be_truthy
expect(CodePointRange.new(100, 120).include?(CodePointRange.new(102, 118))).to be_truthy
end
end
describe ".invert_ranges" do
it "inverts ranges" do
expect(CodePointRange.invert_ranges(
[CodePointRange.new(60, 90),
CodePointRange.new(80, 85),
CodePointRange.new(80, 100),
CodePointRange.new(101),
CodePointRange.new(200, 300)])).to eq [
CodePointRange.new(0, 59),
CodePointRange.new(102, 199),
CodePointRange.new(301, 0xFFFFFFFF)]
expect(CodePointRange.invert_ranges(
[CodePointRange.new(0, 500),
CodePointRange.new(7000, 0xFFFFFFFF)])).to eq [
CodePointRange.new(501, 6999)]
end
end
describe ".first_subrange" do
it "returns the first subrange to split" do
expect(CodePointRange.first_subrange(
[CodePointRange.new(65, 90),
CodePointRange.new(66, 66),
CodePointRange.new(80, 90)])).to eq CodePointRange.new(65)
expect(CodePointRange.first_subrange(
[CodePointRange.new(65, 90)])).to eq CodePointRange.new(65, 90)
expect(CodePointRange.first_subrange(
[CodePointRange.new(65, 90),
CodePointRange.new(80, 90)])).to eq CodePointRange.new(65, 79)
expect(CodePointRange.first_subrange(
[CodePointRange.new(65, 90),
CodePointRange.new(65, 100),
CodePointRange.new(65, 95)])).to eq CodePointRange.new(65, 90)
expect(CodePointRange.first_subrange(
[CodePointRange.new(100, 120),
CodePointRange.new(70, 90)])).to eq CodePointRange.new(70, 90)
end
end
end
end
+121
View File
@@ -0,0 +1,121 @@
class TestLexer
def initialize(token_dfa)
@token_dfa = token_dfa
end
def lex(input)
input_chars = input.chars
output = []
while lexed_token = lex_token(input_chars)
output << lexed_token
input_chars.slice!(0, lexed_token[1].size)
end
unless input_chars.empty?
raise "Unmatched input #{input_chars.join(" ")}"
end
output
end
def lex_token(input_chars)
return nil if input_chars.empty?
s = ""
current_state = @token_dfa.start_state
last_accepts = nil
last_s = nil
input_chars.each_with_index do |input_char, index|
if next_state = transition(current_state, input_char)
s += input_char
current_state = next_state
if current_state.accepts
last_accepts = current_state.accepts
last_s = s
end
else
break
end
end
if last_accepts
[last_accepts.name, last_s]
end
end
def transition(state, input_char)
state.transitions.each do |transition|
if transition.code_point_range.include?(input_char.ord)
return transition.destination
end
end
nil
end
end
def run(grammar, input)
propane = Propane.new(grammar)
token_dfa = Propane::Lexer::DFA.new(propane.instance_variable_get(:@tokens))
test_lexer = TestLexer.new(token_dfa)
test_lexer.lex(input)
end
describe Propane::Lexer::DFA do
it "lexes a simple token" do
expect(run(<<EOF, "foo")).to eq [["foo", "foo"]]
token foo
EOF
end
it "lexes two tokens" do
expected = [
["foo", "foo"],
["bar", "bar"],
]
expect(run(<<EOF, "foobar")).to eq expected
token foo
token bar
EOF
end
it "lexes the longer of multiple options" do
expected = [
["identifier", "foobar"],
]
expect(run(<<EOF, "foobar")).to eq expected
token foo
token bar
token identifier [a-z]+
EOF
expected = [
["plusplus", "++"],
["plus", "+"],
]
expect(run(<<EOF, "+++")).to eq expected
token plus \\+
token plusplus \\+\\+
EOF
end
it "lexes whitespace" do
expected = [
["foo", "foo"],
["WS", " \t"],
["bar", "bar"],
]
expect(run(<<EOF, "foo \tbar")).to eq expected
token foo
token bar
token WS \\s+
EOF
end
it "allows dropping a matched pattern" do
expected = [
["foo", "foo"],
[nil, " \t"],
["bar", "bar"],
]
expect(run(<<EOF, "foo \tbar")).to eq expected
token foo
token bar
drop \\s+
EOF
end
end
+19
View File
@@ -0,0 +1,19 @@
class Propane
class Parser
describe Item do
it "operates properly with a set" do
rule = Object.new
item1 = Item.new(rule, 2)
item2 = Item.new(rule, 2)
expect(item1).to eq item2
expect(item1.eql?(item2)).to be_truthy
set = Set.new([item1, item2])
expect(set.size).to eq 1
end
end
end
end
+333
View File
@@ -0,0 +1,333 @@
class Propane
RSpec.describe Regex do
it "parses an empty expression" do
regex = Regex.new("")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0].size).to eq 0
end
it "parses a single character unit expression" do
regex = Regex.new("a")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
end
it "parses a group with a single character unit expression" do
regex = Regex.new("(a)")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::AlternatesUnit
alt_unit = seq_unit[0]
expect(alt_unit.alternates.size).to eq 1
expect(alt_unit.alternates[0]).to be_a Regex::SequenceUnit
expect(alt_unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
end
it "parses a *" do
regex = Regex.new("a*")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 0
expect(m_unit.max_count).to be_nil
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
end
it "parses a +" do
regex = Regex.new("a+")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 1
expect(m_unit.max_count).to be_nil
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
end
it "parses a ?" do
regex = Regex.new("a?")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 0
expect(m_unit.max_count).to eq 1
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
end
it "parses a multiplicity count" do
regex = Regex.new("a{5}")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 5
expect(m_unit.max_count).to eq 5
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
end
it "parses a minimum-only multiplicity count" do
regex = Regex.new("a{5,}")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 5
expect(m_unit.max_count).to be_nil
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
end
it "parses a minimum and maximum multiplicity count" do
regex = Regex.new("a{5,8}")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::MultiplicityUnit
m_unit = seq_unit[0]
expect(m_unit.min_count).to eq 5
expect(m_unit.max_count).to eq 8
expect(m_unit.unit).to be_a Regex::CharacterRangeUnit
expect(m_unit.unit.first).to eq "a".ord
end
it "parses an escaped *" do
regex = Regex.new("a\\*")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 2
expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[0].first).to eq "a".ord
expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[1].first).to eq "*".ord
end
it "parses an escaped +" do
regex = Regex.new("a\\+")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 2
expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[0].first).to eq "a".ord
expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[1].first).to eq "+".ord
end
it "parses an escaped \\" do
regex = Regex.new("\\\\d")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 2
expect(seq_unit[0]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[0].first).to eq "\\".ord
expect(seq_unit[1]).to be_a Regex::CharacterRangeUnit
expect(seq_unit[1].first).to eq "d".ord
end
it "parses a character class" do
regex = Regex.new("[a-z_]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.negate).to be_falsey
expect(ccu.size).to eq 2
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "a".ord
expect(ccu[0].last).to eq "z".ord
expect(ccu[1]).to be_a Regex::CharacterRangeUnit
expect(ccu[1].first).to eq "_".ord
end
it "parses a negated character class" do
regex = Regex.new("[^xyz]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.negate).to be_truthy
expect(ccu.size).to eq 3
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "x".ord
end
it "parses - as a plain character at beginning of a character class" do
regex = Regex.new("[-9]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.size).to eq 2
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "-".ord
end
it "parses - as a plain character at end of a character class" do
regex = Regex.new("[0-]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.size).to eq 2
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "0".ord
expect(ccu[1]).to be_a Regex::CharacterRangeUnit
expect(ccu[1].first).to eq "-".ord
end
it "parses - as a plain character at beginning of a negated character class" do
regex = Regex.new("[^-9]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.negate).to be_truthy
expect(ccu.size).to eq 2
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "-".ord
end
it "parses . as a plain character in a character class" do
regex = Regex.new("[.]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.negate).to be_falsey
expect(ccu.size).to eq 1
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq ".".ord
end
it "parses - as a plain character when escaped in middle of character class" do
regex = Regex.new("[0\\-9]")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
seq_unit = regex.unit.alternates[0]
expect(seq_unit.size).to eq 1
expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
ccu = seq_unit[0]
expect(ccu.negate).to be_falsey
expect(ccu.size).to eq 3
expect(ccu[0]).to be_a Regex::CharacterRangeUnit
expect(ccu[0].first).to eq "0".ord
expect(ccu[1]).to be_a Regex::CharacterRangeUnit
expect(ccu[1].first).to eq "-".ord
expect(ccu[2]).to be_a Regex::CharacterRangeUnit
expect(ccu[2].first).to eq "9".ord
end
it "parses alternates" do
regex = Regex.new("ab|c")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 2
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[1]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[0].size).to eq 2
expect(regex.unit.alternates[1].size).to eq 1
end
it "parses a ." do
regex = Regex.new("a.b")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 1
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[0][1]).to be_a Regex::CharacterClassUnit
expect(regex.unit.alternates[0][1].units.size).to eq 2
expect(regex.unit.alternates[0][2]).to be_a Regex::CharacterRangeUnit
end
it "parses something complex" do
regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
expect(regex.unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates.size).to eq 4
expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[0].size).to eq 1
expect(regex.unit.alternates[0][0]).to be_a Regex::MultiplicityUnit
expect(regex.unit.alternates[0][0].min_count).to eq 0
expect(regex.unit.alternates[0][0].max_count).to be_nil
expect(regex.unit.alternates[0][0].unit).to be_a Regex::AlternatesUnit
expect(regex.unit.alternates[0][0].unit.alternates.size).to eq 2
expect(regex.unit.alternates[0][0].unit.alternates[0]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[0][0].unit.alternates[0].size).to eq 1
expect(regex.unit.alternates[0][0].unit.alternates[0][0]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[0][0].unit.alternates[1]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[0][0].unit.alternates[1].size).to eq 0
expect(regex.unit.alternates[1]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[1].size).to eq 1
expect(regex.unit.alternates[1][0]).to be_a Regex::CharacterClassUnit
expect(regex.unit.alternates[1][0].negate).to be_truthy
expect(regex.unit.alternates[1][0].size).to eq 1
expect(regex.unit.alternates[1][0][0]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[2]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[2].size).to eq 2
expect(regex.unit.alternates[2][0]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[2][0].first).to eq "|".ord
expect(regex.unit.alternates[2][1]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[2][1].first).to eq "v".ord
expect(regex.unit.alternates[3]).to be_a Regex::SequenceUnit
expect(regex.unit.alternates[3].size).to eq 1
expect(regex.unit.alternates[3][0]).to be_a Regex::MultiplicityUnit
expect(regex.unit.alternates[3][0].min_count).to eq 1
expect(regex.unit.alternates[3][0].max_count).to be_nil
expect(regex.unit.alternates[3][0].unit).to be_a Regex::CharacterClassUnit
expect(regex.unit.alternates[3][0].unit.size).to eq 1
expect(regex.unit.alternates[3][0].unit[0]).to be_a Regex::CharacterRangeUnit
expect(regex.unit.alternates[3][0].unit[0].first).to eq "x".ord
expect(regex.unit.alternates[3][0].unit[0].last).to eq "y".ord
end
end
end