diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 4736949..0d16a9f 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -981,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start { if (token == INVALID_TOKEN_ID) { - size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info); + size_t lexer_result = <%= lex_fn %>(context, &token_info); if (lexer_result != P_SUCCESS) { result = lexer_result; diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 64b3dd7..df5d4e9 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -1037,7 +1037,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star { if (token == INVALID_TOKEN_ID) { - size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info); + size_t lexer_result = <%= lex_fn %>(context, &token_info); if (lexer_result != P_SUCCESS) { return lexer_result; diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 087df31..d2b357c 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -354,6 +354,14 @@ class Propane code end + # Get the lex function to use. + # + # @return [String] + # Lex function to use. + def lex_fn + @grammar.custom_lex_fn || "#{@grammar.prefix}lex" + end + # Get the parser value type for the start rule. # # @return [Array] diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index 095a418..10de26c 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -6,6 +6,7 @@ class Propane IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/ attr_reader :context_user_fields + attr_reader :custom_lex_fn attr_reader :tree attr_reader :tree_prefix attr_reader :tree_suffix @@ -69,6 +70,7 @@ class Propane elsif parse_comment_line! elsif @modeline.nil? && parse_mode_label! elsif parse_context_user_fields_statement! + elsif parse_custom_lex_fn! elsif parse_tree_statement! elsif parse_tree_prefix_statement! elsif parse_tree_suffix_statement! @@ -117,6 +119,12 @@ class Propane end end + def parse_custom_lex_fn! + if md = consume!(/custom_lex_fn\b\s*(\w+)\s*;/) + @custom_lex_fn = $1 + end + end + def parse_tree_statement! if consume!(/tree\s*;/) @tree = true diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index bdf4b2f..41d5c45 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1699,6 +1699,52 @@ EOF results = run_test(language: language) expect(results.status).to eq 0 end + + it "allows a custom lex function" do + if language == "d" + write_grammar <> + +tree; + +token a << $$ = 1; >> +token b << $$ = 2; >> +token c << $$ = 3; >> +Start -> << $$ = 0; >> +Start -> ID Start << $$ = ($1 << 4) | $2; >> +ID -> a << $$ = $1; >> +ID -> b << $$ = $1; >> +ID -> c << $$ = $1; >> +EOF + else + write_grammar <> + +tree; + +token a << $$ = 1; >> +token b << $$ = 2; >> +token c << $$ = 3; >> +Start -> << $$ = 0; >> +Start -> ID Start << $$ = ($1 << 4) | $2; >> +ID -> a << $$ = $1; >> +ID -> b << $$ = $1; >> +ID -> c << $$ = $1; >> +EOF + end + run_propane(language: language) + compile("spec/test_custom_lex_fn.#{language}", language: language) + results = run_test(language: language) + expect(results.status).to eq 0 + end end end end