wip

assets/parser.c.erb

@@ -981,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;

assets/parser.d.erb

@@ -1037,7 +1037,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;

lib/propane/generator.rb

@@ -354,6 +354,14 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.custom_lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :custom_lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -69,6 +70,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_custom_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -117,6 +119,12 @@ class Propane
       end
     end
 
+    def parse_custom_lex_fn!
+      if md = consume!(/custom_lex_fn\b\s*(\w+)\s*;/)
+        @custom_lex_fn = $1
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true

spec/propane_spec.rb

@@ -1699,6 +1699,52 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+}
+>>
+
+tree;
+
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> << $$ = 0; >>
+Start -> ID Start << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+}
+>>
+
+tree;
+
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+Start -> << $$ = 0; >>
+Start -> ID Start << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
     end
   end
 end