Allow drop patterns to execute lexer user code blocks

doc/user_guide.md

@@ -151,8 +151,35 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
+#### C/C++
+
+The lexer code block is passed the following arguments:
+
+  * `match` - a pointer points to the text matched by the lexer pattern.
+  * `match_length` - length of the matched text.
+
 Example:
 
+```
+ptype long;
+
+token integer /\d+/ <<
+  long v = 0;
+  for (size_t i = 0u; i < match_length; i++)
+  {
+    v *= 10;
+    v += (match[i] - '0');
+  }
+  $$ = v;
+>>
+```
+
+#### D
+
+The lexer code block is passed the following arguments:
+
+  * `match` - a slice containing the text matched by the lexer pattern.
+
 ```
 ptype ulong;
 
@@ -167,7 +194,7 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `token` or pattern expression.
+Lexer code blocks appear following a `drop`, `token`, or pattern expression.
 User code in a lexer code block will be executed when the lexer matches the
 given pattern.
 Assignment to the `$$` symbol will associate a parser value with the lexed

lib/propane/grammar.rb

@@ -195,8 +195,10 @@ class Propane
           raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
         end
         consume!(/\s+/)
-        consume!(/;/, "expected `;'")
+        unless code = parse_code_block!
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
+          consume!(/;/, "expected `;' or code block")
+        end
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline)
         @modeline = nil
         true
       end
@@ -285,6 +287,8 @@ class Propane
             end
           elsif md = consume!(%r{(.)})
             pattern += md[1]
+          elsif @input == "" || @input.start_with?("\n")
+            raise Error.new("Line #{@line_number}: Unterminated pattern; expected `/`")
           end
         end
         pattern

spec/propane_spec.rb

@@ -1483,7 +1483,7 @@ EOF
 
       if %w[c cpp].include?(language)
         it "allows a user function to free token node memory in tree mode" do
-    write_grammar <<EOF
+          write_grammar <<EOF
 <<
 static void free_token(Token * token)
 {
@@ -1510,6 +1510,40 @@ EOF
           expect(results.status).to eq 0
         end
       end
+
+      it "executes code blocks associated with drop statements" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  stderr.write("comment: ", match);
+>>
+token a;
+Start -> a;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+  fprintf(stderr, "comment: %.*s", (int)match_length, match);
+>>
+token a;
+Start -> a;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_drop_code_block.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to match %r{comment: # comment 1\n.*comment: #    comment 2}m
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_drop_code_block.c

@@ -0,0 +1,14 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_drop_code_block.d

@@ -0,0 +1,16 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    # comment 1\n#    comment 2\na\n";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}