Allow user-defined context fields

assets/parser.d.erb

@@ -183,6 +183,8 @@ public struct <%= @grammar.prefix %>context_t
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_code %>
 }
 
 /**************************************************************************

assets/parser.h.erb

@@ -172,6 +172,8 @@ typedef struct
 
     /** User terminate code. */
     size_t user_terminate_code;
+
+<%= @grammar.context_user_code %>
 } <%= @grammar.prefix %>context_t;
 
 /**************************************************************************

doc/user_guide.md

@@ -221,6 +221,41 @@ Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
 
+### Context code blocks: the `context` statement
+
+Propane uses a context structure for lexer and parser operations.
+Custom fields may be added to the context structure by using the grammar
+`context` statement.
+This allows lexer pattern or parser rule code blocks to access user-defined
+fields within the context structure.
+
+Example:
+
+```
+context <<
+    int mycontextval;
+>>
+```
+
+Lexer user code blocks or parser user code blocks can access user-defined
+context fields by using the `${context.<field>}` syntax.
+
+C++ example:
+
+```
+context <<
+    std::string comments;
+>>
+drop /#(.*)\n/ <<
+    /* Accumulate comments before the next parser tree node. */
+    ${context.comments} += std::string((const char *)match, match_length);
+>>
+```
+
+If a pointer to any allocated memory is stored in a user-defined context field,
+it is up to the user to free any memory when the program is finished using the
+context structure.
+
 ##> Tree generation mode - the `tree` statement
 
 To activate tree generation mode, place the `tree` statement in your grammar file:

lib/propane/generator.rb

@@ -265,6 +265,15 @@ class Propane
           "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
         end
       end
+      code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
+        fieldname = $1
+        case @language
+        when "c"
+          "context->#{fieldname}"
+        when "d"
+          "context.#{fieldname}"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
           case @language

lib/propane/grammar.rb

@@ -5,6 +5,7 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :context_user_code
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -34,6 +35,7 @@ class Propane
       @tree_prefix = ""
       @tree_suffix = ""
       @free_token_node = nil
+      @context_user_code = ""
       parse_grammar!
       @start_rules << "Start" if @start_rules.empty?
     end
@@ -62,6 +64,7 @@ class Propane
       if parse_white_space!
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
+      elsif parse_context_statement!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -98,6 +101,15 @@ class Propane
       consume!(/#.*\n/)
     end
 
+    def parse_context_statement!
+      if md = consume!(/context\b\s*/)
+        unless code = parse_code_block!
+          raise Error.new("Line #{@line_number}: expected code block")
+        end
+        @context_user_code += code
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true

spec/propane_spec.rb

@@ -1544,6 +1544,66 @@ EOF
         expect(results.stderr).to match %r{comment: # comment 1\n.*comment: #    comment 2}m
         expect(results.status).to eq 0
       end
+
+      it "allows user-defined context fields" do
+        if language == "d"
+          write_grammar <<EOF
+context <<
+    string comments;
+    uint acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    ${context.comments} ~= match;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <string.h>
+#include <stdlib.h>
+>>
+context <<
+    char * comments;
+    unsigned int acount;
+>>
+drop /\\s+/;
+drop /#(.*)\\n/ <<
+    size_t cur_len = 0u;
+    if (${context.comments} != NULL)
+        cur_len = strlen(${context.comments});
+    char * commentsnew = (char *)malloc(cur_len + match_length + 1);
+    if (${context.comments} != NULL)
+        memcpy(commentsnew, ${context.comments}, cur_len);
+    memcpy(&commentsnew[cur_len], match, match_length);
+    commentsnew[cur_len + match_length] = '\\0';
+    if (${context.comments} != NULL)
+    {
+        free(${context.comments});
+    }
+    ${context.comments} = commentsnew;
+>>
+token a <<
+    ${context.acount}++;
+>>
+Start -> As;
+As -> ;
+As -> a As;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_user_context_fields.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to include %r{comments: # comment 1\n#    comment 2}
+        expect(results.stderr).to include %r{acount: 11\n}
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_user_context_fields.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+int main()
+{
+    char const * input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    fprintf(stderr, "comments: %s", context.comments);
+    fprintf(stderr, "acount: %u\n", context.acount);
+    free(context.comments);
+
+    return 0;
+}

spec/test_user_context_fields.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aaa\n\n\na\n    # comment 1\na  a    aa\n\naa\n#    comment 2\na\n";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    stderr.writeln("comments: ", context.comments);
+    stderr.writeln("acount: ", context.acount);
+}