From 78adf86103b24a612f68fff1d57f93f0cbd053f5 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 13 Feb 2026 23:53:18 -0500 Subject: [PATCH] Allow user-defined context fields --- assets/parser.d.erb | 2 ++ assets/parser.h.erb | 2 ++ doc/user_guide.md | 35 +++++++++++++++++++ lib/propane/generator.rb | 9 +++++ lib/propane/grammar.rb | 12 +++++++ spec/propane_spec.rb | 60 +++++++++++++++++++++++++++++++++ spec/test_user_context_fields.c | 19 +++++++++++ spec/test_user_context_fields.d | 19 +++++++++++ 8 files changed, 158 insertions(+) create mode 100644 spec/test_user_context_fields.c create mode 100644 spec/test_user_context_fields.d diff --git a/assets/parser.d.erb b/assets/parser.d.erb index b00217f..809ce4a 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -183,6 +183,8 @@ public struct <%= @grammar.prefix %>context_t /** User terminate code. */ size_t user_terminate_code; + +<%= @grammar.context_user_code %> } /************************************************************************** diff --git a/assets/parser.h.erb b/assets/parser.h.erb index a4e022a..878271f 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -172,6 +172,8 @@ typedef struct /** User terminate code. */ size_t user_terminate_code; + +<%= @grammar.context_user_code %> } <%= @grammar.prefix %>context_t; /************************************************************************** diff --git a/doc/user_guide.md b/doc/user_guide.md index 8a7635f..b1d1e85 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -221,6 +221,41 @@ Parser rule code blocks are not available in tree generation mode. In tree generation mode, a full parse tree is automatically constructed in memory for user code to traverse after parsing is complete. +### Context code blocks: the `context` statement + +Propane uses a context structure for lexer and parser operations. +Custom fields may be added to the context structure by using the grammar +`context` statement. +This allows lexer pattern or parser rule code blocks to access user-defined +fields within the context structure. + +Example: + +``` +context << + int mycontextval; +>> +``` + +Lexer user code blocks or parser user code blocks can access user-defined +context fields by using the `${context.}` syntax. + +C++ example: + +``` +context << + std::string comments; +>> +drop /#(.*)\n/ << + /* Accumulate comments before the next parser tree node. */ + ${context.comments} += std::string((const char *)match, match_length); +>> +``` + +If a pointer to any allocated memory is stored in a user-defined context field, +it is up to the user to free any memory when the program is finished using the +context structure. + ##> Tree generation mode - the `tree` statement To activate tree generation mode, place the `tree` statement in your grammar file: diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index bd2e41f..e91990d 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -265,6 +265,15 @@ class Propane "context.user_terminate_code = (#{user_terminate_code}); return #{retval};" end end + code = code.gsub(/\$\{context\.(\w+)\}/) do |match| + fieldname = $1 + case @language + when "c" + "context->#{fieldname}" + when "d" + "context.#{fieldname}" + end + end if parser code = code.gsub(/\$\$/) do |match| case @language diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index 9804d36..ffe57dc 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -5,6 +5,7 @@ class Propane # Reserve identifiers beginning with a double-underscore for internal use. IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/ + attr_reader :context_user_code attr_reader :tree attr_reader :tree_prefix attr_reader :tree_suffix @@ -34,6 +35,7 @@ class Propane @tree_prefix = "" @tree_suffix = "" @free_token_node = nil + @context_user_code = "" parse_grammar! @start_rules << "Start" if @start_rules.empty? end @@ -62,6 +64,7 @@ class Propane if parse_white_space! elsif parse_comment_line! elsif @modeline.nil? && parse_mode_label! + elsif parse_context_statement! elsif parse_tree_statement! elsif parse_tree_prefix_statement! elsif parse_tree_suffix_statement! @@ -98,6 +101,15 @@ class Propane consume!(/#.*\n/) end + def parse_context_statement! + if md = consume!(/context\b\s*/) + unless code = parse_code_block! + raise Error.new("Line #{@line_number}: expected code block") + end + @context_user_code += code + end + end + def parse_tree_statement! if consume!(/tree\s*;/) @tree = true diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 72561f2..3dd5177 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1544,6 +1544,66 @@ EOF expect(results.stderr).to match %r{comment: # comment 1\n.*comment: # comment 2}m expect(results.status).to eq 0 end + + it "allows user-defined context fields" do + if language == "d" + write_grammar <> +drop /\\s+/; +drop /#(.*)\\n/ << + ${context.comments} ~= match; +>> +token a << + ${context.acount}++; +>> +Start -> As; +As -> ; +As -> a As; +EOF + else + write_grammar < +#include +>> +context << + char * comments; + unsigned int acount; +>> +drop /\\s+/; +drop /#(.*)\\n/ << + size_t cur_len = 0u; + if (${context.comments} != NULL) + cur_len = strlen(${context.comments}); + char * commentsnew = (char *)malloc(cur_len + match_length + 1); + if (${context.comments} != NULL) + memcpy(commentsnew, ${context.comments}, cur_len); + memcpy(&commentsnew[cur_len], match, match_length); + commentsnew[cur_len + match_length] = '\\0'; + if (${context.comments} != NULL) + { + free(${context.comments}); + } + ${context.comments} = commentsnew; +>> +token a << + ${context.acount}++; +>> +Start -> As; +As -> ; +As -> a As; +EOF + end + run_propane(language: language) + compile("spec/test_user_context_fields.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to include %r{comments: # comment 1\n# comment 2} + expect(results.stderr).to include %r{acount: 11\n} + expect(results.status).to eq 0 + end end end end diff --git a/spec/test_user_context_fields.c b/spec/test_user_context_fields.c new file mode 100644 index 0000000..003cc23 --- /dev/null +++ b/spec/test_user_context_fields.c @@ -0,0 +1,19 @@ +#include "testparser.h" +#include +#include +#include +#include + +int main() +{ + char const * input = "aaa\n\n\na\n # comment 1\na a aa\n\naa\n# comment 2\na\n"; + p_context_t context; + p_context_init(&context, (uint8_t const *)input, strlen(input)); + assert(p_parse(&context) == P_SUCCESS); + + fprintf(stderr, "comments: %s", context.comments); + fprintf(stderr, "acount: %u\n", context.acount); + free(context.comments); + + return 0; +} diff --git a/spec/test_user_context_fields.d b/spec/test_user_context_fields.d new file mode 100644 index 0000000..aabb10f --- /dev/null +++ b/spec/test_user_context_fields.d @@ -0,0 +1,19 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + string input = "aaa\n\n\na\n # comment 1\na a aa\n\naa\n# comment 2\na\n"; + p_context_t context; + p_context_init(&context, input); + assert(p_parse(&context) == P_SUCCESS); + + stderr.writeln("comments: ", context.comments); + stderr.writeln("acount: ", context.acount); +}