From 0e5b0b1f8ab8d8438c7b8452868ccb45a1d9a670 Mon Sep 17 00:00:00 2001 From: Josh Holtrop Date: Fri, 10 Apr 2026 16:03:45 -0400 Subject: [PATCH] complete adding lex_fn --- assets/parser.d.erb | 8 ++++++ assets/parser.h.erb | 8 ++++++ lib/propane/generator.rb | 2 +- lib/propane/grammar.rb | 10 +++---- spec/propane_spec.rb | 56 ++++++++++++++++++++++++++++++++------- spec/test_custom_lex_fn.c | 15 +++++++++++ spec/test_custom_lex_fn.d | 18 +++++++++++++ 7 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 spec/test_custom_lex_fn.c create mode 100644 spec/test_custom_lex_fn.d diff --git a/assets/parser.d.erb b/assets/parser.d.erb index df5d4e9..99217ad 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -86,6 +86,14 @@ public union <%= @grammar.prefix %>value_t <%= typestring %> v_<%= name %>; <% end %> } + +/** Parser value constructor(s). */ +<% @grammar.ptypes.each do |name, typestring| %> +public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v) +{ + return <%= @grammar.prefix %>value_t(v_<%= name %>: v); +} +<% end %> <% end %> <% if @grammar.tree %> diff --git a/assets/parser.h.erb b/assets/parser.h.erb index 606d0be..32f3139 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -69,6 +69,14 @@ typedef union <%= typestring %> v_<%= name %>; <% end %> } <%= @grammar.prefix %>value_t; + +/** Parser value constructor(s). */ +<% @grammar.ptypes.each do |name, typestring| %> +static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v) +{ + return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v}; +} +<% end %> <% end %> <% if @grammar.tree %> diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index d2b357c..c20483c 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -359,7 +359,7 @@ class Propane # @return [String] # Lex function to use. def lex_fn - @grammar.custom_lex_fn || "#{@grammar.prefix}lex" + @grammar.lex_fn || "#{@grammar.prefix}lex" end # Get the parser value type for the start rule. diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index 10de26c..f8e6569 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -6,7 +6,7 @@ class Propane IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/ attr_reader :context_user_fields - attr_reader :custom_lex_fn + attr_reader :lex_fn attr_reader :tree attr_reader :tree_prefix attr_reader :tree_suffix @@ -70,7 +70,7 @@ class Propane elsif parse_comment_line! elsif @modeline.nil? && parse_mode_label! elsif parse_context_user_fields_statement! - elsif parse_custom_lex_fn! + elsif parse_lex_fn! elsif parse_tree_statement! elsif parse_tree_prefix_statement! elsif parse_tree_suffix_statement! @@ -119,9 +119,9 @@ class Propane end end - def parse_custom_lex_fn! - if md = consume!(/custom_lex_fn\b\s*(\w+)\s*;/) - @custom_lex_fn = $1 + def parse_lex_fn! + if md = consume!(/lex_fn\b\s*(\w+)\s*;/) + @lex_fn = md[1] end end diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 41d5c45..384d8d6 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1706,16 +1706,34 @@ EOF << private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) { + static size_t count; + size_t result = P_SUCCESS; + if (count > 0) + { + out_token_info.token = TOKEN_a; + out_token_info.pvalue = p_value(count); + count--; + } + else + { + result = p_lex(context, out_token_info); + if (out_token_info.token == TOKEN_c) + { + count = 3; + } + } + return result; } >> -tree; +ptype size_t; +lex_fn mylexfn; -token a << $$ = 1; >> -token b << $$ = 2; >> -token c << $$ = 3; >> +token a << $$ = 7; >> +token b << $$ = 8; >> +token c << $$ = 9; >> Start -> << $$ = 0; >> -Start -> ID Start << $$ = ($1 << 4) | $2; >> +Start -> Start ID << $$ = ($1 << 4) | $2; >> ID -> a << $$ = $1; >> ID -> b << $$ = $1; >> ID -> c << $$ = $1; >> @@ -1725,16 +1743,34 @@ EOF << static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) { + static size_t count; + size_t result = P_SUCCESS; + if (count > 0) + { + out_token_info->token = TOKEN_a; + out_token_info->pvalue = p_value(count); + count--; + } + else + { + result = p_lex(context, out_token_info); + if (out_token_info->token == TOKEN_c) + { + count = 3; + } + } + return result; } >> -tree; +ptype size_t; +lex_fn mylexfn; -token a << $$ = 1; >> -token b << $$ = 2; >> -token c << $$ = 3; >> +token a << $$ = 7; >> +token b << $$ = 8; >> +token c << $$ = 9; >> Start -> << $$ = 0; >> -Start -> ID Start << $$ = ($1 << 4) | $2; >> +Start -> Start ID << $$ = ($1 << 4) | $2; >> ID -> a << $$ = $1; >> ID -> b << $$ = $1; >> ID -> c << $$ = $1; >> diff --git a/spec/test_custom_lex_fn.c b/spec/test_custom_lex_fn.c new file mode 100644 index 0000000..7b4439d --- /dev/null +++ b/spec/test_custom_lex_fn.c @@ -0,0 +1,15 @@ +#include "testparser.h" +#include "testutils.h" +#include + +int main() +{ + char const * input = "cbacba"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(context)); + size_t result = p_result(context); + assert_eq(0x932187932187, result); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_custom_lex_fn.d b/spec/test_custom_lex_fn.d new file mode 100644 index 0000000..76db2e3 --- /dev/null +++ b/spec/test_custom_lex_fn.d @@ -0,0 +1,18 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + string input = "cbacba"; + p_context_t * context = p_context_new(input); + assert_eq(P_SUCCESS, p_parse(context)); + size_t result = p_result(context); + assert_eq(0x932187932187, result); + p_context_delete(context); +}