diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 4736949..0d16a9f 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -981,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start { if (token == INVALID_TOKEN_ID) { - size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info); + size_t lexer_result = <%= lex_fn %>(context, &token_info); if (lexer_result != P_SUCCESS) { result = lexer_result; diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 64b3dd7..99217ad 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -86,6 +86,14 @@ public union <%= @grammar.prefix %>value_t <%= typestring %> v_<%= name %>; <% end %> } + +/** Parser value constructor(s). */ +<% @grammar.ptypes.each do |name, typestring| %> +public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v) +{ + return <%= @grammar.prefix %>value_t(v_<%= name %>: v); +} +<% end %> <% end %> <% if @grammar.tree %> @@ -1037,7 +1045,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star { if (token == INVALID_TOKEN_ID) { - size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info); + size_t lexer_result = <%= lex_fn %>(context, &token_info); if (lexer_result != P_SUCCESS) { return lexer_result; diff --git a/assets/parser.h.erb b/assets/parser.h.erb index 606d0be..32f3139 100644 --- a/assets/parser.h.erb +++ b/assets/parser.h.erb @@ -69,6 +69,14 @@ typedef union <%= typestring %> v_<%= name %>; <% end %> } <%= @grammar.prefix %>value_t; + +/** Parser value constructor(s). */ +<% @grammar.ptypes.each do |name, typestring| %> +static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v) +{ + return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v}; +} +<% end %> <% end %> <% if @grammar.tree %> diff --git a/doc/user_guide.md b/doc/user_guide.md index 3608941..fe4bf3a 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -316,6 +316,61 @@ free_token_node << The `free_token_node` statement user code block is not emitted for D language since D has a garbage collector. +##> `lex_fn` statement - specifying a custom lexer function + +Propane generates both a lexer and a parser. +By default, the parser uses the generated `p_lex()` function directly to +return information for the next lexed token from the input stream. + +However, the user can specify a custom lex function. +This function may or may not use the Propane generated `p_lex()` function under +the hood. +For example, a token sequence could be injected or repeated from a previously +saved macro definition. + +Example (C/C++): + +``` +<< +static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info) +{ + static size_t count; + size_t result = P_SUCCESS; + if (count > 0) + { + out_token_info->token = TOKEN_a; + out_token_info->pvalue = p_value(count); + count--; + } + else + { + result = p_lex(context, out_token_info); + if (out_token_info->token == TOKEN_c) + { + count = 3; + } + } + return result; +} +>> + +lex_fn mylexfn; +``` + +The `lex_fn` statement takes one argument specifying the name of the custom +lexer function. +The user must supply a value for the `token` field of the `p_token_info_t` +output structure so that the parser knows what token was lexed. + +Additionally, if the parser user code makes use of the token's pvalue, then +the lexer function must supply a value for the `pvalue` field of the +`p_token_info_t` structure. +The `p_value()` generated API function could be useful for specifying +parser values to associate with the lexed token when tree generation is not +enabled. +When tree generation is enabled, the `pvalue` field can be set to an instance +of whatever type the user has defined as the `ptype` type. + ##> `module` statement - specifying the generated parser module name The `module` statement can be used to specify the module name for a generated @@ -1007,6 +1062,17 @@ For C targets, the `p_position_t` structure can be checked for validity by calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure instance. +### `p_value_t` + +If tree generation mode is enabled, the `p_value_t` type is defined to be the +type given to the `ptype` statement in the grammar file. + +If tree generation mode is not enabled, there could be more than one `ptype` +given, so the `p_value_t` type is a union of all possible `ptype` types. +In this case, the API functions `p_value()` and `p_value_XXX()` for each given +`ptype` name `XXX` are generated to return `p_value_t` instances holding the +corresponding `ptype`. + ### `p_token_info_t` The `p_token_info_t` structure contains the following fields: @@ -1336,6 +1402,20 @@ p_tree_delete_Statement(statement_tree); In this case, Propane will free a `Statement` tree structure returned by the `p_parse_Statement(context)` function. +### `p_value` + +When tree generation mode is not active, the `p_value_t` union can hold one of +several different possible value types. + +The `p_value(v)` function returns an instance of the `p_value_t` with the +default member set to the value of `v`. + +A `p_value_XXX(v)` function is set for each user-defined `ptype` name with +the user-given name in place of the `XXX`. + +These functions are useful for custom lexer functions which need to return a +parser value corresponding to a lexed token. + ##> Data ### `p_token_names` diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 087df31..c20483c 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -354,6 +354,14 @@ class Propane code end + # Get the lex function to use. + # + # @return [String] + # Lex function to use. + def lex_fn + @grammar.lex_fn || "#{@grammar.prefix}lex" + end + # Get the parser value type for the start rule. # # @return [Array] diff --git a/lib/propane/grammar.rb b/lib/propane/grammar.rb index 095a418..f8e6569 100644 --- a/lib/propane/grammar.rb +++ b/lib/propane/grammar.rb @@ -6,6 +6,7 @@ class Propane IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/ attr_reader :context_user_fields + attr_reader :lex_fn attr_reader :tree attr_reader :tree_prefix attr_reader :tree_suffix @@ -69,6 +70,7 @@ class Propane elsif parse_comment_line! elsif @modeline.nil? && parse_mode_label! elsif parse_context_user_fields_statement! + elsif parse_lex_fn! elsif parse_tree_statement! elsif parse_tree_prefix_statement! elsif parse_tree_suffix_statement! @@ -117,6 +119,12 @@ class Propane end end + def parse_lex_fn! + if md = consume!(/lex_fn\b\s*(\w+)\s*;/) + @lex_fn = md[1] + end + end + def parse_tree_statement! if consume!(/tree\s*;/) @tree = true diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index bdf4b2f..384d8d6 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1699,6 +1699,88 @@ EOF results = run_test(language: language) expect(results.status).to eq 0 end + + it "allows a custom lex function" do + if language == "d" + write_grammar < 0) + { + out_token_info.token = TOKEN_a; + out_token_info.pvalue = p_value(count); + count--; + } + else + { + result = p_lex(context, out_token_info); + if (out_token_info.token == TOKEN_c) + { + count = 3; + } + } + return result; +} +>> + +ptype size_t; +lex_fn mylexfn; + +token a << $$ = 7; >> +token b << $$ = 8; >> +token c << $$ = 9; >> +Start -> << $$ = 0; >> +Start -> Start ID << $$ = ($1 << 4) | $2; >> +ID -> a << $$ = $1; >> +ID -> b << $$ = $1; >> +ID -> c << $$ = $1; >> +EOF + else + write_grammar < 0) + { + out_token_info->token = TOKEN_a; + out_token_info->pvalue = p_value(count); + count--; + } + else + { + result = p_lex(context, out_token_info); + if (out_token_info->token == TOKEN_c) + { + count = 3; + } + } + return result; +} +>> + +ptype size_t; +lex_fn mylexfn; + +token a << $$ = 7; >> +token b << $$ = 8; >> +token c << $$ = 9; >> +Start -> << $$ = 0; >> +Start -> Start ID << $$ = ($1 << 4) | $2; >> +ID -> a << $$ = $1; >> +ID -> b << $$ = $1; >> +ID -> c << $$ = $1; >> +EOF + end + run_propane(language: language) + compile("spec/test_custom_lex_fn.#{language}", language: language) + results = run_test(language: language) + expect(results.status).to eq 0 + end end end end diff --git a/spec/test_custom_lex_fn.c b/spec/test_custom_lex_fn.c new file mode 100644 index 0000000..7b4439d --- /dev/null +++ b/spec/test_custom_lex_fn.c @@ -0,0 +1,15 @@ +#include "testparser.h" +#include "testutils.h" +#include + +int main() +{ + char const * input = "cbacba"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + assert_eq(P_SUCCESS, p_parse(context)); + size_t result = p_result(context); + assert_eq(0x932187932187, result); + p_context_delete(context); + + return 0; +} diff --git a/spec/test_custom_lex_fn.d b/spec/test_custom_lex_fn.d new file mode 100644 index 0000000..76db2e3 --- /dev/null +++ b/spec/test_custom_lex_fn.d @@ -0,0 +1,18 @@ +import testparser; +import std.stdio; +import testutils; + +int main() +{ + return 0; +} + +unittest +{ + string input = "cbacba"; + p_context_t * context = p_context_new(input); + assert_eq(P_SUCCESS, p_parse(context)); + size_t result = p_result(context); + assert_eq(0x932187932187, result); + p_context_delete(context); +}