josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

complete adding lex_fn

Browse Source
This commit is contained in:
Josh Holtrop 2026-04-10 16:03:45 -04:00
parent e2db9c95c5
commit 0e5b0b1f8a
7 changed files with 101 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
assets/parser.d.erb
View File

@ -86,6 +86,14 @@ public union <%= @grammar.prefix %>value_t
<%= typestring %> v_<%= name %>;
<% end %>
}
/** Parser value constructor(s). */
<% @grammar.ptypes.each do |name, typestring| %>
public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
{
return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
}
<% end %>
<% end %>
<% if @grammar.tree %>

8
assets/parser.h.erb
View File

@ -69,6 +69,14 @@ typedef union
<%= typestring %> v_<%= name %>;
<% end %>
} <%= @grammar.prefix %>value_t;
/** Parser value constructor(s). */
<% @grammar.ptypes.each do |name, typestring| %>
static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
{
return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
}
<% end %>
<% end %>
<% if @grammar.tree %>

2
lib/propane/generator.rb
View File

@ -359,7 +359,7 @@ class Propane
# @return [String]
# Lex function to use.
def lex_fn
@grammar.custom_lex_fn || "#{@grammar.prefix}lex"
@grammar.lex_fn || "#{@grammar.prefix}lex"
end
# Get the parser value type for the start rule.

10
lib/propane/grammar.rb
View File

@ -6,7 +6,7 @@ class Propane
IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
attr_reader :context_user_fields
attr_reader :custom_lex_fn
attr_reader :lex_fn
attr_reader :tree
attr_reader :tree_prefix
attr_reader :tree_suffix
@ -70,7 +70,7 @@ class Propane
elsif parse_comment_line!
elsif @modeline.nil? && parse_mode_label!
elsif parse_context_user_fields_statement!
elsif parse_custom_lex_fn!
elsif parse_lex_fn!
elsif parse_tree_statement!
elsif parse_tree_prefix_statement!
elsif parse_tree_suffix_statement!
@ -119,9 +119,9 @@ class Propane
end
end
def parse_custom_lex_fn!
if md = consume!(/custom_lex_fn\b\s*(\w+)\s*;/)
@custom_lex_fn = $1
def parse_lex_fn!
if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
@lex_fn = md[1]
end
end

56
spec/propane_spec.rb
View File

@ -1706,16 +1706,34 @@ EOF
<<
private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
{
static size_t count;
size_t result = P_SUCCESS;
if (count > 0)
{
out_token_info.token = TOKEN_a;
out_token_info.pvalue = p_value(count);
count--;
}
else
{
result = p_lex(context, out_token_info);
if (out_token_info.token == TOKEN_c)
{
count = 3;
}
}
return result;
}
>>
tree;
ptype size_t;
lex_fn mylexfn;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
token a << $$ = 7; >>
token b << $$ = 8; >>
token c << $$ = 9; >>
Start -> << $$ = 0; >>
Start -> ID Start << $$ = ($1 << 4) | $2; >>
Start -> Start ID << $$ = ($1 << 4) | $2; >>
ID -> a << $$ = $1; >>
ID -> b << $$ = $1; >>
ID -> c << $$ = $1; >>
@ -1725,16 +1743,34 @@ EOF
<<
static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
{
static size_t count;
size_t result = P_SUCCESS;
if (count > 0)
{
out_token_info->token = TOKEN_a;
out_token_info->pvalue = p_value(count);
count--;
}
else
{
result = p_lex(context, out_token_info);
if (out_token_info->token == TOKEN_c)
{
count = 3;
}
}
return result;
}
>>
tree;
ptype size_t;
lex_fn mylexfn;
token a << $$ = 1; >>
token b << $$ = 2; >>
token c << $$ = 3; >>
token a << $$ = 7; >>
token b << $$ = 8; >>
token c << $$ = 9; >>
Start -> << $$ = 0; >>
Start -> ID Start << $$ = ($1 << 4) | $2; >>
Start -> Start ID << $$ = ($1 << 4) | $2; >>
ID -> a << $$ = $1; >>
ID -> b << $$ = $1; >>
ID -> c << $$ = $1; >>

15
spec/test_custom_lex_fn.c Normal file
View File

@ -0,0 +1,15 @@
#include "testparser.h"
#include "testutils.h"
#include <string.h>
int main()
{
char const * input = "cbacba";
p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
assert_eq(P_SUCCESS, p_parse(context));
size_t result = p_result(context);
assert_eq(0x932187932187, result);
p_context_delete(context);
return 0;
}

18
spec/test_custom_lex_fn.d Normal file
View File

@ -0,0 +1,18 @@
import testparser;
import std.stdio;
import testutils;
int main()
{
return 0;
}
unittest
{
string input = "cbacba";
p_context_t * context = p_context_new(input);
assert_eq(P_SUCCESS, p_parse(context));
size_t result = p_result(context);
assert_eq(0x932187932187, result);
p_context_delete(context);
}