josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

allow one line user code blocks - close #21

Browse Source
This commit is contained in:
Josh Holtrop 2024-04-02 17:44:15 -04:00
parent 918dc7b2bb
commit 3eaf0d3d49
3 changed files with 89 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
doc/user_guide.md
View File

@ -77,33 +77,15 @@ token rparen /\\)/;
# Drop whitespace. # Drop whitespace.
drop /\\s+/; drop /\\s+/;
Start -> E1 << Start -> E1 << $$ = $1; >>
$$ = $1; E1 -> E2 << $$ = $1; >>
>> E1 -> E1 plus E2 << $$ = $1 + $3; >>
E1 -> E2 << E2 -> E3 << $$ = $1; >>
$$ = $1; E2 -> E2 times E3 << $$ = $1 * $3; >>
>> E3 -> E4 << $$ = $1; >>
E1 -> E1 plus E2 << E3 -> E3 power E4 << $$ = pow($1, $3); >>
$$ = $1 + $3; E4 -> integer << $$ = $1; >>
>> E4 -> lparen E1 rparen << $$ = $2; >>
E2 -> E3 <<
$$ = $1;
>>
E2 -> E2 times E3 <<
$$ = $1 * $3;
>>
E3 -> E4 <<
$$ = $1;
>>
E3 -> E3 power E4 <<
$$ = pow($1, $3);
>>
E4 -> integer <<
$$ = $1;
>>
E4 -> lparen E1 rparen <<
$$ = $2;
>>
``` ```
Grammar files can contain comment lines beginning with `#` which are ignored. Grammar files can contain comment lines beginning with `#` which are ignored.
@ -117,8 +99,8 @@ lowercase character and beginning a rule name with an uppercase character.
##> User Code Blocks ##> User Code Blocks
User code blocks begin with the line following a "<<" token and end with the User code blocks begin following a "<<" token and end with a ">>" token found
line preceding a grammar line consisting of solely the ">>" token. at the end of a line.
All text lines in the code block are copied verbatim into the output file. All text lines in the code block are copied verbatim into the output file.
### Standalone Code Blocks ### Standalone Code Blocks
@ -189,9 +171,7 @@ This parser value can then be used later in a parser rule.
Example: Example:
``` ```
E1 -> E1 plus E2 << E1 -> E1 plus E2 << $$ = $1 + $3; >>
$$ = $1 + $3;
>>
``` ```
Parser rule code blocks appear following a rule expression. Parser rule code blocks appear following a rule expression.
@ -238,9 +218,7 @@ lexer.
Example: Example:
``` ```
token if << token if << writeln("'if' keyword lexed"); >>
writeln("'if' keyword lexed");
>>
``` ```
The `token` statement is actually a shortcut statement for a combination of a The `token` statement is actually a shortcut statement for a combination of a
@ -277,9 +255,7 @@ code but may not result in a matched token.
Example: Example:
``` ```
/foo+/ << /foo+/ << writeln("saw a foo pattern"); >>
writeln("saw a foo pattern");
>>
``` ```
This can be especially useful with ${#Lexer modes}. This can be especially useful with ${#Lexer modes}.
@ -388,9 +364,7 @@ tokenid str;
mystringvalue = ""; mystringvalue = "";
$mode(string); $mode(string);
>> >>
string: /[^"]+/ << string: /[^"]+/ << mystringvalue += match; >>
mystringvalue += match;
>>
string: /"/ << string: /"/ <<
$mode(default); $mode(default);
return $token(str); return $token(str);
@ -447,20 +421,12 @@ ptype Value;
ptype array = Value[]; ptype array = Value[];
ptype dict = Value[string]; ptype dict = Value[string];
Object -> lbrace rbrace << Object -> lbrace rbrace << $$ = new Value(); >>
$$ = new Value();
>>
Values (array) -> Value << Values (array) -> Value << $$ = [$1]; >>
$$ = [$1]; Values -> Values comma Value << $$ = $1 ~ [$3]; >>
>>
Values -> Values comma Value <<
$$ = $1 ~ [$3];
>>
KeyValue (dict) -> string colon Value << KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
$$ = [$1: $3];
>>
``` ```
In this example, the default parser value type is `Value`. In this example, the default parser value type is `Value`.
@ -493,12 +459,8 @@ Example:
``` ```
ptype ulong; ptype ulong;
token word /[a-z]+/ << token word /[a-z]+/ << $$ = match.length; >>
$$ = match.length; Start -> word << $$ = $1; >>
>>
Start -> word <<
$$ = $1;
>>
``` ```
In the above example the `Start` rule is defined to match a single `word` In the above example the `Start` rule is defined to match a single `word`
@ -507,33 +469,15 @@ token.
Example: Example:
``` ```
Start -> E1 << Start -> E1 << $$ = $1; >>
$$ = $1; E1 -> E2 << $$ = $1; >>
>> E1 -> E1 plus E2 << $$ = $1 + $3; >>
E1 -> E2 << E2 -> E3 << $$ = $1; >>
$$ = $1; E2 -> E2 times E3 << $$ = $1 * $3; >>
>> E3 -> E4 << $$ = $1; >>
E1 -> E1 plus E2 << E3 -> E3 power E4 << $$ = pow($1, $3); >>
$$ = $1 + $3; E4 -> integer << $$ = $1; >>
>> E4 -> lparen E1 rparen << $$ = $2; >>
E2 -> E3 <<
$$ = $1;
>>
E2 -> E2 times E3 <<
$$ = $1 * $3;
>>
E3 -> E4 <<
$$ = $1;
>>
E3 -> E3 power E4 <<
$$ = pow($1, $3);
>>
E4 -> integer <<
$$ = $1;
>>
E4 -> lparen E1 rparen <<
$$ = $2;
>>
``` ```
A parser rule has zero or more terms on the right side of its definition. A parser rule has zero or more terms on the right side of its definition.
@ -596,9 +540,7 @@ To terminate parsing from a lexer or parser user code block, use the
For example: For example:
``` ```
NewExpression -> new Expression << NewExpression -> new Expression << $terminate(42); >>
$terminate(42);
>>
``` ```
The value passed to the `$terminate()` function is known as the "user terminate The value passed to the `$terminate()` function is known as the "user terminate

11
lib/propane/grammar.rb
View File

@ -183,8 +183,10 @@ class Propane
end end
def parse_code_block_statement! def parse_code_block_statement!
if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m) if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
name, code = md[1..2] name, code = md[1..2]
code.sub!(/\A\n/, "")
code += "\n" unless code.end_with?("\n")
if @code_blocks[name] if @code_blocks[name]
@code_blocks[name] += code @code_blocks[name] += code
else else
@ -222,8 +224,11 @@ class Propane
end end
def parse_code_block! def parse_code_block!
if md = consume!(/<<\n(.*?)^>>\n/m) if md = consume!(/<<(.*?)>>\n/m)
md[1] code = md[1]
code.sub!(/\A\n/, "")
code += "\n" unless code.end_with?("\n")
code
end end
end end

198
spec/propane_spec.rb
View File

@ -123,10 +123,8 @@ token plus /\\+/;
token times /\\*/; token times /\\*/;
drop /\\s+/; drop /\\s+/;
Start -> Foo; Start -> Foo;
Foo -> int << Foo -> int <<>>
>> Foo -> plus <<>>
Foo -> plus <<
>>
EOF EOF
build_parser(language: language) build_parser(language: language)
compile("spec/test_lexer.#{language}", language: language) compile("spec/test_lexer.#{language}", language: language)
@ -149,9 +147,7 @@ token int /\\d+/ <<
} }
$$ = v; $$ = v;
>> >>
Start -> int << Start -> int << $$ = $1; >>
$$ = $1;
>>
EOF EOF
when "d" when "d"
write_grammar <<EOF write_grammar <<EOF
@ -165,9 +161,7 @@ token int /\\d+/ <<
} }
$$ = v; $$ = v;
>> >>
Start -> int << Start -> int << $$ = $1; >>
$$ = $1;
>>
EOF EOF
end end
build_parser(language: language) build_parser(language: language)
@ -219,33 +213,15 @@ token lparen /\\(/;
token rparen /\\)/; token rparen /\\)/;
drop /\\s+/; drop /\\s+/;
Start -> E1 << Start -> E1 << $$ = $1; >>
$$ = $1; E1 -> E2 << $$ = $1; >>
>> E1 -> E1 plus E2 << $$ = $1 + $3; >>
E1 -> E2 << E2 -> E3 << $$ = $1; >>
$$ = $1; E2 -> E2 times E3 << $$ = $1 * $3; >>
>> E3 -> E4 << $$ = $1; >>
E1 -> E1 plus E2 << E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
$$ = $1 + $3; E4 -> integer << $$ = $1; >>
>> E4 -> lparen E1 rparen << $$ = $2; >>
E2 -> E3 <<
$$ = $1;
>>
E2 -> E2 times E3 <<
$$ = $1 * $3;
>>
E3 -> E4 <<
$$ = $1;
>>
E3 -> E3 power E4 <<
$$ = (size_t)pow($1, $3);
>>
E4 -> integer <<
$$ = $1;
>>
E4 -> lparen E1 rparen <<
$$ = $2;
>>
EOF EOF
when "d" when "d"
write_grammar <<EOF write_grammar <<EOF
@ -271,33 +247,15 @@ token lparen /\\(/;
token rparen /\\)/; token rparen /\\)/;
drop /\\s+/; drop /\\s+/;
Start -> E1 << Start -> E1 << $$ = $1; >>
$$ = $1; E1 -> E2 << $$ = $1; >>
>> E1 -> E1 plus E2 << $$ = $1 + $3; >>
E1 -> E2 << E2 -> E3 << $$ = $1; >>
$$ = $1; E2 -> E2 times E3 << $$ = $1 * $3; >>
>> E3 -> E4 << $$ = $1; >>
E1 -> E1 plus E2 << E3 -> E3 power E4 << $$ = pow($1, $3); >>
$$ = $1 + $3; E4 -> integer << $$ = $1; >>
>> E4 -> lparen E1 rparen << $$ = $2; >>
E2 -> E3 <<
$$ = $1;
>>
E2 -> E2 times E3 <<
$$ = $1 * $3;
>>
E3 -> E4 <<
$$ = $1;
>>
E3 -> E3 power E4 <<
$$ = pow($1, $3);
>>
E4 -> integer <<
$$ = $1;
>>
E4 -> lparen E1 rparen <<
$$ = $2;
>>
EOF EOF
end end
build_parser(language: language) build_parser(language: language)
@ -408,9 +366,7 @@ EOF
import std.stdio; import std.stdio;
>> >>
token abc; token abc;
/def/ << /def/ << writeln("def!"); >>
writeln("def!");
>>
Start -> abc; Start -> abc;
EOF EOF
end end
@ -435,9 +391,7 @@ EOF
#include <stdio.h> #include <stdio.h>
>> >>
token abc; token abc;
/def/ << /def/ << printf("def!\\n"); >>
printf("def!\\n");
>>
/ghi/ << /ghi/ <<
printf("ghi!\\n"); printf("ghi!\\n");
return $token(abc); return $token(abc);
@ -450,9 +404,7 @@ EOF
import std.stdio; import std.stdio;
>> >>
token abc; token abc;
/def/ << /def/ << writeln("def!"); >>
writeln("def!");
>>
/ghi/ << /ghi/ <<
writeln("ghi!"); writeln("ghi!");
return $token(abc); return $token(abc);
@ -541,15 +493,9 @@ EOF
>> >>
token a; token a;
token b; token b;
Start -> A B << Start -> A B << printf("Start!\\n"); >>
printf("Start!\\n"); A -> a << printf("A!\\n"); >>
>> B -> b << printf("B!\\n"); >>
A -> a <<
printf("A!\\n");
>>
B -> b <<
printf("B!\\n");
>>
EOF EOF
when "d" when "d"
write_grammar <<EOF write_grammar <<EOF
@ -558,15 +504,9 @@ import std.stdio;
>> >>
token a; token a;
token b; token b;
Start -> A B << Start -> A B << writeln("Start!"); >>
writeln("Start!"); A -> a << writeln("A!"); >>
>> B -> b << writeln("B!"); >>
A -> a <<
writeln("A!");
>>
B -> b <<
writeln("B!");
>>
EOF EOF
end end
build_parser(language: language) build_parser(language: language)
@ -584,15 +524,9 @@ EOF
write_grammar <<EOF write_grammar <<EOF
ptype #{language == "c" ? "uint32_t" : "uint"}; ptype #{language == "c" ? "uint32_t" : "uint"};
token a; token a;
Start -> As << Start -> As << $$ = $1; >>
$$ = $1; As -> << $$ = 0u; >>
>> As -> As a << $$ = $1 + 1u; >>
As -> <<
$$ = 0u;
>>
As -> As a <<
$$ = $1 + 1u;
>>
EOF EOF
build_parser(language: language) build_parser(language: language)
compile("spec/test_parsing_lists.#{language}", language: language) compile("spec/test_parsing_lists.#{language}", language: language)
@ -756,9 +690,7 @@ token b;
token c; token c;
Start -> Any; Start -> Any;
Any -> a Any; Any -> a Any;
Any -> b Any << Any -> b Any << $terminate(4200); >>
$terminate(4200);
>>
Any -> c Any; Any -> c Any;
Any -> ; Any -> ;
EOF EOF
@ -777,30 +709,14 @@ EOF
#include <stdio.h> #include <stdio.h>
>> >>
tokenid t; tokenid t;
/\\a/ << /\\a/ << printf("A\\n"); >>
printf("A\\n"); /\\b/ << printf("B\\n"); >>
>> /\\t/ << printf("T\\n"); >>
/\\b/ << /\\n/ << printf("N\\n"); >>
printf("B\\n"); /\\v/ << printf("V\\n"); >>
>> /\\f/ << printf("F\\n"); >>
/\\t/ << /\\r/ << printf("R\\n"); >>
printf("T\\n"); /t/ << return $token(t); >>
>>
/\\n/ <<
printf("N\\n");
>>
/\\v/ <<
printf("V\\n");
>>
/\\f/ <<
printf("F\\n");
>>
/\\r/ <<
printf("R\\n");
>>
/t/ <<
return $token(t);
>>
Start -> t; Start -> t;
EOF EOF
when "d" when "d"
@ -809,27 +725,13 @@ EOF
import std.stdio; import std.stdio;
>> >>
tokenid t; tokenid t;
/\\a/ << /\\a/ << writeln("A"); >>
writeln("A"); /\\b/ << writeln("B"); >>
>> /\\t/ << writeln("T"); >>
/\\b/ << /\\n/ << writeln("N"); >>
writeln("B"); /\\v/ << writeln("V"); >>
>> /\\f/ << writeln("F"); >>
/\\t/ << /\\r/ << writeln("R"); >>
writeln("T");
>>
/\\n/ <<
writeln("N");
>>
/\\v/ <<
writeln("V");
>>
/\\f/ <<
writeln("F");
>>
/\\r/ <<
writeln("R");
>>
/t/ << /t/ <<
return $token(t); return $token(t);
>> >>