josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Rename match to match_text for lexer user code blocks

Browse Source
This commit is contained in:
Josh Holtrop 2026-08-19 23:30:27 -04:00
parent 47ad55ee2c
commit d596d47cef
23 changed files with 124 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -6,6 +6,11 @@
### API Changes
- The matched text argument passed to lexer user code blocks is now named
`match_text` for every target language. It was previously named `match` for
C, C++, and D, and `match_` for Rust. Any lexer user code block which
references the matched text must be updated to use the new name. The
`match_length` argument (C, C++, and Rust) is unchanged.
- Tree generation mode now stores all tree nodes in a compact arena owned by
the parser context (a flat node array plus a shared child-link array).
This replaces the previous design of one heap allocation per node with

2
README.md
View File

@ -69,7 +69,7 @@ token times /\*/;
token power /\*\*/;
token integer /\d+/ <<
ulong v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');

15
UPGRADING.md
View File

@ -2,7 +2,20 @@
The generated API for tree generation mode (`tree;`) has been changed
significantly for this version.
The lexer/parser value APIs for non-tree grammars are unchanged.
Aside from the lexer user code block matched text rename described below, the
lexer/parser value APIs for non-tree grammars are unchanged.
### Lexer user code block matched text
The matched text argument passed to lexer user code blocks has been renamed
from `match` to `match_text` for all target languages.
- C, C++, and D: rename references to `match` in lexer user code blocks to
`match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
The `match_length` argument (C, C++) is unchanged.
This rename only affects lexer user code blocks; parser rule user code blocks
never had a matched text argument.
### Tree memory management

8
assets/parser.c.erb
View File

@ -319,7 +319,7 @@ static lexer_mode_t lexer_mode_table[] = {
* Lexer/parser context structure.
* @param code_id
* The ID of the user code block to execute.
* @param match
* @param match_text
* Matched text for this pattern.
* @param match_length
* Matched text length.
@ -330,7 +330,7 @@ static lexer_mode_t lexer_mode_table[] = {
* not explicitly return a token.
*/
static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
lexer_user_code_id_t code_id, uint8_t const * match,
lexer_user_code_id_t code_id, uint8_t const * match_text,
size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
{
switch (code_id)
@ -534,9 +534,9 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
}
if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
{
uint8_t const * match = &context->input[context->input_index];
uint8_t const * match_text = &context->input[context->input_index];
<%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
match_info.accepting_state->code_id, match, match_info.length, &token_info);
match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
* that the user code is requesting to terminate the lexer. */
if (user_code_token == TERMINATE_TOKEN_ID)

8
assets/parser.d.erb
View File

@ -560,7 +560,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
* Lexer/parser context structure.
* @param code_id
* The ID of the user code block to execute.
* @param match
* @param match_text
* Matched text for this pattern.
* @param out_token_info
* Lexer token info in progress.
@ -569,7 +569,7 @@ private immutable lexer_mode_t[] lexer_mode_table = [
* not explicitly return a token.
*/
private <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
lexer_user_code_id_t code_id, string match,
lexer_user_code_id_t code_id, string match_text,
<%= @grammar.prefix %>token_info_t * out_token_info)
{
switch (code_id)
@ -765,9 +765,9 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
}
if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
{
string match = context.input[context.input_index..(context.input_index + match_info.length)];
string match_text = context.input[context.input_index..(context.input_index + match_info.length)];
<%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
match_info.accepting_state.code_id, match, &token_info);
match_info.accepting_state.code_id, match_text, &token_info);
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
* that the user code is requesting to terminate the lexer. */
if (user_code_token == TERMINATE_TOKEN_ID)

6
assets/parser.rs.erb
View File

@ -482,8 +482,10 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
* Lexer/parser context structure.
* @param code_id
* The ID of the user code block to execute.
* @param match
* @param match_text
* Matched text for this pattern.
* @param match_length
* Matched text length.
* @param out_token_info
* Lexer token info in progress.
*
@ -491,7 +493,7 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [
* not explicitly return a token.
*/
fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t,
code_id: <%= get_type_for(user_code_id_count) %>, match_: &[u8],
code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8],
match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t {
match code_id {
<% @grammar.patterns.each do |pattern| %>

38
doc/user_guide.md
View File

@ -90,7 +90,7 @@ token times /\*/;
token power /\*\*/;
token integer /\d+/ <<
ulong v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@ -197,7 +197,7 @@ Example:
token integer /\d+/ <<
printf("integer token on row %d, col %d\n",
${position}.row, ${position}.col);
$$ = parse_integer(match, match_length);
$$ = parse_integer(match_text, match_length);
>>
```
@ -205,7 +205,7 @@ token integer /\d+/ <<
The lexer code block is passed the following arguments:
* `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
* `match_text` (`uint8_t const *`) - points to the text matched by the lexer pattern.
* `match_length` (`size_t`) - length of the matched text.
Example:
@ -218,7 +218,7 @@ token integer /\d+/ <<
for (size_t i = 0u; i < match_length; i++)
{
v *= 10;
v += (match[i] - '0');
v += (match_text[i] - '0');
}
$$ = v;
>>
@ -228,14 +228,14 @@ token integer /\d+/ <<
The lexer code block is passed the following arguments:
* `match` (`string`) - a slice containing the text matched by the lexer pattern.
* `match_text` (`string`) - a slice containing the text matched by the lexer pattern.
```
ptype ulong;
token integer /\d+/ <<
ulong v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@ -248,20 +248,18 @@ token integer /\d+/ <<
The lexer code block is passed the following arguments:
* `match_` (`&[u8]`) - a slice containing the text matched by the lexer pattern.
* `match_text` (`&[u8]`) - a slice containing the text matched by the lexer pattern.
* `match_length` (`usize`) - length of the matched text.
The argument is named `match_` rather than `match` because `match` is a Rust
keyword.
It is a byte slice rather than a string; use `std::str::from_utf8()` or
`String::from_utf8_lossy()` to view the matched text as a string.
The matched text is a byte slice rather than a string; use
`std::str::from_utf8()` or `String::from_utf8_lossy()` to view it as a string.
```
ptype i64;
token integer /\d+/ <<
let mut v: i64 = 0;
for c in match_
for c in match_text
{
v *= 10;
v += (c - b'0') as i64;
@ -423,7 +421,7 @@ context_user_fields <<
>>
drop /#(.*)\n/ <<
/* Accumulate comments before the next parser tree node. */
${context.comments} += std::string((const char *)match, match_length);
${context.comments} += std::string((const char *)match_text, match_length);
>>
```
@ -435,7 +433,7 @@ context_user_fields <<
>>
drop /#(.*)\n/ <<
/* Accumulate comments before the next parser tree node. */
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
```
@ -723,7 +721,7 @@ on_token_node <<
>>
drop /#(.*)\n/ <<
/* Accumulate comments before the next parser tree node. */
${context.comments} += std::string((const char *)match, match_length);
${context.comments} += std::string((const char *)match_text, match_length);
>>
```
@ -741,7 +739,7 @@ on_token_node <<
>>
drop /#(.*)\n/ <<
/* Accumulate comments before the next parser tree node. */
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
```
@ -988,7 +986,7 @@ on_token_node <<
>>
drop /#(.*)\n/ <<
/* Accumulate comments before the next parser tree node. */
${context.comments} += std::string((const char *)match, match_length);
${context.comments} += std::string((const char *)match_text, match_length);
>>
```
@ -1260,7 +1258,7 @@ tokenid str;
mystringvalue = "";
$mode(string);
>>
string: /[^"]+/ << mystringvalue ~= match; >>
string: /[^"]+/ << mystringvalue ~= match_text; >>
string: /"/ <<
$mode(default);
return $token(str);
@ -1300,7 +1298,7 @@ ptype char;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match[0];
$$ = match_text[0];
$mode(default);
return $token(ident);
>>
@ -1334,7 +1332,7 @@ Example:
```
ptype ulong;
start Top;
token word /[a-z]+/ << $$ = match.length; >>
token word /[a-z]+/ << $$ = match_text.length; >>
Top -> word << $$ = $1; >>
```

26
spec/json_parser.c.propane
View File

@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
double n = 0.0;
bool negative = false;
size_t i = 0u;
if (match[i] == '-')
if (match_text[i] == '-')
{
negative = true;
i++;
}
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n *= 10.0;
n += (match[i] - '0');
n += (match_text[i] - '0');
i++;
}
if (match[i] == '.')
if (match_text[i] == '.')
{
i++;
double mult = 0.1;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n += mult * (match[i] - '0');
n += mult * (match_text[i] - '0');
mult /= 10.0;
i++;
}
}
if (match[i] == 'e' || match[i] == 'E')
if (match_text[i] == 'e' || match_text[i] == 'E')
{
bool exp_negative = false;
i++;
if (match[i] == '-')
if (match_text[i] == '-')
{
exp_negative = true;
i++;
}
else if (match[i] == '+')
else if (match_text[i] == '+')
{
i++;
}
long exp = 0.0;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
exp *= 10;
exp += (match[i] - '0');
exp += (match_text[i] - '0');
i++;
}
if (exp_negative)
@ -120,11 +120,11 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
str_append(&string_value, s);
>>
string: /[^\\]/ <<
char s[] = {(char)match[0], 0};
char s[] = {(char)match_text[0], 0};
str_append(&string_value, s);
>>
Start -> Value <<

26
spec/json_parser.d.propane
View File

@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
double n;
bool negative;
size_t i = 0u;
if (match[i] == '-')
if (match_text[i] == '-')
{
negative = true;
i++;
}
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n *= 10.0;
n += (match[i] - '0');
n += (match_text[i] - '0');
i++;
}
if (match[i] == '.')
if (match_text[i] == '.')
{
i++;
double mult = 0.1;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n += mult * (match[i] - '0');
n += mult * (match_text[i] - '0');
mult /= 10.0;
i++;
}
}
if (match[i] == 'e' || match[i] == 'E')
if (match_text[i] == 'e' || match_text[i] == 'E')
{
bool exp_negative;
i++;
if (match[i] == '-')
if (match_text[i] == '-')
{
exp_negative = true;
i++;
}
else if (match[i] == '+')
else if (match_text[i] == '+')
{
i++;
}
long exp;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
exp *= 10;
exp += (match[i] - '0');
exp += (match_text[i] - '0');
i++;
}
if (exp_negative)
@ -117,10 +117,10 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
string_value ~= "{" ~ match[2..6] ~ "}";
string_value ~= "{" ~ match_text[2..6] ~ "}";
>>
string: /[^\\]/ <<
string_value ~= match;
string_value ~= match_text;
>>
Start -> Value <<
$$ = $1;

6
spec/json_parser.rust.propane
View File

@ -60,7 +60,7 @@ token rbracket /\]/;
token comma /,/;
token colon /:/;
token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
$$ = JSONValue::Number(n);
>>
token true <<
@ -106,11 +106,11 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
${context.string_value}.push_str(&s);
>>
string: /[^\\]/ <<
${context.string_value}.push(match_[0] as char);
${context.string_value}.push(match_text[0] as char);
>>
Start -> Value <<
$$ = $1;

2
spec/macros.c.propane
View File

@ -14,7 +14,7 @@ token rbrace /\}/;
token plus /\+/;
token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Statements;
Statements -> ;

2
spec/macros.d.propane
View File

@ -14,7 +14,7 @@ token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ <<
int n = 0;
foreach (c; match)
foreach (c; match_text)
{
n *= 10;
n += (c - '0');

2
spec/macros.rust.propane
View File

@ -67,7 +67,7 @@ token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>

2
spec/parse_inner_nested.c.propane
View File

@ -12,7 +12,7 @@ drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Expr << $$ = $1; >>
Expr -> num << $$ = $1; >>

2
spec/parse_inner_nested.d.propane
View File

@ -10,7 +10,7 @@ drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
Start -> Expr << $$ = $1; >>
Expr -> num << $$ = $1; >>

2
spec/parse_inner_nested.rust.propane
View File

@ -32,7 +32,7 @@ token rparen /\)/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>

58
spec/propane_spec.rb
View File

@ -343,7 +343,7 @@ token int /\\d+/ <<
for (size_t i = 0u; i < match_length; i++)
{
v *= 10;
v += (match[i] - '0');
v += (match_text[i] - '0');
}
$$ = v;
>>
@ -354,7 +354,7 @@ EOF
ptype int;
token int /\\d+/ <<
int v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@ -368,7 +368,7 @@ EOF
ptype i64;
token int /\\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> int << $$ = $1; >>
@ -415,7 +415,7 @@ token integer /\\d+/ <<
for (size_t i = 0u; i < match_length; i++)
{
v *= 10;
v += (match[i] - '0');
v += (match_text[i] - '0');
}
$$ = v;
>>
@ -446,7 +446,7 @@ token times /\\*/;
token power /\\*\\*/;
token integer /\\d+/ <<
ulong v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@ -475,7 +475,7 @@ token times /\\*/;
token power /\\*\\*/;
token integer /\\d+/ <<
let mut v: u64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as u64; }
for c in match_text { v = v * 10 + (*c - b'0') as u64; }
$$ = v;
>>
token lparen /\\(/;
@ -775,7 +775,7 @@ ptype char;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match[0];
$$ = match_text[0];
$mode(default);
return $token(ident);
>>
@ -796,7 +796,7 @@ ptype char;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match[0];
$$ = match_text[0];
$mode(default);
>>
token dot /\\./ <<
@ -813,7 +813,7 @@ ptype u8;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match_[0];
$$ = match_text[0];
$mode(default);
return $token(ident);
>>
@ -1064,7 +1064,7 @@ EOF
>>
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * t = (char *)malloc(match_length + 1);
strncpy(t, (char *)match, match_length);
strncpy(t, (char *)match_text, match_length);
printf("Matched token is %s\\n", t);
free(t);
>>
@ -1076,14 +1076,14 @@ EOF
import std.stdio;
>>
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
writeln("Matched token is ", match);
writeln("Matched token is ", match_text);
>>
Start -> id;
EOF
when "rust"
write_grammar <<EOF
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
println!("Matched token is {}", std::str::from_utf8(match_text).unwrap());
>>
Start -> id;
EOF
@ -1114,7 +1114,7 @@ EOF
write_grammar <<EOF
ptype ulong;
token word /[a-z]+/ <<
$$ = match.length;
$$ = match_text.length;
>>
Start -> word <<
$$ = $1;
@ -1793,7 +1793,7 @@ EOF
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
$$ = std::str::from_utf8(match_text).unwrap().to_string();
>>
drop /\\s+/;
Start -> id:first id:second <<
@ -1808,7 +1808,7 @@ import std.stdio;
>>
ptype string;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = match;
$$ = match_text;
>>
drop /\\s+/;
Start -> id:first id:second <<
@ -1825,7 +1825,7 @@ EOF
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
strncpy(s, (char const *)match_text, match_length);
s[match_length] = 0;
$$ = s;
>>
@ -1851,7 +1851,7 @@ EOF
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
$$ = std::str::from_utf8(match_text).unwrap().to_string();
>>
drop /\\s+/;
Start -> id;
@ -1869,7 +1869,7 @@ import std.stdio;
>>
ptype string;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = match;
$$ = match_text;
>>
drop /\\s+/;
Start -> id;
@ -1889,7 +1889,7 @@ EOF
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
strncpy(s, (char const *)match_text, match_length);
s[match_length] = 0;
$$ = s;
>>
@ -2087,7 +2087,7 @@ EOF
write_grammar <<EOF
drop /\\s+/;
drop /#(.*)\\n/ <<
eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
eprint!("comment: {}", std::str::from_utf8(match_text).unwrap());
>>
token a;
Start -> a;
@ -2099,7 +2099,7 @@ import std.stdio;
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
stderr.write("comment: ", match);
stderr.write("comment: ", match_text);
>>
token a;
Start -> a;
@ -2112,7 +2112,7 @@ EOF
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
fprintf(stderr, "comment: %.*s", (int)match_length, match);
fprintf(stderr, "comment: %.*s", (int)match_length, match_text);
>>
token a;
Start -> a;
@ -2134,7 +2134,7 @@ context_user_fields <<
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
token a <<
${context.acount} += 1;
@ -2151,7 +2151,7 @@ context_user_fields <<
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} ~= match;
${context.comments} ~= match_text;
>>
token a <<
${context.acount}++;
@ -2178,7 +2178,7 @@ drop /#(.*)\\n/ <<
char * commentsnew = (char *)malloc(cur_len + match_length + 1);
if (${context.comments} != NULL)
memcpy(commentsnew, ${context.comments}, cur_len);
memcpy(&commentsnew[cur_len], match, match_length);
memcpy(&commentsnew[cur_len], match_text, match_length);
commentsnew[cur_len + match_length] = '\\0';
if (${context.comments} != NULL)
{
@ -2217,7 +2217,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
token id /\\w+/;
Start -> IDs;
@ -2239,7 +2239,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} ~= match;
${context.comments} ~= match_text;
>>
token id /\\w+/;
Start -> IDs;
@ -2275,7 +2275,7 @@ drop /#(.*)\\n/ <<
char * commentsnew = (char *)malloc(cur_len + match_length + 1);
if (${context.comments} != NULL)
memcpy(commentsnew, ${context.comments}, cur_len);
memcpy(&commentsnew[cur_len], match, match_length);
memcpy(&commentsnew[cur_len], match_text, match_length);
commentsnew[cur_len + match_length] = '\\0';
if (${context.comments} != NULL)
{
@ -2306,7 +2306,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::string((const char *)match, match_length);
${context.comments} += std::string((const char *)match_text, match_length);
>>
token id /\\w+/;
Start -> IDs;

2
spec/rewind.c.propane
View File

@ -14,7 +14,7 @@ token repeat /repeat/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Statements;
Statements -> ;

2
spec/rewind.d.propane
View File

@ -11,7 +11,7 @@ token repeat /repeat/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
Start -> Statements;
Statements -> ;

2
spec/rewind.rust.propane
View File

@ -56,7 +56,7 @@ token rbrace /\}/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>

2
spec/tree_node_memory_remains.c.propane
View File

@ -46,7 +46,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
$$.s = match;
$$.s = match_text;
$mode(default);
return $token(ident);
>>

10
spec/tree_node_memory_remains.d.propane
View File

@ -42,8 +42,8 @@ token semicolon /;/;
# Integer literals.
token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
$$.bi = BigInt(match[0..3]);
foreach (c; match[3..$])
$$.bi = BigInt(match_text[0..3]);
foreach (c; match_text[3..$])
{
if (('0' <= c) && (c <= '9'))
{
@ -65,13 +65,13 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
if (match[0] == '$')
if (match_text[0] == '$')
{
$$.s = match[1..$];
$$.s = match_text[1..$];
}
else
{
$$.s = match;
$$.s = match_text;
}
$mode(default);
return $token(ident);

2
spec/tree_node_memory_remains.rust.propane
View File

@ -43,7 +43,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
$$.s = std::str::from_utf8(match_).unwrap().to_string();
$$.s = std::str::from_utf8(match_text).unwrap().to_string();
$mode(default);
return $token(ident);
>>