Rename match to match_text for lexer user code blocks

This commit is contained in:
josh committed 2026-08-19 23:30:27 -04:00
1 parent 47ad55ee2c
commit d596d47cef
23 files changed
+124 -106

No files matched your search

+13 -13
View File
@@ -21,46 +21,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
double n = 0.0;
bool negative = false;
size_t i = 0u;
if (match[i] == '-')
if (match_text[i] == '-')
{
negative = true;
i++;
}
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n *= 10.0;
n += (match[i] - '0');
n += (match_text[i] - '0');
i++;
}
if (match[i] == '.')
if (match_text[i] == '.')
{
i++;
double mult = 0.1;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n += mult * (match[i] - '0');
n += mult * (match_text[i] - '0');
mult /= 10.0;
i++;
}
}
if (match[i] == 'e' || match[i] == 'E')
if (match_text[i] == 'e' || match_text[i] == 'E')
{
bool exp_negative = false;
i++;
if (match[i] == '-')
if (match_text[i] == '-')
{
exp_negative = true;
i++;
}
else if (match[i] == '+')
else if (match_text[i] == '+')
{
i++;
}
long exp = 0.0;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
exp *= 10;
exp += (match[i] - '0');
exp += (match_text[i] - '0');
i++;
}
if (exp_negative)
@@ -120,11 +120,11 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
str_append(&string_value, s);
>>
string: /[^\\]/ <<
char s[] = {(char)match[0], 0};
char s[] = {(char)match_text[0], 0};
str_append(&string_value, s);
>>
Start -> Value <<
+13 -13
View File
@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
double n;
bool negative;
size_t i = 0u;
if (match[i] == '-')
if (match_text[i] == '-')
{
negative = true;
i++;
}
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n *= 10.0;
n += (match[i] - '0');
n += (match_text[i] - '0');
i++;
}
if (match[i] == '.')
if (match_text[i] == '.')
{
i++;
double mult = 0.1;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
n += mult * (match[i] - '0');
n += mult * (match_text[i] - '0');
mult /= 10.0;
i++;
}
}
if (match[i] == 'e' || match[i] == 'E')
if (match_text[i] == 'e' || match_text[i] == 'E')
{
bool exp_negative;
i++;
if (match[i] == '-')
if (match_text[i] == '-')
{
exp_negative = true;
i++;
}
else if (match[i] == '+')
else if (match_text[i] == '+')
{
i++;
}
long exp;
while ('0' <= match[i] && match[i] <= '9')
while ('0' <= match_text[i] && match_text[i] <= '9')
{
exp *= 10;
exp += (match[i] - '0');
exp += (match_text[i] - '0');
i++;
}
if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
string_value ~= "{" ~ match[2..6] ~ "}";
string_value ~= "{" ~ match_text[2..6] ~ "}";
>>
string: /[^\\]/ <<
string_value ~= match;
string_value ~= match_text;
>>
Start -> Value <<
$$ = $1;
+3 -3
View File
@@ -60,7 +60,7 @@ token rbracket /\]/;
token comma /,/;
token colon /:/;
token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
let n: f64 = std::str::from_utf8(match_).unwrap().parse().unwrap();
let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
$$ = JSONValue::Number(n);
>>
token true <<
@@ -106,11 +106,11 @@ string: /\\t/ <<
>>
string: /\\u[0-9a-fA-F]{4}/ <<
/* Not actually going to encode the code point for this example... */
let s: String = ['{', match_[2] as char, match_[3] as char, match_[4] as char, match_[5] as char, '}'].iter().collect();
let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
${context.string_value}.push_str(&s);
>>
string: /[^\\]/ <<
${context.string_value}.push(match_[0] as char);
${context.string_value}.push(match_text[0] as char);
>>
Start -> Value <<
$$ = $1;
+1 -1
View File
@@ -14,7 +14,7 @@ token rbrace /\}/;
token plus /\+/;
token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ << char b[100]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Statements;
Statements -> ;
+1 -1
View File
@@ -14,7 +14,7 @@ token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ <<
int n = 0;
foreach (c; match)
foreach (c; match_text)
{
n *= 10;
n += (c - '0');
+1 -1
View File
@@ -67,7 +67,7 @@ token macro;
token macroname /@[a-zA-Z_]\w*/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
+1 -1
View File
@@ -12,7 +12,7 @@ drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Expr << $$ = $1; >>
Expr -> num << $$ = $1; >>
+1 -1
View File
@@ -10,7 +10,7 @@ drop /\s+/;
token lparen /\(/;
token rparen /\)/;
token plus /\+/;
token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
Start -> Expr << $$ = $1; >>
Expr -> num << $$ = $1; >>
+1 -1
View File
@@ -32,7 +32,7 @@ token rparen /\)/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
+29 -29
View File
@@ -343,7 +343,7 @@ token int /\\d+/ <<
for (size_t i = 0u; i < match_length; i++)
{
v *= 10;
v += (match[i] - '0');
v += (match_text[i] - '0');
}
$$ = v;
>>
@@ -354,7 +354,7 @@ EOF
ptype int;
token int /\\d+/ <<
int v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@@ -368,7 +368,7 @@ EOF
ptype i64;
token int /\\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
Start -> int << $$ = $1; >>
@@ -415,7 +415,7 @@ token integer /\\d+/ <<
for (size_t i = 0u; i < match_length; i++)
{
v *= 10;
v += (match[i] - '0');
v += (match_text[i] - '0');
}
$$ = v;
>>
@@ -446,7 +446,7 @@ token times /\\*/;
token power /\\*\\*/;
token integer /\\d+/ <<
ulong v;
foreach (c; match)
foreach (c; match_text)
{
v *= 10;
v += (c - '0');
@@ -475,7 +475,7 @@ token times /\\*/;
token power /\\*\\*/;
token integer /\\d+/ <<
let mut v: u64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as u64; }
for c in match_text { v = v * 10 + (*c - b'0') as u64; }
$$ = v;
>>
token lparen /\\(/;
@@ -775,7 +775,7 @@ ptype char;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match[0];
$$ = match_text[0];
$mode(default);
return $token(ident);
>>
@@ -796,7 +796,7 @@ ptype char;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match[0];
$$ = match_text[0];
$mode(default);
>>
token dot /\\./ <<
@@ -813,7 +813,7 @@ ptype u8;
token abc;
token def;
default, identonly: token ident /[a-z]+/ <<
$$ = match_[0];
$$ = match_text[0];
$mode(default);
return $token(ident);
>>
@@ -1064,7 +1064,7 @@ EOF
>>
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * t = (char *)malloc(match_length + 1);
strncpy(t, (char *)match, match_length);
strncpy(t, (char *)match_text, match_length);
printf("Matched token is %s\\n", t);
free(t);
>>
@@ -1076,14 +1076,14 @@ EOF
import std.stdio;
>>
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
writeln("Matched token is ", match);
writeln("Matched token is ", match_text);
>>
Start -> id;
EOF
when "rust"
write_grammar <<EOF
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
println!("Matched token is {}", std::str::from_utf8(match_).unwrap());
println!("Matched token is {}", std::str::from_utf8(match_text).unwrap());
>>
Start -> id;
EOF
@@ -1114,7 +1114,7 @@ EOF
write_grammar <<EOF
ptype ulong;
token word /[a-z]+/ <<
$$ = match.length;
$$ = match_text.length;
>>
Start -> word <<
$$ = $1;
@@ -1793,7 +1793,7 @@ EOF
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
$$ = std::str::from_utf8(match_text).unwrap().to_string();
>>
drop /\\s+/;
Start -> id:first id:second <<
@@ -1808,7 +1808,7 @@ import std.stdio;
>>
ptype string;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = match;
$$ = match_text;
>>
drop /\\s+/;
Start -> id:first id:second <<
@@ -1825,7 +1825,7 @@ EOF
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
strncpy(s, (char const *)match_text, match_length);
s[match_length] = 0;
$$ = s;
>>
@@ -1851,7 +1851,7 @@ EOF
write_grammar <<EOF
ptype String;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = std::str::from_utf8(match_).unwrap().to_string();
$$ = std::str::from_utf8(match_text).unwrap().to_string();
>>
drop /\\s+/;
Start -> id;
@@ -1869,7 +1869,7 @@ import std.stdio;
>>
ptype string;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
$$ = match;
$$ = match_text;
>>
drop /\\s+/;
Start -> id;
@@ -1889,7 +1889,7 @@ EOF
ptype char *;
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
char * s = (char *)malloc(match_length + 1);
strncpy(s, (char const *)match, match_length);
strncpy(s, (char const *)match_text, match_length);
s[match_length] = 0;
$$ = s;
>>
@@ -2087,7 +2087,7 @@ EOF
write_grammar <<EOF
drop /\\s+/;
drop /#(.*)\\n/ <<
eprint!("comment: {}", std::str::from_utf8(match_).unwrap());
eprint!("comment: {}", std::str::from_utf8(match_text).unwrap());
>>
token a;
Start -> a;
@@ -2099,7 +2099,7 @@ import std.stdio;
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
stderr.write("comment: ", match);
stderr.write("comment: ", match_text);
>>
token a;
Start -> a;
@@ -2112,7 +2112,7 @@ EOF
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
fprintf(stderr, "comment: %.*s", (int)match_length, match);
fprintf(stderr, "comment: %.*s", (int)match_length, match_text);
>>
token a;
Start -> a;
@@ -2134,7 +2134,7 @@ context_user_fields <<
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
token a <<
${context.acount} += 1;
@@ -2151,7 +2151,7 @@ context_user_fields <<
>>
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} ~= match;
${context.comments} ~= match_text;
>>
token a <<
${context.acount}++;
@@ -2178,7 +2178,7 @@ drop /#(.*)\\n/ <<
char * commentsnew = (char *)malloc(cur_len + match_length + 1);
if (${context.comments} != NULL)
memcpy(commentsnew, ${context.comments}, cur_len);
memcpy(&commentsnew[cur_len], match, match_length);
memcpy(&commentsnew[cur_len], match_text, match_length);
commentsnew[cur_len + match_length] = '\\0';
if (${context.comments} != NULL)
{
@@ -2217,7 +2217,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::str::from_utf8(match_).unwrap();
${context.comments} += std::str::from_utf8(match_text).unwrap();
>>
token id /\\w+/;
Start -> IDs;
@@ -2239,7 +2239,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} ~= match;
${context.comments} ~= match_text;
>>
token id /\\w+/;
Start -> IDs;
@@ -2275,7 +2275,7 @@ drop /#(.*)\\n/ <<
char * commentsnew = (char *)malloc(cur_len + match_length + 1);
if (${context.comments} != NULL)
memcpy(commentsnew, ${context.comments}, cur_len);
memcpy(&commentsnew[cur_len], match, match_length);
memcpy(&commentsnew[cur_len], match_text, match_length);
commentsnew[cur_len + match_length] = '\\0';
if (${context.comments} != NULL)
{
@@ -2306,7 +2306,7 @@ on_token_node <<
tree;
drop /\\s+/;
drop /#(.*)\\n/ <<
${context.comments} += std::string((const char *)match, match_length);
${context.comments} += std::string((const char *)match_text, match_length);
>>
token id /\\w+/;
Start -> IDs;
+1 -1
View File
@@ -14,7 +14,7 @@ token repeat /repeat/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token num /\d+/ << char b[32]; memcpy(b, match, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
Start -> Statements;
Statements -> ;
+1 -1
View File
@@ -11,7 +11,7 @@ token repeat /repeat/;
token lbrace /\{/;
token rbrace /\}/;
token plus /\+/;
token num /\d+/ << int n = 0; foreach (ch; match) { n *= 10; n += (ch - '0'); } $$ = n; >>
token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
Start -> Statements;
Statements -> ;
+1 -1
View File
@@ -56,7 +56,7 @@ token rbrace /\}/;
token plus /\+/;
token num /\d+/ <<
let mut v: i64 = 0;
for c in match_ { v = v * 10 + (*c - b'0') as i64; }
for c in match_text { v = v * 10 + (*c - b'0') as i64; }
$$ = v;
>>
+1 -1
View File
@@ -46,7 +46,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
$$.s = match;
$$.s = match_text;
$mode(default);
return $token(ident);
>>
+5 -5
View File
@@ -42,8 +42,8 @@ token semicolon /;/;
# Integer literals.
token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
$$.bi = BigInt(match[0..3]);
foreach (c; match[3..$])
$$.bi = BigInt(match_text[0..3]);
foreach (c; match_text[3..$])
{
if (('0' <= c) && (c <= '9'))
{
@@ -65,13 +65,13 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
if (match[0] == '$')
if (match_text[0] == '$')
{
$$.s = match[1..$];
$$.s = match_text[1..$];
}
else
{
$$.s = match;
$$.s = match_text;
}
$mode(default);
return $token(ident);
+1 -1
View File
@@ -43,7 +43,7 @@ token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
# Identifier.
token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
$$.s = std::str::from_utf8(match_).unwrap().to_string();
$$.s = std::str::from_utf8(match_text).unwrap().to_string();
$mode(default);
return $token(ident);
>>