josh/propane
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Switch to new API - close #8

Browse Source 
The new API is more C-like and will allow consistency across all future
supported language targets.
This commit is contained in:
Josh Holtrop 2023-07-12 15:46:13 -04:00
parent e0e5e87338
commit 7a1b4064c1
15 changed files with 611 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

533
assets/parser.d.erb
View File

@ -91,12 +91,40 @@ public static struct TokenInfo
ParserValue pvalue;
}
/**
* Lexer and parser context.
*
* The user must allocate an instance of this structure and pass it to any
* public API function.
*/
public struct p_context_t
{
/* Lexer context data. */
/** Input text. */
string input;
/** Input text index (byte offset). */
size_t input_index;
/** Input text position (row/column). */
Position input_position;
/** Current lexer mode. */
size_t mode;
/* Parser context data. */
/** Parse result value. */
ParserValue parse_result;
}
/**************************************************************************
* Public data
*************************************************************************/
/** Token names. */
public static immutable string[] token_names = [
public static immutable string[] p_token_names = [
<% @grammar.tokens.each_with_index do |token, index| %>
"<%= token.name %>",
<% end %>
@ -109,23 +137,52 @@ public static immutable string[] token_names = [
/* An invalid ID value. */
private enum size_t INVALID_ID = cast(size_t)-1;
/**************************************************************************
* State initialization
*************************************************************************/
/**
* Initialize lexer/parser context structure.
*
* @param[out] context
* Lexer/parser context structure.
* @param input
* Text input.
*/
public void p_context_init(p_context_t * context, string input)
{
/* New default-initialized context structure. */
p_context_t newcontext;
/* Lexer initialization. */
newcontext.input = input;
newcontext.mode = <%= @lexer.mode_id("default") %>;
/* Copy to the user's context structure. */
*context = newcontext;
}
/**************************************************************************
* Decoder
*************************************************************************/
public static class Decoder
{
/**
* Decode a UTF-8 code point.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_EOF
* @param input
* Text input to decode.
* @param[out] out_code_point
* The decoded code point is stored here if the return value is P_SUCCESS.
* @param[out] out_code_point_length
* The number of bytes the code point used is stored here if the return value
* is P_SUCCESS.
*
* @retval P_SUCCESS on a successful code point decode
* @retval P_DECODE_ERROR when an encoding error is observed
* @retval P_EOF when the end of the text input is reached
*/
static size_t decode_code_point(string input,
CodePoint * out_code_point,
ubyte * out_code_point_length)
public size_t p_decode_code_point(string input,
CodePoint * out_code_point, ubyte * out_code_point_length)
{
if (input.length == 0u)
{
@ -190,7 +247,6 @@ public static class Decoder
*out_code_point_length = code_point_length;
return P_SUCCESS;
}
}
/**************************************************************************
* Lexer
@ -223,6 +279,13 @@ private struct Mode
uint state_table_offset;
}
private struct MatchInfo
{
size_t length;
Position delta_position;
const(LexerState) * accepting_state;
}
private static immutable Transition[] lexer_transitions = [
<% @lexer.transition_table.each do |transition_table_entry| %>
Transition(<%= transition_table_entry[:first] %>u,
@ -255,50 +318,22 @@ private static immutable Mode[] modes = [
<% end %>
];
public static class Lexer
{
private string m_input;
private size_t m_input_index;
private Position m_input_position;
private size_t m_mode;
this(string input)
{
m_input = input;
m_mode = <%= @lexer.mode_id("default") %>;
}
/**
* Lex the next token in the input stream.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
*/
size_t lex_token(TokenInfo * out_token_info)
{
for (;;)
{
size_t result = attempt_lex_token(out_token_info);
if (result != P_DROP)
{
return result;
}
}
}
/**
* Execute user code associated with a lexer pattern.
*
* @param code_id The ID of the user code block to execute.
* @param match Matched text for this pattern.
* @param out_token_info Lexer token info in progress.
* @param context
* Lexer/parser context structure.
* @param code_id
* The ID of the user code block to execute.
* @param match
* Matched text for this pattern.
* @param out_token_info
* Lexer token info in progress.
*
* @return Token to accept, or invalid token if the user code does
* not explicitly return a token.
*/
private Token user_code(UserCodeID code_id, string match, TokenInfo * out_token_info)
private Token lexer_user_code(p_context_t * context, UserCodeID code_id, string match, TokenInfo * out_token_info)
{
switch (code_id)
{
@ -316,103 +351,66 @@ public static class Lexer
}
/**
* Attempt to lex the next token in the input stream.
* Check if there is a transition from the current lexer state to another
* based on the given input code point.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
* - P_DROP
* @param current_state
* Current lexer state.
* @param code_point
* Input code point.
*
* @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
*/
private size_t attempt_lex_token(TokenInfo * out_token_info)
private LexerStateID check_lexer_transition(uint current_state, uint code_point)
{
TokenInfo token_info;
token_info.position = m_input_position;
token_info.token = INVALID_TOKEN_ID;
*out_token_info = token_info; // TODO: remove
MatchInfo match_info;
size_t unexpected_input_length;
size_t result = find_longest_match(&match_info, &unexpected_input_length);
switch (result)
uint transition_table_index = lexer_states[current_state].transition_table_index;
for (uint i = 0u; i < lexer_states[current_state].n_transitions; i++)
{
case P_SUCCESS:
Token token_to_accept = match_info.accepting_state.token;
if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
if ((lexer_transitions[transition_table_index + i].first <= code_point) &&
(code_point <= lexer_transitions[transition_table_index + i].last))
{
Token user_code_token = user_code(match_info.accepting_state.code_id, m_input[m_input_index..(m_input_index + match_info.length)], &token_info);
/* An invalid Token from user_code() means that the user
* code did not explicitly return a token. So only override
* the token to return if the user code does explicitly
* return a token. */
if (user_code_token != INVALID_TOKEN_ID)
{
token_to_accept = user_code_token;
return lexer_transitions[transition_table_index + i].destination_state;
}
}
/* Update the input position tracking. */
m_input_index += match_info.length;
m_input_position.row += match_info.delta_position.row;
if (match_info.delta_position.row != 0u)
{
m_input_position.col = match_info.delta_position.col;
}
else
{
m_input_position.col += match_info.delta_position.col;
}
if (token_to_accept == INVALID_TOKEN_ID)
{
return P_DROP;
}
token_info.token = token_to_accept;
token_info.length = match_info.length;
*out_token_info = token_info;
return P_SUCCESS;
case P_EOF:
token_info.token = TOKEN___EOF;
*out_token_info = token_info;
return P_SUCCESS;
default:
return result;
}
}
private struct MatchInfo
{
size_t length;
Position delta_position;
const(LexerState) * accepting_state;
return INVALID_LEXER_STATE_ID;
}
/**
* Find the longest lexer pattern match at the current position.
*
* Returns one of:
* - P_SUCCESS
* - P_DECODE_ERROR
* - P_UNEXPECTED_INPUT
* - P_EOF
* @param context
* Lexer/parser context structure.
* @param[out] out_token_info
* The lexed token information is stored here if the return value is
* P_SUCCESS.
*
* @reval P_SUCCESS
* A token was successfully lexed.
* @reval P_DECODE_ERROR
* The decoder encountered invalid text encoding.
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
* @retval P_EOF
* The end of the text input was reached.
*/
private size_t find_longest_match(
p_context_t * context,
MatchInfo * out_match_info,
size_t * out_unexpected_input_length)
{
MatchInfo longest_match;
MatchInfo attempt_match;
uint current_state = modes[m_mode].state_table_offset;
uint current_state = modes[context.mode].state_table_offset;
for (;;)
{
string input = m_input[(m_input_index + attempt_match.length)..(m_input.length)];
string input = context.input[(context.input_index + attempt_match.length)..(context.input.length)];
CodePoint code_point;
ubyte code_point_length;
size_t result = Decoder.decode_code_point(input, &code_point, &code_point_length);
size_t result = p_decode_code_point(input, &code_point, &code_point_length);
switch (result)
{
case P_SUCCESS:
LexerStateID transition_state = transition(current_state, code_point);
LexerStateID transition_state = check_lexer_transition(current_state, code_point);
if (transition_state != INVALID_LEXER_STATE_ID)
{
attempt_match.length += code_point_length;
@ -471,18 +469,105 @@ public static class Lexer
}
}
private LexerStateID transition(uint current_state, uint code_point)
/**
* Attempt to lex the next token in the input stream.
*
* @param context
* Lexer/parser context structure.
* @param[out] out_token_info
* The lexed token information is stored here if the return value is
* P_SUCCESS.
*
* @reval P_SUCCESS
* A token was successfully lexed.
* @reval P_DECODE_ERROR
* The decoder encountered invalid text encoding.
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
* @retval P_DROP
* A drop pattern was matched so the lexer should continue.
*/
private size_t attempt_lex_token(p_context_t * context, TokenInfo * out_token_info)
{
uint transition_table_index = lexer_states[current_state].transition_table_index;
for (uint i = 0u; i < lexer_states[current_state].n_transitions; i++)
TokenInfo token_info;
token_info.position = context.input_position;
token_info.token = INVALID_TOKEN_ID;
*out_token_info = token_info; // TODO: remove
MatchInfo match_info;
size_t unexpected_input_length;
size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
switch (result)
{
if ((lexer_transitions[transition_table_index + i].first <= code_point) &&
(code_point <= lexer_transitions[transition_table_index + i].last))
case P_SUCCESS:
Token token_to_accept = match_info.accepting_state.token;
if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
{
return lexer_transitions[transition_table_index + i].destination_state;
Token user_code_token = lexer_user_code(context, match_info.accepting_state.code_id, context.input[context.input_index..(context.input_index + match_info.length)], &token_info);
/* An invalid Token from lexer_user_code() means that the user
* code did not explicitly return a token. So only override
* the token to return if the user code does explicitly
* return a token. */
if (user_code_token != INVALID_TOKEN_ID)
{
token_to_accept = user_code_token;
}
}
return INVALID_LEXER_STATE_ID;
/* Update the input position tracking. */
context.input_index += match_info.length;
context.input_position.row += match_info.delta_position.row;
if (match_info.delta_position.row != 0u)
{
context.input_position.col = match_info.delta_position.col;
}
else
{
context.input_position.col += match_info.delta_position.col;
}
if (token_to_accept == INVALID_TOKEN_ID)
{
return P_DROP;
}
token_info.token = token_to_accept;
token_info.length = match_info.length;
*out_token_info = token_info;
return P_SUCCESS;
case P_EOF:
token_info.token = TOKEN___EOF;
*out_token_info = token_info;
return P_SUCCESS;
default:
return result;
}
}
/**
* Lex the next token in the input stream.
*
* @param context
* Lexer/parser context structure.
* @param[out] out_token_info
* The lexed token information is stored here if the return value is
* P_SUCCESS.
*
* @reval P_SUCCESS
* A token was successfully lexed.
* @reval P_DECODE_ERROR
* The decoder encountered invalid text encoding.
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
*/
public size_t p_lex(p_context_t * context, TokenInfo * out_token_info)
{
for (;;)
{
size_t result = attempt_lex_token(context, out_token_info);
if (result != P_DROP)
{
return result;
}
}
}
@ -551,18 +636,115 @@ private static immutable ParserState[] parser_states = [
<% end %>
];
public static class Parser
/**
* Execute user code associated with a parser rule.
*
* @param rule The ID of the rule.
*
* @return Parse value.
*/
private ParserValue parser_user_code(uint rule, StateValue[] statevalues, uint n_states)
{
private Lexer m_lexer;
ParserValue _pvalue;
private ParserValue parse_result;
this(string input)
switch (rule)
{
m_lexer = new Lexer(input);
<% @grammar.rules.each do |rule| %>
<% if rule.code %>
case <%= rule.id %>u: {
<%= expand_code(rule.code, true, rule, nil) %>
} break;
<% end %>
<% end %>
default: break;
}
size_t parse()
return _pvalue;
}
/**
* Check if the parser should shift to a new state.
*
* @param state
* Parser state ID.
* @param symbol
* Incoming token/rule set ID.
*
* @return State to shift to, or INVALID_ID if none.
*/
private size_t check_shift(size_t state, size_t symbol)
{
uint start = parser_states[state].shift_table_index;
uint end = start + parser_states[state].n_shift_entries;
for (uint i = start; i < end; i++)
{
if (parser_shifts[i].symbol == symbol)
{
// if (symbol != INVALID_TOKEN_ID)
// {
// writeln("Shifting ", p_token_names[symbol]);
// }
// else
// {
// writeln("Shifting rule set ", symbol);
// }
return parser_shifts[i].state;
}
}
return INVALID_ID;
}
/**
* Check if the parser should reduce to a new state.
*
* @param state
* Parser state ID.
* @param token
* Incoming token ID.
*
* @return State to reduce to, or INVALID_ID if none.
*/
private size_t check_reduce(size_t state, Token token)
{
size_t start = parser_states[state].reduce_table_index;
size_t end = start + parser_states[state].n_reduce_entries;
for (size_t i = start; i < end; i++)
{
if ((parser_reduces[i].token == token) ||
(parser_reduces[i].token == INVALID_TOKEN_ID))
{
// write("Reducing rule ", parser_reduces[i].rule, ", rule set ", parser_reduces[i].rule_set, " lookahead ");
// if (token != INVALID_TOKEN_ID)
// {
// writeln(p_token_names[token]);
// }
// else
// {
// writeln("{other}");
// }
return i;
}
}
return INVALID_ID;
}
/**
* Run the parser.
*
* @param context
* Lexer/parser context structure.
*
* @retval P_SUCCESS
* The parser successfully matched the input text. The parse result value
* can be accessed with p_result().
* @retval P_UNEXPECTED_TOKEN
* An unexpected token was encountered that does not match any grammar rule.
* @reval P_DECODE_ERROR
* The decoder encountered invalid text encoding.
* @reval P_UNEXPECTED_INPUT
* Input text does not match any lexer pattern.
*/
public size_t p_parse(p_context_t * context)
{
TokenInfo token_info;
Token token = INVALID_TOKEN_ID;
@ -573,7 +755,7 @@ public static class Parser
{
if (token == INVALID_TOKEN_ID)
{
size_t lexer_result = m_lexer.lex_token(&token_info);
size_t lexer_result = p_lex(context, &token_info);
if (lexer_result != P_SUCCESS)
{
return lexer_result;
@ -591,7 +773,7 @@ public static class Parser
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
{
/* Successful parse. */
parse_result = statevalues[$-1].pvalue;
context.parse_result = statevalues[$-1].pvalue;
return P_SUCCESS;
}
}
@ -620,7 +802,7 @@ public static class Parser
if (reduce_index != INVALID_ID)
{
/* We have something to reduce. */
reduced_parser_value = user_code(parser_reduces[reduce_index].rule, statevalues, parser_reduces[reduce_index].n_states);
reduced_parser_value = parser_user_code(parser_reduces[reduce_index].rule, statevalues, parser_reduces[reduce_index].n_states);
reduced_rule_set = parser_reduces[reduce_index].rule_set;
statevalues.length -= parser_reduces[reduce_index].n_states;
continue;
@ -630,7 +812,7 @@ public static class Parser
write("Unexpected token ");
if (token != INVALID_TOKEN_ID)
{
writeln(token_names[token]);
writeln(p_token_names[token]);
}
else
{
@ -640,80 +822,15 @@ public static class Parser
}
}
@property <%= start_rule_type[1] %> result()
{
return parse_result.v_<%= start_rule_type[0] %>;
}
private size_t check_shift(size_t state, size_t symbol)
{
uint start = parser_states[state].shift_table_index;
uint end = start + parser_states[state].n_shift_entries;
for (uint i = start; i < end; i++)
{
if (parser_shifts[i].symbol == symbol)
{
// if (symbol != INVALID_TOKEN_ID)
// {
// writeln("Shifting ", token_names[symbol]);
// }
// else
// {
// writeln("Shifting rule set ", symbol);
// }
return parser_shifts[i].state;
}
}
return INVALID_ID;
}
private size_t check_reduce(size_t state, Token token)
{
size_t start = parser_states[state].reduce_table_index;
size_t end = start + parser_states[state].n_reduce_entries;
for (size_t i = start; i < end; i++)
{
if ((parser_reduces[i].token == token) ||
(parser_reduces[i].token == INVALID_TOKEN_ID))
{
// write("Reducing rule ", parser_reduces[i].rule, ", rule set ", parser_reduces[i].rule_set, " lookahead ");
// if (token != INVALID_TOKEN_ID)
// {
// writeln(token_names[token]);
// }
// else
// {
// writeln("{other}");
// }
return i;
}
}
return INVALID_ID;
}
/**
* Execute user code associated with a parser rule.
* Get the parse result value.
*
* @param rule The ID of the rule.
* @param context
* Lexer/parser context structure.
*
* @return Parse value.
* @return Parse result value.
*/
private ParserValue user_code(uint rule, StateValue[] statevalues, uint n_states)
public <%= start_rule_type[1] %> p_result(p_context_t * context)
{
ParserValue _pvalue;
switch (rule)
{
<% @grammar.rules.each do |rule| %>
<% if rule.code %>
case <%= rule.id %>u: {
<%= expand_code(rule.code, true, rule, nil) %>
} break;
<% end %>
<% end %>
default: break;
}
return _pvalue;
}
return context.parse_result.v_<%= start_rule_type[0] %>;
}

2
lib/propane/generator.rb
View File

@ -203,7 +203,7 @@ class Propane
unless mode_id
raise Error.new("Lexer mode '#{mode_name}' not found")
end
"m_mode = #{mode_id}u"
"context.mode = #{mode_id}u"
end
end
code

37
spec/test_d_lexer.d
View File

@ -12,31 +12,31 @@ unittest
CodePoint code_point;
ubyte code_point_length;
result = Decoder.decode_code_point("5", &code_point, &code_point_length);
result = p_decode_code_point("5", &code_point, &code_point_length);
assert(result == P_SUCCESS);
assert(code_point == '5');
assert(code_point_length == 1u);
result = Decoder.decode_code_point("", &code_point, &code_point_length);
result = p_decode_code_point("", &code_point, &code_point_length);
assert(result == P_EOF);
result = Decoder.decode_code_point("\xC2\xA9", &code_point, &code_point_length);
result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
assert(result == P_SUCCESS);
assert(code_point == 0xA9u);
assert(code_point_length == 2u);
result = Decoder.decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
assert(result == P_SUCCESS);
assert(code_point == 0x1F9E1u);
assert(code_point_length == 4u);
result = Decoder.decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR);
result = Decoder.decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR);
result = Decoder.decode_code_point("\xfe", &code_point, &code_point_length);
result = p_decode_code_point("\xfe", &code_point, &code_point_length);
assert(result == P_DECODE_ERROR);
}
@ -44,25 +44,26 @@ unittest
{
TokenInfo token_info;
string input = "5 + 4 * \n677 + 567";
Lexer lexer = new Lexer(input);
assert(lexer.lex_token(&token_info) == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 0), 1, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 2), 1, TOKEN_plus));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 4), 1, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 6), 1, TOKEN_times));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 0), 3, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 4), 1, TOKEN_plus));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 6), 3, TOKEN_int));
assert(lexer.lex_token(&token_info) == P_SUCCESS);
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(1, 9), 0, TOKEN___EOF));
lexer = new Lexer("");
assert(lexer.lex_token(&token_info) == P_SUCCESS);
p_context_init(&context, "");
assert(p_lex(&context, &token_info) == P_SUCCESS);
assert(token_info == TokenInfo(Position(0, 0), 0, TOKEN___EOF));
}

9
spec/test_d_parser_identical_rules_lookahead.d
View File

@ -9,10 +9,11 @@ int main()
unittest
{
string input = "aba";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
input = "abb";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
}

13
spec/test_d_parser_rule_from_multiple_states.d
View File

@ -9,14 +9,15 @@ int main()
unittest
{
string input = "a";
auto parser = new Parser(input);
assert(parser.parse() == P_UNEXPECTED_TOKEN);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
input = "a b";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
input = "bb";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
}

5
spec/test_lexer_match_text.d
View File

@ -9,7 +9,8 @@ int main()
unittest
{
string input = `identifier_123`;
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1");
}

9
spec/test_lexer_modes.d
View File

@ -9,12 +9,13 @@ int main()
unittest
{
string input = `abc "a string" def`;
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1");
input = `abc "abc def" def`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass2");
}

13
spec/test_lexer_result_value.d
View File

@ -9,12 +9,13 @@ int main()
unittest
{
string input = `x`;
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 1u);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 1u);
input = `fabulous`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 8u);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 8u);
}

11
spec/test_lexer_unknown_character.d
View File

@ -9,11 +9,12 @@ int main()
unittest
{
string input = `x`;
auto parser = new Parser(input);
assert(parser.parse() == P_UNEXPECTED_INPUT);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
input = `123`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 123u);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 123u);
}

5
spec/test_parser_rule_user_code.d
View File

@ -9,6 +9,7 @@ int main()
unittest
{
string input = "ab";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
}

49
spec/test_parsing_json.d
View File

@ -10,44 +10,45 @@ int main()
unittest
{
string input = ``;
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
input = `{}`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONObject)parser.result);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)p_result(&context));
input = `[]`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONArray)parser.result);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONArray)p_result(&context));
input = `-45.6`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONNumber)parser.result);
assert((cast(JSONNumber)parser.result).value == -45.6);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONNumber)p_result(&context));
assert((cast(JSONNumber)p_result(&context)).value == -45.6);
input = `2E-2`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONNumber)parser.result);
assert((cast(JSONNumber)parser.result).value == 0.02);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONNumber)p_result(&context));
assert((cast(JSONNumber)p_result(&context)).value == 0.02);
input = `{"hi":true}`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONObject)parser.result);
JSONObject o = cast(JSONObject)parser.result;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)p_result(&context));
JSONObject o = cast(JSONObject)p_result(&context);
assert(o.value["hi"]);
assert(cast(JSONTrue)o.value["hi"]);
input = `{"ff": false, "nn": null}`;
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(cast(JSONObject)parser.result);
o = cast(JSONObject)parser.result;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(cast(JSONObject)p_result(&context));
o = cast(JSONObject)p_result(&context);
assert(o.value["ff"]);
assert(cast(JSONFalse)o.value["ff"]);
assert(o.value["nn"]);

19
spec/test_parsing_lists.d
View File

@ -9,17 +9,18 @@ int main()
unittest
{
string input = "a";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 1u);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 1u);
input = "";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 0u);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 0u);
input = "aaaaaaaaaaaaaaaa";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
assert(parser.result == 16u);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
assert(p_result(&context) == 16u);
}

9
spec/test_pattern.d
View File

@ -9,12 +9,13 @@ int main()
unittest
{
string input = "abcdef";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1");
input = "defabcdef";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass2");
}

5
spec/test_return_token_from_pattern.d
View File

@ -9,6 +9,7 @@ int main()
unittest
{
string input = "defghidef";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
}

9
spec/test_user_code.d
View File

@ -9,12 +9,13 @@ int main()
unittest
{
string input = "abcdef";
auto parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_t context;
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass1");
input = "abcabcdef";
parser = new Parser(input);
assert(parser.parse() == P_SUCCESS);
p_context_init(&context, input);
assert(p_parse(&context) == P_SUCCESS);
writeln("pass2");
}