diff --git a/assets/parser.c.erb b/assets/parser.c.erb index 9294ba4..6451924 100644 --- a/assets/parser.c.erb +++ b/assets/parser.c.erb @@ -486,6 +486,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @ case P_SUCCESS: { <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token; + /* Calculate the token length and start/end positions before invoking + * the lexer user code so that the user code can access them. The + * context input text position tracking is not updated until after the + * user code has run so that it is left unchanged if the user code + * requests to terminate the lexer. */ + token_info.length = match_info.length; + if (match_info.end_delta_position.row != 0u) + { + token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row; + token_info.end_position.col = match_info.end_delta_position.col; + } + else + { + token_info.end_position.row = token_info.position.row; + token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col; + } if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID) { uint8_t const * match = &context->input[context->input_index]; @@ -524,17 +540,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @ return P_DROP; } token_info.token = token_to_accept; - token_info.length = match_info.length; - if (match_info.end_delta_position.row != 0u) - { - token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row; - token_info.end_position.col = match_info.end_delta_position.col; - } - else - { - token_info.end_position.row = token_info.position.row; - token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col; - } *out_token_info = token_info; } return P_SUCCESS; diff --git a/assets/parser.d.erb b/assets/parser.d.erb index 5154937..91b5217 100644 --- a/assets/parser.d.erb +++ b/assets/parser.d.erb @@ -665,6 +665,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= { case P_SUCCESS: <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token; + /* Calculate the token length and start/end positions before invoking + * the lexer user code so that the user code can access them. The + * context input text position tracking is not updated until after the + * user code has run so that it is left unchanged if the user code + * requests to terminate the lexer. */ + token_info.length = match_info.length; + if (match_info.end_delta_position.row != 0u) + { + token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row; + token_info.end_position.col = match_info.end_delta_position.col; + } + else + { + token_info.end_position.row = token_info.position.row; + token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col; + } if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID) { string match = context.input[context.input_index..(context.input_index + match_info.length)]; @@ -703,17 +719,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= return P_DROP; } token_info.token = token_to_accept; - token_info.length = match_info.length; - if (match_info.end_delta_position.row != 0u) - { - token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row; - token_info.end_position.col = match_info.end_delta_position.col; - } - else - { - token_info.end_position.row = token_info.position.row; - token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col; - } *out_token_info = token_info; return P_SUCCESS; diff --git a/doc/user_guide.md b/doc/user_guide.md index 3225fe0..1e025db 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -151,12 +151,37 @@ needed by a `ptype` directive, for example: ### Lexer pattern code blocks -#### C/C++ +Lexer code blocks appear between `<<` and `>>` markers following a `drop`, +`token`, or pattern expression. +User code in a lexer code block will be executed when the lexer matches the +given pattern. +Assignment to the `$$` symbol will associate a parser value with the lexed +token. +This parser value can then be used later in a parser rule. + +The input text positions of the matched token can also be accessed from within +a lexer code block. +Each of these positions is an instance of the `p_position_t` structure (see +`${#p_position_t}`), which contains 1-based `row` and `col` fields. +The start position of the matched token is accessed with `${position}`, and +the end position of the matched token is accessed with `${end_position}`. + +Example: + +``` +token integer /\d+/ << + printf("integer token on row %d, col %d\n", + ${position}.row, ${position}.col); + $$ = parse_integer(match, match_length); +>> +``` + +#### C/C++ lexer code block arguments The lexer code block is passed the following arguments: - * `match` - a pointer points to the text matched by the lexer pattern. - * `match_length` - length of the matched text. + * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern. + * `match_length` (`size_t`) - length of the matched text. Example: @@ -174,11 +199,11 @@ token integer /\d+/ << >> ``` -#### D +#### D lexer code block arguments The lexer code block is passed the following arguments: - * `match` - a slice containing the text matched by the lexer pattern. + * `match` (`string`) - a slice containing the text matched by the lexer pattern. ``` ptype ulong; @@ -194,13 +219,6 @@ token integer /\d+/ << >> ``` -Lexer code blocks appear following a `drop`, `token`, or pattern expression. -User code in a lexer code block will be executed when the lexer matches the -given pattern. -Assignment to the `$$` symbol will associate a parser value with the lexed -token. -This parser value can then be used later in a parser rule. - ### Parser rule code blocks Example: diff --git a/lib/propane/generator.rb b/lib/propane/generator.rb index 2d88ed6..3df5c31 100644 --- a/lib/propane/generator.rb +++ b/lib/propane/generator.rb @@ -351,6 +351,22 @@ class Propane end end end + code = code.gsub(/\$\{position\}/) do |match| + case @language + when "c" + "out_token_info->position" + when "d" + "out_token_info.position" + end + end + code = code.gsub(/\$\{end_position\}/) do |match| + case @language + when "c" + "out_token_info->end_position" + when "d" + "out_token_info.end_position" + end + end code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match| mode_name = $1 mode_id = @lexer.mode_id(mode_name) diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 0e4ad53..4fa9256 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -910,6 +910,57 @@ EOF expect(results.status).to eq 0 end + it "allows lexer code blocks to access the matched token positions" do + case language + when "c", "cpp" + write_grammar < +>> +context_user_fields << + p_position_t last_start; + p_position_t last_end; +>> +drop /\\s+/; +token word /[a-z]+/ << + ${context.last_start} = ${position}; + ${context.last_end} = ${end_position}; +>> +token stop /!/ << + $terminate(42); +>> +Start -> Words; +Words -> ; +Words -> word Words; +Words -> stop Words; +EOF + when "d" + write_grammar <> +drop /\\s+/; +token word /[a-z]+/ << + ${context.last_start} = ${position}; + ${context.last_end} = ${end_position}; +>> +token stop /!/ << + $terminate(42); +>> +Start -> Words; +Words -> ; +Words -> word Words; +Words -> stop Words; +EOF + end + run_propane(language: language) + compile("spec/test_lexer_positions.#{language}", language: language) + results = run_test(language: language) + expect(results.stderr).to eq "" + expect(results.status).to eq 0 + end + it "allows the user to terminate the parser" do write_grammar < +#include + +int main() +{ + char const * input = "abc\n defg hi\n!"; + p_context_t * context = p_context_new((uint8_t const *)input, strlen(input)); + p_token_info_t token_info; + + /* First token "abc" on row 1, cols 1-3. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context->last_start.row == 1u); + assert(context->last_start.col == 1u); + assert(context->last_end.row == 1u); + assert(context->last_end.col == 3u); + /* The lexer code block observed the same positions reported to the caller. */ + assert(context->last_start.row == token_info.position.row); + assert(context->last_start.col == token_info.position.col); + assert(context->last_end.row == token_info.end_position.row); + assert(context->last_end.col == token_info.end_position.col); + + /* Second token "defg" on row 2, cols 3-6. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context->last_start.row == 2u); + assert(context->last_start.col == 3u); + assert(context->last_end.row == 2u); + assert(context->last_end.col == 6u); + + /* Third token "hi" on row 2, cols 8-9. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context->last_start.row == 2u); + assert(context->last_start.col == 8u); + assert(context->last_end.row == 2u); + assert(context->last_end.col == 9u); + + /* The "!" stop token terminates the lexer. The context input text position + * must not be updated when the lexer user code requests termination, so it + * still points at the "!" token on row 3, col 1. */ + assert(p_lex(context, &token_info) == P_USER_TERMINATED); + assert(p_user_terminate_code(context) == 42u); + assert(context->text_position.row == 3u); + assert(context->text_position.col == 1u); + + p_context_delete(context); + return 0; +} diff --git a/spec/test_lexer_positions.d b/spec/test_lexer_positions.d new file mode 100644 index 0000000..88b8b50 --- /dev/null +++ b/spec/test_lexer_positions.d @@ -0,0 +1,42 @@ +import testparser; +import std.stdio; + +int main() +{ + return 0; +} + +unittest +{ + string input = "abc\n defg hi\n!"; + p_context_t * context = p_context_new(input); + p_token_info_t token_info; + + /* First token "abc" on row 1, cols 1-3. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context.last_start == p_position_t(1, 1)); + assert(context.last_end == p_position_t(1, 3)); + /* The lexer code block observed the same positions reported to the caller. */ + assert(context.last_start == token_info.position); + assert(context.last_end == token_info.end_position); + + /* Second token "defg" on row 2, cols 3-6. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context.last_start == p_position_t(2, 3)); + assert(context.last_end == p_position_t(2, 6)); + + /* Third token "hi" on row 2, cols 8-9. */ + assert(p_lex(context, &token_info) == P_SUCCESS); + assert(token_info.token == TOKEN_word); + assert(context.last_start == p_position_t(2, 8)); + assert(context.last_end == p_position_t(2, 9)); + + /* The "!" stop token terminates the lexer. The context input text position + * must not be updated when the lexer user code requests termination, so it + * still points at the "!" token on row 3, col 1. */ + assert(p_lex(context, &token_info) == P_USER_TERMINATED); + assert(p_user_terminate_code(context) == 42u); + assert(context.text_position == p_position_t(3, 1)); +}