diff --git a/CHANGELOG.md b/CHANGELOG.md index 346cb77..cb5aefd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ `match_text` for every target language. It was previously named `match` for C, C++, and D, and `match_` for Rust. Any lexer user code block which references the matched text must be updated to use the new name. The - `match_length` argument (C, C++, and Rust) is unchanged. + `match_length` argument (C and C++) is unchanged. - Tree generation mode now stores all tree nodes in a compact arena owned by the parser context (a flat node array plus a shared child-link array). This replaces the previous design of one heap allocation per node with diff --git a/assets/parser.rs.erb b/assets/parser.rs.erb index 3b9243e..5542966 100644 --- a/assets/parser.rs.erb +++ b/assets/parser.rs.erb @@ -484,8 +484,6 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [ * The ID of the user code block to execute. * @param match_text * Matched text for this pattern. - * @param match_length - * Matched text length. * @param out_token_info * Lexer token info in progress. * @@ -494,7 +492,7 @@ static lexer_mode_table: [lexer_mode_t; <%= @lexer.mode_table.size %>] = [ */ fn lexer_user_code(context: &mut <%= @grammar.prefix %>context_t, code_id: <%= get_type_for(user_code_id_count) %>, match_text: &[u8], - match_length: usize, out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t { + out_token_info: &mut <%= @grammar.prefix %>token_info_t) -> <%= @grammar.prefix %>token_t { match code_id { <% @grammar.patterns.each do |pattern| %> <% if pattern.code_id %> @@ -668,7 +666,7 @@ fn attempt_lex_token(context: &mut <%= @grammar.prefix %>context_t, out_token_in let match_start = context.input_index; let match_slice = context.input[match_start..(match_start + match_info.length)].to_vec(); let user_code_token = lexer_user_code(context, - lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, match_info.length, &mut token_info); + lexer_state_table[match_info.accepting_state as usize].code_id, &match_slice, &mut token_info); /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means * that the user code is requesting to terminate the lexer. */ if user_code_token == TERMINATE_TOKEN_ID { diff --git a/doc/user_guide.md b/doc/user_guide.md index 9e49654..e21d57f 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -249,7 +249,6 @@ token integer /\d+/ << The lexer code block is passed the following arguments: * `match_text` (`&[u8]`) - a slice containing the text matched by the lexer pattern. - * `match_length` (`usize`) - length of the matched text. The matched text is a byte slice rather than a string; use `std::str::from_utf8()` or `String::from_utf8_lossy()` to view it as a string. diff --git a/spec/propane_spec.rb b/spec/propane_spec.rb index 5e1db97..4bfd0d6 100644 --- a/spec/propane_spec.rb +++ b/spec/propane_spec.rb @@ -1124,7 +1124,7 @@ EOF write_grammar <> Start -> word << $$ = $1;