Store token end position - #27

assets/parser.c.erb

@@ -226,7 +226,10 @@ typedef struct
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta. */
+    /** Input text position delta to end of token. */
+    <%= @grammar.prefix %>position_t end_delta_position;
+
+    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -358,6 +361,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
+                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
@@ -490,11 +494,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
         }
         token_info.token = token_to_accept;
         token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
+        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -952,6 +967,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                 token_ast_node->token = token;
                 token_ast_node->pvalue = token_info.pvalue;
                 token_ast_node->position = token_info.position;
+                token_ast_node->end_position = token_info.end_position;
                 state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;