v4.6.0

CHANGELOG.md

@@ -1,3 +1,14 @@
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
 ## v4.5.0
 
 ### New Features
@@ -9,6 +20,7 @@
 
 - Fix #line reset directives
 - Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
 
 ## v4.4.0
 

assets/parser.c.erb

@@ -486,6 +486,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     case P_SUCCESS:
         {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
@@ -524,17 +540,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         }
         return P_SUCCESS;
@@ -707,12 +712,14 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 } state_value_t;
 
@@ -866,6 +873,57 @@ static void state_values_stack_free(state_values_stack_t * stack)
 }
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -971,6 +1029,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
@@ -1013,7 +1073,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         {
             /* We have something to shift. */
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
@@ -1030,9 +1091,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
+                new_state_info->tree_node = token_tree_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
             }
@@ -1040,9 +1103,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
+                new_state_info->tree_node = reduced_parser_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
@@ -1113,6 +1178,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);

assets/parser.d.erb

@@ -59,10 +59,10 @@ public alias <%= @grammar.prefix %>code_point_t = uint;
  */
 public struct <%= @grammar.prefix %>position_t
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint col;
 
     /** Invalid position value. */
@@ -665,6 +665,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
@@ -703,17 +719,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -882,12 +887,14 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** Tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 
     this(size_t state_id)
@@ -941,6 +948,53 @@ private immutable parser_state_t[] parser_state_table = [
 ];
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -1047,6 +1101,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     for (;;)
@@ -1091,6 +1147,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
@@ -1102,6 +1160,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
 <% end %>
@@ -1174,6 +1234,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;

assets/parser.h.erb

@@ -45,10 +45,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
  */
 typedef struct
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint32_t row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint32_t col;
 } <%= @grammar.prefix %>position_t;
 

doc/user_guide.md

@@ -151,12 +151,37 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
-#### C/C++
+Lexer code blocks appear between `<<` and `>>` markers following a `drop`,
+`token`, or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+The input text positions of the matched token can also be accessed from within
+a lexer code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+The start position of the matched token is accessed with `${position}`, and
+the end position of the matched token is accessed with `${end_position}`.
+
+Example:
+
+```
+token integer /\d+/ <<
+  printf("integer token on row %d, col %d\n",
+      ${position}.row, ${position}.col);
+  $$ = parse_integer(match, match_length);
+>>
+```
+
+#### C/C++ lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a pointer points to the text matched by the lexer pattern.
-  * `match_length` - length of the matched text.
+  * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
+  * `match_length` (`size_t`) - length of the matched text.
 
 Example:
 
@@ -174,11 +199,11 @@ token integer /\d+/ <<
 >>
 ```
 
-#### D
+#### D lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a slice containing the text matched by the lexer pattern.
+  * `match` (`string`) - a slice containing the text matched by the lexer pattern.
 
 ```
 ptype ulong;
@@ -194,13 +219,6 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `drop`, `token`, or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
 ### Parser rule code blocks
 
 Example:
@@ -217,6 +235,41 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
+The input text positions for the reduced rule and for the individual rule
+components can also be accessed from within a parser rule code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+
+The start position of the overall reduced rule is accessed with
+`${$.position}`, and the end position of the overall reduced rule is accessed
+with `${$.end_position}`.
+
+The start and end positions of an individual rule component are accessed
+positionally with `${N.position}` and `${N.end_position}`, where `N` is the
+1-based index of the component (`${1.position}` for the first component,
+`${2.position}` for the second, and so on).
+
+Example:
+
+```
+Assignment -> ident equals Expr <<
+    printf("assignment on row %d, col %d\n",
+        ${$.position}.row, ${$.position}.col);
+    printf("target identifier ends on row %d, col %d\n",
+        ${1.end_position}.row, ${1.end_position}.col);
+    printf("expression starts on row %d, col %d\n",
+        ${3.position}.row, ${3.position}.col);
+>>
+```
+
+A rule or rule component that allows for an empty match may not have valid
+positions.
+In this case the position should be checked for validity before its `row` and
+`col` fields are used (see `${#p_position_valid}`).
+For C targets this can be accomplished with
+`if (p_position_valid(${$.position}))` and for D targets this can be
+accomplished with `if (${$.position}.valid)`.
+
 Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
@@ -1346,7 +1399,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 
@@ -1474,7 +1527,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
         p_token_names[context->token],
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 

extra/vim/syntax/propane.vim

@@ -24,12 +24,19 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword context_user_fields drop free_token_node lex_fn module noline on_token_node prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
 hi def link propaneFieldOperator Operator

lib/propane/generator.rb

@@ -312,6 +312,14 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
+        end
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
@@ -343,6 +351,22 @@ class Propane
             end
           end
         end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.4.0"
+  VERSION = "4.6.0"
 end

spec/propane_spec.rb

@@ -910,6 +910,57 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows lexer code blocks to access the matched token positions" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        when "d"
+          write_grammar <<EOF
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows the user to terminate the parser" do
         write_grammar <<EOF
 token a;
@@ -1814,6 +1865,132 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "allows accessing rule and component text positions" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Start -> Empty <<
+    writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+token pct /%/;
+Start -> ident Num <<
+    printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to eq <<EOF
+ident start: 1, 5
+ident end: 1, 9
+Num start: 3, 9
+Num end: 3, 12
+Start start: 1, 5
+Start end: 3, 12
+
+Empty start: 0, 0
+Empty end: 0, 0
+tok2 start: 2, 2
+tok2 end: 2, 5
+R start: 2, 2
+R end: 2, 5
+StartR start: 2, 2
+StartR end: 2, 5
+
+tok1 start: 1, 3
+tok1 end: 1, 6
+Empty start: 0, 0
+Empty end: 0, 0
+R2 start: 1, 3
+R2 end: 1, 6
+StartR start: 1, 3
+StartR end: 1, 6
+EOF
+      end
     end
   end
 end

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}