v4.6.0

Allow user code to access specific pvalue values, especially when
multiple ptypes are in use.

CHANGELOG.md

@@ -1,3 +1,33 @@
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
 ## v4.3.0
 
 ### New Features

Gemfile.lock

@@ -6,7 +6,7 @@ GEM
     diff-lcs (1.6.2)
     docile (1.4.1)
     erb (6.0.4)
-    psych (5.3.1)
+    psych (5.4.0)
       date
       stringio
     rake (13.4.2)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-  4.0.11
+  4.0.14

assets/parser.c.erb

@@ -486,6 +486,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     case P_SUCCESS:
         {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
@@ -524,17 +540,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         }
         return P_SUCCESS;
@@ -707,12 +712,14 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 } state_value_t;
 
@@ -866,6 +873,57 @@ static void state_values_stack_free(state_values_stack_t * stack)
 }
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -971,6 +1029,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
@@ -1013,7 +1073,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         {
             /* We have something to shift. */
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
@@ -1030,9 +1091,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
+                new_state_info->tree_node = token_tree_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
             }
@@ -1040,9 +1103,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
+                new_state_info->tree_node = reduced_parser_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
@@ -1113,6 +1178,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);

assets/parser.d.erb

@@ -59,10 +59,10 @@ public alias <%= @grammar.prefix %>code_point_t = uint;
  */
 public struct <%= @grammar.prefix %>position_t
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint col;
 
     /** Invalid position value. */
@@ -94,6 +94,14 @@ public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "def
     return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
 }
 <%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -657,6 +665,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
@@ -695,17 +719,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -874,12 +887,14 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** Tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 
     this(size_t state_id)
@@ -933,6 +948,53 @@ private immutable parser_state_t[] parser_state_table = [
 ];
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -1039,6 +1101,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     for (;;)
@@ -1083,6 +1147,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
@@ -1094,6 +1160,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
 <% end %>
@@ -1166,6 +1234,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;

assets/parser.h.erb

@@ -45,10 +45,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
  */
 typedef struct
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint32_t row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint32_t col;
 } <%= @grammar.prefix %>position_t;
 
@@ -77,6 +77,14 @@ static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name
     return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
 }
 <%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -151,12 +151,37 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
-#### C/C++
+Lexer code blocks appear between `<<` and `>>` markers following a `drop`,
+`token`, or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+The input text positions of the matched token can also be accessed from within
+a lexer code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+The start position of the matched token is accessed with `${position}`, and
+the end position of the matched token is accessed with `${end_position}`.
+
+Example:
+
+```
+token integer /\d+/ <<
+  printf("integer token on row %d, col %d\n",
+      ${position}.row, ${position}.col);
+  $$ = parse_integer(match, match_length);
+>>
+```
+
+#### C/C++ lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a pointer points to the text matched by the lexer pattern.
+  * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
-  * `match_length` - length of the matched text.
+  * `match_length` (`size_t`) - length of the matched text.
 
 Example:
 
@@ -174,11 +199,11 @@ token integer /\d+/ <<
 >>
 ```
 
-#### D
+#### D lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a slice containing the text matched by the lexer pattern.
+  * `match` (`string`) - a slice containing the text matched by the lexer pattern.
 
 ```
 ptype ulong;
@@ -194,13 +219,6 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `drop`, `token`, or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
 ### Parser rule code blocks
 
 Example:
@@ -217,6 +235,41 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
+The input text positions for the reduced rule and for the individual rule
+components can also be accessed from within a parser rule code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+
+The start position of the overall reduced rule is accessed with
+`${$.position}`, and the end position of the overall reduced rule is accessed
+with `${$.end_position}`.
+
+The start and end positions of an individual rule component are accessed
+positionally with `${N.position}` and `${N.end_position}`, where `N` is the
+1-based index of the component (`${1.position}` for the first component,
+`${2.position}` for the second, and so on).
+
+Example:
+
+```
+Assignment -> ident equals Expr <<
+    printf("assignment on row %d, col %d\n",
+        ${$.position}.row, ${$.position}.col);
+    printf("target identifier ends on row %d, col %d\n",
+        ${1.end_position}.row, ${1.end_position}.col);
+    printf("expression starts on row %d, col %d\n",
+        ${3.position}.row, ${3.position}.col);
+>>
+```
+
+A rule or rule component that allows for an empty match may not have valid
+positions.
+In this case the position should be checked for validity before its `row` and
+`col` fields are used (see `${#p_position_valid}`).
+For C targets this can be accomplished with
+`if (p_position_valid(${$.position}))` and for D targets this can be
+accomplished with `if (${$.position}.valid)`.
+
 Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
@@ -383,6 +436,27 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
+##> `noline` statement - disabling `#line` directives
+
+By default, Propane emits `#line` directives into the generated output around
+user code blocks.
+These directives instruct the compiler to report any warnings or errors in the
+user code using the file name and line number of the original grammar file,
+rather than the generated output file.
+This makes it easier to locate the source of a compiler diagnostic in the
+grammar file.
+
+The `noline` statement disables the emission of these `#line` directives.
+
+```
+noline;
+```
+
+When the `noline` statement is present, user code blocks are copied to the
+generated output without any surrounding `#line` directives.
+This can be useful when debugging the generated parser itself, or when the
+`#line` directives interfere with other tooling.
+
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -1082,6 +1156,9 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
+  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
+  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
+  `ptype` values.
 
 ### Tree Node Types
 
@@ -1322,7 +1399,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 
@@ -1404,18 +1481,35 @@ In this case, Propane will free a `Statement` tree structure returned by the
 
 ### `p_value`
 
-When tree generation mode is not active, the `p_value_t` union can hold one of
+The `p_value(v)` function builds an instance of the `p_value_t` with the
-several different possible value types.
-
-The `p_value(v)` function returns an instance of the `p_value_t` with the
 default member set to the value of `v`.
 
-A `p_value_XXX(v)` function is set for each user-defined `ptype` name with
+A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
 the user-given name in place of the `XXX`.
 
 These functions are useful for custom lexer functions which need to return a
 parser value corresponding to a lexed token.
 
+They are especially useful when tree generation mode is not active.
+In that case, the `p_value_t` union can hold one of several different possible
+value types.
+
+### `p_value_get`
+
+The `p_value_get()` accessor functions can be used to extract a user value
+from a `p_value_t` union.
+
+The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
+returns the value of its default member.
+
+A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
+name with the user-given name in place of the `XXX`, returning the value of the
+corresponding member.
+
+These functions are the counterpart to `p_value()` constructor functions and
+are useful for reading the parser value associated with a lexed token, for
+example `p_value_get(&token_info.pvalue)`.
+
 ##> Data
 
 ### `p_token_names`
@@ -1433,7 +1527,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
         p_token_names[context->token],
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 

extra/vim/syntax/propane.vim

@@ -8,7 +8,11 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -20,12 +24,19 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
 hi def link propaneFieldOperator Operator

lib/propane/generator.rb

@@ -40,7 +40,7 @@ class Propane
         erb = ERB.new(template, trim_mode: "<>")
         result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
           if line == "#linereset\n"
-            "#line #{i + 2}\n"
+            %[#line #{i + 2} "#{output_file}"\n]
           else
             line
           end
@@ -312,6 +312,14 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
+        end
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
@@ -343,6 +351,22 @@ class Propane
             end
           end
         end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)

lib/propane/grammar.rb

@@ -88,6 +88,7 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -306,10 +307,12 @@ class Propane
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
         code = code.chomp
-        if code.start_with?("\n")
+        unless @noline
-          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          if code.start_with?("\n")
-        else
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
         end
         if @code_blocks[name]
           @code_blocks[name] += code
@@ -328,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -352,10 +362,12 @@ class Propane
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
         code = md[1].chomp
-        if code.start_with?("\n")
+        unless @noline
-          code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          if code.start_with?("\n")
-        else
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
         end
         code
       end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.3.0"
+  VERSION = "4.6.0"
 end

spec/propane/grammar_spec.rb

@@ -133,6 +133,41 @@ EOF
       expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
     end
 
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
+    end
+
     it "supports mode labels" do
       input = <<EOF
 token a;

spec/propane_spec.rb

@@ -910,6 +910,57 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows lexer code blocks to access the matched token positions" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        when "d"
+          write_grammar <<EOF
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows the user to terminate the parser" do
         write_grammar <<EOF
 token a;
@@ -1781,6 +1832,165 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "provides accessor functions to extract user values from a parser value" do
+        if language == "d"
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = string;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello"; >>
+Start -> num flt str;
+EOF
+        else
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = (char *)"hello"; >>
+Start -> num flt str;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_value_accessors.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows accessing rule and component text positions" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Start -> Empty <<
+    writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+token pct /%/;
+Start -> ident Num <<
+    printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to eq <<EOF
+ident start: 1, 5
+ident end: 1, 9
+Num start: 3, 9
+Num end: 3, 12
+Start start: 1, 5
+Start end: 3, 12
+
+Empty start: 0, 0
+Empty end: 0, 0
+tok2 start: 2, 2
+tok2 end: 2, 5
+R start: 2, 2
+R end: 2, 5
+StartR start: 2, 2
+StartR end: 2, 5
+
+tok1 start: 1, 3
+tok1 end: 1, 6
+Empty start: 0, 0
+Empty end: 0, 0
+R2 start: 1, 3
+R2 end: 1, 6
+StartR start: 1, 3
+StartR end: 1, 6
+EOF
+      end
     end
   end
 end

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_value_accessors.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    p_token_info_t token_info;
+    char const * input = "42 f s";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(strcmp(p_value_get_string(&token_info.pvalue), "hello") == 0);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_value_accessors.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "42 f s";
+    p_context_t * context;
+    context = p_context_new(input);
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(p_value_get_string(&token_info.pvalue) == "hello");
+}