v4.6.0

Allow user code to access specific pvalue values, especially when
multiple ptypes are in use.

CHANGELOG.md

@@ -1,3 +1,51 @@
+## v4.6.0
+
+### New Features
+
+- Add lexer user code API to access matched input text positions
+- Track rule component text positions and add parser user code API to access
+
+### Fixes
+
+- Fixed a few user guide and source comments related to text input positions
+
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+- Fix propane.vim keyword detection
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
+## v4.1.0
+
+### New Features
+
+- Add `p_context_delete()` and `p_tree_delete()` for D targets.
+
 ## v4.0.0
 
 ### New Features

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
-    psych (5.3.1)
+    erb (6.0.4)
+    psych (5.4.0)
       date
       stringio
-    rake (13.3.1)
+    rake (13.4.2)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.14

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.c.erb

@@ -486,6 +486,22 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     case P_SUCCESS:
         {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
             uint8_t const * match = &context->input[context->input_index];
@@ -524,17 +540,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         }
         return P_SUCCESS;
@@ -707,12 +712,14 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 } state_value_t;
 
@@ -866,6 +873,57 @@ static void state_values_stack_free(state_values_stack_t * stack)
 }
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                int stack_index = -1;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
+                    {
+                        return sv->end_position;
+                    }
+                }
+            }
+            else
+            {
+                int stack_index = -(int)n_states;
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
+                    if (<%= @grammar.prefix %>position_valid(sv->position))
+                    {
+                        return sv->position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
+            }
+            else
+            {
+                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
+            }
+        }
+    }
+    <%= @grammar.prefix %>position_t empty_pos;
+    memset(&empty_pos, 0, sizeof(empty_pos));
+    return empty_pos;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -971,6 +1029,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     state_values_stack_init(&statevalues);
@@ -981,7 +1041,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -1013,7 +1073,8 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
         {
             /* We have something to shift. */
             state_values_stack_push(&statevalues);
-            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
+            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
@@ -1030,9 +1091,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 token_tree_node->token = token;
                 token_tree_node->pvalue = token_info.pvalue;
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                state_values_stack_index(&statevalues, -1)->tree_node = token_tree_node;
+                new_state_info->tree_node = token_tree_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+                new_state_info->position = token_info.position;
+                new_state_info->end_position = token_info.end_position;
+                new_state_info->pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
             }
@@ -1040,9 +1103,11 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             {
                 /* We shifted a RuleSet. */
 <% if @grammar.tree %>
-                state_values_stack_index(&statevalues, -1)->tree_node = reduced_parser_node;
+                new_state_info->tree_node = reduced_parser_node;
 <% else %>
-                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
+                new_state_info->pvalue = reduced_parser_value;
+                new_state_info->position = reduced_position;
+                new_state_info->end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
@@ -1113,6 +1178,16 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                memset(&reduced_position, 0, sizeof(reduced_position));
+                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);

assets/parser.d.erb

@@ -9,7 +9,7 @@ module <%= @grammar.modulename %>;
 <% end %>
 
 import core.memory;
-import core.stdc.stdlib : malloc;
+import core.stdc.stdlib : malloc, free;
 
 /**************************************************************************
  * User code blocks
@@ -59,10 +59,10 @@ public alias <%= @grammar.prefix %>code_point_t = uint;
  */
 public struct <%= @grammar.prefix %>position_t
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint col;
 
     /** Invalid position value. */
@@ -86,6 +86,22 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -94,6 +110,8 @@ private struct TreeNode
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     void *[0] fields;
 }
 
@@ -103,6 +121,8 @@ public struct <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>
     /* TreeNode fields must be present in the same order here. */
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
 <%= @grammar.token_user_fields %>
@@ -115,6 +135,8 @@ public struct <%= @grammar.tree_prefix %><%= name %><%= @grammar.tree_suffix %>
 {
     <%= @grammar.prefix %>position_t position;
     <%= @grammar.prefix %>position_t end_position;
+    ushort n_fields;
+    bool is_token;
 <%     rule_set.tree_fields.each do |fields| %>
     union
     {
@@ -225,6 +247,7 @@ private enum size_t INVALID_ID = cast(size_t)-1;
  *************************************************************************/
 
 /**
+ * Allocate and initialize lexer/parser context structure.
  *
  * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
@@ -249,6 +272,16 @@ private enum size_t INVALID_ID = cast(size_t)-1;
     return context;
 }
 
+/**
+ * Deinitialize and deallocate lexer/parser context structure.
+ *
+ * @param context
+ *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
+ */
+void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
+{
+}
+
 /**************************************************************************
  * Decoder
  *************************************************************************/
@@ -632,6 +665,22 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     {
     case P_SUCCESS:
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state.token;
+        /* Calculate the token length and start/end positions before invoking
+         * the lexer user code so that the user code can access them. The
+         * context input text position tracking is not updated until after the
+         * user code has run so that it is left unchanged if the user code
+         * requests to terminate the lexer. */
+        token_info.length = match_info.length;
+        if (match_info.end_delta_position.row != 0u)
+        {
+            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
+            token_info.end_position.col = match_info.end_delta_position.col;
+        }
+        else
+        {
+            token_info.end_position.row = token_info.position.row;
+            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
+        }
         if (match_info.accepting_state.code_id != INVALID_USER_CODE_ID)
         {
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
@@ -670,17 +719,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             return P_DROP;
         }
         token_info.token = token_to_accept;
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -849,12 +887,14 @@ private struct state_value_t
     /** Parser state ID. */
     size_t state_id;
 
-    /** Parser value from this state. */
-    <%= @grammar.prefix %>value_t pvalue;
-
 <% if @grammar.tree %>
     /** Tree node. */
     void * tree_node;
+<% else %>
+    <%= @grammar.prefix %>position_t position;
+    <%= @grammar.prefix %>position_t end_position;
+    /** Parser value from this state. */
+    <%= @grammar.prefix %>value_t pvalue;
 <% end %>
 
     this(size_t state_id)
@@ -908,6 +948,53 @@ private immutable parser_state_t[] parser_state_table = [
 ];
 
 <% unless @grammar.tree %>
+/**
+ * Get the rule position (start or end) for the currently matched rule.
+ */
+private <%= @grammar.prefix %>position_t get_rule_position(state_value_t[] statevalues, size_t i, size_t n_states, bool get_end)
+{
+    if (n_states > 0u)
+    {
+        if (i == 0u)
+        {
+            if (get_end)
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-1-j];
+                    if (sv.end_position.valid)
+                    {
+                        return sv.end_position;
+                    }
+                }
+            }
+            else
+            {
+                for (size_t j = 0u; j < n_states; j++)
+                {
+                    state_value_t * sv = &statevalues[$-n_states+j];
+                    if (sv.position.valid)
+                    {
+                        return sv.position;
+                    }
+                }
+            }
+        }
+        else
+        {
+            if (get_end)
+            {
+                return statevalues[$-1-n_states+i].end_position;
+            }
+            else
+            {
+                return statevalues[$-1-n_states+i].position;
+            }
+        }
+    }
+    return <%= @grammar.prefix %>position_t.INVALID;
+}
+
 /**
  * Execute user code associated with a parser rule.
  *
@@ -1014,13 +1101,15 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
 <% if @grammar.tree %>
     void * reduced_parser_node;
 <% else %>
+    <%= @grammar.prefix %>position_t reduced_position;
+    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
 <% end %>
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;
@@ -1054,10 +1143,12 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.tree %>
-                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, token, token_info.pvalue);
+                <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %> * token_tree_node = new <%= @grammar.tree_prefix %>Token<%= @grammar.tree_suffix %>(token_info.position, token_info.end_position, 0u, true, token, token_info.pvalue);
 <%= expand_code(@grammar.on_token_node, false, nil, nil) %>
                 statevalues[$-1].tree_node = token_tree_node;
 <% else %>
+                statevalues[$-1].position = token_info.position;
+                statevalues[$-1].end_position = token_info.end_position;
                 statevalues[$-1].pvalue = token_info.pvalue;
 <% end %>
                 token = INVALID_TOKEN_ID;
@@ -1069,6 +1160,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 statevalues[$-1].tree_node = reduced_parser_node;
 <% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
+                statevalues[$-1].position = reduced_position;
+                statevalues[$-1].end_position = reduced_end_position;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
 <% end %>
@@ -1094,6 +1187,8 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
+                node.n_fields = cast(ushort)n_fields;
+                node.is_token = false;
                 foreach (i; 0..n_fields)
                 {
                     node.fields[i] = null;
@@ -1139,6 +1234,16 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+            if (parser_reduce_table[reduce_index].n_states > 0u)
+            {
+                reduced_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
+                reduced_end_position = get_rule_position(statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
+            }
+            else
+            {
+                reduced_position = <%= @grammar.prefix %>position_t.INVALID;
+                reduced_end_position = <%= @grammar.prefix %>position_t.INVALID;
+            }
 <% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
@@ -1201,6 +1306,35 @@ public <%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule
 <%   end %>
 <% end %>
 
+<% if @grammar.tree %>
+private void tree_delete(TreeNode * node)
+{
+    if (!node.is_token)
+    {
+        for (size_t i = 0u; i < node.n_fields; i++)
+        {
+            if (node.fields[i])
+            {
+                tree_delete(cast(TreeNode *)node.fields[i]);
+            }
+        }
+        GC.removeRange(node);
+        free(node);
+    }
+}
+
+void <%= @grammar.prefix %>tree_delete(<%= @grammar.tree_prefix %><%= @grammar.start_rules[0] %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
+void <%= @grammar.prefix %>tree_delete_<%= start_rule %>(<%= @grammar.tree_prefix %><%= start_rule %><%= @grammar.tree_suffix %> * tree)
+{
+    tree_delete(cast(TreeNode *)tree);
+}
+<%   end %>
+<% end %>
+
 /**
  * Get the current text input position.
  *

assets/parser.h.erb

@@ -45,10 +45,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
  */
 typedef struct
 {
-    /** Input text row (0-based). */
+    /** Input text row (1-based). */
     uint32_t row;
 
-    /** Input text column (0-based). */
+    /** Input text column (1-based). */
     uint32_t col;
 } <%= @grammar.prefix %>position_t;
 
@@ -69,6 +69,22 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -151,12 +151,37 @@ needed by a `ptype` directive, for example:
 
 ### Lexer pattern code blocks
 
-#### C/C++
+Lexer code blocks appear between `<<` and `>>` markers following a `drop`,
+`token`, or pattern expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+The input text positions of the matched token can also be accessed from within
+a lexer code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+The start position of the matched token is accessed with `${position}`, and
+the end position of the matched token is accessed with `${end_position}`.
+
+Example:
+
+```
+token integer /\d+/ <<
+  printf("integer token on row %d, col %d\n",
+      ${position}.row, ${position}.col);
+  $$ = parse_integer(match, match_length);
+>>
+```
+
+#### C/C++ lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a pointer points to the text matched by the lexer pattern.
-  * `match_length` - length of the matched text.
+  * `match` (`uint8_t const`) - the pointer points to the text matched by the lexer pattern.
+  * `match_length` (`size_t`) - length of the matched text.
 
 Example:
 
@@ -174,11 +199,11 @@ token integer /\d+/ <<
 >>
 ```
 
-#### D
+#### D lexer code block arguments
 
 The lexer code block is passed the following arguments:
 
-  * `match` - a slice containing the text matched by the lexer pattern.
+  * `match` (`string`) - a slice containing the text matched by the lexer pattern.
 
 ```
 ptype ulong;
@@ -194,13 +219,6 @@ token integer /\d+/ <<
 >>
 ```
 
-Lexer code blocks appear following a `drop`, `token`, or pattern expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
 ### Parser rule code blocks
 
 Example:
@@ -217,6 +235,41 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
+The input text positions for the reduced rule and for the individual rule
+components can also be accessed from within a parser rule code block.
+Each of these positions is an instance of the `p_position_t` structure (see
+`${#p_position_t}`), which contains 1-based `row` and `col` fields.
+
+The start position of the overall reduced rule is accessed with
+`${$.position}`, and the end position of the overall reduced rule is accessed
+with `${$.end_position}`.
+
+The start and end positions of an individual rule component are accessed
+positionally with `${N.position}` and `${N.end_position}`, where `N` is the
+1-based index of the component (`${1.position}` for the first component,
+`${2.position}` for the second, and so on).
+
+Example:
+
+```
+Assignment -> ident equals Expr <<
+    printf("assignment on row %d, col %d\n",
+        ${$.position}.row, ${$.position}.col);
+    printf("target identifier ends on row %d, col %d\n",
+        ${1.end_position}.row, ${1.end_position}.col);
+    printf("expression starts on row %d, col %d\n",
+        ${3.position}.row, ${3.position}.col);
+>>
+```
+
+A rule or rule component that allows for an empty match may not have valid
+positions.
+In this case the position should be checked for validity before its `row` and
+`col` fields are used (see `${#p_position_valid}`).
+For C targets this can be accomplished with
+`if (p_position_valid(${$.position}))` and for D targets this can be
+accomplished with `if (${$.position}.valid)`.
+
 Parser rule code blocks are not available in tree generation mode.
 In tree generation mode, a full parse tree is automatically constructed in
 memory for user code to traverse after parsing is complete.
@@ -316,6 +369,61 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -328,6 +436,27 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
+##> `noline` statement - disabling `#line` directives
+
+By default, Propane emits `#line` directives into the generated output around
+user code blocks.
+These directives instruct the compiler to report any warnings or errors in the
+user code using the file name and line number of the original grammar file,
+rather than the generated output file.
+This makes it easier to locate the source of a compiler diagnostic in the
+grammar file.
+
+The `noline` statement disables the emission of these `#line` directives.
+
+```
+noline;
+```
+
+When the `noline` statement is present, user code blocks are copied to the
+generated output without any surrounding `#line` directives.
+This can be useful when debugging the generated parser itself, or when the
+`#line` directives interfere with other tooling.
+
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -1007,6 +1136,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1016,6 +1156,9 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
+  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
+  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
+  `ptype` values.
 
 ### Tree Node Types
 
@@ -1130,8 +1273,6 @@ p_context_t * context = p_context_new(input);
 The `p_context_delete()` function must be called to deinitialize and deallocate
 a context structure allocated by `p_context_init()`.
 
-This function is not available for D language since D has a garbage collector.
-
 ### `p_lex`
 
 The `p_lex()` function is the main entry point to the lexer.
@@ -1258,7 +1399,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
 {
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token at row %u column %u\n",
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 
@@ -1320,8 +1461,6 @@ assert(code_point_length == 4u);
 The `p_tree_delete()` function can be used to free the memory used by the tree.
 It should be passed the same value that is returned by `p_result()`.
 
-The `p_tree_delete()` function is only available for C/C++ output targets.
-
 Note that if any lexer user code block allocates memory to store in a token's
 `pvalue`, in order to properly free this memory the `free_token_node` statement
 should be used to provide a code block that frees this memory.
@@ -1340,6 +1479,37 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
+### `p_value`
+
+The `p_value(v)` function builds an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
+
+They are especially useful when tree generation mode is not active.
+In that case, the `p_value_t` union can hold one of several different possible
+value types.
+
+### `p_value_get`
+
+The `p_value_get()` accessor functions can be used to extract a user value
+from a `p_value_t` union.
+
+The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
+returns the value of its default member.
+
+A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
+name with the user-given name in place of the `XXX`, returning the value of the
+corresponding member.
+
+These functions are the counterpart to `p_value()` constructor functions and
+are useful for reading the parser value associated with a lexed token, for
+example `p_value_get(&token_info.pvalue)`.
+
 ##> Data
 
 ### `p_token_names`
@@ -1357,7 +1527,7 @@ if (p_parse(context) == P_UNEXPECTED_TOKEN)
     p_position_t error_position = p_position(context);
     fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
         p_token_names[context->token],
-        error_position.row + 1, error_position.col + 1);
+        error_position.row, error_position.col);
 }
 ```
 

extra/vim/syntax/propane.vim

@@ -8,7 +8,11 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -20,12 +24,19 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+" Keywords that introduce a user-defined name. The name is consumed by
+" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
+" is not highlighted as a keyword. These must be a match (not syn keyword)
+" because a syn keyword always wins over a contained nextgroup match.
+syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
+syn match propaneName "\<\h\w*\>" contained
+syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
+hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
 hi def link propaneFieldOperator Operator

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -38,7 +38,13 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            %[#line #{i + 2} "#{output_file}"\n]
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -306,6 +312,14 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
+        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, false)"
+        end
+        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+          index = $1.to_i
+          "get_rule_position(statevalues, #{index}, n_states, true)"
+        end
         code = code.gsub(/\$\{(\w+)\}/) do |match|
           aliasname = $1
           if index = rule.aliases[aliasname]
@@ -337,6 +351,22 @@ class Propane
             end
           end
         end
+        code = code.gsub(/\$\{position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->position"
+          when "d"
+            "out_token_info.position"
+          end
+        end
+        code = code.gsub(/\$\{end_position\}/) do |match|
+          case @language
+          when "c"
+            "out_token_info->end_position"
+          when "d"
+            "out_token_info.end_position"
+          end
+        end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
           mode_name = $1
           mode_id = @lexer.mode_id(mode_name)
@@ -354,6 +384,14 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -21,7 +22,8 @@ class Propane
     attr_reader :on_token_node
     attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -69,6 +71,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -85,6 +88,7 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -117,6 +121,12 @@ class Propane
       end
     end
 
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true
@@ -296,8 +306,14 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -315,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -338,9 +361,14 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.0.0"
+  VERSION = "4.6.0"
 end

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,42 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+    end
+
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -144,7 +179,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +232,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -95,15 +95,13 @@ EOF
     File.binwrite("spec/run/.stdout", stdout)
     stderr.sub!(/^.*modules passed unittests\n/, "")
     results = Results.new(stdout, stderr, status)
-    if %w[c cpp].include?(options[:language])
-      stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
-      vgout = stdout + stderr
-      File.binwrite("spec/run/.vgout", vgout)
-      vgout.scan(/lost: (\d+) bytes/) do |match|
-        bytes = $1.to_i
-        if bytes > 0
-          raise "Valgrind detected memory leak"
-        end
+    stdout, stderr, status = Open3.capture3("valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes --verbose spec/run/testparser")
+    vgout = stdout + stderr
+    File.binwrite("spec/run/.vgout", vgout)
+    vgout.scan(/(?:definitely|indirectly) lost: (\d+) bytes/) do |match|
+      bytes = $1.to_i
+      if bytes > 0
+        raise "Valgrind detected memory leak"
       end
     end
     results
@@ -912,6 +910,57 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows lexer code blocks to access the matched token positions" do
+        case language
+        when "c", "cpp"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        when "d"
+          write_grammar <<EOF
+context_user_fields <<
+    p_position_t last_start;
+    p_position_t last_end;
+>>
+drop /\\s+/;
+token word /[a-z]+/ <<
+    ${context.last_start} = ${position};
+    ${context.last_end} = ${end_position};
+>>
+token stop /!/ <<
+    $terminate(42);
+>>
+Start -> Words;
+Words -> ;
+Words -> word Words;
+Words -> stop Words;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows the user to terminate the parser" do
         write_grammar <<EOF
 token a;
@@ -1431,6 +1480,9 @@ EOF
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
 
+      # D garbage collector was freeing memory for nodes when it should not
+      # have been. This was due to pointers being in the TreeNode that was
+      # allocated by C malloc() function instead of allocated using D.
       it "does not free memory allocated for tree nodes" do
         ext = language == "cpp" ? "c" : language
         write_grammar(File.read("spec/tree_node_memory_remains.#{ext}.propane"))
@@ -1623,7 +1675,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         elsif language == "c"
           write_grammar <<EOF
@@ -1665,7 +1717,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         else # C++
           write_grammar <<EOF
@@ -1690,7 +1742,7 @@ drop /#(.*)\\n/ <<
 token id /\\w+/;
 Start -> IDs;
 IDs -> ;
-IDs -> id IDs;
+IDs -> id:id IDs;
 EOF
         end
         run_propane(language: language)
@@ -1698,6 +1750,247 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
+
+      it "provides accessor functions to extract user values from a parser value" do
+        if language == "d"
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = string;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello"; >>
+Start -> num flt str;
+EOF
+        else
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = (char *)"hello"; >>
+Start -> num flt str;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_value_accessors.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows accessing rule and component text positions" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+Start -> ident Num <<
+    writeln("ident start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("ident end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Num start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Num end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("Start start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("Start end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    writeln("Empty start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("Empty end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("tok2 start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("tok2 end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    writeln("tok1 start: ", ${1.position}.row, ", ", ${1.position}.col);
+    writeln("tok1 end: ", ${1.end_position}.row, ", ", ${1.end_position}.col);
+    writeln("Empty start: ", ${2.position}.row, ", ", ${2.position}.col);
+    writeln("Empty end: ", ${2.end_position}.row, ", ", ${2.end_position}.col);
+    writeln("R2 start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("R2 end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    writeln("StartR start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartR end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+Start -> Empty <<
+    writeln("StartEmpty start: ", ${$.position}.row, ", ", ${$.position}.col);
+    writeln("StartEmpty end: ", ${$.end_position}.row, ", ", ${$.end_position}.col);
+>>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+drop /\\s+/;
+token tok1;
+token tok2;
+token ident /[a-zA-Z_]\\w*/;
+token num /\\d+/;
+Num -> num;
+token pct /%/;
+Start -> ident Num <<
+    printf("ident start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("ident end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Num start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Num end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("Start start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("Start end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> Empty tok2 <<
+    printf("Empty start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("Empty end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("tok2 start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("tok2 end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+R -> tok1 Empty <<
+    printf("tok1 start: %d, %d\\n", ${1.position}.row, ${1.position}.col);
+    printf("tok1 end: %d, %d\\n", ${1.end_position}.row, ${1.end_position}.col);
+    printf("Empty start: %d, %d\\n", ${2.position}.row, ${2.position}.col);
+    printf("Empty end: %d, %d\\n", ${2.end_position}.row, ${2.end_position}.col);
+    printf("R2 start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("R2 end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Empty -> ;
+Start -> R <<
+    printf("StartR start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartR end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+Start -> Empty <<
+    printf("StartEmpty start: %d, %d\\n", ${$.position}.row, ${$.position}.col);
+    printf("StartEmpty end: %d, %d\\n", ${$.end_position}.row, ${$.end_position}.col);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_positions.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to eq <<EOF
+ident start: 1, 5
+ident end: 1, 9
+Num start: 3, 9
+Num end: 3, 12
+Start start: 1, 5
+Start end: 3, 12
+
+Empty start: 0, 0
+Empty end: 0, 0
+tok2 start: 2, 2
+tok2 end: 2, 5
+R start: 2, 2
+R end: 2, 5
+StartR start: 2, 2
+StartR end: 2, 5
+
+tok1 start: 1, 3
+tok1 end: 1, 6
+Empty start: 0, 0
+Empty end: 0, 0
+R2 start: 1, 3
+R2 end: 1, 6
+StartR start: 1, 3
+StartR end: 1, 6
+EOF
+      end
     end
   end
 end

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_lexer_positions.c

@@ -0,0 +1,50 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 1u);
+    assert(context->last_start.col == 1u);
+    assert(context->last_end.row == 1u);
+    assert(context->last_end.col == 3u);
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context->last_start.row == token_info.position.row);
+    assert(context->last_start.col == token_info.position.col);
+    assert(context->last_end.row == token_info.end_position.row);
+    assert(context->last_end.col == token_info.end_position.col);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 3u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 6u);
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context->last_start.row == 2u);
+    assert(context->last_start.col == 8u);
+    assert(context->last_end.row == 2u);
+    assert(context->last_end.col == 9u);
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context->text_position.row == 3u);
+    assert(context->text_position.col == 1u);
+
+    p_context_delete(context);
+    return 0;
+}

spec/test_lexer_positions.d

@@ -0,0 +1,42 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc\n  defg hi\n!";
+    p_context_t * context = p_context_new(input);
+    p_token_info_t token_info;
+
+    /* First token "abc" on row 1, cols 1-3. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(1, 1));
+    assert(context.last_end == p_position_t(1, 3));
+    /* The lexer code block observed the same positions reported to the caller. */
+    assert(context.last_start == token_info.position);
+    assert(context.last_end == token_info.end_position);
+
+    /* Second token "defg" on row 2, cols 3-6. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 3));
+    assert(context.last_end == p_position_t(2, 6));
+
+    /* Third token "hi" on row 2, cols 8-9. */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_word);
+    assert(context.last_start == p_position_t(2, 8));
+    assert(context.last_end == p_position_t(2, 9));
+
+    /* The "!" stop token terminates the lexer. The context input text position
+     * must not be updated when the lexer user code requests termination, so it
+     * still points at the "!" token on row 3, col 1. */
+    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(context) == 42u);
+    assert(context.text_position == p_position_t(3, 1));
+}

spec/test_named_optional_rule_component_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.pToken1 is null);
@@ -21,6 +20,8 @@ unittest
     assert(start.pR is null);
     assert(start.r is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -35,6 +36,8 @@ unittest
     assert(start.pR == start.r);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -43,4 +46,6 @@ unittest
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_optional_rule_component_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.pToken1 is null);
@@ -20,6 +19,8 @@ unittest
     assert(start.pR3 is null);
     assert(start.pR is null);
 
+    p_tree_delete(start);
+
     input = "abcd";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -32,6 +33,8 @@ unittest
     assert(start.pR == start.pR3);
     assert_eq(TOKEN_c, start.pR.pToken1.token);
 
+    p_tree_delete(start);
+
     input = "bdc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -40,4 +43,6 @@ unittest
     assert(start.pToken2 !is null);
     assert(start.pR !is null);
     assert_eq(TOKEN_d, start.pR.pToken1.token);
+
+    p_tree_delete(start);
 }

spec/test_positions.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "\n tok2";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    printf("\n");
+
+    input = "  tok1";
+    context = p_context_new((uint8_t const *)input, strlen(input));
+    assert(p_parse(context) == P_SUCCESS);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_positions.d

@@ -0,0 +1,27 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "    Hello\n\n        4200\n";
+    p_context_t * context;
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "\n tok2";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+
+    writeln();
+
+    input = "  tok1";
+    context = p_context_new(input);
+    assert(p_parse(context) == P_SUCCESS);
+}

spec/test_starting_rules_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "bbbb";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
     assert(start.bs);
@@ -20,6 +19,8 @@ unittest
     assert(start.bs.bs.bs.b);
     assert(start.bs.bs.bs.bs.b);
 
+    p_tree_delete(start);
+
     context = p_context_new(input);
     assert(p_parse_Bs(context) == P_SUCCESS);
     Bs * bs = p_result_Bs(context);
@@ -28,9 +29,13 @@ unittest
     assert(bs.bs.bs.b);
     assert(bs.bs.bs.bs.b);
 
+    p_tree_delete_Bs(bs);
+
     input = "c";
     context = p_context_new(input);
     assert(p_parse_R(context) == P_SUCCESS);
     R * r = p_result_R(context);
     assert(r.c);
+
+    p_tree_delete_R(r);
 }

spec/test_token_user_fields.c

@@ -19,6 +19,22 @@ int main()
     context = p_context_new((uint8_t const *)input, strlen(input));
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
+    assert(start->pIDs);
+    assert(start->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->id->comments == "# c1\n#  c2\n");
+#else
+    assert(start->pIDs->id->comments);
+    assert(strcmp(start->pIDs->id->comments, "# c1\n#  c2\n") == 0);
+#endif
+    assert(start->pIDs->pIDs);
+    assert(start->pIDs->pIDs->id);
+#ifdef __cplusplus
+    assert(start->pIDs->pIDs->id->comments == "# s1\n#   s2\n");
+#else
+    assert(start->pIDs->pIDs->id->comments);
+    assert(strcmp(start->pIDs->pIDs->id->comments, "# s1\n#   s2\n") == 0);
+#endif
 
 #ifndef __cplusplus
     free(context->comments);

spec/test_token_user_fields.d

@@ -17,8 +17,15 @@ unittest
         "  # s1\n" ~
         "   #   s2\n" ~
         "second\n";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
+    assert(start.pIDs);
+    assert(start.pIDs.id);
+    assert(start.pIDs.id.comments == "# c1\n#  c2\n");
+    assert(start.pIDs.pIDs);
+    assert(start.pIDs.pIDs.id);
+    assert(start.pIDs.pIDs.id.comments == "# s1\n#   s2\n");
+
+    p_tree_delete(start);
 }

spec/test_tree.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     Start * start = p_result(context);
     assert(start.pItems1 !is null);
@@ -37,12 +36,16 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_field_aliases.d

@@ -10,12 +10,13 @@ int main()
 unittest
 {
     string input = "\na\nb\nc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
     assert_eq(TOKEN_a, start.first.pToken.token);
     assert_eq(TOKEN_b, start.second.pToken.token);
     assert_eq(TOKEN_c, start.third.pToken.token);
+
+    p_tree_delete(start);
 }

spec/test_tree_invalid_positions.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "\na\n  bb ccc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
@@ -34,6 +33,8 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(8, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nbb";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -58,6 +59,8 @@ unittest
     assert_eq(2, start.end_position.row);
     assert_eq(2, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a\nc\nc";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -82,6 +85,8 @@ unittest
     assert_eq(3, start.end_position.row);
     assert_eq(1, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "a";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -101,4 +106,6 @@ unittest
     assert_eq(1, start.position.col);
     assert_eq(1, start.end_position.row);
     assert_eq(1, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_tree_node_memory_remains.d

@@ -405,4 +405,5 @@ def main() -> int
             stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
         }
     }
+    p_tree_delete(pmod);
 }

spec/test_tree_ps.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "a, ((b)), b";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     PStartS * start = p_result(context);
     assert(start.pItems1 !is null);
@@ -37,12 +36,16 @@ unittest
     assert_eq(22, itemsmore.pItem.pToken1.pvalue);
     assert(itemsmore.pItemsMore is null);
 
+    p_tree_delete(start);
+
     input = "";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
     start = p_result(context);
     assert(start.pItems is null);
 
+    p_tree_delete(start);
+
     input = "2 1";
     context = p_context_new(input);
     assert_eq(P_SUCCESS, p_parse(context));
@@ -54,4 +57,6 @@ unittest
     assert(start.pItems.pItem.pDual.pOne2 !is null);
     assert(start.pItems.pItem.pDual.pTwo2 is null);
     assert(start.pItems.pItem.pDual.pOne1 is null);
+
+    p_tree_delete(start);
 }

spec/test_tree_token_positions.d

@@ -10,8 +10,7 @@ int main()
 unittest
 {
     string input = "abbccc";
-    p_context_t * context;
-    context = p_context_new(input);
+    p_context_t * context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
     Start * start = p_result(context);
 
@@ -47,6 +46,8 @@ unittest
     assert_eq(1, start.end_position.row);
     assert_eq(6, start.end_position.col);
 
+    p_tree_delete(start);
+
     input = "\n\n  bb\nc\ncc\n\n     a";
     context = p_context_new(input);
     assert(p_parse(context) == P_SUCCESS);
@@ -83,4 +84,6 @@ unittest
     assert_eq(3, start.position.col);
     assert_eq(7, start.end_position.row);
     assert_eq(6, start.end_position.col);
+
+    p_tree_delete(start);
 }

spec/test_value_accessors.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    p_token_info_t token_info;
+    char const * input = "42 f s";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(strcmp(p_value_get_string(&token_info.pvalue), "hello") == 0);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_value_accessors.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "42 f s";
+    p_context_t * context;
+    context = p_context_new(input);
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(p_value_get_string(&token_info.pvalue) == "hello");
+}