v1.4.0

Add ast_prefix and ast_suffix grammar statements.

CHANGELOG.md

@@ -1,3 +1,21 @@
+## v1.4.0
+
+### New Features
+
+- Allow user to specify AST node name prefix or suffix
+- Allow specifying the start rule name
+- Allow rule terms to be marked as optional
+
+### Improvements
+
+- Give a better error message when a referenced ptype has not been declared
+
+## v1.3.0
+
+### New Features
+
+- Add AST generation (#22)
+
 ## v1.2.0
 
 ### New Features

README.md

@@ -6,7 +6,8 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * target C or D language outputs
+  * targets C or D language outputs
+  * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
@@ -17,6 +18,7 @@ can be copied into and versioned in a project's source tree.
 The only requirement to run Propane is that the system has a Ruby interpreter
 installed.
 The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+
 Simply copy the `propane` executable script into the desired location within
 the project to be built (typically the root of the repository) and mark it
 executable.
@@ -55,10 +57,10 @@ import std.math;
 ptype ulong;
 
 # A few basic arithmetic operators.
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
+token plus /\+/;
+token times /\*/;
+token power /\*\*/;
+token integer /\d+/ <<
   ulong v;
   foreach (c; match)
   {
@@ -67,38 +69,22 @@ token integer /\\d+/ <<
   }
   $$ = v;
 >>
-token lparen /\\(/;
-token rparen /\\)/;
+token lparen /\(/;
+token rparen /\)/;
 # Drop whitespace.
-drop /\\s+/;
+drop /\s+/;
 
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
 E3 -> E3 power E4 <<
   $$ = pow($1, $3);
 >>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
 Grammar files can contain comment lines beginning with `#` which are ignored.

assets/parser.c.erb

@@ -320,9 +320,12 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
  *
  * @param context
  *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
  *
  * @reval P_SUCCESS
  *   A token was successfully lexed.
@@ -608,6 +611,25 @@ typedef struct
      * reduce action.
      */
     parser_state_id_t n_states;
+<% if @grammar.ast %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    uint16_t const * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set AST node fields.
+     */
+    uint16_t rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
+<% end %>
 } reduce_t;
 
 /** Parser state entry. */
@@ -638,6 +660,11 @@ typedef struct
 
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
+
+<% if @grammar.ast %>
+    /** AST node. */
+    void * ast_node;
+<% end %>
 } state_value_t;
 
 /** Parser shift table. */
@@ -647,10 +674,28 @@ static const shift_t parser_shift_table[] = {
 <%   end %>
 };
 
+<% if @grammar.ast %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
+    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+<%     if @grammar.ast %>
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+             , NULL
+<%       else %>
+             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+<%       end %>
+             , <%= reduce[:rule].rule_set.ast_fields.size %>
+             , <%= reduce[:propagate_optional_target] %>
+<%     end %>
+    },
 <%   end %>
 };
 
@@ -754,6 +799,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
+<% unless @grammar.ast %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -768,18 +814,19 @@ static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t
 {
     switch (rule)
     {
-<% @grammar.rules.each do |rule| %>
-<%   if rule.code %>
+<%   @grammar.rules.each do |rule| %>
+<%     if rule.code %>
     case <%= rule.id %>u: {
 <%= expand_code(rule.code, true, rule, nil) %>
     } break;
+<%     end %>
 <%   end %>
-<% end %>
     default: break;
     }
 
     return P_SUCCESS;
 }
+<% end %>
 
 /**
  * Check if the parser should shift to a new state.
@@ -853,7 +900,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.ast %>
+    void * reduced_parser_node;
+<% else %>
     <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
     state_values_stack_init(&statevalues);
     state_values_stack_push(&statevalues);
     size_t result;
@@ -880,7 +931,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
+<% if @grammar.ast %>
+                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
+<% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
                 result = P_SUCCESS;
                 break;
             }
@@ -893,15 +948,26 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
+<% if @grammar.ast %>
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
+                token_ast_node->token = token;
+                token_ast_node->pvalue = token_info.pvalue;
+                state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
+<% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+<% end %>
+                token = INVALID_TOKEN_ID;
             }
             else
             {
                 /* We shifted a RuleSet. */
+<% if @grammar.ast %>
+                state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node;
+<% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
                 <%= @grammar.prefix %>value_t new_parse_result = {0};
                 reduced_parser_value = new_parse_result;
+<% end %>
                 reduced_rule_set = INVALID_ID;
             }
             continue;
@@ -911,12 +977,42 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
+<% if @grammar.ast %>
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->ast_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                void ** node_fields = calloc(parser_reduce_table[reduce_index].rule_set_node_field_array_size, sizeof(void *));
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node_fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                    }
+                }
+                else
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                    }
+                }
+                reduced_parser_node = node_fields;
+            }
+            else
+            {
+                reduced_parser_node = NULL;
+            }
+<% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
             {
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+<% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
@@ -944,9 +1040,17 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  *
  * @return Parse result value.
  */
+<% if @grammar.ast %>
+<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% end %>
 {
+<% if @grammar.ast %>
+    return context->parse_result;
+<% else %>
     return context->parse_result.v_<%= start_rule_type[0] %>;
+<% end %>
 }
 
 /**

assets/parser.d.erb

@@ -49,13 +49,45 @@ public enum : <%= @grammar.prefix %>token_t
 /** Code point type. */
 public alias <%= @grammar.prefix %>code_point_t = uint;
 
+<% if @grammar.ast %>
+/** Parser values type. */
+public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
+<% else %>
 /** Parser values type(s). */
 public union <%= @grammar.prefix %>value_t
 {
-<% @grammar.ptypes.each do |name, typestring| %>
+<%   @grammar.ptypes.each do |name, typestring| %>
     <%= typestring %> v_<%= name %>;
-<% end %>
+<%   end %>
 }
+<% end %>
+
+<% if @grammar.ast %>
+/** AST node types. @{ */
+public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
+{
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+}
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
+{
+<%     rule_set.ast_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        <%= type %> * <%= field_name %>;
+<%       end %>
+    }
+<%     end %>
+}
+
+<%   end %>
+/** @} */
+<% end %>
 
 /**
  * A structure to keep track of parser position.
@@ -112,7 +144,11 @@ public struct <%= @grammar.prefix %>context_t
     /* Parser context data. */
 
     /** Parse result value. */
+<% if @grammar.ast %>
+    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+<% else %>
     <%= @grammar.prefix %>value_t parse_result;
+<% end %>
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
@@ -428,9 +464,12 @@ private lexer_state_id_t check_lexer_transition(uint current_state, uint code_po
  *
  * @param context
  *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
  *
  * @reval P_SUCCESS
  *   A token was successfully lexed.
@@ -714,6 +753,25 @@ private struct reduce_t
      * reduce action.
      */
     parser_state_id_t n_states;
+<% if @grammar.ast %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    immutable(ushort) * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set AST node fields.
+     */
+    ushort rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
+<% end %>
 }
 
 /** Parser state entry. */
@@ -745,6 +803,11 @@ private struct state_value_t
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
 
+<% if @grammar.ast %>
+    /** AST node. */
+    void * ast_node;
+<% end %>
+
     this(size_t state_id)
     {
         this.state_id = state_id;
@@ -758,10 +821,28 @@ private immutable shift_t[] parser_shift_table = [
 <%   end %>
 ];
 
+<% if @grammar.ast %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
+    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+<%     if @grammar.ast %>
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+             , null
+<%       else %>
+             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+<%       end %>
+             , <%= reduce[:rule].rule_set.ast_fields.size %>
+             , <%= reduce[:propagate_optional_target] %>
+<%     end %>
+            ),
 <%   end %>
 ];
 
@@ -772,6 +853,7 @@ private immutable parser_state_t[] parser_state_table = [
 <%   end %>
 ];
 
+<% unless @grammar.ast %>
 /**
  * Execute user code associated with a parser rule.
  *
@@ -786,18 +868,19 @@ private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint ru
 {
     switch (rule)
     {
-<% @grammar.rules.each do |rule| %>
-<%   if rule.code %>
+<%   @grammar.rules.each do |rule| %>
+<%     if rule.code %>
     case <%= rule.id %>u: {
 <%= expand_code(rule.code, true, rule, nil) %>
     } break;
+<%     end %>
 <%   end %>
-<% end %>
     default: break;
     }
 
     return P_SUCCESS;
 }
+<% end %>
 
 /**
  * Check if the parser should shift to a new state.
@@ -871,7 +954,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_value_t[] statevalues = new state_value_t[](1);
     size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.ast %>
+    void * reduced_parser_node;
+<% else %>
     <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
@@ -894,7 +981,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
+<% if @grammar.ast %>
+                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
+<% else %>
                 context.parse_result = statevalues[$-1].pvalue;
+<% end %>
                 return P_SUCCESS;
             }
         }
@@ -905,15 +996,24 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
+<% if @grammar.ast %>
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token, token_info.pvalue);
+                statevalues[$-1].ast_node = token_ast_node;
+<% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;
+<% end %>
+                token = INVALID_TOKEN_ID;
             }
             else
             {
                 /* We shifted a RuleSet. */
+<% if @grammar.ast %>
+                statevalues[$-1].ast_node = reduced_parser_node;
+<% else %>
                 statevalues[$-1].pvalue = reduced_parser_value;
                 <%= @grammar.prefix %>value_t new_parse_result;
                 reduced_parser_value = new_parse_result;
+<% end %>
                 reduced_rule_set = INVALID_ID;
             }
             continue;
@@ -923,12 +1023,46 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
+<% if @grammar.ast %>
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = statevalues[$ - 1].ast_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                void *[] node_fields = new void *[parser_reduce_table[reduce_index].rule_set_node_field_array_size];
+                foreach (i; 0..parser_reduce_table[reduce_index].rule_set_node_field_array_size)
+                {
+                    node_fields[i] = null;
+                }
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node_fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                    }
+                }
+                else
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                    }
+                }
+                reduced_parser_node = node_fields.ptr;
+            }
+            else
+            {
+                reduced_parser_node = null;
+            }
+<% else %>
             <%= @grammar.prefix %>value_t reduced_parser_value2;
             if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
             {
                 return P_USER_TERMINATED;
             }
             reduced_parser_value = reduced_parser_value2;
+<% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
             continue;
@@ -953,9 +1087,17 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  *
  * @return Parse result value.
  */
+<% if @grammar.ast %>
+public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% end %>
 {
+<% if @grammar.ast %>
+    return context.parse_result;
+<% else %>
     return context.parse_result.v_<%= start_rule_type[0] %>;
+<% end %>
 }
 
 /**

assets/parser.h.erb

@@ -41,13 +41,51 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>
 
+<% if @grammar.ast %>
+/** Parser values type. */
+typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
+<% else %>
 /** Parser values type(s). */
 typedef union
 {
-<% @grammar.ptypes.each do |name, typestring| %>
+<%   @grammar.ptypes.each do |name, typestring| %>
     <%= typestring %> v_<%= name %>;
-<% end %>
+<%   end %>
 } <%= @grammar.prefix %>value_t;
+<% end %>
+
+<% if @grammar.ast %>
+/** AST node types. @{ */
+typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
+{
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+struct <%= name %>;
+<%   end %>
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
+typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
+{
+<%     rule_set.ast_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        struct <%= type %> * <%= field_name %>;
+<%       end %>
+    };
+<%     end %>
+} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>;
+
+<%   end %>
+/** @} */
+<% end %>
 
 /**
  * A structure to keep track of parser position.
@@ -107,7 +145,11 @@ typedef struct
     /* Parser context data. */
 
     /** Parse result value. */
+<% if @grammar.ast %>
+    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
+<% else %>
     <%= @grammar.prefix %>value_t parse_result;
+<% end %>
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
@@ -132,7 +174,11 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 
+<% if @grammar.ast %>
+<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% end %>
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 

doc/user_guide.md

@@ -14,6 +14,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
   * targets C or D language outputs
+  * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
@@ -182,6 +183,143 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...
 
+Parser rule code blocks are not available in AST generation mode.
+In AST generation mode, a full parse tree is automatically constructed in
+memory for user code to traverse after parsing is complete.
+
+##> AST generation mode - the `ast` statement
+
+To activate AST generation mode, place the `ast` statement in your grammar file:
+
+```
+ast;
+```
+
+It is recommended to place this statement early in the grammar.
+
+In AST generation mode various aspects of propane's behavior are changed:
+
+  * Only one `ptype` is allowed.
+  * Parser user code blocks are not supported.
+  * Structure types are generated to represent the parsed tokens and rules as
+  defined in the grammar.
+  * The parse result from `p_result()` points to a `Start` struct containing
+  the entire parse tree for the input. If the user has changed the start rule
+  with the `start` grammar statement, the name of the start struct will be
+  given by the user-specified start rule instead of `Start`.
+
+Example AST generation grammar:
+
+```
+ast;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+```
+
+The following unit test describes the fields that will be present for an
+example parse:
+
+```
+string input = "a, ((b)), b";
+p_context_t context;
+p_context_init(&context, input);
+assert_eq(P_SUCCESS, p_parse(&context));
+Start * start = p_result(&context);
+assert(start.pItems1 !is null);
+assert(start.pItems !is null);
+Items * items = start.pItems;
+assert(items.pItem !is null);
+assert(items.pItem.pToken1 !is null);
+assert_eq(TOKEN_a, items.pItem.pToken1.token);
+assert_eq(11, items.pItem.pToken1.pvalue);
+assert(items.pItemsMore !is null);
+ItemsMore * itemsmore = items.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+assert(itemsmore.pItemsMore !is null);
+itemsmore = itemsmore.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+assert(itemsmore.pItemsMore is null);
+```
+
+## `ast_prefix` and `ast_suffix` statements
+
+In AST generation mode, structure types are defined and named based on the
+rules in the grammar.
+Additionally, a structure type called `Token` is generated to hold parsed
+token information.
+
+These structure names can be modified by using the `ast_prefix` or `ast_suffix`
+statements in the grammar file.
+The field names that point to instances of the structures are not affected by
+the `ast_prefix` or `ast_suffix` values.
+
+For example, if the following two lines were added to the example above:
+
+```
+ast_prefix ABC;
+ast_suffix XYZ;
+```
+
+Then the types would be used as such instead:
+
+```
+string input = "a, ((b)), b";
+p_context_t context;
+p_context_init(&context, input);
+assert_eq(P_SUCCESS, p_parse(&context));
+ABCStartXYZ * start = p_result(&context);
+assert(start.pItems1 !is null);
+assert(start.pItems !is null);
+ABCItemsXYZ * items = start.pItems;
+assert(items.pItem !is null);
+assert(items.pItem.pToken1 !is null);
+assert_eq(TOKEN_a, items.pItem.pToken1.token);
+assert_eq(11, items.pItem.pToken1.pvalue);
+assert(items.pItemsMore !is null);
+ABCItemsMoreXYZ * itemsmore = items.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+```
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -442,6 +580,12 @@ In this example:
   * a reduced `Values`'s parser value has a type of `Value[]`.
   * a reduced `KeyValue`'s parser value has a type of `Value[string]`.
 
+When AST generation mode is active, the `ptype` functionality works differently.
+In this mode, only one `ptype` is used by the parser.
+Lexer user code blocks may assign a parse value to the generated `Token` node
+by assigning to `$$` within a lexer code block.
+The type of the parse value `$$` is given by the global `ptype` type.
+
 ##> Specifying a parser rule - the rule statement
 
 Rule statements create parser rules which define the grammar that will be
@@ -452,21 +596,24 @@ Rules with the same name define a rule set for that name and act as
 alternatives that the parser can accept when attempting to match a reference to
 that rule.
 
-The grammar file must define a rule with the name `Start` which will be used as
-the top-level starting rule that the parser attempts to reduce.
+The default start rule name is `Start`.
+This can be changed with the `start` statement.
+The grammar file must define a rule with the name of the start rule name which
+will be used as the top-level starting rule that the parser attempts to reduce.
 
 Example:
 
 ```
 ptype ulong;
+start Top;
 token word /[a-z]+/ << $$ = match.length; >>
-Start -> word << $$ = $1; >>
+Top -> word << $$ = $1; >>
 ```
 
-In the above example the `Start` rule is defined to match a single `word`
+In the above example the `Top` rule is defined to match a single `word`
 token.
 
-Example:
+Another example:
 
 ```
 Start -> E1 << $$ = $1; >>
@@ -480,8 +627,24 @@ E4 -> integer << $$ = $1; >>
 E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
+This example uses the default start rule name of `Start`.
+
 A parser rule has zero or more terms on the right side of its definition.
 Each of these terms is either a token name or a rule name.
+A term can be immediately followed by a `?` character to signify that it is
+optional.
+Another example:
+
+```
+token public;
+token private;
+token int;
+token ident /[a-zA-Z_][a-zA-Z_0-9]*/;
+token semicolon /;/;
+IntegerDeclaration -> Visibility? int ident semicolon;
+Visibility -> public;
+Visibility -> private;
+```
 
 In a parser rule code block, parser values for the right side terms are
 accessible as `$1` for the first term's parser value, `$2` for the second
@@ -490,6 +653,19 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.
 
+Parser rule code blocks are not allowed and not used when AST generation mode
+is active.
+
+##> Specifying the parser start rule name - the `start` statement
+
+The start rule can be changed from the default of `Start` by using the `start`
+statement.
+Example:
+
+```
+start MyStartRule;
+```
+
 ##> Specifying the parser module name - the `module` statement
 
 The `module` statement can be used to specify the module name for a generated
@@ -586,6 +762,67 @@ A pointer to this instance is passed to the generated functions.
 The `p_position_t` structure contains two fields `row` and `col`.
 These fields contain the 0-based row and column describing a parser position.
 
+### AST Node Types
+
+If AST generation mode is enabled, a structure type for each rule will be
+generated.
+The name of the structure type is given by the name of the rule.
+Additionally a structure type called `Token` is generated to represent an
+AST node which refers to a raw parser token rather than a composite rule.
+
+#### AST Node Fields
+
+A `Token` node has two fields:
+
+  * `token` which specifies which token was parsed (one of `TOKEN_*`)
+  * `pvalue` which specifies the parser value for the token. If a lexer user
+  code block assigned to `$$`, the assigned value will be stored here.
+
+The other generated AST node structures have fields generated based on the
+right hand side components specified for all rules of a given name.
+
+In this example:
+
+```
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+```
+
+The `Start` structure will have a field called `pItems` and another field of
+the same name but with a positional suffix (`pItems1`) which both point to the
+parsed `Items` node.
+Their value will be null if the parsed `Items` rule was empty.
+
+The `Items` structure will have fields:
+
+  * `pItem` and `pItem1` which point to the parsed `Item` structure.
+  * `pItemsMore` and `pItemsMore2` which point to the parsed `ItemsMore` structure.
+
+If a rule can be empty (for example in the second `Items` rule above), then
+an instance of a pointer to that rule's generated AST node will be null if the
+parser matches the empty rule definition.
+
+The non-positional AST node field pointer will not be generated if there are
+multiple positions in which an instance of the node it points to could be
+present.
+For example, in the below rules:
+
+```
+Dual -> One Two;
+Dual -> Two One;
+```
+
+The generated `Dual` structure will contain `pOne1`, `pTwo2`, `pTwo1`, and
+`pOne2` fields.
+However, a `pOne` field and `pTwo` field will not be generated since it would
+be ambiguous which one was matched.
+
+If the first rule is matched, then `pOne1` and `pTwo2` will be non-null while
+`pTwo1` and `pOne2` will be null.
+If the second rule is matched instead, then the opposite would be the case.
+
 ##> Functions
 
 ### `p_context_init`
@@ -639,6 +876,9 @@ if (p_parse(&context) == P_SUCCESS)
 }
 ```
 
+If AST generation mode is active, then the `p_result()` function returns a
+`Start *` pointing to the `Start` AST structure.
+
 ### `p_position`
 
 The `p_position()` function can be used to retrieve the parser position where

lib/propane/cli.rb

@@ -24,7 +24,7 @@ EOF
               log_file = args[i]
             end
           when "--version"
-            puts "propane v#{VERSION}"
+            puts "propane version #{VERSION}"
             return 0
           when "-h", "--help"
             puts USAGE

lib/propane/generator.rb

@@ -51,6 +51,7 @@ class Propane
       unless found_default
         raise Error.new("No patterns found for default mode")
       end
+      check_ptypes!
       # Add EOF token.
       @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
@@ -66,11 +67,14 @@ class Propane
         tokens_by_name[token.name] = token
       end
       # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == "Start"}
-        raise Error.new("Start rule not found")
+      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
+        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
       end
       # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
+      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
+      # Generate and add rules for optional components.
+      generate_optional_component_rules!(tokens_by_name)
+      # Build rule sets.
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|
@@ -119,12 +123,57 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
+      rule_sets.each do |name, rule_set|
+        rule_set.finalize(@grammar)
+      end
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
       @parser = Parser.new(@grammar, rule_sets, @log)
     end
 
+    # Check that any referenced ptypes have been defined.
+    def check_ptypes!
+      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
+        if potor.ptypename
+          unless @grammar.ptypes.include?(potor.ptypename)
+            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
+          end
+        end
+      end
+    end
+
+    # Generate and add rules for any optional components.
+    def generate_optional_component_rules!(tokens_by_name)
+      optional_rules_added = Set.new
+      @grammar.rules.each do |rule|
+        rule.components.each do |component|
+          if component =~ /^(.*)\?$/
+            c = $1
+            unless optional_rules_added.include?(component)
+              # Create two rules for the optional component: one empty and
+              # one just matching the component.
+              # We need to find the ptypename for the optional component in
+              # order to copy it to the generated rules.
+              if tokens_by_name[c]
+                # The optional component is a token.
+                ptypename = tokens_by_name[c].ptypename
+              else
+                # The optional component must be a rule, so find any instance
+                # of that rule that specifies a ptypename.
+                ptypename = @grammar.rules.reduce(nil) do |result, rule|
+                  rule.name == c && rule.ptypename ? rule.ptypename : result
+                end
+              end
+              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
+              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optional_rules_added << component
+            end
+          end
+        end
+      end
+    end
+
     # Determine which grammar rules could expand to empty sequences.
     #
     # @param rule_sets [Hash]
@@ -228,11 +277,20 @@ class Propane
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "out_token_info->pvalue.v_#{pattern.ptypename}"
-          when "d"
-            "out_token_info.pvalue.v_#{pattern.ptypename}"
+          if @grammar.ast
+            case @language
+            when "c"
+              "out_token_info->pvalue"
+            when "d"
+              "out_token_info.pvalue"
+            end
+          else
+            case @language
+            when "c"
+              "out_token_info->pvalue.v_#{pattern.ptypename}"
+            when "d"
+              "out_token_info.pvalue.v_#{pattern.ptypename}"
+            end
           end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
@@ -258,7 +316,7 @@ class Propane
     #   Start rule parser value type name and type string.
     def start_rule_type
       start_rule = @grammar.rules.find do |rule|
-        rule.name == "Start"
+        rule.name == @grammar.start_rule
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -5,9 +5,13 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
+    attr_reader :ast
+    attr_reader :ast_prefix
+    attr_reader :ast_suffix
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
+    attr_reader :start_rule
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
@@ -15,6 +19,7 @@ class Propane
 
     def initialize(input)
       @patterns = []
+      @start_rule = "Start"
       @tokens = []
       @rules = []
       @code_blocks = {}
@@ -24,6 +29,9 @@ class Propane
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
+      @ast = false
+      @ast_prefix = ""
+      @ast_suffix = ""
       parse_grammar!
     end
 
@@ -51,9 +59,13 @@ class Propane
       if parse_white_space!
       elsif parse_comment_line!
       elsif @mode.nil? && parse_mode_label!
+      elsif parse_ast_statement!
+      elsif parse_ast_prefix_statement!
+      elsif parse_ast_suffix_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
+      elsif parse_start_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
@@ -82,6 +94,24 @@ class Propane
       consume!(/#.*\n/)
     end
 
+    def parse_ast_statement!
+      if consume!(/ast\s*;/)
+        @ast = true
+      end
+    end
+
+    def parse_ast_prefix_statement!
+      if md = consume!(/ast_prefix\s+(\w+)\s*;/)
+        @ast_prefix = md[1]
+      end
+    end
+
+    def parse_ast_suffix_statement!
+      if md = consume!(/ast_suffix\s+(\w+)\s*;/)
+        @ast_suffix = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -96,6 +126,9 @@ class Propane
       if consume!(/ptype\s+/)
         name = "default"
         if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           name = md[1]
         end
         md = consume!(/([^;]+);/, "expected parser result type expression")
@@ -108,12 +141,15 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         pattern = parse_pattern! || name
         consume!(/\s+/)
         unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+          consume!(/;/, "expected `;' or code block")
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
@@ -129,6 +165,9 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         consume!(/;/, "expected `;'");
@@ -156,10 +195,17 @@ class Propane
     def parse_rule_statement!
       if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
         rule_name, ptypename = *md[1, 2]
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        if @ast && ptypename
+          raise Error.new("Multiple ptypes are unsupported in AST mode")
+        end
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
-        unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+        if @ast
+          consume!(/;/, "expected `;'")
+        else
+          unless code = parse_code_block!
+            consume!(/;/, "expected `;' or code block")
+          end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
         @mode = nil
@@ -171,6 +217,9 @@ class Propane
       if pattern = parse_pattern!
         consume!(/\s+/)
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
           ptypename = md[1]
         end
         unless code = parse_code_block!
@@ -182,6 +231,12 @@ class Propane
       end
     end
 
+    def parse_start_statement!
+      if md = consume!(/start\s+(\w+)\s*;/)
+        @start_rule = md[1]
+      end
+    end
+
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]

lib/propane/parser.rb

@@ -63,12 +63,14 @@ class Propane
         reduce_entries =
           case ra = item_set.reduce_actions
           when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
+            [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
+              rule_set_id: ra.rule_set.id, n_states: ra.components.size,
+              propagate_optional_target: ra.optional? && ra.components.size == 1}]
           when Hash
             ra.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
+              {token_id: token.id, rule_id: rule.id, rule: rule,
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
           else
             []

lib/propane/rule.rb

@@ -30,6 +30,11 @@ class Propane
     #   The RuleSet that this Rule is a part of.
     attr_accessor :rule_set
 
+    # @return [Array<Integer>]
+    #   Map this rule's components to their positions in the parent RuleSet's
+    #   node field pointer array. This is used for AST construction.
+    attr_accessor :rule_set_node_field_index_map
+
     # Construct a Rule.
     #
     # @param name [String]
@@ -45,6 +50,7 @@ class Propane
     def initialize(name, components, code, ptypename, line_number)
       @name = name
       @components = components
+      @rule_set_node_field_index_map = components.map {0}
       @code = code
       @ptypename = ptypename
       @line_number = line_number
@@ -60,6 +66,14 @@ class Propane
       @components.empty?
     end
 
+    # Return whether this is an optional Rule.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional Rule.
+    def optional?
+      @name.end_with?("?")
+    end
+
     # Represent the Rule as a String.
     #
     # @return [String]
@@ -68,6 +82,17 @@ class Propane
       "#{@name} -> #{@components.map(&:name).join(" ")}"
     end
 
+    # Check whether the rule set node field index map is just a 1:1 mapping.
+    #
+    # @return [Boolean]
+    #   Boolean indicating whether the rule set node field index map is just a
+    #   1:1 mapping.
+    def flat_rule_set_node_field_index_map?
+      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
+        v == i
+      end
+    end
+
   end
 
 end

lib/propane/rule_set.rb

@@ -1,7 +1,12 @@
 class Propane
 
+  # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
+    # @return [Array<Hash>]
+    #   AST fields.
+    attr_reader :ast_fields
+
     # @return [Integer]
     #   ID of the RuleSet.
     attr_reader :id
@@ -51,6 +56,24 @@ class Propane
       @could_be_empty
     end
 
+    # Return whether this is an optional RuleSet.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional RuleSet.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # For optional rule sets, return the underlying component that is optional.
+    def option_target
+      @rules.each do |rule|
+        if rule.components.size > 0
+          return rule.components[0]
+        end
+      end
+      raise "Optional rule target not found"
+    end
+
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -75,6 +98,58 @@ class Propane
       @_start_token_set
     end
 
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize(grammar)
+      build_ast_fields(grammar)
+    end
+
+    private
+
+    # Build the set of AST fields for this RuleSet.
+    #
+    # This is an Array of Hashes. Each entry in the Array corresponds to a
+    # field location in the AST node. The entry is a Hash. It could have one or
+    # two keys. It will always have the field name with a positional suffix as
+    # a key. It may also have the field name without the positional suffix if
+    # that field only exists in one position across all Rules in the RuleSet.
+    #
+    # @return [void]
+    def build_ast_fields(grammar)
+      field_ast_node_indexes = {}
+      field_indexes_across_all_rules = {}
+      @ast_fields = []
+      @rules.each do |rule|
+        rule.components.each_with_index do |component, i|
+          if component.is_a?(RuleSet) && component.optional?
+            component = component.option_target
+          end
+          if component.is_a?(Token)
+            node_name = "Token"
+          else
+            node_name = component.name
+          end
+          struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+          field_name = "p#{node_name}#{i + 1}"
+          unless field_ast_node_indexes[field_name]
+            field_ast_node_indexes[field_name] = @ast_fields.size
+            @ast_fields << {field_name => struct_name}
+          end
+          field_indexes_across_all_rules[node_name] ||= Set.new
+          field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
+          rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
+        end
+      end
+      field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
+        if indexes_across_all_rules.size == 1
+          # If this field was only seen in one position across all rules,
+          # then add an alias to the positional field name that does not
+          # include the position.
+          @ast_fields[indexes_across_all_rules.first]["p#{node_name}"] =
+            "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
+        end
+      end
+    end
+
   end
 
 end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.2.0"
+  VERSION = "1.4.0"
 end

spec/propane_spec.rb

@@ -13,7 +13,7 @@ describe Propane do
     File.write("spec/run/testparser#{options[:name]}.propane", grammar)
   end
 
-  def build_parser(options = {})
+  def run_propane(options = {})
     @statics[:build_test_id] ||= 0
     @statics[:build_test_id] += 1
     if ENV["dist_specs"]
@@ -49,7 +49,11 @@ ENV["TERM"] = nil
 EOF
       end
     end
-    command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+    if options[:args]
+      command += options[:args]
+    else
+      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+    end
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -74,7 +78,7 @@ EOF
     expect(result).to be_truthy
   end
 
-  def run
+  def run_test
     stdout, stderr, status = Open3.capture3("spec/run/testparser")
     File.binwrite("spec/run/.stderr", stderr)
     File.binwrite("spec/run/.stdout", stdout)
@@ -112,6 +116,74 @@ EOF
     FileUtils.mkdir_p("spec/run")
   end
 
+  it "reports its version" do
+    results = run_propane(args: %w[--version], capture: true)
+    expect(results.stdout).to match /propane version \d+\.\d+/
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "shows help usage" do
+    results = run_propane(args: %w[-h], capture: true)
+    expect(results.stdout).to match /Usage/i
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "errors with unknown option" do
+    results = run_propane(args: %w[-i], capture: true)
+    expect(results.stderr).to match /Error: unknown option -i/
+    expect(results.status).to_not eq 0
+  end
+
+  it "errors when input and output files are not specified" do
+    results = run_propane(args: [], capture: true)
+    expect(results.stderr).to match /Error: specify input and output files/
+    expect(results.status).to_not eq 0
+  end
+
+  it "errors when input file is not readable" do
+    results = run_propane(args: %w[nope.txt out.d], capture: true)
+    expect(results.stderr).to match /Error: cannot read nope.txt/
+    expect(results.status).to_not eq 0
+  end
+
+  it "raises an error when a pattern referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+/foo/ (yes) <<
+>>
+/bar/ (no) <<
+>>
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 4: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
+  it "raises an error when a token referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+token foo (yes);
+token bar (no);
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 3: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
+  it "raises an error when a rule referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+token xyz;
+foo (yes) -> bar;
+bar (no) -> xyz;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 4: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
   %w[d c].each do |language|
 
     context "#{language.upcase} language" do
@@ -126,9 +198,9 @@ Start -> Foo;
 Foo -> int <<>>
 Foo -> plus <<>>
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_lexer.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -164,9 +236,9 @@ token int /\\d+/ <<
 Start -> int << $$ = $1; >>
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -184,7 +256,7 @@ E -> B;
 B -> zero;
 B -> one;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
       end
 
       it "generates a parser that does basic math - user guide example" do
@@ -258,9 +330,9 @@ E4 -> integer << $$ = $1; >>
 E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -272,7 +344,7 @@ Start -> E;
 E -> one E;
 E -> one;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
       end
 
       it "distinguishes between multiple identical rules with lookahead symbol" do
@@ -284,9 +356,9 @@ Start -> R2 b;
 R1 -> a b;
 R2 -> a b;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
       end
 
@@ -299,9 +371,9 @@ Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
       end
 
@@ -334,9 +406,9 @@ Abcs -> ;
 Abcs -> abc Abcs;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_user_code.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "abc!",
@@ -370,9 +442,9 @@ token abc;
 Start -> abc;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_pattern.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -412,9 +484,9 @@ token abc;
 Start -> abc;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "def!",
@@ -470,9 +542,9 @@ string: /"/ <<
 Start -> abc string def;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "begin string mode",
@@ -509,9 +581,9 @@ A -> a << writeln("A!"); >>
 B -> b << writeln("B!"); >>
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "A!",
@@ -528,9 +600,9 @@ Start -> As << $$ = $1; >>
 As -> << $$ = 0u; >>
 As -> As a << $$ = $1 + 1u; >>
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         expect(results.stderr).to eq ""
       end
@@ -549,7 +621,7 @@ Start -> b E d;
 E -> e;
 F -> e;
 EOF
-        results = build_parser(capture: true, language: language)
+        results = run_propane(capture: true, language: language)
         expect(results.status).to_not eq 0
         expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
       end
@@ -581,9 +653,9 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
 Start -> id;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
           "Matched token is identifier_123",
@@ -614,9 +686,9 @@ Start -> word <<
 >>
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -629,16 +701,16 @@ drop /\\s+/;
 Start -> a num Start;
 Start -> a num;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_error_positions.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
 
       it "allows creating a JSON parser" do
         write_grammar(File.read("spec/json_parser.#{language}.propane"))
-        build_parser(language: language)
+        run_propane(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end
 
@@ -650,16 +722,16 @@ token num /\\d+/;
 drop /\\s+/;
 Start -> a num;
 EOF
-        build_parser(name: "myp1", language: language)
+        run_propane(name: "myp1", language: language)
         write_grammar(<<EOF, name: "myp2")
 prefix myp2_;
 token b;
 token c;
 Start -> b c b;
 EOF
-        build_parser(name: "myp2", language: language)
+        run_propane(name: "myp2", language: language)
         compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -676,9 +748,9 @@ Any -> a;
 Any -> b;
 Any -> c;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_user_terminate_lexer.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -694,9 +766,9 @@ Any -> b Any << $terminate(4200); >>
 Any -> c Any;
 Any -> ;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_user_terminate.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
@@ -738,9 +810,9 @@ tokenid t;
 Start -> t;
 EOF
         end
-        build_parser(language: language)
+        run_propane(language: language)
         compile("spec/test_match_backslashes.#{language}", language: language)
-        results = run
+        results = run_test
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
         verify_lines(results.stdout, [
@@ -765,7 +837,219 @@ Start -> ;
 Opt -> two;
 Opt -> ;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
+      end
+
+      it "generates an AST" do
+        write_grammar <<EOF
+ast;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "supports AST node prefix and suffix" do
+        write_grammar <<EOF
+ast;
+ast_prefix P ;
+ast_suffix  S;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_ps.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows specifying a different start rule" do
+        write_grammar <<EOF
+token hi;
+start Top;
+Top -> hi;
+EOF
+        run_propane(language: language)
+        compile("spec/test_start_rule.#{language}", language: language)
+      end
+
+      it "allows specifying a different start rule with AST generation" do
+        write_grammar <<EOF
+ast;
+token hi;
+start Top;
+Top -> hi;
+EOF
+        run_propane(language: language)
+        compile("spec/test_start_rule_ast.#{language}", language: language)
+      end
+
+      it "allows marking a rule component as optional" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+
+ptype int;
+ptype float = float;
+ptype string = string;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  writeln("a: ", $1);
+  writeln("b: ", $2);
+  writeln("R: ", $3);
+>>
+R -> c d << $$ = "cd"; >>
+R (string) -> d c << $$ = "dc"; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  printf("a: %.1f\\n", $1);
+  printf("b: %d\\n", $2);
+  printf("R: %s\\n", $3 == NULL ? "" : $3);
+>>
+R -> c d << $$ = "cd"; >>
+R (string) -> d c << $$ = "dc"; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_optional_rule_component.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "a: 0#{language == "d" ? "" : ".0"}",
+          "b: 2",
+          "R: ",
+          "a: 1.5",
+          "b: 2",
+          "R: cd",
+          "a: 1.5",
+          "b: 2",
+          "R: dc",
+        ])
+      end
+
+      it "allows marking a rule component as optional in AST generation mode" do
+        if language == "d"
+          write_grammar <<EOF
+ast;
+
+<<
+import std.stdio;
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        else
+          write_grammar <<EOF
+ast;
+
+<<
+#include <stdio.h>
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_optional_rule_component_ast.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
       end
     end
   end

spec/test_ast.c

@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    Items * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    ItemsMore * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}

spec/test_ast.d

@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    Items * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    ItemsMore * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}

spec/test_ast_ps.c

@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    PItemsS * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    PItemsMoreS * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}

spec/test_ast_ps.d

@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    PItemsS * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    PItemsMoreS * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}

spec/test_optional_rule_component.c

@@ -0,0 +1,22 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
+

spec/test_optional_rule_component.d

@@ -0,0 +1,23 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_optional_rule_component_ast.c

@@ -0,0 +1,42 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->pR == start->pR3);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->pR != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    return 0;
+}
+

spec/test_optional_rule_component_ast.d

@@ -0,0 +1,43 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.pR == start.pR3);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_start_rule.c

@@ -0,0 +1,9 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule.d

@@ -0,0 +1,8 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule_ast.c

@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "hi";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
+
+    return 0;
+}

spec/test_start_rule_ast.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "hi";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top.pToken !is null);
+    assert_eq(TOKEN_hi, top.pToken.token);
+}