v1.3.0

Update CHANGELOG for v1.3.0
Add documentation for AST generation mode - close #22
2024-04-23 00:31:56 -04:00 · 2024-04-23 00:21:28 -04:00 · 2024-04-23 00:15:19 -04:00 · 2024-04-22 21:50:26 -04:00 · 2024-04-22 20:51:27 -04:00 · 2024-04-06 14:37:15 -04:00
24 changed files with 1240 additions and 314 deletions
@@ -1,3 +1,31 @@
+## v1.3.0
+
+### New Features
+
+- Add AST generation (#22)
+
+## v1.2.0
+
+### New Features
+
+- Allow one line user code blocks (#21)
+- Add backslash escape codes (#19)
+- Add API to access unexpected token found (#18)
+- Add token_names API (#17)
+- Add D example to user guide for p_context_init() (#16)
+- Allow user termination from lexer code blocks (#15)
+
+### Fixes
+
+- Fix generator hang when state transition cycle is present (#20)
+
+## v1.1.0
+
+### New Features
+
+- Add user parser terminations (#13)
+- Document generated parser API in user guide (#14)
+
 ## v1.0.0

 - Initial release
@@ -6,7 +6,8 @@ Propane is a LALR Parser Generator (LPG) which:
  * generates a built-in lexer to tokenize input
  * supports UTF-8 lexer inputs
  * generates a table-driven shift/reduce parser to parse input in linear time
-  * target C or D language outputs
+  * targets C or D language outputs
+  * optionally supports automatic full AST generation
  * is MIT-licensed
  * is distributable as a standalone Ruby script

@@ -17,6 +18,7 @@ can be copied into and versioned in a project's source tree.
 The only requirement to run Propane is that the system has a Ruby interpreter
 installed.
 The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
+
 Simply copy the `propane` executable script into the desired location within
 the project to be built (typically the root of the repository) and mark it
 executable.
@@ -55,10 +57,10 @@ import std.math;
 ptype ulong;

 # A few basic arithmetic operators.
-token plus /\\+/;
-token times /\\*/;
-token power /\\*\\*/;
-token integer /\\d+/ <<
+token plus /\+/;
+token times /\*/;
+token power /\*\*/;
+token integer /\d+/ <<
  ulong v;
  foreach (c; match)
  {
@@ -67,38 +69,22 @@ token integer /\\d+/ <<
  }
  $$ = v;
 >>
-token lparen /\\(/;
-token rparen /\\)/;
+token lparen /\(/;
+token rparen /\)/;
 # Drop whitespace.
-drop /\\s+/;
+drop /\s+/;

-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
 E3 -> E3 power E4 <<
  $$ = pow($1, $3);
 >>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```

 Grammar files can contain comment lines beginning with `#` which are ignored.
@@ -3,6 +3,17 @@
 #include <stdlib.h>
 #include <string.h>

+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+const char * <%= @grammar.prefix %>token_names[] = {
+<% @grammar.tokens.each_with_index do |token, index| %>
+    "<%= token.name %>",
+<% end %>
+};
+
 /**************************************************************************
 * User code blocks
 *************************************************************************/
@@ -309,9 +320,12 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 *
 * @param context
 *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
 *
 * @reval P_SUCCESS
 *   A token was successfully lexed.
@@ -422,6 +436,8 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
 *   Input text does not match any lexer pattern.
 * @retval P_DROP
 *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
 */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -441,6 +457,12 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
            uint8_t const * match = &context->input[context->input_index];
            <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                match_info.accepting_state->code_id, match, match_info.length, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
            /* An invalid token returned from lexer_user_code() means that the
             * user code did not explicitly return a token. So only override
             * the token to return if the user code does explicitly return a
@@ -511,6 +533,8 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
 *   The decoder encountered invalid text encoding.
 * @reval P_UNEXPECTED_INPUT
 *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
 */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -587,6 +611,18 @@ typedef struct
     * reduce action.
     */
    parser_state_id_t n_states;
+<% if @grammar.ast %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    uint16_t const * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set AST node fields.
+     */
+    uint16_t rule_set_node_field_array_size;
+<% end %>
 } reduce_t;

 /** Parser state entry. */
@@ -617,6 +653,11 @@ typedef struct

    /** Parser value from this state. */
    <%= @grammar.prefix %>value_t pvalue;
+
+<% if @grammar.ast %>
+    /** AST node. */
+    void * ast_node;
+<% end %>
 } state_value_t;

 /** Parser shift table. */
@@ -626,10 +667,27 @@ static const shift_t parser_shift_table[] = {
 <%   end %>
 };

+<% if @grammar.ast %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
+    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+<%     if @grammar.ast %>
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+             , NULL
+<%       else %>
+             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+<%       end %>
+             , <%= reduce[:rule].rule_set.ast_fields.size %>
+<%     end %>
+    },
 <%   end %>
 };

@@ -733,6 +791,7 @@ static void state_values_stack_free(state_values_stack_t * stack)
     free(stack->entries);
 }

+<% unless @grammar.ast %>
 /**
 * Execute user code associated with a parser rule.
 *
@@ -759,6 +818,7 @@ static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t

    return P_SUCCESS;
 }
+<% end %>

 /**
 * Check if the parser should shift to a new state.
@@ -820,7 +880,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 *   can be accessed with <%= @grammar.prefix %>result().
 * @retval P_UNEXPECTED_TOKEN
 *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context->token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
 * @reval P_DECODE_ERROR
 *   The decoder encountered invalid text encoding.
 * @reval P_UNEXPECTED_INPUT
@@ -832,7 +892,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
    <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
    state_values_stack_t statevalues;
    size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.ast %>
+    void * reduced_parser_node;
+<% else %>
    <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
    state_values_stack_init(&statevalues);
    state_values_stack_push(&statevalues);
    size_t result;
@@ -859,7 +923,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
            if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
            {
                /* Successful parse. */
+<% if @grammar.ast %>
+                context->parse_result = (Start *)state_values_stack_index(&statevalues, -1)->ast_node;
+<% else %>
                context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
+<% end %>
                result = P_SUCCESS;
                break;
            }
@@ -872,15 +940,26 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
            if (reduced_rule_set == INVALID_ID)
            {
                /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
+<% if @grammar.ast %>
+                Token * token_ast_node = malloc(sizeof(Token));
+                token_ast_node->token = token;
+                token_ast_node->pvalue = token_info.pvalue;
+                state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
+<% else %>
                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
+<% end %>
+                token = INVALID_TOKEN_ID;
            }
            else
            {
                /* We shifted a RuleSet. */
+<% if @grammar.ast %>
+                state_values_stack_index(&statevalues, -1)->ast_node = reduced_parser_node;
+<% else %>
                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
                <%= @grammar.prefix %>value_t new_parse_result = {0};
                reduced_parser_value = new_parse_result;
+<% end %>
                reduced_rule_set = INVALID_ID;
            }
            continue;
@@ -890,12 +969,38 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
        if (reduce_index != INVALID_ID)
        {
            /* We have something to reduce. */
+<% if @grammar.ast %>
+            if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                void ** node_fields = calloc(parser_reduce_table[reduce_index].rule_set_node_field_array_size, sizeof(void *));
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node_fields[i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                    }
+                }
+                else
+                {
+                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
+                    {
+                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->ast_node;
+                    }
+                }
+                reduced_parser_node = node_fields;
+            }
+            else
+            {
+                reduced_parser_node = NULL;
+            }
+<% else %>
            <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
            {
                return P_USER_TERMINATED;
            }
            reduced_parser_value = reduced_parser_value2;
+<% end %>
            reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
            state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
            continue;
@@ -923,9 +1028,17 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 *
 * @return Parse result value.
 */
+<% if @grammar.ast %>
+Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% end %>
 {
+<% if @grammar.ast %>
+    return context->parse_result;
+<% else %>
    return context->parse_result.v_<%= start_rule_type[0] %>;
+<% end %>
 }

 /**
@@ -953,3 +1066,13 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
    return context->user_terminate_code;
 }
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context->token;
+}
@@ -31,7 +31,7 @@ public enum : size_t
 }

 /** Token type. */
-public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;

 /** Token IDs. */
 public enum : <%= @grammar.prefix %>token_t
@@ -43,11 +43,16 @@ public enum : <%= @grammar.prefix %>token_t
 <%   end %>
 <% end %>
    INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
+    TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
 }

 /** Code point type. */
 public alias <%= @grammar.prefix %>code_point_t = uint;

+<% if @grammar.ast %>
+/** Parser values type. */
+public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
+<% else %>
 /** Parser values type(s). */
 public union <%= @grammar.prefix %>value_t
 {
@@ -55,6 +60,33 @@ public union <%= @grammar.prefix %>value_t
    <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+<% end %>
+
+<% if @grammar.ast %>
+/** AST node types. @{ */
+public struct Token
+{
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+}
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+public struct <%= name %>
+{
+<%     rule_set.ast_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        <%= type %> * <%= field_name %>;
+<%       end %>
+    }
+<%     end %>
+}
+
+<%   end %>
+/** @} */
+<% end %>

 /**
 * A structure to keep track of parser position.
@@ -111,7 +143,11 @@ public struct <%= @grammar.prefix %>context_t
    /* Parser context data. */

    /** Parse result value. */
+<% if @grammar.ast %>
+    Start * parse_result;
+<% else %>
    <%= @grammar.prefix %>value_t parse_result;
+<% end %>

    /** Unexpected token received. */
    <%= @grammar.prefix %>token_t token;
@@ -427,9 +463,12 @@ private lexer_state_id_t check_lexer_transition(uint current_state, uint code_po
 *
 * @param context
 *   Lexer/parser context structure.
- * @param[out] out_token_info
- *   The lexed token information is stored here if the return value is
- *   P_SUCCESS.
+ * @param[out] out_match_info
+ *   The longest match information is stored here if the return value is
+ *   P_SUCCESS or P_DECODE_ERROR.
+ * @param[out] out_unexpected_input_length
+ *   The unexpected input length is stored here if the return value is
+ *   P_UNEXPECTED_INPUT.
 *
 * @reval P_SUCCESS
 *   A token was successfully lexed.
@@ -538,6 +577,8 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
 *   Input text does not match any lexer pattern.
 * @retval P_DROP
 *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
 */
 private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -557,6 +598,12 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
            string match = context.input[context.input_index..(context.input_index + match_info.length)];
            <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                match_info.accepting_state.code_id, match, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
            /* An invalid token returned from lexer_user_code() means that the
             * user code did not explicitly return a token. So only override
             * the token to return if the user code does explicitly return a
@@ -627,6 +674,8 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
 *   The decoder encountered invalid text encoding.
 * @reval P_UNEXPECTED_INPUT
 *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
 */
 public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -703,6 +752,18 @@ private struct reduce_t
     * reduce action.
     */
    parser_state_id_t n_states;
+<% if @grammar.ast %>
+
+    /**
+     * Map of rule components to rule set child fields.
+     */
+    immutable(ushort) * rule_set_node_field_index_map;
+
+    /**
+     * Number of rule set AST node fields.
+     */
+    ushort rule_set_node_field_array_size;
+<% end %>
 }

 /** Parser state entry. */
@@ -734,6 +795,11 @@ private struct state_value_t
    /** Parser value from this state. */
    <%= @grammar.prefix %>value_t pvalue;

+<% if @grammar.ast %>
+    /** AST node. */
+    void * ast_node;
+<% end %>
+
    this(size_t state_id)
    {
        this.state_id = state_id;
@@ -747,10 +813,27 @@ private immutable shift_t[] parser_shift_table = [
 <%   end %>
 ];

+<% if @grammar.ast %>
+<%   @grammar.rules.each do |rule| %>
+<%     unless rule.flat_rule_set_node_field_index_map? %>
+immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map = [<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>];
+<%     end %>
+<%   end %>
+<% end %>
+
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u),
+    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+<%     if @grammar.ast %>
+<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
+             , null
+<%       else %>
+             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+<%       end %>
+             , <%= reduce[:rule].rule_set.ast_fields.size %>
+<%     end %>
+            ),
 <%   end %>
 ];

@@ -761,6 +844,7 @@ private immutable parser_state_t[] parser_state_table = [
 <%   end %>
 ];

+<% unless @grammar.ast %>
 /**
 * Execute user code associated with a parser rule.
 *
@@ -787,6 +871,7 @@ private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint ru

    return P_SUCCESS;
 }
+<% end %>

 /**
 * Check if the parser should shift to a new state.
@@ -848,7 +933,7 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
 *   can be accessed with <%= @grammar.prefix %>result().
 * @retval P_UNEXPECTED_TOKEN
 *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context.token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
 * @reval P_DECODE_ERROR
 *   The decoder encountered invalid text encoding.
 * @reval P_UNEXPECTED_INPUT
@@ -860,7 +945,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
    <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
    state_value_t[] statevalues = new state_value_t[](1);
    size_t reduced_rule_set = INVALID_ID;
+<% if @grammar.ast %>
+    void * reduced_parser_node;
+<% else %>
    <%= @grammar.prefix %>value_t reduced_parser_value;
+<% end %>
    for (;;)
    {
        if (token == INVALID_TOKEN_ID)
@@ -883,7 +972,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
            if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
            {
                /* Successful parse. */
+<% if @grammar.ast %>
+                context.parse_result = cast(Start *)statevalues[$-1].ast_node;
+<% else %>
                context.parse_result = statevalues[$-1].pvalue;
+<% end %>
                return P_SUCCESS;
            }
        }
@@ -894,15 +987,24 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
            if (reduced_rule_set == INVALID_ID)
            {
                /* We shifted a token, mark it consumed. */
-                token = INVALID_TOKEN_ID;
+<% if @grammar.ast %>
+                Token * token_ast_node = new Token(token, token_info.pvalue);
+                statevalues[$-1].ast_node = token_ast_node;
+<% else %>
                statevalues[$-1].pvalue = token_info.pvalue;
+<% end %>
+                token = INVALID_TOKEN_ID;
            }
            else
            {
                /* We shifted a RuleSet. */
+<% if @grammar.ast %>
+                statevalues[$-1].ast_node = reduced_parser_node;
+<% else %>
                statevalues[$-1].pvalue = reduced_parser_value;
                <%= @grammar.prefix %>value_t new_parse_result;
                reduced_parser_value = new_parse_result;
+<% end %>
                reduced_rule_set = INVALID_ID;
            }
            continue;
@@ -912,12 +1014,42 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
        if (reduce_index != INVALID_ID)
        {
            /* We have something to reduce. */
+<% if @grammar.ast %>
+            if (parser_reduce_table[reduce_index].n_states > 0)
+            {
+                void *[] node_fields = new void *[parser_reduce_table[reduce_index].rule_set_node_field_array_size];
+                foreach (i; 0..parser_reduce_table[reduce_index].rule_set_node_field_array_size)
+                {
+                    node_fields[i] = null;
+                }
+                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map is null)
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node_fields[i] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                    }
+                }
+                else
+                {
+                    foreach (i; 0..parser_reduce_table[reduce_index].n_states)
+                    {
+                        node_fields[parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = statevalues[$ - parser_reduce_table[reduce_index].n_states + i].ast_node;
+                    }
+                }
+                reduced_parser_node = node_fields.ptr;
+            }
+            else
+            {
+                reduced_parser_node = null;
+            }
+<% else %>
            <%= @grammar.prefix %>value_t reduced_parser_value2;
            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
            {
                return P_USER_TERMINATED;
            }
            reduced_parser_value = reduced_parser_value2;
+<% end %>
            reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
            statevalues.length -= parser_reduce_table[reduce_index].n_states;
            continue;
@@ -942,9 +1074,17 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
 *
 * @return Parse result value.
 */
+<% if @grammar.ast %>
+public Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<% end %>
 {
+<% if @grammar.ast %>
+    return context.parse_result;
+<% else %>
    return context.parse_result.v_<%= start_rule_type[0] %>;
+<% end %>
 }

 /**
@@ -972,3 +1112,13 @@ public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>co
 {
    return context.user_terminate_code;
 }
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context.token;
+}
@@ -23,7 +23,7 @@
 #define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u

 /** Token type. */
-typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;

 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -33,6 +33,7 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u

 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
@@ -40,6 +41,10 @@ typedef uint32_t <%= @grammar.prefix %>code_point_t;
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>

+<% if @grammar.ast %>
+/** Parser values type. */
+typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
+<% else %>
 /** Parser values type(s). */
 typedef union
 {
@@ -47,6 +52,38 @@ typedef union
    <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+<% end %>
+
+<% if @grammar.ast %>
+/** AST node types. @{ */
+typedef struct Token
+{
+    <%= @grammar.prefix %>token_t token;
+    <%= @grammar.prefix %>value_t pvalue;
+} Token;
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+struct <%= name %>;
+<%   end %>
+
+<%   @parser.rule_sets.each do |name, rule_set| %>
+<%     next if name.start_with?("$") %>
+typedef struct <%= name %>
+{
+<%     rule_set.ast_fields.each do |fields| %>
+    union
+    {
+<%       fields.each do |field_name, type| %>
+        struct <%= type %> * <%= field_name %>;
+<%       end %>
+    };
+<%     end %>
+} <%= name %>;
+
+<%   end %>
+/** @} */
+<% end %>

 /**
 * A structure to keep track of parser position.
@@ -106,7 +143,11 @@ typedef struct
    /* Parser context data. */

    /** Parse result value. */
+<% if @grammar.ast %>
+    Start * parse_result;
+<% else %>
    <%= @grammar.prefix %>value_t parse_result;
+<% end %>

    /** Unexpected token received. */
    <%= @grammar.prefix %>token_t token;
@@ -115,6 +156,13 @@ typedef struct
    size_t user_terminate_code;
 } <%= @grammar.prefix %>context_t;

+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+extern const char * <%= @grammar.prefix %>token_names[];
+
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);

 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
@@ -124,8 +172,14 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=

 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);

+<% if @grammar.ast %>
+Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<% end %>

 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);

 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);
@@ -14,6 +14,7 @@ Propane is a LALR Parser Generator (LPG) which:
  * supports UTF-8 lexer inputs
  * generates a table-driven shift/reduce parser to parse input in linear time
  * targets C or D language outputs
+  * optionally supports automatic full AST generation
  * is MIT-licensed
  * is distributable as a standalone Ruby script

@@ -77,33 +78,15 @@ token rparen /\\)/;
 # Drop whitespace.
 drop /\\s+/;

-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```

 Grammar files can contain comment lines beginning with `#` which are ignored.
@@ -117,8 +100,8 @@ lowercase character and beginning a rule name with an uppercase character.

 ##> User Code Blocks

-User code blocks begin with the line following a "<<" token and end with the
-line preceding a grammar line consisting of solely the ">>" token.
+User code blocks begin following a "<<" token and end with a ">>" token found
+at the end of a line.
 All text lines in the code block are copied verbatim into the output file.

 ### Standalone Code Blocks
@@ -189,9 +172,7 @@ This parser value can then be used later in a parser rule.
 Example:

 ```
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
 ```

 Parser rule code blocks appear following a rule expression.
@@ -202,6 +183,99 @@ rule.
 Parser values for the rules or tokens in the rule pattern can be accessed
 positionally with tokens `$1`, `$2`, `$3`, etc...

+Parser rule code blocks are not available in AST generation mode.
+In AST generation mode, a full parse tree is automatically constructed in
+memory for user code to traverse after parsing is complete.
+
+##> AST generation mode - the `ast` statement
+
+To activate AST generation mode, place the `ast` statement in your grammar file:
+
+```
+ast;
+```
+
+It is recommended to place this statement early in the grammar.
+
+In AST generation mode various aspects of propane's behavior are changed:
+
+  * Only one `ptype` is allowed.
+  * Parser user code blocks are not supported.
+  * Structure types are generated to represent the parsed tokens and rules as
+  defined in the grammar.
+  * The parse result from `p_result()` points to a `Start` structure containing
+  the entire parse tree for the input.
+
+Example AST generation grammar:
+
+```
+ast;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+```
+
+The following unit test describes the fields that will be present for an
+example parse:
+
+```
+string input = "a, ((b)), b";
+p_context_t context;
+p_context_init(&context, input);
+assert_eq(P_SUCCESS, p_parse(&context));
+Start * start = p_result(&context);
+assert(start.pItems1 !is null);
+assert(start.pItems !is null);
+Items * items = start.pItems;
+assert(items.pItem !is null);
+assert(items.pItem.pToken1 !is null);
+assert_eq(TOKEN_a, items.pItem.pToken1.token);
+assert_eq(11, items.pItem.pToken1.pvalue);
+assert(items.pItemsMore !is null);
+ItemsMore * itemsmore = items.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+assert(itemsmore.pItemsMore !is null);
+itemsmore = itemsmore.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pToken1 !is null);
+assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+assert(itemsmore.pItemsMore is null);
+```
+
 ##> Specifying tokens - the `token` statement

 The `token` statement allows defining a lexer token and a pattern to match that
@@ -238,9 +312,7 @@ lexer.
 Example:

 ```
-token if <<
-  writeln("'if' keyword lexed");
->>
+token if << writeln("'if' keyword lexed"); >>
 ```

 The `token` statement is actually a shortcut statement for a combination of a
@@ -277,9 +349,7 @@ code but may not result in a matched token.
 Example:

 ```
-/foo+/ <<
-  writeln("saw a foo pattern");
->>
+/foo+/ << writeln("saw a foo pattern"); >>
 ```

 This can be especially useful with ${#Lexer modes}.
@@ -325,9 +395,16 @@ Regular expressions can include many special characters:
  * The `(` character begins a matching group.
  * The `{` character begins a count qualifier.
  * The `\` character escapes the following character and changes its meaning:
+    * The `\a` sequence matches an ASCII bell character (0x07).
+    * The `\b` sequence matches an ASCII backspace character (0x08).
    * The `\d` sequence matches any character `0` through `9`.
+    * The `\f` sequence matches an ASCII form feed character (0x0C).
+    * The `\n` sequence matches an ASCII new line character (0x0A).
+    * The `\r` sequence matches an ASCII carriage return character (0x0D).
    * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
    `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * The `\t` sequence matches an ASCII tab character (0x09).
+    * The `\v` sequence matches an ASCII vertical tab character (0x0B).
    * Any other character matches itself.
  * The `|` character creates an alternate match.

@@ -381,9 +458,7 @@ tokenid str;
  mystringvalue = "";
  $mode(string);
 >>
-string: /[^"]+/ <<
-  mystringvalue += match;
->>
+string: /[^"]+/ << mystringvalue += match; >>
 string: /"/ <<
  $mode(default);
  return $token(str);
@@ -440,20 +515,12 @@ ptype Value;
 ptype array = Value[];
 ptype dict = Value[string];

-Object -> lbrace rbrace <<
-  $$ = new Value();
->>
+Object -> lbrace rbrace << $$ = new Value(); >>

-Values (array) -> Value <<
-  $$ = [$1];
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>
+Values (array) -> Value << $$ = [$1]; >>
+Values -> Values comma Value << $$ = $1 ~ [$3]; >>

-KeyValue (dict) -> string colon Value <<
-  $$ = [$1: $3];
->>
+KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
 ```

 In this example, the default parser value type is `Value`.
@@ -469,6 +536,12 @@ In this example:
  * a reduced `Values`'s parser value has a type of `Value[]`.
  * a reduced `KeyValue`'s parser value has a type of `Value[string]`.

+When AST generation mode is active, the `ptype` functionality works differently.
+In this mode, only one `ptype` is used by the parser.
+Lexer user code blocks may assign a parse value to the generated `Token` node
+by assigning to `$$` within a lexer code block.
+The type of the parse value `$$` is given by the global `ptype` type.
+
 ##> Specifying a parser rule - the rule statement

 Rule statements create parser rules which define the grammar that will be
@@ -486,12 +559,8 @@ Example:

 ```
 ptype ulong;
-token word /[a-z]+/ <<
-  $$ = match.length;
->>
-Start -> word <<
-  $$ = $1;
->>
+token word /[a-z]+/ << $$ = match.length; >>
+Start -> word << $$ = $1; >>
 ```

 In the above example the `Start` rule is defined to match a single `word`
@@ -500,33 +569,15 @@ token.
 Example:

 ```
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```

 A parser rule has zero or more terms on the right side of its definition.
@@ -539,6 +590,9 @@ The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
 can be used to produce the parser value for the accepted rule.

+Parser rule code blocks are not allowed and not used when AST generation mode
+is active.
+
 ##> Specifying the parser module name - the `module` statement

 The `module` statement can be used to specify the module name for a generated
@@ -574,23 +628,22 @@ default.
 It can also be used when generating multiple lexers/parsers to be used in the
 same program to avoid symbol collisions.

-##> User termination of the parser
+##> User termination of the lexer or parser

-Propane supports allowing parser user code blocks to terminate execution of the
-parser.
-One example use of this functionality is to detect and report an error before
-the parser continues parsing the remainder of the input.
-Another use of this features is to begin parsing input and determine whether a
-different parser should be used instead.
+Propane supports allowing lexer or parser user code blocks to terminate
+execution of the parser.
+Some example uses of this functionality could be to:

-To terminate parsing from a parser user code block, use the `$terminate(code)`
-function, passing an integer expression argument.
+  * Detect integer overflow when lexing an integer literal constant.
+  * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
+  * Determine whether parsing should stop and instead be performed using a different parser version.
+
+To terminate parsing from a lexer or parser user code block, use the
+`$terminate(code)` function, passing an integer expression argument.
 For example:

 ```
-NewExpression -> new Expression <<
-  $terminate(42);
->>
+NewExpression -> new Expression << $terminate(42); >>
 ```

 The value passed to the `$terminate()` function is known as the "user terminate
@@ -636,6 +689,67 @@ A pointer to this instance is passed to the generated functions.
 The `p_position_t` structure contains two fields `row` and `col`.
 These fields contain the 0-based row and column describing a parser position.

+### AST Node Types
+
+If AST generation mode is enabled, a structure type for each rule will be
+generated.
+The name of the structure type is given by the name of the rule.
+Additionally a structure type called `Token` is generated to represent an
+AST node which refers to a raw parser token rather than a composite rule.
+
+#### AST Node Fields
+
+A `Token` node has two fields:
+
+  * `token` which specifies which token was parsed (one of `TOKEN_*`)
+  * `pvalue` which specifies the parser value for the token. If a lexer user
+  code block assigned to `$$`, the assigned value will be stored here.
+
+The other generated AST node structures have fields generated based on the
+right hand side components specified for all rules of a given name.
+
+In this example:
+
+```
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+```
+
+The `Start` structure will have a field called `pItems` and another field of
+the same name but with a positional suffix (`pItems1`) which both point to the
+parsed `Items` node.
+Their value will be null if the parsed `Items` rule was empty.
+
+The `Items` structure will have fields:
+
+  * `pItem` and `pItem1` which point to the parsed `Item` structure.
+  * `pItemsMore` and `pItemsMore2` which point to the parsed `ItemsMore` structure.
+
+If a rule can be empty (for example in the second `Items` rule above), then
+an instance of a pointer to that rule's generated AST node will be null if the
+parser matches the empty rule definition.
+
+The non-positional AST node field pointer will not be generated if there are
+multiple positions in which an instance of the node it points to could be
+present.
+For example, in the below rules:
+
+```
+Dual -> One Two;
+Dual -> Two One;
+```
+
+The generated `Dual` structure will contain `pOne1`, `pTwo2`, `pTwo1`, and
+`pOne2` fields.
+However, a `pOne` field and `pTwo` field will not be generated since it would
+be ambiguous which one was matched.
+
+If the first rule is matched, then `pOne1` and `pTwo2` will be non-null while
+`pTwo1` and `pOne2` will be null.
+If the second rule is matched instead, then the opposite would be the case.
+
 ##> Functions

 ### `p_context_init`
@@ -645,13 +759,20 @@ structure.
 The input to be used for lexing/parsing is passed in when initializing the
 context structure.

-Example:
+C example:

 ```
 p_context_t context;
 p_context_init(&context, input, input_length);
 ```

+D example:
+
+```
+p_context_t context;
+p_context_init(&context, input);
+```
+
 ### `p_parse`

 The `p_parse()` function is the main entry point to the parser.
@@ -682,6 +803,9 @@ if (p_parse(&context) == P_SUCCESS)
 }
 ```

+If AST generation mode is active, then the `p_result()` function returns a
+`Start *` pointing to the `Start` AST structure.
+
 ### `p_position`

 The `p_position()` function can be used to retrieve the parser position where
@@ -718,6 +842,44 @@ if (p_parse(&context) == P_USER_TERMINATED)
 }
 ```

+### `p_token`
+
+The `p_token()` function can be used to retrieve the current parse token.
+This is useful after `p_parse()` returns a `P_UNEXPECTED_TOKEN` value.
+terminate code after `p_parse()` returns a `P_USER_TERMINATED` value to
+indicate what token the parser was not expecting.
+
+Example:
+
+```
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_token_t unexpected_token = p_token(&context);
+}
+```
+
+##> Data
+
+### `p_token_names`
+
+The `p_token_names` array contains the grammar-specified token names.
+It is indexed by the token ID.
+
+C example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_position_t error_position = p_position(&context);
+    fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
+        p_token_names[context.token],
+        error_position.row + 1, error_position.col + 1);
+}
+```
+
 #> License

 Propane is licensed under the terms of the MIT License:
@@ -24,7 +24,7 @@ EOF
              log_file = args[i]
            end
          when "--version"
-            puts "propane v#{VERSION}"
+            puts "propane version #{VERSION}"
            return 0
          when "-h", "--help"
            puts USAGE
@@ -119,6 +119,9 @@ class Propane
        end
      end
      determine_possibly_empty_rulesets!(rule_sets)
+      rule_sets.each do |name, rule_set|
+        rule_set.finalize
+      end
      # Generate the lexer.
      @lexer = Lexer.new(@grammar)
      # Generate the parser.
@@ -198,6 +201,16 @@ class Propane
      code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
        "TOKEN_#{Token.code_name($1)}"
      end
+      code = code.gsub(/\$terminate\((.*)\);/) do |match|
+        user_terminate_code = $1
+        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
+        case @language
+        when "c"
+          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "d"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        end
+      end
      if parser
        code = code.gsub(/\$\$/) do |match|
          case @language
@@ -216,17 +229,16 @@ class Propane
            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
          end
        end
-        code = code.gsub(/\$terminate\((.*)\);/) do |match|
-          user_terminate_code = $1
-          case @language
-          when "c"
-            "context->user_terminate_code = (#{user_terminate_code}); return P_USER_TERMINATED;"
-          when "d"
-            "context.user_terminate_code = (#{user_terminate_code}); return P_USER_TERMINATED;"
-          end
-        end
      else
        code = code.gsub(/\$\$/) do |match|
+          if @grammar.ast
+            case @language
+            when "c"
+              "out_token_info->pvalue"
+            when "d"
+              "out_token_info.pvalue"
+            end
+          else
            case @language
            when "c"
              "out_token_info->pvalue.v_#{pattern.ptypename}"
@@ -234,6 +246,7 @@ class Propane
              "out_token_info.pvalue.v_#{pattern.ptypename}"
            end
          end
+        end
        code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
          mode_name = $1
          mode_id = @lexer.mode_id(mode_name)
@@ -5,6 +5,7 @@ class Propane
    # Reserve identifiers beginning with a double-underscore for internal use.
    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/

+    attr_reader :ast
    attr_reader :modulename
    attr_reader :patterns
    attr_reader :rules
@@ -24,6 +25,7 @@ class Propane
      @input = input.gsub("\r\n", "\n")
      @ptypes = {"default" => "void *"}
      @prefix = "p_"
+      @ast = false
      parse_grammar!
    end

@@ -35,6 +37,10 @@ class Propane
      @tokens.size
    end

+    def terminate_token_id
+      @tokens.size + 1
+    end
+
    private

    def parse_grammar!
@@ -47,6 +53,7 @@ class Propane
      if parse_white_space!
      elsif parse_comment_line!
      elsif @mode.nil? && parse_mode_label!
+      elsif parse_ast_statement!
      elsif parse_module_statement!
      elsif parse_ptype_statement!
      elsif parse_pattern_statement!
@@ -78,6 +85,12 @@ class Propane
      consume!(/#.*\n/)
    end

+    def parse_ast_statement!
+      if consume!(/ast\s*;/)
+        @ast = true
+      end
+    end
+
    def parse_module_statement!
      if consume!(/module\s+/)
        md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -92,6 +105,9 @@ class Propane
      if consume!(/ptype\s+/)
        name = "default"
        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
          name = md[1]
        end
        md = consume!(/([^;]+);/, "expected parser result type expression")
@@ -104,12 +120,15 @@ class Propane
        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
        name = md[1]
        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
          ptypename = md[1]
        end
        pattern = parse_pattern! || name
        consume!(/\s+/)
        unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+          consume!(/;/, "expected `;' or code block")
        end
        token = Token.new(name, ptypename, @line_number)
        @tokens << token
@@ -125,6 +144,9 @@ class Propane
        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
        name = md[1]
        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
          ptypename = md[1]
        end
        consume!(/;/, "expected `;'");
@@ -152,10 +174,17 @@ class Propane
    def parse_rule_statement!
      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
        rule_name, ptypename = *md[1, 2]
+        if @ast && ptypename
+          raise Error.new("Multiple ptypes are unsupported in AST mode")
+        end
        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
        components = md[1].strip.split(/\s+/)
+        if @ast
+          consume!(/;/, "expected `;'")
+        else
          unless code = parse_code_block!
-          consume!(/;/, "expected pattern or `;' or code block")
+            consume!(/;/, "expected `;' or code block")
+          end
        end
        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
        @mode = nil
@@ -167,6 +196,9 @@ class Propane
      if pattern = parse_pattern!
        consume!(/\s+/)
        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
+          if @ast
+            raise Error.new("Multiple ptypes are unsupported in AST mode")
+          end
          ptypename = md[1]
        end
        unless code = parse_code_block!
@@ -179,8 +211,10 @@ class Propane
    end

    def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
        name, code = md[1..2]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
        if @code_blocks[name]
          @code_blocks[name] += code
        else
@@ -218,8 +252,11 @@ class Propane
    end

    def parse_code_block!
-      if md = consume!(/<<\n(.*?)^>>\n/m)
-        md[1]
+      if md = consume!(/<<(.*?)>>\n/m)
+        code = md[1]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
+        code
      end
    end

@@ -63,11 +63,11 @@ class Propane
        reduce_entries =
          case ra = item_set.reduce_actions
          when Rule
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
+            [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
          when Hash
            ra.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id,
+              {token_id: token.id, rule_id: rule.id, rule: rule,
               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
            end
          else
@@ -134,7 +134,7 @@ class Propane
      # possible following tokens, we will only look in the item sets leading
      # up to this one. This restriction gives us a more precise lookahead set,
      # and allows us to parse LALR grammars.
-      item_sets = item_set.leading_item_sets
+      item_sets = Set[item_set] + item_set.leading_item_sets
      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
        lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
        lookahead_tokens_for_rule.each do |lookahead_token|
@@ -87,15 +87,25 @@ class Propane

      # Set of ItemSets that lead to this ItemSet.
      #
-      # This set includes this ItemSet.
-      #
      # @return [Set<ItemSet>]
      #   Set of all ItemSets that lead up to this ItemSet.
      def leading_item_sets
-        @in_sets.reduce(Set[self]) do |result, item_set|
-          result + item_set.leading_item_sets
+        result = Set.new
+        eval_sets = Set[self]
+        evaled = Set.new
+        while eval_sets.size > 0
+          eval_set = eval_sets.first
+          eval_sets.delete(eval_set)
+          evaled << eval_set
+          eval_set.in_sets.each do |in_set|
+            result << in_set
+            unless evaled.include?(in_set)
+              eval_sets << in_set
            end
          end
+        end
+        result
+      end

      # Represent the ItemSet as a String.
      #
@@ -134,8 +134,18 @@ class Propane
      else
        c = @pattern.slice!(0)
        case c
+        when "a"
+          CharacterRangeUnit.new("\a", "\a")
+        when "b"
+          CharacterRangeUnit.new("\b", "\b")
        when "d"
          CharacterRangeUnit.new("0", "9")
+        when "f"
+          CharacterRangeUnit.new("\f", "\f")
+        when "n"
+          CharacterRangeUnit.new("\n", "\n")
+        when "r"
+          CharacterRangeUnit.new("\r", "\r")
        when "s"
          ccu = CharacterClassUnit.new
          ccu << CharacterRangeUnit.new(" ")
@@ -145,6 +155,10 @@ class Propane
          ccu << CharacterRangeUnit.new("\f")
          ccu << CharacterRangeUnit.new("\v")
          ccu
+        when "t"
+          CharacterRangeUnit.new("\t", "\t")
+        when "v"
+          CharacterRangeUnit.new("\v", "\v")
        else
          CharacterRangeUnit.new(c)
        end
@@ -30,6 +30,11 @@ class Propane
    #   The RuleSet that this Rule is a part of.
    attr_accessor :rule_set

+    # @return [Array<Integer>]
+    #   Map this rule's components to their positions in the parent RuleSet's
+    #   node field pointer array. This is used for AST construction.
+    attr_accessor :rule_set_node_field_index_map
+
    # Construct a Rule.
    #
    # @param name [String]
@@ -45,6 +50,7 @@ class Propane
    def initialize(name, components, code, ptypename, line_number)
      @name = name
      @components = components
+      @rule_set_node_field_index_map = components.map {0}
      @code = code
      @ptypename = ptypename
      @line_number = line_number
@@ -68,6 +74,17 @@ class Propane
      "#{@name} -> #{@components.map(&:name).join(" ")}"
    end

+    # Check whether the rule set node field index map is just a 1:1 mapping.
+    #
+    # @return [Boolean]
+    #   Boolean indicating whether the rule set node field index map is just a
+    #   1:1 mapping.
+    def flat_rule_set_node_field_index_map?
+      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
+        v == i
+      end
+    end
+
  end

 end
@@ -1,5 +1,6 @@
 class Propane

+  # A RuleSet collects all grammar rules of the same name.
  class RuleSet

    # @return [Integer]
@@ -75,6 +76,56 @@ class Propane
      @_start_token_set
    end

+    # Build the set of AST fields for this RuleSet.
+    #
+    # This is an Array of Hashes. Each entry in the Array corresponds to a
+    # field location in the AST node. The entry is a Hash. It could have one or
+    # two keys. It will always have the field name with a positional suffix as
+    # a key. It may also have the field name without the positional suffix if
+    # that field only exists in one position across all Rules in the RuleSet.
+    #
+    # @return [Array<Hash>]
+    #   AST fields.
+    def ast_fields
+      @_ast_fields ||=
+        begin
+          field_ast_node_indexes = {}
+          field_indexes_across_all_rules = {}
+          ast_node_fields = []
+          @rules.each do |rule|
+            rule.components.each_with_index do |component, i|
+              if component.is_a?(Token)
+                node_name = "Token"
+              else
+                node_name = component.name
+              end
+              field_name = "p#{node_name}#{i + 1}"
+              unless field_ast_node_indexes[field_name]
+                field_ast_node_indexes[field_name] = ast_node_fields.size
+                ast_node_fields << {field_name => node_name}
+              end
+              field_indexes_across_all_rules[node_name] ||= Set.new
+              field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
+              rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
+            end
+          end
+          field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
+            if indexes_across_all_rules.size == 1
+              # If this field was only seen in one position across all rules,
+              # then add an alias to the positional field name that does not
+              # include the position.
+              ast_node_fields[indexes_across_all_rules.first]["p#{node_name}"] = node_name
+            end
+          end
+          ast_node_fields
+        end
+    end
+
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize
+      ast_fields
+    end
+
  end

 end
@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.1.0"
+  VERSION = "1.3.0"
 end
@@ -13,7 +13,7 @@ describe Propane do
    File.write("spec/run/testparser#{options[:name]}.propane", grammar)
  end

-  def build_parser(options = {})
+  def run_propane(options = {})
    @statics[:build_test_id] ||= 0
    @statics[:build_test_id] += 1
    if ENV["dist_specs"]
@@ -49,7 +49,11 @@ ENV["TERM"] = nil
 EOF
      end
    end
+    if options[:args]
+      command += options[:args]
+    else
      command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
+    end
    if (options[:capture])
      stdout, stderr, status = Open3.capture3(*command)
      Results.new(stdout, stderr, status)
@@ -74,7 +78,7 @@ EOF
    expect(result).to be_truthy
  end

-  def run
+  def run_test
    stdout, stderr, status = Open3.capture3("spec/run/testparser")
    File.binwrite("spec/run/.stderr", stderr)
    File.binwrite("spec/run/.stdout", stdout)
@@ -112,6 +116,38 @@ EOF
    FileUtils.mkdir_p("spec/run")
  end

+  it "reports its version" do
+    results = run_propane(args: %w[--version], capture: true)
+    expect(results.stdout).to match /propane version \d+\.\d+/
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "shows help usage" do
+    results = run_propane(args: %w[-h], capture: true)
+    expect(results.stdout).to match /Usage/i
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+  end
+
+  it "errors with unknown option" do
+    results = run_propane(args: %w[-i], capture: true)
+    expect(results.stderr).to match /Error: unknown option -i/
+    expect(results.status).to_not eq 0
+  end
+
+  it "errors when input and output files are not specified" do
+    results = run_propane(args: [], capture: true)
+    expect(results.stderr).to match /Error: specify input and output files/
+    expect(results.status).to_not eq 0
+  end
+
+  it "errors when input file is not readable" do
+    results = run_propane(args: %w[nope.txt out.d], capture: true)
+    expect(results.stderr).to match /Error: cannot read nope.txt/
+    expect(results.status).to_not eq 0
+  end
+
  %w[d c].each do |language|

    context "#{language.upcase} language" do
@@ -123,14 +159,12 @@ token plus /\\+/;
 token times /\\*/;
 drop /\\s+/;
 Start -> Foo;
-Foo -> int <<
->>
-Foo -> plus <<
->>
+Foo -> int <<>>
+Foo -> plus <<>>
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_lexer.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -149,9 +183,7 @@ token int /\\d+/ <<
  }
  $$ = v;
 >>
-Start -> int <<
-  $$ = $1;
->>
+Start -> int << $$ = $1; >>
 EOF
        when "d"
          write_grammar <<EOF
@@ -165,14 +197,12 @@ token int /\\d+/ <<
  }
  $$ = v;
 >>
-Start -> int <<
-  $$ = $1;
->>
+Start -> int << $$ = $1; >>
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_lexer_unknown_character.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -190,7 +220,7 @@ E -> B;
 B -> zero;
 B -> one;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
      end

      it "generates a parser that does basic math - user guide example" do
@@ -219,33 +249,15 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;

-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = (size_t)pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
        when "d"
          write_grammar <<EOF
@@ -271,38 +283,20 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;

-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_basic_math_grammar.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -314,7 +308,7 @@ Start -> E;
 E -> one E;
 E -> one;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
      end

      it "distinguishes between multiple identical rules with lookahead symbol" do
@@ -326,9 +320,9 @@ Start -> R2 b;
 R1 -> a b;
 R2 -> a b;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
      end

@@ -341,9 +335,9 @@ Start -> a R1;
 Start -> b R1;
 R1 -> b;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
      end

@@ -376,9 +370,9 @@ Abcs -> ;
 Abcs -> abc Abcs;
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_user_code.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "abc!",
@@ -408,15 +402,13 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
-  writeln("def!");
->>
+/def/ << writeln("def!"); >>
 Start -> abc;
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_pattern.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "def!",
@@ -435,9 +427,7 @@ EOF
 #include <stdio.h>
 >>
 token abc;
-/def/ <<
-  printf("def!\\n");
->>
+/def/ << printf("def!\\n"); >>
 /ghi/ <<
  printf("ghi!\\n");
  return $token(abc);
@@ -450,9 +440,7 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
-  writeln("def!");
->>
+/def/ << writeln("def!"); >>
 /ghi/ <<
  writeln("ghi!");
  return $token(abc);
@@ -460,9 +448,9 @@ token abc;
 Start -> abc;
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_return_token_from_pattern.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "def!",
@@ -518,9 +506,9 @@ string: /"/ <<
 Start -> abc string def;
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_lexer_modes.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "begin string mode",
@@ -541,15 +529,9 @@ EOF
 >>
 token a;
 token b;
-Start -> A B <<
-  printf("Start!\\n");
->>
-A -> a <<
-  printf("A!\\n");
->>
-B -> b <<
-  printf("B!\\n");
->>
+Start -> A B << printf("Start!\\n"); >>
+A -> a << printf("A!\\n"); >>
+B -> b << printf("B!\\n"); >>
 EOF
        when "d"
          write_grammar <<EOF
@@ -558,20 +540,14 @@ import std.stdio;
 >>
 token a;
 token b;
-Start -> A B <<
-  writeln("Start!");
->>
-A -> a <<
-  writeln("A!");
->>
-B -> b <<
-  writeln("B!");
->>
+Start -> A B << writeln("Start!"); >>
+A -> a << writeln("A!"); >>
+B -> b << writeln("B!"); >>
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_parser_rule_user_code.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "A!",
@@ -584,19 +560,13 @@ EOF
        write_grammar <<EOF
 ptype #{language == "c" ? "uint32_t" : "uint"};
 token a;
-Start -> As <<
-  $$ = $1;
->>
-As -> <<
-  $$ = 0u;
->>
-As -> As a <<
-  $$ = $1 + 1u;
->>
+Start -> As << $$ = $1; >>
+As -> << $$ = 0u; >>
+As -> As a << $$ = $1 + 1u; >>
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_parsing_lists.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        expect(results.stderr).to eq ""
      end
@@ -615,7 +585,7 @@ Start -> b E d;
 E -> e;
 F -> e;
 EOF
-        results = build_parser(capture: true, language: language)
+        results = run_propane(capture: true, language: language)
        expect(results.status).to_not eq 0
        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
      end
@@ -647,9 +617,9 @@ token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
 Start -> id;
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_lexer_match_text.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.status).to eq 0
        verify_lines(results.stdout, [
          "Matched token is identifier_123",
@@ -680,9 +650,9 @@ Start -> word <<
 >>
 EOF
        end
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_lexer_result_value.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -695,16 +665,16 @@ drop /\\s+/;
 Start -> a num Start;
 Start -> a num;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_error_positions.#{language}", language: language)
-        results = run
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end

      it "allows creating a JSON parser" do
        write_grammar(File.read("spec/json_parser.#{language}.propane"))
-        build_parser(language: language)
+        run_propane(language: language)
        compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
      end

@@ -716,16 +686,35 @@ token num /\\d+/;
 drop /\\s+/;
 Start -> a num;
 EOF
-        build_parser(name: "myp1", language: language)
+        run_propane(name: "myp1", language: language)
        write_grammar(<<EOF, name: "myp2")
 prefix myp2_;
 token b;
 token c;
 Start -> b c b;
 EOF
-        build_parser(name: "myp2", language: language)
+        run_propane(name: "myp2", language: language)
        compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
-        results = run
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows the user to terminate the lexer" do
+        write_grammar <<EOF
+token a;
+token b <<
+  $terminate(8675309);
+>>
+token c;
+Start -> Any;
+Any -> a;
+Any -> b;
+Any -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_user_terminate_lexer.#{language}", language: language)
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -737,15 +726,122 @@ token b;
 token c;
 Start -> Any;
 Any -> a Any;
-Any -> b Any <<
-  $terminate(4200);
->>
+Any -> b Any << $terminate(4200); >>
 Any -> c Any;
 Any -> ;
 EOF
-        build_parser(language: language)
+        run_propane(language: language)
        compile("spec/test_user_terminate.#{language}", language: language)
-        results = run
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "matches backslash escape sequences" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+  #include <stdio.h>
+>>
+tokenid t;
+/\\a/ << printf("A\\n"); >>
+/\\b/ << printf("B\\n"); >>
+/\\t/ << printf("T\\n"); >>
+/\\n/ << printf("N\\n"); >>
+/\\v/ << printf("V\\n"); >>
+/\\f/ << printf("F\\n"); >>
+/\\r/ << printf("R\\n"); >>
+/t/ << return $token(t); >>
+Start -> t;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+  import std.stdio;
+>>
+tokenid t;
+/\\a/ << writeln("A"); >>
+/\\b/ << writeln("B"); >>
+/\\t/ << writeln("T"); >>
+/\\n/ << writeln("N"); >>
+/\\v/ << writeln("V"); >>
+/\\f/ << writeln("F"); >>
+/\\r/ << writeln("R"); >>
+/t/ <<
+  return $token(t);
+>>
+Start -> t;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_match_backslashes.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "A",
+          "B",
+          "T",
+          "N",
+          "V",
+          "F",
+          "R",
+        ])
+      end
+
+      it "handles when an item set leads to itself" do
+        write_grammar <<EOF
+token one;
+token two;
+
+Start -> Opt one Start;
+Start -> ;
+
+Opt -> two;
+Opt -> ;
+EOF
+        run_propane(language: language)
+      end
+
+      it "generates an AST" do
+        write_grammar <<EOF
+ast;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast.#{language}", language: language)
+        results = run_test
        expect(results.stderr).to eq ""
        expect(results.status).to eq 0
      end
@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    Items * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    ItemsMore * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}
@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Start * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    Items * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    ItemsMore * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}
@@ -14,14 +14,14 @@ int main()
    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
    assert(p_position(&context).row == 2);
    assert(p_position(&context).col == 3);
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);

    input = "12";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
    assert(p_position(&context).row == 0);
    assert(p_position(&context).col == 0);
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);

    input = "a 12\n\nab";
    p_context_init(&context, (uint8_t const *)input, strlen(input));
@@ -35,5 +35,8 @@ int main()
    assert(p_position(&context).row == 5);
    assert(p_position(&context).col == 4);

+    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
+    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+
    return 0;
 }
@@ -17,13 +17,13 @@ unittest
    p_context_init(&context, input);
    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
    assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);

    input = "12";
    p_context_init(&context, input);
    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
    assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);

    input = "a 12\n\nab";
    p_context_init(&context, input);
@@ -34,4 +34,7 @@ unittest
    p_context_init(&context, input);
    assert(p_parse(&context) == P_DECODE_ERROR);
    assert(p_position(&context) == p_position_t(5, 4));
+
+    assert(p_token_names[TOKEN_a] == "a");
+    assert(p_token_names[TOKEN_num] == "num");
 }
@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}
@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+
+    return 0;
+}
@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+}
Author	SHA1	Message	Date
josh	d0f542cbd7	v1.3.0	2024-04-23 00:31:56 -04:00
josh	786c78b635	Update CHANGELOG for v1.3.0	2024-04-23 00:21:28 -04:00
josh	f0bd8d8663	Add documentation for AST generation mode - close #22	2024-04-23 00:15:19 -04:00
josh	c7a18ef821	Add AST node field name with no suffix when unique - #22	2024-04-22 21:50:26 -04:00
josh	cb06a56f81	Add AST generation - #22	2024-04-22 20:51:27 -04:00
josh	2b28ef622d	Add specs to fully cover cli.rb	2024-04-06 14:37:15 -04:00
josh	19c32b58dc	Fix README example grammar	2024-04-06 14:16:27 -04:00
josh	3a8dcac55f	v1.2.0	2024-04-02 21:42:33 -04:00
josh	632ab2fe6f	Update CHANGELOG for v1.2.0	2024-04-02 21:42:18 -04:00
josh	3eaf0d3d49	allow one line user code blocks - close #21	2024-04-02 17:44:15 -04:00
josh	918dc7b2bb	fix generator hang when state transition cycle is present - close #20	2024-04-02 14:27:08 -04:00
josh	5b2cbe53e6	Add backslash escape codes - close #19	2024-03-29 16:45:54 -04:00
josh	1d1590dfda	Add API to access unexpected token found - close #18	2024-03-29 15:58:56 -04:00
josh	1c91dcd298	Add token_names API - close #17	2024-03-29 15:02:01 -04:00
josh	5dfd62b756	Add D example to user guide for p_context_init() - close #16	2024-03-29 13:52:16 -04:00
josh	fad7f4fb36	Allow user termination from lexer code blocks - close #15	2024-03-29 13:45:08 -04:00
josh	d55c5e0080	Update CHANGELOG for v1.1.0	2024-01-07 17:48:47 -05:00