v1.4.0

Add ast_prefix and ast_suffix grammar statements.

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v1.4.0
+
+### New Features
+
+- Allow user to specify AST node name prefix or suffix
+- Allow specifying the start rule name
+- Allow rule terms to be marked as optional
+
+### Improvements
+
+- Give a better error message when a referenced ptype has not been declared
+
 ## v1.3.0
 
 ### New Features

assets/parser.c.erb

@@ -622,6 +622,13 @@ typedef struct
      * Number of rule set AST node fields.
      */
     uint16_t rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
 <% end %>
 } reduce_t;
 
@@ -686,6 +693,7 @@ static const reduce_t parser_reduce_table[] = {
              , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
 <%       end %>
              , <%= reduce[:rule].rule_set.ast_fields.size %>
+             , <%= reduce[:propagate_optional_target] %>
 <%     end %>
     },
 <%   end %>
@@ -924,7 +932,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context->parse_result = (Start *)state_values_stack_index(&statevalues, -1)->ast_node;
+                context->parse_result = (<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)state_values_stack_index(&statevalues, -1)->ast_node;
 <% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
 <% end %>
@@ -941,7 +949,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                Token * token_ast_node = malloc(sizeof(Token));
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
                 token_ast_node->token = token;
                 token_ast_node->pvalue = token_info.pvalue;
                 state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
@@ -970,7 +978,11 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         {
             /* We have something to reduce. */
 <% if @grammar.ast %>
-            if (parser_reduce_table[reduce_index].n_states > 0)
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = state_values_stack_index(&statevalues, -1)->ast_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 void ** node_fields = calloc(parser_reduce_table[reduce_index].rule_set_node_field_array_size, sizeof(void *));
                 if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
@@ -1029,7 +1041,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>

assets/parser.d.erb

@@ -64,7 +64,7 @@ public union <%= @grammar.prefix %>value_t
 
 <% if @grammar.ast %>
 /** AST node types. @{ */
-public struct Token
+public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
@@ -72,7 +72,8 @@ public struct Token
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
-public struct <%= name %>
+<%     next if rule_set.optional? %>
+public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
 <%     rule_set.ast_fields.each do |fields| %>
     union
@@ -144,7 +145,7 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    Start * parse_result;
+    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -763,6 +764,13 @@ private struct reduce_t
      * Number of rule set AST node fields.
      */
     ushort rule_set_node_field_array_size;
+
+    /**
+     * Whether this rule was a generated optional rule that matched the
+     * optional target. In this case, propagate the matched target node up
+     * instead of making a new node for this rule.
+     */
+    bool propagate_optional_target;
 <% end %>
 }
 
@@ -832,6 +840,7 @@ private immutable reduce_t[] parser_reduce_table = [
              , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
 <%       end %>
              , <%= reduce[:rule].rule_set.ast_fields.size %>
+             , <%= reduce[:propagate_optional_target] %>
 <%     end %>
             ),
 <%   end %>
@@ -973,7 +982,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* Successful parse. */
 <% if @grammar.ast %>
-                context.parse_result = cast(Start *)statevalues[$-1].ast_node;
+                context.parse_result = cast(<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> *)statevalues[$-1].ast_node;
 <% else %>
                 context.parse_result = statevalues[$-1].pvalue;
 <% end %>
@@ -988,7 +997,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                Token * token_ast_node = new Token(token, token_info.pvalue);
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token, token_info.pvalue);
                 statevalues[$-1].ast_node = token_ast_node;
 <% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;
@@ -1015,7 +1024,11 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         {
             /* We have something to reduce. */
 <% if @grammar.ast %>
-            if (parser_reduce_table[reduce_index].n_states > 0)
+            if (parser_reduce_table[reduce_index].propagate_optional_target)
+            {
+                reduced_parser_node = statevalues[$ - 1].ast_node;
+            }
+            else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 void *[] node_fields = new void *[parser_reduce_table[reduce_index].rule_set_node_field_array_size];
                 foreach (i; 0..parser_reduce_table[reduce_index].rule_set_node_field_array_size)
@@ -1075,7 +1088,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
  * @return Parse result value.
  */
 <% if @grammar.ast %>
-public Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
+public <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% else %>
 public <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 <% end %>

assets/parser.h.erb

@@ -56,20 +56,22 @@ typedef union
 
 <% if @grammar.ast %>
 /** AST node types. @{ */
-typedef struct Token
+typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
-} Token;
+} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
+<%     next if rule_set.optional? %>
 struct <%= name %>;
 <%   end %>
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
 <%     next if name.start_with?("$") %>
-typedef struct <%= name %>
+<%     next if rule_set.optional? %>
+typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 {
 <%     rule_set.ast_fields.each do |fields| %>
     union
@@ -79,7 +81,7 @@ typedef struct <%= name %>
 <%       end %>
     };
 <%     end %>
-} <%= name %>;
+} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>;
 
 <%   end %>
 /** @} */
@@ -144,7 +146,7 @@ typedef struct
 
     /** Parse result value. */
 <% if @grammar.ast %>
-    Start * parse_result;
+    <%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * parse_result;
 <% else %>
     <%= @grammar.prefix %>value_t parse_result;
 <% end %>
@@ -173,7 +175,7 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 
 <% if @grammar.ast %>
-Start * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
+<%= @grammar.ast_prefix %><%= @grammar.start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 <% end %>

doc/user_guide.md

@@ -203,8 +203,10 @@ In AST generation mode various aspects of propane's behavior are changed:
   * Parser user code blocks are not supported.
   * Structure types are generated to represent the parsed tokens and rules as
   defined in the grammar.
-  * The parse result from `p_result()` points to a `Start` structure containing
+  * The parse result from `p_result()` points to a `Start` struct containing
-  the entire parse tree for the input.
+  the entire parse tree for the input. If the user has changed the start rule
+  with the `start` grammar statement, the name of the start struct will be
+  given by the user-specified start rule instead of `Start`.
 
 Example AST generation grammar:
 
@@ -276,6 +278,48 @@ assert_eq(22, itemsmore.pItem.pToken1.pvalue);
 assert(itemsmore.pItemsMore is null);
 ```
 
+## `ast_prefix` and `ast_suffix` statements
+
+In AST generation mode, structure types are defined and named based on the
+rules in the grammar.
+Additionally, a structure type called `Token` is generated to hold parsed
+token information.
+
+These structure names can be modified by using the `ast_prefix` or `ast_suffix`
+statements in the grammar file.
+The field names that point to instances of the structures are not affected by
+the `ast_prefix` or `ast_suffix` values.
+
+For example, if the following two lines were added to the example above:
+
+```
+ast_prefix ABC;
+ast_suffix XYZ;
+```
+
+Then the types would be used as such instead:
+
+```
+string input = "a, ((b)), b";
+p_context_t context;
+p_context_init(&context, input);
+assert_eq(P_SUCCESS, p_parse(&context));
+ABCStartXYZ * start = p_result(&context);
+assert(start.pItems1 !is null);
+assert(start.pItems !is null);
+ABCItemsXYZ * items = start.pItems;
+assert(items.pItem !is null);
+assert(items.pItem.pToken1 !is null);
+assert_eq(TOKEN_a, items.pItem.pToken1.token);
+assert_eq(11, items.pItem.pToken1.pvalue);
+assert(items.pItemsMore !is null);
+ABCItemsMoreXYZ * itemsmore = items.pItemsMore;
+assert(itemsmore.pItem !is null);
+assert(itemsmore.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem !is null);
+assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+```
+
 ##> Specifying tokens - the `token` statement
 
 The `token` statement allows defining a lexer token and a pattern to match that
@@ -552,21 +596,24 @@ Rules with the same name define a rule set for that name and act as
 alternatives that the parser can accept when attempting to match a reference to
 that rule.
 
-The grammar file must define a rule with the name `Start` which will be used as
+The default start rule name is `Start`.
-the top-level starting rule that the parser attempts to reduce.
+This can be changed with the `start` statement.
+The grammar file must define a rule with the name of the start rule name which
+will be used as the top-level starting rule that the parser attempts to reduce.
 
 Example:
 
 ```
 ptype ulong;
+start Top;
 token word /[a-z]+/ << $$ = match.length; >>
-Start -> word << $$ = $1; >>
+Top -> word << $$ = $1; >>
 ```
 
-In the above example the `Start` rule is defined to match a single `word`
+In the above example the `Top` rule is defined to match a single `word`
 token.
 
-Example:
+Another example:
 
 ```
 Start -> E1 << $$ = $1; >>
@@ -580,8 +627,24 @@ E4 -> integer << $$ = $1; >>
 E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
+This example uses the default start rule name of `Start`.
+
 A parser rule has zero or more terms on the right side of its definition.
 Each of these terms is either a token name or a rule name.
+A term can be immediately followed by a `?` character to signify that it is
+optional.
+Another example:
+
+```
+token public;
+token private;
+token int;
+token ident /[a-zA-Z_][a-zA-Z_0-9]*/;
+token semicolon /;/;
+IntegerDeclaration -> Visibility? int ident semicolon;
+Visibility -> public;
+Visibility -> private;
+```
 
 In a parser rule code block, parser values for the right side terms are
 accessible as `$1` for the first term's parser value, `$2` for the second
@@ -593,6 +656,16 @@ can be used to produce the parser value for the accepted rule.
 Parser rule code blocks are not allowed and not used when AST generation mode
 is active.
 
+##> Specifying the parser start rule name - the `start` statement
+
+The start rule can be changed from the default of `Start` by using the `start`
+statement.
+Example:
+
+```
+start MyStartRule;
+```
+
 ##> Specifying the parser module name - the `module` statement
 
 The `module` statement can be used to specify the module name for a generated

lib/propane/generator.rb

@@ -51,6 +51,7 @@ class Propane
       unless found_default
         raise Error.new("No patterns found for default mode")
       end
+      check_ptypes!
       # Add EOF token.
       @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
@@ -66,11 +67,14 @@ class Propane
         tokens_by_name[token.name] = token
       end
       # Check for user start rule.
-      unless @grammar.rules.find {|rule| rule.name == "Start"}
+      unless @grammar.rules.find {|rule| rule.name == @grammar.start_rule}
-        raise Error.new("Start rule not found")
+        raise Error.new("Start rule `#{@grammar.start_rule}` not found")
       end
       # Add "real" start rule.
-      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
+      @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
+      # Generate and add rules for optional components.
+      generate_optional_component_rules!(tokens_by_name)
+      # Build rule sets.
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|
@@ -120,7 +124,7 @@ class Propane
       end
       determine_possibly_empty_rulesets!(rule_sets)
       rule_sets.each do |name, rule_set|
-        rule_set.finalize
+        rule_set.finalize(@grammar)
       end
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
@@ -128,6 +132,48 @@ class Propane
       @parser = Parser.new(@grammar, rule_sets, @log)
     end
 
+    # Check that any referenced ptypes have been defined.
+    def check_ptypes!
+      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
+        if potor.ptypename
+          unless @grammar.ptypes.include?(potor.ptypename)
+            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
+          end
+        end
+      end
+    end
+
+    # Generate and add rules for any optional components.
+    def generate_optional_component_rules!(tokens_by_name)
+      optional_rules_added = Set.new
+      @grammar.rules.each do |rule|
+        rule.components.each do |component|
+          if component =~ /^(.*)\?$/
+            c = $1
+            unless optional_rules_added.include?(component)
+              # Create two rules for the optional component: one empty and
+              # one just matching the component.
+              # We need to find the ptypename for the optional component in
+              # order to copy it to the generated rules.
+              if tokens_by_name[c]
+                # The optional component is a token.
+                ptypename = tokens_by_name[c].ptypename
+              else
+                # The optional component must be a rule, so find any instance
+                # of that rule that specifies a ptypename.
+                ptypename = @grammar.rules.reduce(nil) do |result, rule|
+                  rule.name == c && rule.ptypename ? rule.ptypename : result
+                end
+              end
+              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
+              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optional_rules_added << component
+            end
+          end
+        end
+      end
+    end
+
     # Determine which grammar rules could expand to empty sequences.
     #
     # @param rule_sets [Hash]
@@ -270,7 +316,7 @@ class Propane
     #   Start rule parser value type name and type string.
     def start_rule_type
       start_rule = @grammar.rules.find do |rule|
-        rule.name == "Start"
+        rule.name == @grammar.start_rule
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end

lib/propane/grammar.rb

@@ -6,9 +6,12 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :ast
+    attr_reader :ast_prefix
+    attr_reader :ast_suffix
     attr_reader :modulename
     attr_reader :patterns
     attr_reader :rules
+    attr_reader :start_rule
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
@@ -16,6 +19,7 @@ class Propane
 
     def initialize(input)
       @patterns = []
+      @start_rule = "Start"
       @tokens = []
       @rules = []
       @code_blocks = {}
@@ -26,6 +30,8 @@ class Propane
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
       @ast = false
+      @ast_prefix = ""
+      @ast_suffix = ""
       parse_grammar!
     end
 
@@ -54,9 +60,12 @@ class Propane
       elsif parse_comment_line!
       elsif @mode.nil? && parse_mode_label!
       elsif parse_ast_statement!
+      elsif parse_ast_prefix_statement!
+      elsif parse_ast_suffix_statement!
       elsif parse_module_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
+      elsif parse_start_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
@@ -91,6 +100,18 @@ class Propane
       end
     end
 
+    def parse_ast_prefix_statement!
+      if md = consume!(/ast_prefix\s+(\w+)\s*;/)
+        @ast_prefix = md[1]
+      end
+    end
+
+    def parse_ast_suffix_statement!
+      if md = consume!(/ast_suffix\s+(\w+)\s*;/)
+        @ast_suffix = md[1]
+      end
+    end
+
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
@@ -177,7 +198,7 @@ class Propane
         if @ast && ptypename
           raise Error.new("Multiple ptypes are unsupported in AST mode")
         end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         if @ast
           consume!(/;/, "expected `;'")
@@ -210,6 +231,12 @@ class Propane
       end
     end
 
+    def parse_start_statement!
+      if md = consume!(/start\s+(\w+)\s*;/)
+        @start_rule = md[1]
+      end
+    end
+
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]

lib/propane/parser.rb

@@ -64,11 +64,13 @@ class Propane
           case ra = item_set.reduce_actions
           when Rule
             [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
+              rule_set_id: ra.rule_set.id, n_states: ra.components.size,
+              propagate_optional_target: ra.optional? && ra.components.size == 1}]
           when Hash
             ra.map do |token, rule|
               {token_id: token.id, rule_id: rule.id, rule: rule,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
           else
             []

lib/propane/rule.rb

@@ -66,6 +66,14 @@ class Propane
       @components.empty?
     end
 
+    # Return whether this is an optional Rule.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional Rule.
+    def optional?
+      @name.end_with?("?")
+    end
+
     # Represent the Rule as a String.
     #
     # @return [String]

lib/propane/rule_set.rb

@@ -3,6 +3,10 @@ class Propane
   # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
+    # @return [Array<Hash>]
+    #   AST fields.
+    attr_reader :ast_fields
+
     # @return [Integer]
     #   ID of the RuleSet.
     attr_reader :id
@@ -52,6 +56,24 @@ class Propane
       @could_be_empty
     end
 
+    # Return whether this is an optional RuleSet.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional RuleSet.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # For optional rule sets, return the underlying component that is optional.
+    def option_target
+      @rules.each do |rule|
+        if rule.components.size > 0
+          return rule.components[0]
+        end
+      end
+      raise "Optional rule target not found"
+    end
+
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -76,6 +98,13 @@ class Propane
       @_start_token_set
     end
 
+    # Finalize a RuleSet after adding all Rules to it.
+    def finalize(grammar)
+      build_ast_fields(grammar)
+    end
+
+    private
+
     # Build the set of AST fields for this RuleSet.
     #
     # This is an Array of Hashes. Each entry in the Array corresponds to a
@@ -84,25 +113,26 @@ class Propane
     # a key. It may also have the field name without the positional suffix if
     # that field only exists in one position across all Rules in the RuleSet.
     #
-    # @return [Array<Hash>]
+    # @return [void]
-    #   AST fields.
+    def build_ast_fields(grammar)
-    def ast_fields
-      @_ast_fields ||=
-        begin
       field_ast_node_indexes = {}
       field_indexes_across_all_rules = {}
-          ast_node_fields = []
+      @ast_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
+          if component.is_a?(RuleSet) && component.optional?
+            component = component.option_target
+          end
           if component.is_a?(Token)
             node_name = "Token"
           else
             node_name = component.name
           end
+          struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
           field_name = "p#{node_name}#{i + 1}"
           unless field_ast_node_indexes[field_name]
-                field_ast_node_indexes[field_name] = ast_node_fields.size
+            field_ast_node_indexes[field_name] = @ast_fields.size
-                ast_node_fields << {field_name => node_name}
+            @ast_fields << {field_name => struct_name}
           end
           field_indexes_across_all_rules[node_name] ||= Set.new
           field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
@@ -114,16 +144,10 @@ class Propane
           # If this field was only seen in one position across all rules,
           # then add an alias to the positional field name that does not
           # include the position.
-              ast_node_fields[indexes_across_all_rules.first]["p#{node_name}"] = node_name
+          @ast_fields[indexes_across_all_rules.first]["p#{node_name}"] =
+            "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
         end
       end
-          ast_node_fields
-        end
-    end
-
-    # Finalize a RuleSet after adding all Rules to it.
-    def finalize
-      ast_fields
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

spec/propane_spec.rb

@@ -148,6 +148,42 @@ EOF
     expect(results.status).to_not eq 0
   end
 
+  it "raises an error when a pattern referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+/foo/ (yes) <<
+>>
+/bar/ (no) <<
+>>
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 4: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
+  it "raises an error when a token referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+token foo (yes);
+token bar (no);
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 3: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
+  it "raises an error when a rule referenced ptype has not been defined" do
+    write_grammar <<EOF
+ptype yes = int;
+token xyz;
+foo (yes) -> bar;
+bar (no) -> xyz;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to match /Error: Line 4: ptype no not declared\. Declare with `ptype` statement\./
+    expect(results.status).to_not eq 0
+  end
+
   %w[d c].each do |language|
 
     context "#{language.upcase} language" do
@@ -845,6 +881,176 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "supports AST node prefix and suffix" do
+        write_grammar <<EOF
+ast;
+ast_prefix P ;
+ast_suffix  S;
+
+ptype int;
+
+token a << $$ = 11; >>
+token b << $$ = 22; >>
+token one /1/;
+token two /2/;
+token comma /,/ <<
+  $$ = 42;
+>>
+token lparen /\\(/;
+token rparen /\\)/;
+drop /\\s+/;
+
+Start -> Items;
+
+Items -> Item ItemsMore;
+Items -> ;
+
+ItemsMore -> comma Item ItemsMore;
+ItemsMore -> ;
+
+Item -> a;
+Item -> b;
+Item -> lparen Item rparen;
+Item -> Dual;
+
+Dual -> One Two;
+Dual -> Two One;
+One -> one;
+Two -> two;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_ps.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows specifying a different start rule" do
+        write_grammar <<EOF
+token hi;
+start Top;
+Top -> hi;
+EOF
+        run_propane(language: language)
+        compile("spec/test_start_rule.#{language}", language: language)
+      end
+
+      it "allows specifying a different start rule with AST generation" do
+        write_grammar <<EOF
+ast;
+token hi;
+start Top;
+Top -> hi;
+EOF
+        run_propane(language: language)
+        compile("spec/test_start_rule_ast.#{language}", language: language)
+      end
+
+      it "allows marking a rule component as optional" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+
+ptype int;
+ptype float = float;
+ptype string = string;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  writeln("a: ", $1);
+  writeln("b: ", $2);
+  writeln("R: ", $3);
+>>
+R -> c d << $$ = "cd"; >>
+R (string) -> d c << $$ = "dc"; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+token a (float) << $$ = 1.5; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
+token d << $$ = 4; >>
+Start -> a? b R? <<
+  printf("a: %.1f\\n", $1);
+  printf("b: %d\\n", $2);
+  printf("R: %s\\n", $3 == NULL ? "" : $3);
+>>
+R -> c d << $$ = "cd"; >>
+R (string) -> d c << $$ = "dc"; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_optional_rule_component.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "a: 0#{language == "d" ? "" : ".0"}",
+          "b: 2",
+          "R: ",
+          "a: 1.5",
+          "b: 2",
+          "R: cd",
+          "a: 1.5",
+          "b: 2",
+          "R: dc",
+        ])
+      end
+
+      it "allows marking a rule component as optional in AST generation mode" do
+        if language == "d"
+          write_grammar <<EOF
+ast;
+
+<<
+import std.stdio;
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        else
+          write_grammar <<EOF
+ast;
+
+<<
+#include <stdio.h>
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a? b R?;
+R -> c d;
+R -> d c;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_optional_rule_component_ast.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_ast_ps.c

@@ -0,0 +1,55 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start->pItems1 != NULL);
+    assert(start->pItems != NULL);
+    PItemsS * items = start->pItems;
+    assert(items->pItem != NULL);
+    assert(items->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_a, items->pItem->pToken1->token);
+    assert_eq(11, items->pItem->pToken1->pvalue);
+    assert(items->pItemsMore != NULL);
+    PItemsMoreS * itemsmore = items->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem != NULL);
+    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore != NULL);
+    itemsmore = itemsmore->pItemsMore;
+    assert(itemsmore->pItem != NULL);
+    assert(itemsmore->pItem->pToken1 != NULL);
+    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
+    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
+    assert(itemsmore->pItemsMore == NULL);
+
+    input = "";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems == NULL);
+
+    input = "2 1";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start->pItems != NULL);
+    assert(start->pItems->pItem != NULL);
+    assert(start->pItems->pItem->pDual != NULL);
+    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
+    assert(start->pItems->pItem->pDual->pOne2 != NULL);
+    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
+    assert(start->pItems->pItem->pDual->pOne1 == NULL);
+
+    return 0;
+}

spec/test_ast_ps.d

@@ -0,0 +1,57 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a, ((b)), b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    PStartS * start = p_result(&context);
+    assert(start.pItems1 !is null);
+    assert(start.pItems !is null);
+    PItemsS * items = start.pItems;
+    assert(items.pItem !is null);
+    assert(items.pItem.pToken1 !is null);
+    assert_eq(TOKEN_a, items.pItem.pToken1.token);
+    assert_eq(11, items.pItem.pToken1.pvalue);
+    assert(items.pItemsMore !is null);
+    PItemsMoreS * itemsmore = items.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem !is null);
+    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore !is null);
+    itemsmore = itemsmore.pItemsMore;
+    assert(itemsmore.pItem !is null);
+    assert(itemsmore.pItem.pToken1 !is null);
+    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
+    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
+    assert(itemsmore.pItemsMore is null);
+
+    input = "";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems is null);
+
+    input = "2 1";
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    start = p_result(&context);
+    assert(start.pItems !is null);
+    assert(start.pItems.pItem !is null);
+    assert(start.pItems.pItem.pDual !is null);
+    assert(start.pItems.pItem.pDual.pTwo1 !is null);
+    assert(start.pItems.pItem.pDual.pOne2 !is null);
+    assert(start.pItems.pItem.pDual.pTwo2 is null);
+    assert(start.pItems.pItem.pDual.pOne1 is null);
+}

spec/test_optional_rule_component.c

@@ -0,0 +1,22 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}
+

spec/test_optional_rule_component.d

@@ -0,0 +1,23 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_optional_rule_component_ast.c

@@ -0,0 +1,42 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->pR == start->pR3);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->pToken1 == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->pR != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    return 0;
+}
+

spec/test_optional_rule_component_ast.d

@@ -0,0 +1,43 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.pR == start.pR3);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_start_rule.c

@@ -0,0 +1,9 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule.d

@@ -0,0 +1,8 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}

spec/test_start_rule_ast.c

@@ -0,0 +1,17 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "hi";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top->pToken != NULL);
+    assert_eq(TOKEN_hi, top->pToken->token);
+
+    return 0;
+}

spec/test_start_rule_ast.d

@@ -0,0 +1,19 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "hi";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert_eq(P_SUCCESS, p_parse(&context));
+    Top * top = p_result(&context);
+    assert(top.pToken !is null);
+    assert_eq(TOKEN_hi, top.pToken.token);
+}