Allow rule terms to be marked as optional

lib/propane/generator.rb

@@ -71,6 +71,9 @@ class Propane
       end
       # Add "real" start rule.
       @grammar.rules.unshift(Rule.new("$Start", [@grammar.start_rule, "$EOF"], nil, nil, nil))
+      # Generate and add rules for optional components.
+      generate_optional_component_rules!(tokens_by_name)
+      # Build rule sets.
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|
@@ -128,6 +131,37 @@ class Propane
       @parser = Parser.new(@grammar, rule_sets, @log)
     end
 
+    # Generate and add rules for any optional components.
+    def generate_optional_component_rules!(tokens_by_name)
+      optional_rules_added = Set.new
+      @grammar.rules.each do |rule|
+        rule.components.each do |component|
+          if component =~ /^(.*)\?$/
+            c = $1
+            unless optional_rules_added.include?(component)
+              # Create two rules for the optional component: one empty and
+              # one just matching the component.
+              # We need to find the ptypename for the optional component in
+              # order to copy it to the generated rules.
+              if tokens_by_name[c]
+                # The optional component is a token.
+                ptypename = tokens_by_name[c].ptypename
+              else
+                # The optional component must be a rule, so find any instance
+                # of that rule that specifies a ptypename.
+                ptypename = @grammar.rules.reduce(nil) do |result, rule|
+                  rule.name == c && rule.ptypename ? rule.ptypename : result
+                end
+              end
+              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
+              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
+              optional_rules_added << component
+            end
+          end
+        end
+      end
+    end
+
     # Determine which grammar rules could expand to empty sequences.
     #
     # @param rule_sets [Hash]

lib/propane/grammar.rb

@@ -198,7 +198,7 @@ class Propane
         if @ast && ptypename
           raise Error.new("Multiple ptypes are unsupported in AST mode")
         end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         if @ast
           consume!(/;/, "expected `;'")

lib/propane/parser.rb

@@ -64,11 +64,13 @@ class Propane
           case ra = item_set.reduce_actions
           when Rule
             [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
+              rule_set_id: ra.rule_set.id, n_states: ra.components.size,
+              propagate_optional_target: ra.optional? && ra.components.size == 1}]
           when Hash
             ra.map do |token, rule|
               {token_id: token.id, rule_id: rule.id, rule: rule,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
           else
             []

lib/propane/rule.rb

@@ -66,6 +66,14 @@ class Propane
       @components.empty?
     end
 
+    # Return whether this is an optional Rule.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional Rule.
+    def optional?
+      @name.end_with?("?")
+    end
+
     # Represent the Rule as a String.
     #
     # @return [String]

lib/propane/rule_set.rb

@@ -56,6 +56,24 @@ class Propane
       @could_be_empty
     end
 
+    # Return whether this is an optional RuleSet.
+    #
+    # @return [Boolean]
+    #   Whether this is an optional RuleSet.
+    def optional?
+      @name.end_with?("?")
+    end
+
+    # For optional rule sets, return the underlying component that is optional.
+    def option_target
+      @rules.each do |rule|
+        if rule.components.size > 0
+          return rule.components[0]
+        end
+      end
+      raise "Optional rule target not found"
+    end
+
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -102,6 +120,9 @@ class Propane
       @ast_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
+          if component.is_a?(RuleSet) && component.optional?
+            component = component.option_target
+          end
           if component.is_a?(Token)
             node_name = "Token"
           else