CHANGELOG.md

@@ -1,21 +1,8 @@
-## v1.5.1
-
-### Improvements
-
-- Improve performance (#28)
-
 ## v1.5.0
 
 ### New Features
 
-- Track start and end text positions for tokens and rules in AST node structures (#27)
-- Add warnings for shift/reduce conflicts to log file (#25)
-- Add -w command line switch to treat warnings as errors and output to stderr (#26)
-- Add rule field aliases (#24)
-
-### Improvements
-
-- Show line numbers of rules on conflict (#23)
+- Track token position in AST Token node
 
 ## v1.4.0
 

README.md

@@ -31,14 +31,9 @@ Propane is typically invoked from the command-line as `./propane`.
 
     Usage: ./propane [options] <input-file> <output-file>
     Options:
-      -h, --help  Show this usage and exit.
-      --log LOG   Write log file. This will show all parser states and their
-                  associated shifts and reduces. It can be helpful when
-                  debugging a grammar.
-      --version   Show program version and exit.
-      -w          Treat warnings as errors. This option will treat shift/reduce
-                  conflicts as fatal errors and will print them to stderr in
-                  addition to the log file.
+      --log LOG   Write log file
+      --version   Show program version and exit
+      -h, --help  Show this usage and exit
 
 The user must specify the path to a Propane input grammar file and a path to an
 output file.

extra/vim/syntax/propane.vim

@@ -17,9 +17,6 @@ syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contai
 
 syn match propaneComment "#.*"
 syn match propaneOperator "->"
-syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
-syn match propaneFieldOperator ":" contained
-syn match propaneOperator "?"
 syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
 
 syn region propaneRegex start="/" end="/" skip="\\/"
@@ -28,6 +25,4 @@ hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
-hi def link propaneFieldOperator Operator
 hi def link propaneDelimiter Delimiter
-hi def link propaneFieldAlias Identifier

lib/propane/parser.rb

@@ -39,6 +39,7 @@ class Propane
       end
 
       build_reduce_actions!
+      build_follow_sets!
       build_tables!
       write_log!
       if @warnings.size > 0 && @options[:warnings_as_errors]
@@ -65,10 +66,10 @@ class Propane
             state_id: state_id,
           }
         end
-        unless item_set.reduce_rules.empty?
+        if item_set.reduce_actions
           shift_entries.each do |shift_entry|
             token = shift_entry[:symbol]
-            if get_lookahead_reduce_actions_for_item_set(item_set).include?(token)
+            if item_set.reduce_actions.include?(token)
               rule = item_set.reduce_actions[token]
               @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
             end
@@ -114,7 +115,7 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
-        build_reduce_actions_for_item_set(item_set)
+        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
       end
     end
 
@@ -123,36 +124,27 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [void]
+    # @return [nil, Hash]
+    #   If no reduce actions are possible for the given item set, nil.
+    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
       # "complete" items, i.e., items where the parse position is at the
       # end of a rule. These are the only rules that are candidates for
       # reduction in the current ItemSet.
-      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      if item_set.reduce_rules.size == 1
-        item_set.reduce_rule = item_set.reduce_rules.first
+      if reduce_rules.size == 1
+        item_set.reduce_rule = reduce_rules.first
       end
 
-      if item_set.reduce_rules.size > 1
-        # Force item_set.reduce_actions to be built to store the lookahead
-        # tokens for the possible reduce rules if there is more than one.
-        get_lookahead_reduce_actions_for_item_set(item_set)
+      if reduce_rules.size == 0
+        nil
+      else
+        build_lookahead_reduce_actions_for_item_set(item_set)
       end
     end
 
-    # Get the reduce actions for a single item set (parser state).
-    #
-    # @param item_set [ItemSet]
-    #   ItemSet (parser state)
-    #
-    # @return [Hash]
-    #   Mapping of lookahead Tokens to the Rules to reduce.
-    def get_lookahead_reduce_actions_for_item_set(item_set)
-      item_set.reduce_actions ||= build_lookahead_reduce_actions_for_item_set(item_set)
-    end
-
     # Build the reduce actions for a single item set (parser state).
     #
     # @param item_set [ItemSet]
@@ -161,13 +153,15 @@ class Propane
     # @return [Hash]
     #   Mapping of lookahead Tokens to the Rules to reduce.
     def build_lookahead_reduce_actions_for_item_set(item_set)
+      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
       item_sets = Set[item_set] + item_set.leading_item_sets
-      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
@@ -239,6 +233,51 @@ class Propane
       lookahead_tokens
     end
 
+    # Build the follow sets for each ItemSet.
+    #
+    # @return [void]
+    def build_follow_sets!
+      @item_sets.each do |item_set|
+        item_set.follow_set = build_follow_set_for_item_set(item_set)
+      end
+    end
+
+    # Build the follow set for the given ItemSet.
+    #
+    # @param item_set [ItemSet]
+    #   The ItemSet to build the follow set for.
+    #
+    # @return [Set]
+    #   Follow set for the given ItemSet.
+    def build_follow_set_for_item_set(item_set)
+      follow_set = Set.new
+      rule_sets_to_check_after = Set.new
+      item_set.items.each do |item|
+        (1..).each do |offset|
+          case symbol = item.next_symbol(offset)
+          when nil
+            rule_sets_to_check_after << item.rule.rule_set
+            break
+          when Token
+            follow_set << symbol
+            break
+          when RuleSet
+            follow_set += symbol.start_token_set
+            unless symbol.could_be_empty?
+              break
+            end
+          end
+        end
+      end
+      reduce_lookaheads = build_lookahead_reduce_actions_for_item_set(item_set)
+      reduce_lookaheads.each do |token, rule_set|
+        if rule_sets_to_check_after.include?(rule_set)
+          follow_set << token
+        end
+      end
+      follow_set
+    end
+
     def write_log!
       @log.puts Util.banner("Parser Rules")
       @grammar.rules.each do |rule|

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing. This is equivalent to a parser state.
+    # parser could currently be parsing.
     class ItemSet
 
       # @return [Set<Item>]
@@ -25,15 +25,15 @@ class Propane
       #   Rule to reduce if there is only one possibility.
       attr_accessor :reduce_rule
 
-      # @return [Set<Rule>]
-      #   Set of rules that could be reduced in this parser state.
-      attr_accessor :reduce_rules
-
       # @return [nil, Hash]
       #   Reduce actions, mapping lookahead tokens to rules, if there is
       #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
+      # @return [Set<Token>]
+      #   Follow set for the ItemSet.
+      attr_accessor :follow_set
+
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
@@ -50,7 +50,7 @@ class Propane
       # @return [Set<Token, RuleSet>]
       #   Set of next symbols for all Items in this ItemSet.
       def next_symbols
-        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
+        Set.new(@items.map(&:next_symbol).compact)
       end
 
       # Build a next ItemSet for the given next symbol.
@@ -99,24 +99,21 @@ class Propane
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @_leading_item_sets ||=
-          begin
-            result = Set.new
-            eval_sets = Set[self]
-            evaled = Set.new
-            while eval_sets.size > 0
-              eval_set = eval_sets.first
-              eval_sets.delete(eval_set)
-              evaled << eval_set
-              eval_set.in_sets.each do |in_set|
-                result << in_set
-                unless evaled.include?(in_set)
-                  eval_sets << in_set
-                end
-              end
+        result = Set.new
+        eval_sets = Set[self]
+        evaled = Set.new
+        while eval_sets.size > 0
+          eval_set = eval_sets.first
+          eval_sets.delete(eval_set)
+          evaled << eval_set
+          eval_set.in_sets.each do |in_set|
+            result << in_set
+            unless evaled.include?(in_set)
+              eval_sets << in_set
             end
-            result
           end
+        end
+        result
       end
 
       # Represent the ItemSet as a String.

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.5.1"
+  VERSION = "1.4.0"
 end