Show line numbers of rules upon conflict - close #23

Even if they won't be needed for the generated parser, they'll be useful
to detect shift/reduce conflicts.

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v1.5.0
+
+### New Features
+
+- Track token position in AST Token node
+
 ## v1.4.0
 
 ### New Features

assets/parser.c.erb

@@ -444,7 +444,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
     <%= @grammar.prefix %>token_info_t token_info = {0};
     token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
@@ -670,7 +669,7 @@ typedef struct
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
-    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
+    {<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u},
 <%   end %>
 };
 
@@ -952,6 +951,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
                 <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = malloc(sizeof(<%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>));
                 token_ast_node->token = token;
                 token_ast_node->pvalue = token_info.pvalue;
+                token_ast_node->position = token_info.position;
                 state_values_stack_index(&statevalues, -1)->ast_node = token_ast_node;
 <% else %>
                 state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;

assets/parser.d.erb

@@ -49,6 +49,20 @@ public enum : <%= @grammar.prefix %>token_t
 /** Code point type. */
 public alias <%= @grammar.prefix %>code_point_t = uint;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+public struct <%= @grammar.prefix %>position_t
+{
+    /** Input text row (0-based). */
+    uint row;
+
+    /** Input text column (0-based). */
+    uint col;
+}
+
 <% if @grammar.ast %>
 /** Parser values type. */
 public alias <%= @grammar.prefix %>value_t = <%= @grammar.ptype %>;
@@ -68,6 +82,7 @@ public struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
+    <%= @grammar.prefix %>position_t position;
 }
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -89,20 +104,6 @@ public struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 /** @} */
 <% end %>
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-public struct <%= @grammar.prefix %>position_t
-{
-    /** Input text row (0-based). */
-    uint row;
-
-    /** Input text column (0-based). */
-    uint col;
-}
-
 /** Lexed token information. */
 public struct <%= @grammar.prefix %>token_info_t
 {
@@ -586,7 +587,6 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
     <%= @grammar.prefix %>token_info_t token_info;
     token_info.position = context.text_position;
     token_info.token = INVALID_TOKEN_ID;
-    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
@@ -817,7 +817,7 @@ private struct state_value_t
 /** Parser shift table. */
 private immutable shift_t[] parser_shift_table = [
 <%   @parser.shift_table.each do |shift| %>
-    shift_t(<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u),
+    shift_t(<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u),
 <%   end %>
 ];
 
@@ -997,7 +997,7 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             {
                 /* We shifted a token, mark it consumed. */
 <% if @grammar.ast %>
-                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token, token_info.pvalue);
+                <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %> * token_ast_node = new <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>(token, token_info.pvalue, token_info.position);
                 statevalues[$-1].ast_node = token_ast_node;
 <% else %>
                 statevalues[$-1].pvalue = token_info.pvalue;

assets/parser.h.erb

@@ -38,6 +38,20 @@ typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>t
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
+/**
+ * A structure to keep track of input position.
+ *
+ * This is useful for reporting errors, etc...
+ */
+typedef struct
+{
+    /** Input text row (0-based). */
+    uint32_t row;
+
+    /** Input text column (0-based). */
+    uint32_t col;
+} <%= @grammar.prefix %>position_t;
+
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>
 
@@ -60,6 +74,7 @@ typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
 {
     <%= @grammar.prefix %>token_t token;
     <%= @grammar.prefix %>value_t pvalue;
+    <%= @grammar.prefix %>position_t position;
 } <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
 
 <%   @parser.rule_sets.each do |name, rule_set| %>
@@ -87,20 +102,6 @@ typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
 /** @} */
 <% end %>
 
-/**
- * A structure to keep track of parser position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
-{
-    /** Input text row (0-based). */
-    uint32_t row;
-
-    /** Input text column (0-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
-
 /** Lexed token information. */
 typedef struct
 {

extra/vim/ftdetect/propane.vim

@@ -0,0 +1 @@
+au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -0,0 +1,28 @@
+" Vim syntax file for Propane
+" Language: propane
+" Maintainer: Josh Holtrop
+" URL: https://github.com/holtrop/propane
+
+if exists("b:current_syntax")
+  finish
+endif
+
+if !exists("b:propane_subtype")
+  let b:propane_subtype = "d"
+endif
+
+exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
+
+syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
+
+syn match propaneComment "#.*"
+syn match propaneOperator "->"
+syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
+
+syn region propaneRegex start="/" end="/" skip="\\/"
+
+hi def link propaneComment Comment
+hi def link propaneKeyword Keyword
+hi def link propaneRegex String
+hi def link propaneOperator Operator
+hi def link propaneDelimiter Delimiter

lib/propane.rb

@@ -31,10 +31,10 @@ class Propane
 
   class << self
 
-    def run(input_file, output_file, log_file)
+    def run(input_file, output_file, log_file, options)
       begin
         grammar = Grammar.new(File.read(input_file))
-        generator = Generator.new(grammar, output_file, log_file)
+        generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e
         $stderr.puts e.message

lib/propane/cli.rb

@@ -4,15 +4,21 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  --log LOG   Write log file
+  --log LOG   Write log file. This will show all parser states and their
-  --version   Show program version and exit
+              associated shifts and reduces. It can be helpful when
-  -h, --help  Show this usage and exit
+              debugging a grammar.
+  --version   Show program version and exit.
+  -h, --help  Show this usage and exit.
+  -w          Treat warnings as errors. This option will treat shift/reduce
+              conflicts as fatal errors and will print them to stderr in
+              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
+        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -29,6 +35,8 @@ EOF
           when "-h", "--help"
             puts USAGE
             return 0
+          when "-w"
+            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -45,7 +53,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file)
+        Propane.run(*params, log_file, options)
       end
 
     end

lib/propane/generator.rb

@@ -2,7 +2,7 @@ class Propane
 
   class Generator
 
-    def initialize(grammar, output_file, log_file)
+    def initialize(grammar, output_file, log_file, options)
       @grammar = grammar
       @output_file = output_file
       if log_file
@@ -16,6 +16,7 @@ class Propane
         else
           "d"
         end
+      @options = options
       process_grammar!
     end
 
@@ -129,7 +130,7 @@ class Propane
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, @log)
+      @parser = Parser.new(@grammar, rule_sets, @log, @options)
     end
 
     # Check that any referenced ptypes have been defined.

lib/propane/parser.rb

@@ -7,12 +7,14 @@ class Propane
     attr_reader :reduce_table
     attr_reader :rule_sets
 
-    def initialize(grammar, rule_sets, log)
+    def initialize(grammar, rule_sets, log, options)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
       @item_sets = []
       @item_sets_set = {}
+      @warnings = Set.new
+      @options = options
       start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
 
@@ -23,10 +25,10 @@ class Propane
           item_set.id = @item_sets.size
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
-          item_set.following_symbols.each do |following_symbol|
+          item_set.next_symbols.each do |next_symbol|
-            unless following_symbol.name == "$EOF"
+            unless next_symbol.name == "$EOF"
-              following_set = item_set.build_following_item_set(following_symbol)
+              next_item_set = item_set.build_next_item_set(next_symbol)
-              eval_item_sets << following_set
+              eval_item_sets << next_item_set
             end
           end
         end
@@ -37,8 +39,12 @@ class Propane
       end
 
       build_reduce_actions!
-      write_log!
+      build_follow_sets!
       build_tables!
+      write_log!
+      if @warnings.size > 0 && @options[:warnings_as_errors]
+        raise Error.new("Fatal errors (-w):\n" + @warnings.join("\n"))
+      end
     end
 
     private
@@ -48,26 +54,34 @@ class Propane
       @shift_table = []
       @reduce_table = []
       @item_sets.each do |item_set|
-        shift_entries = item_set.following_symbols.map do |following_symbol|
+        shift_entries = item_set.next_symbols.map do |next_symbol|
           state_id =
-            if following_symbol.name == "$EOF"
+            if next_symbol.name == "$EOF"
               0
             else
-              item_set.following_item_set[following_symbol].id
+              item_set.next_item_set[next_symbol].id
             end
           {
-            symbol_id: following_symbol.id,
+            symbol: next_symbol,
             state_id: state_id,
           }
         end
+        if item_set.reduce_actions
+          shift_entries.each do |shift_entry|
+            token = shift_entry[:symbol]
+            if item_set.reduce_actions.include?(token)
+              rule = item_set.reduce_actions[token]
+              @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+            end
+          end
+        end
         reduce_entries =
-          case ra = item_set.reduce_actions
+          if rule = item_set.reduce_rule
-          when Rule
+            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
-            [{token_id: @grammar.invalid_token_id, rule_id: ra.id, rule: ra,
+              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
-              rule_set_id: ra.rule_set.id, n_states: ra.components.size,
+              propagate_optional_target: rule.optional? && rule.components.size == 1}]
-              propagate_optional_target: ra.optional? && ra.components.size == 1}]
+          elsif reduce_actions = item_set.reduce_actions
-          when Hash
+            reduce_actions.map do |token, rule|
-            ra.map do |token, rule|
               {token_id: token.id, rule_id: rule.id, rule: rule,
                rule_set_id: rule.rule_set.id, n_states: rule.components.size,
                propagate_optional_target: rule.optional? && rule.components.size == 1}
@@ -87,11 +101,11 @@ class Propane
     end
 
     def process_item_set(item_set)
-      item_set.following_symbols.each do |following_symbol|
+      item_set.next_symbols.each do |next_symbol|
-        unless following_symbol.name == "$EOF"
+        unless next_symbol.name == "$EOF"
-          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
+          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
-          item_set.following_item_set[following_symbol] = following_set
+          item_set.next_item_set[next_symbol] = next_item_set
-          following_set.in_sets << item_set
+          next_item_set.in_sets << item_set
         end
       end
     end
@@ -110,10 +124,8 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [nil, Rule, Hash]
+    # @return [nil, Hash]
     #   If no reduce actions are possible for the given item set, nil.
-    #   If only one reduce action is possible for the given item set, the Rule
-    #   to reduce.
     #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
@@ -122,14 +134,26 @@ class Propane
       # reduction in the current ItemSet.
       reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      # If there are no rules to reduce for this ItemSet, we're done here.
+      if reduce_rules.size == 1
-      return nil if reduce_rules.size == 0
+        item_set.reduce_rule = reduce_rules.first
+      end
 
-      # If there is exactly one rule to reduce for this ItemSet, then do not
+      if reduce_rules.size == 0
-      # figure out the lookaheads; just reduce it.
+        nil
-      return reduce_rules.first if reduce_rules.size == 1
+      else
+        build_lookahead_reduce_actions_for_item_set(item_set)
+      end
+    end
 
-      # Otherwise, we have more than one possible rule to reduce.
+    # Build the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def build_lookahead_reduce_actions_for_item_set(item_set)
+      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
@@ -141,7 +165,7 @@ class Propane
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            raise Error.new("Error: reduce/reduce conflict between rule #{existing_reduce_rule.id} (#{existing_reduce_rule.name}) and rule #{reduce_rule.id} (#{reduce_rule.name})")
+            raise Error.new("Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number})")
           end
           reduce_actions[lookahead_token] = reduce_rule
         end
@@ -183,9 +207,9 @@ class Propane
         # tokens to form the lookahead token set.
         item_sets.each do |item_set|
           item_set.items.each do |item|
-            if item.following_symbol == rule_set
+            if item.next_symbol == rule_set
               (1..).each do |offset|
-                case symbol = item.following_symbol(offset)
+                case symbol = item.next_symbol(offset)
                 when nil
                   rule_set = item.rule.rule_set
                   unless checked_rule_sets.include?(rule_set)
@@ -209,6 +233,51 @@ class Propane
       lookahead_tokens
     end
 
+    # Build the follow sets for each ItemSet.
+    #
+    # @return [void]
+    def build_follow_sets!
+      @item_sets.each do |item_set|
+        item_set.follow_set = build_follow_set_for_item_set(item_set)
+      end
+    end
+
+    # Build the follow set for the given ItemSet.
+    #
+    # @param item_set [ItemSet]
+    #   The ItemSet to build the follow set for.
+    #
+    # @return [Set]
+    #   Follow set for the given ItemSet.
+    def build_follow_set_for_item_set(item_set)
+      follow_set = Set.new
+      rule_sets_to_check_after = Set.new
+      item_set.items.each do |item|
+        (1..).each do |offset|
+          case symbol = item.next_symbol(offset)
+          when nil
+            rule_sets_to_check_after << item.rule.rule_set
+            break
+          when Token
+            follow_set << symbol
+            break
+          when RuleSet
+            follow_set += symbol.start_token_set
+            unless symbol.could_be_empty?
+              break
+            end
+          end
+        end
+      end
+      reduce_lookaheads = build_lookahead_reduce_actions_for_item_set(item_set)
+      reduce_lookaheads.each do |token, rule_set|
+        if rule_sets_to_check_after.include?(rule_set)
+          follow_set << token
+        end
+      end
+      follow_set
+    end
+
     def write_log!
       @log.puts Util.banner("Parser Rules")
       @grammar.rules.each do |rule|
@@ -242,20 +311,26 @@ class Propane
         @log.puts
         @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
         @log.puts "  Outgoing states:"
-        item_set.following_item_set.each do |following_symbol, following_item_set|
+        item_set.next_item_set.each do |next_symbol, next_item_set|
-          @log.puts "    #{following_symbol.name} => #{following_item_set.id}"
+          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
         end
         @log.puts
         @log.puts "  Reduce actions:"
-        case item_set.reduce_actions
+        if item_set.reduce_rule
-        when Rule
+          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
-          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
+        elsif item_set.reduce_actions
-        when Hash
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
           end
         end
       end
+      if @warnings.size > 0
+        @log.puts
+        @log.puts "Warnings:"
+        @warnings.each do |warning|
+          @log.puts "  #{warning}"
+        end
+      end
     end
 
   end

lib/propane/parser/item.rb

@@ -56,7 +56,7 @@ class Propane
 
       # Return the set of Items obtained by "closing" the current item.
       #
-      # If the following symbol for the current item is another Rule name, then
+      # If the next symbol for the current item is another Rule name, then
       # this method will return all Items for that Rule with a position of 0.
       # Otherwise, an empty Array is returned.
       #
@@ -81,17 +81,17 @@ class Propane
         @position == @rule.components.size
       end
 
-      # Get the following symbol for the Item.
+      # Get the next symbol for the Item.
       #
-      # That is, the symbol which follows the parse position marker in the
+      # That is, the symbol which is after the parse position marker in the
       # current Item.
       #
       # @param offset [Integer]
       #   Offset from current parse position to examine.
       #
       # @return [Token, RuleSet, nil]
-      #   Following symbol for the Item.
+      #   Next symbol for the Item.
-      def following_symbol(offset = 0)
+      def next_symbol(offset = 0)
         @rule.components[@position + offset]
       end
 
@@ -108,25 +108,25 @@ class Propane
         end
       end
 
-      # Get whether this Item is followed by the provided symbol.
+      # Get whether this Item's next symbol is the given symbol.
       #
       # @param symbol [Token, RuleSet]
       #   Symbol to query.
       #
       # @return [Boolean]
-      #   Whether this Item is followed by the provided symbol.
+      #   Whether this Item's next symbol is the given symbol.
-      def followed_by?(symbol)
+      def next_symbol?(symbol)
-        following_symbol == symbol
+        next_symbol == symbol
       end
 
-      # Get the following item for this Item.
+      # Get the next item for this Item.
       #
       # That is, the Item formed by moving the parse position marker one place
       # forward from its position in this Item.
       #
       # @return [Item]
-      #   The following item for this Item.
+      #   The next item for this Item.
-      def following_item
+      def next_item
         Item.new(@rule, @position + 1)
       end
 

lib/propane/parser/item_set.rb

@@ -14,45 +14,54 @@ class Propane
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a following symbol to its ItemSet.
+      #   Maps a next symbol to its ItemSet.
-      attr_reader :following_item_set
+      attr_reader :next_item_set
 
       # @return [Set<ItemSet>]
       #   ItemSets leading to this item set.
       attr_reader :in_sets
 
-      # @return [nil, Rule, Hash]
+      # @return [nil, Rule]
-      #   Reduce actions, mapping lookahead tokens to rules.
+      #   Rule to reduce if there is only one possibility.
+      attr_accessor :reduce_rule
+
+      # @return [nil, Hash]
+      #   Reduce actions, mapping lookahead tokens to rules, if there is
+      #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
+      # @return [Set<Token>]
+      #   Follow set for the ItemSet.
+      attr_accessor :follow_set
+
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
       #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @following_item_set = {}
+        @next_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      # Get the set of following symbols for all Items in this ItemSet.
+      # Get the set of next symbols for all Items in this ItemSet.
       #
       # @return [Set<Token, RuleSet>]
-      #   Set of following symbols for all Items in this ItemSet.
+      #   Set of next symbols for all Items in this ItemSet.
-      def following_symbols
+      def next_symbols
-        Set.new(@items.map(&:following_symbol).compact)
+        Set.new(@items.map(&:next_symbol).compact)
       end
 
-      # Build a following ItemSet for the given following symbol.
+      # Build a next ItemSet for the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol to build the following ItemSet for.
+      #   Next symbol to build the next ItemSet for.
       #
       # @return [ItemSet]
-      #   Following ItemSet for the given following symbol.
+      #   Next ItemSet for the given next symbol.
-      def build_following_item_set(symbol)
+      def build_next_item_set(symbol)
-        ItemSet.new(items_followed_by(symbol).map(&:following_item))
+        ItemSet.new(items_with_next(symbol).map(&:next_item))
       end
 
       # Hash function.
@@ -137,16 +146,16 @@ class Propane
         end
       end
 
-      # Get the Items followed by the given following symbol.
+      # Get the Items with the given next symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Following symbol.
+      #   Next symbol.
       #
       # @return [Array<Item>]
-      #   Items followed by the given following symbol.
+      #   Items with the given next symbol.
-      def items_followed_by(symbol)
+      def items_with_next(symbol)
         @items.select do |item|
-          item.followed_by?(symbol)
+          item.next_symbol?(symbol)
         end
       end
 

spec/propane_spec.rb

@@ -54,6 +54,7 @@ EOF
     else
       command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
     end
+    command += (options[:extra_args] || [])
     if (options[:capture])
       stdout, stderr, status = Open3.capture3(*command)
       Results.new(stdout, stderr, status)
@@ -184,6 +185,34 @@ EOF
     expect(results.status).to_not eq 0
   end
 
+  it "warns on shift/reduce conflicts" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> As? b?;
+As -> a As2?;
+As2 -> b a As2?;
+EOF
+    results = run_propane(capture: true)
+    expect(results.stderr).to eq ""
+    expect(results.status).to eq 0
+    expect(File.binread("spec/run/testparser.log")).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}
+  end
+
+  it "errors on shift/reduce conflicts with -w" do
+    write_grammar <<EOF
+token a;
+token b;
+Start -> As? b?;
+As -> a As2?;
+As2 -> b a As2?;
+EOF
+    results = run_propane(extra_args: %w[-w], capture: true)
+    expect(results.stderr).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}m
+    expect(results.status).to_not eq 0
+    expect(File.binread("spec/run/testparser.log")).to match %r{Shift/Reduce conflict \(state \d+\) between token b and rule As2\? \(defined on line 4\)}
+  end
+
   %w[d c].each do |language|
 
     context "#{language.upcase} language" do
@@ -623,7 +652,7 @@ F -> e;
 EOF
         results = run_propane(capture: true, language: language)
         expect(results.status).to_not eq 0
-        expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
+        expect(results.stderr).to match %r{Error: reduce/reduce conflict \(state \d+\) between rule E#\d+ \(defined on line 10\) and rule F#\d+ \(defined on line 11\)}
       end
 
       it "provides matched text to user code blocks" do
@@ -1051,6 +1080,26 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "stores the token position in the AST Token node" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> T T T;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_token_positions.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_ast_token_positions.c

@@ -0,0 +1,33 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "abc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert_eq(0, start->pT1->pToken->position.row);
+    assert_eq(0, start->pT1->pToken->position.col);
+    assert_eq(0, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(0, start->pT3->pToken->position.row);
+    assert_eq(2, start->pT3->pToken->position.col);
+
+    input = "\n\n  a\nc\n\n     a";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(2, start->pT1->pToken->position.col);
+    assert_eq(3, start->pT2->pToken->position.row);
+    assert_eq(0, start->pT2->pToken->position.col);
+    assert_eq(5, start->pT3->pToken->position.row);
+    assert_eq(5, start->pT3->pToken->position.col);
+
+    return 0;
+}
+

spec/test_ast_token_positions.d

@@ -0,0 +1,34 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "abc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert_eq(0, start.pT1.pToken.position.row);
+    assert_eq(0, start.pT1.pToken.position.col);
+    assert_eq(0, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(0, start.pT3.pToken.position.row);
+    assert_eq(2, start.pT3.pToken.position.col);
+
+    input = "\n\n  a\nc\n\n     a";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(2, start.pT1.pToken.position.col);
+    assert_eq(3, start.pT2.pToken.position.row);
+    assert_eq(0, start.pT2.pToken.position.col);
+    assert_eq(5, start.pT3.pToken.position.row);
+    assert_eq(5, start.pT3.pToken.position.col);
+}