Version 2.1.0

Lookahead tokens are only need if either:
(1) There is more than one rule that could be reduced in a given parser
state, or
(2) There are shift actions for a state and at least one rule that could
be reduced in the same state (to warn about shift/reduce conflicts).

CHANGELOG.md

@@ -1,8 +1,37 @@
+## v2.0.0
+
+### Improvements
+
+- Log conflicting rules on reduce/reduce conflict (#31)
+- Use 1-based row and column values for position values (#30)
+
+### Fixes
+
+- Fix named optional rules (#29)
+
+### Upgrading
+
+- Adjust all uses of p_position_t row and col values to expect 1-based instead
+of 0-based values.
+
+## v1.5.1
+
+### Improvements
+
+- Improve performance (#28)
+
 ## v1.5.0
 
 ### New Features
 
-- Track token position in AST Token node
+- Track start and end text positions for tokens and rules in AST node structures (#27)
+- Add warnings for shift/reduce conflicts to log file (#25)
+- Add -w command line switch to treat warnings as errors and output to stderr (#26)
+- Add rule field aliases (#24)
+
+### Improvements
+
+- Show line numbers of rules on conflict (#23)
 
 ## v1.4.0
 

Gemfile.lock

@@ -1,34 +1,36 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.5.0)
-    docile (1.4.0)
-    psych (5.1.0)
+    date (3.4.1)
+    diff-lcs (1.5.1)
+    docile (1.4.1)
+    psych (5.2.2)
+      date
       stringio
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.2.1)
+    rdoc (6.10.0)
       psych (>= 4.0.0)
     redcarpet (3.6.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    stringio (3.0.7)
+    stringio (3.1.2)
     syntax (1.2.2)
 
 PLATFORMS

README.md

@@ -31,9 +31,14 @@ Propane is typically invoked from the command-line as `./propane`.
 
     Usage: ./propane [options] <input-file> <output-file>
     Options:
-      --log LOG   Write log file
-      --version   Show program version and exit
-      -h, --help  Show this usage and exit
+      -h, --help  Show this usage and exit.
+      --log LOG   Write log file. This will show all parser states and their
+                  associated shifts and reduces. It can be helpful when
+                  debugging a grammar.
+      --version   Show program version and exit.
+      -w          Treat warnings as errors. This option will treat shift/reduce
+                  conflicts as fatal errors and will print them to stderr in
+                  addition to the log file.
 
 The user must specify the path to a Propane input grammar file and a path to an
 output file.

assets/parser.c.erb

@@ -60,6 +60,8 @@ void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * contex
     /* Lexer initialization. */
     newcontext.input = input;
     newcontext.input_length = input_length;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -365,7 +367,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -567,7 +569,7 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  *************************************************************************/
 
 /** Invalid position value. */
-#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0xFFFFFFFFu, 0xFFFFFFFFu}
+#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
 
 /** Reduce ID type. */
 typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;

assets/parser.d.erb

@@ -65,12 +65,12 @@ public struct <%= @grammar.prefix %>position_t
     uint col;
 
     /** Invalid position value. */
-    enum INVALID = <%= @grammar.prefix %>position_t(0xFFFF_FFFF, 0xFFFF_FFFF);
+    enum INVALID = <%= @grammar.prefix %>position_t(0u, 0u);
 
     /** Return whether the position is valid. */
     public @property bool valid()
     {
-        return row != 0xFFFF_FFFFu;
+        return row != 0u;
     }
 }
 
@@ -235,6 +235,8 @@ public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t *
 
     /* Lexer initialization. */
     newcontext.input = input;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -534,7 +536,7 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {

assets/parser.h.erb

@@ -53,7 +53,7 @@ typedef struct
 } <%= @grammar.prefix %>position_t;
 
 /** Return whether the position is valid. */
-#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0xFFFFFFFFu)
+#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
 
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>

doc/user_guide.md

@@ -641,13 +641,14 @@ This example uses the default start rule name of `Start`.
 
 A parser rule has zero or more fields on the right side of its definition.
 Each of these fields is either a token name or a rule name.
+A field can be immediately followed by a `?` character to signify that it is
+optional.
 A field can optionally be followed by a `:` and then a field alias name.
 If present, the field alias name is used to refer to the field value in user
 code blocks, or if AST mode is active, the field alias name is used as the
 field name in the generated AST node structure.
-A field can be immediately followed by a `?` character to signify that it is
-optional.
-Another example:
+An optional and named field must use the format `field?:name`.
+Example:
 
 ```
 token public;
@@ -655,7 +656,7 @@ token private;
 token int;
 token ident /[a-zA-Z_][a-zA-Z_0-9]*/;
 token semicolon /;/;
-IntegerDeclaration -> Visibility? int ident:name semicolon;
+IntegerDeclaration -> Visibility?:visibility int ident:name semicolon;
 Visibility -> public;
 Visibility -> private;
 ```
@@ -663,7 +664,7 @@ Visibility -> private;
 In a parser rule code block, parser values for the right side fields are
 accessible as `$1` for the first field's parser value, `$2` for the second
 field's parser value, etc...
-For the `IntegerDeclaration` rule, the third field value can also be referred
+For the `IntegerDeclaration` rule, the first field value can also be referred to as `${visibility}` and the third field value can also be referred
 to as `${name}`.
 The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
@@ -776,7 +777,7 @@ A pointer to this instance is passed to the generated functions.
 ### `p_position_t`
 
 The `p_position_t` structure contains two fields `row` and `col`.
-These fields contain the 0-based row and column describing a parser position.
+These fields contain the 1-based row and column describing a parser position.
 
 For D targets, the `p_position_t` structure can be checked for validity by
 querying the `valid` property.

extra/vim/syntax/propane.vim

@@ -17,6 +17,9 @@ syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contai
 
 syn match propaneComment "#.*"
 syn match propaneOperator "->"
+syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
+syn match propaneFieldOperator ":" contained
+syn match propaneOperator "?"
 syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
 
 syn region propaneRegex start="/" end="/" skip="\\/"
@@ -25,4 +28,6 @@ hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
+hi def link propaneFieldOperator Operator
 hi def link propaneDelimiter Delimiter
+hi def link propaneFieldAlias Identifier

lib/propane/grammar.rb

@@ -198,7 +198,7 @@ class Propane
         if @ast && ptypename
           raise Error.new("Multiple ptypes are unsupported in AST mode")
         end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}(?::#{IDENTIFIER_REGEX})?\??\s*)*)\s*/, "expected rule component list")
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         if @ast
           consume!(/;/, "expected `;'")

lib/propane/parser.rb

@@ -14,6 +14,7 @@ class Propane
       @item_sets = []
       @item_sets_set = {}
       @warnings = Set.new
+      @errors = Set.new
       @options = options
       start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
@@ -39,11 +40,20 @@ class Propane
       end
 
       build_reduce_actions!
-      build_follow_sets!
       build_tables!
       write_log!
+      errormessage = ""
+      if @errors.size > 0
+        errormessage += @errors.join("\n")
+      end
       if @warnings.size > 0 && @options[:warnings_as_errors]
-        raise Error.new("Fatal errors (-w):\n" + @warnings.join("\n"))
+        if errormessage != ""
+          errormessage += "\n"
+        end
+        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
+      end
+      if errormessage != ""
+        raise Error.new(errormessage)
       end
     end
 
@@ -66,10 +76,10 @@ class Propane
             state_id: state_id,
           }
         end
-        if item_set.reduce_actions
+        unless item_set.reduce_rules.empty?
           shift_entries.each do |shift_entry|
             token = shift_entry[:symbol]
-            if item_set.reduce_actions.include?(token)
+            if get_lookahead_reduce_actions_for_item_set(item_set).include?(token)
               rule = item_set.reduce_actions[token]
               @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
             end
@@ -115,7 +125,7 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
-        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
+        build_reduce_actions_for_item_set(item_set)
       end
     end
 
@@ -124,27 +134,36 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [nil, Hash]
-    #   If no reduce actions are possible for the given item set, nil.
-    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
+    # @return [void]
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
       # "complete" items, i.e., items where the parse position is at the
       # end of a rule. These are the only rules that are candidates for
       # reduction in the current ItemSet.
-      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      if reduce_rules.size == 1
-        item_set.reduce_rule = reduce_rules.first
+      if item_set.reduce_rules.size == 1
+        item_set.reduce_rule = item_set.reduce_rules.first
       end
 
-      if reduce_rules.size == 0
-        nil
-      else
-        build_lookahead_reduce_actions_for_item_set(item_set)
+      if item_set.reduce_rules.size > 1
+        # Force item_set.reduce_actions to be built to store the lookahead
+        # tokens for the possible reduce rules if there is more than one.
+        get_lookahead_reduce_actions_for_item_set(item_set)
       end
     end
 
+    # Get the reduce actions for a single item set (parser state).
+    #
+    # @param item_set [ItemSet]
+    #   ItemSet (parser state)
+    #
+    # @return [Hash]
+    #   Mapping of lookahead Tokens to the Rules to reduce.
+    def get_lookahead_reduce_actions_for_item_set(item_set)
+      item_set.reduce_actions ||= build_lookahead_reduce_actions_for_item_set(item_set)
+    end
+
     # Build the reduce actions for a single item set (parser state).
     #
     # @param item_set [ItemSet]
@@ -153,19 +172,17 @@ class Propane
     # @return [Hash]
     #   Mapping of lookahead Tokens to the Rules to reduce.
     def build_lookahead_reduce_actions_for_item_set(item_set)
-      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
-
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
       item_sets = Set[item_set] + item_set.leading_item_sets
-      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            raise Error.new("Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number})")
+            @errors << "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
           end
           reduce_actions[lookahead_token] = reduce_rule
         end
@@ -233,51 +250,6 @@ class Propane
       lookahead_tokens
     end
 
-    # Build the follow sets for each ItemSet.
-    #
-    # @return [void]
-    def build_follow_sets!
-      @item_sets.each do |item_set|
-        item_set.follow_set = build_follow_set_for_item_set(item_set)
-      end
-    end
-
-    # Build the follow set for the given ItemSet.
-    #
-    # @param item_set [ItemSet]
-    #   The ItemSet to build the follow set for.
-    #
-    # @return [Set]
-    #   Follow set for the given ItemSet.
-    def build_follow_set_for_item_set(item_set)
-      follow_set = Set.new
-      rule_sets_to_check_after = Set.new
-      item_set.items.each do |item|
-        (1..).each do |offset|
-          case symbol = item.next_symbol(offset)
-          when nil
-            rule_sets_to_check_after << item.rule.rule_set
-            break
-          when Token
-            follow_set << symbol
-            break
-          when RuleSet
-            follow_set += symbol.start_token_set
-            unless symbol.could_be_empty?
-              break
-            end
-          end
-        end
-      end
-      reduce_lookaheads = build_lookahead_reduce_actions_for_item_set(item_set)
-      reduce_lookaheads.each do |token, rule_set|
-        if rule_sets_to_check_after.include?(rule_set)
-          follow_set << token
-        end
-      end
-      follow_set
-    end
-
     def write_log!
       @log.puts Util.banner("Parser Rules")
       @grammar.rules.each do |rule|

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing.
+    # parser could currently be parsing. This is equivalent to a parser state.
     class ItemSet
 
       # @return [Set<Item>]
@@ -25,15 +25,15 @@ class Propane
       #   Rule to reduce if there is only one possibility.
       attr_accessor :reduce_rule
 
+      # @return [Set<Rule>]
+      #   Set of rules that could be reduced in this parser state.
+      attr_accessor :reduce_rules
+
       # @return [nil, Hash]
       #   Reduce actions, mapping lookahead tokens to rules, if there is
       #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
-      # @return [Set<Token>]
-      #   Follow set for the ItemSet.
-      attr_accessor :follow_set
-
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
@@ -50,7 +50,7 @@ class Propane
       # @return [Set<Token, RuleSet>]
       #   Set of next symbols for all Items in this ItemSet.
       def next_symbols
-        Set.new(@items.map(&:next_symbol).compact)
+        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
       end
 
       # Build a next ItemSet for the given next symbol.
@@ -99,21 +99,24 @@ class Propane
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        result = Set.new
-        eval_sets = Set[self]
-        evaled = Set.new
-        while eval_sets.size > 0
-          eval_set = eval_sets.first
-          eval_sets.delete(eval_set)
-          evaled << eval_set
-          eval_set.in_sets.each do |in_set|
-            result << in_set
-            unless evaled.include?(in_set)
-              eval_sets << in_set
+        @_leading_item_sets ||=
+          begin
+            result = Set.new
+            eval_sets = Set[self]
+            evaled = Set.new
+            while eval_sets.size > 0
+              eval_set = eval_sets.first
+              eval_sets.delete(eval_set)
+              evaled << eval_set
+              eval_set.in_sets.each do |in_set|
+                result << in_set
+                unless evaled.include?(in_set)
+                  eval_sets << in_set
+                end
+              end
             end
+            result
           end
-        end
-        result
       end
 
       # Represent the ItemSet as a String.

lib/propane/rule_set.rb

@@ -156,7 +156,7 @@ class Propane
       @rules.each do |rule|
         rule.aliases.each do |alias_name, index|
           if field_aliases[alias_name] && field_aliases[alias_name] != index
-            raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}`")
+            raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
           end
           field_aliases[alias_name] = index
           @ast_fields[index][alias_name] = @ast_fields[index].first[1]

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.4.0"
+  VERSION = "2.1.0"
 end

spec/propane_spec.rb

@@ -689,6 +689,7 @@ EOF
         results = run_propane(capture: true, language: language)
         expect(results.status).to_not eq 0
         expect(results.stderr).to match %r{Error: reduce/reduce conflict \(state \d+\) between rule E#\d+ \(defined on line 10\) and rule F#\d+ \(defined on line 11\)}
+        expect(File.binread("spec/run/testparser.log")).to match %r{Reduce.actions:}
       end
 
       it "provides matched text to user code blocks" do
@@ -1117,6 +1118,47 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows naming an optional rule component in AST generation mode" do
+        if language == "d"
+          write_grammar <<EOF
+ast;
+
+<<
+import std.stdio;
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        else
+          write_grammar <<EOF
+ast;
+
+<<
+#include <stdio.h>
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "stores token and rule positions in AST nodes" do
         write_grammar <<EOF
 ast;

spec/test_ast_invalid_positions.c

@@ -11,92 +11,92 @@ int main()
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(2, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(2, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(7, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(7, start->pT1->end_position.col);
+    assert_eq(3, start->pT1->pA->position.row);
+    assert_eq(3, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(8, start->pT1->pA->end_position.col);
+    assert_eq(2, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(8, start->pT1->end_position.col);
 
-    assert_eq(1, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(7, start->end_position.col);
+    assert_eq(2, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(8, start->end_position.col);
 
     input = "a\nbb";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->pA->position.row);
-    assert_eq(0, start->pT1->pA->position.col);
-    assert_eq(1, start->pT1->pA->end_position.row);
-    assert_eq(1, start->pT1->pA->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(1, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(2, start->pT1->pA->end_position.row);
+    assert_eq(2, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(2, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(1, start->end_position.row);
-    assert_eq(1, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(2, start->end_position.row);
+    assert_eq(2, start->end_position.col);
 
     input = "a\nc\nc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->pA->position.row);
-    assert_eq(0, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(0, start->pT1->pA->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(1, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(0, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(1, start->end_position.col);
 
     input = "a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(!p_position_valid(start->pT1->pA->position));
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(0, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(0, start->end_position.row);
-    assert_eq(0, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(1, start->end_position.col);
 
     return 0;
 }

spec/test_ast_invalid_positions.d

@@ -15,90 +15,90 @@ unittest
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(2, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(2, start.pT1.pA.position.row);
-    assert_eq(2, start.pT1.pA.position.col);
-    assert_eq(2, start.pT1.pA.end_position.row);
-    assert_eq(7, start.pT1.pA.end_position.col);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(7, start.pT1.end_position.col);
+    assert_eq(3, start.pT1.pA.position.row);
+    assert_eq(3, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(8, start.pT1.pA.end_position.col);
+    assert_eq(2, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(8, start.pT1.end_position.col);
 
-    assert_eq(1, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(2, start.end_position.row);
-    assert_eq(7, start.end_position.col);
+    assert_eq(2, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(8, start.end_position.col);
 
     input = "a\nbb";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(1, start.pT1.pA.position.row);
-    assert_eq(0, start.pT1.pA.position.col);
-    assert_eq(1, start.pT1.pA.end_position.row);
-    assert_eq(1, start.pT1.pA.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(1, start.pT1.end_position.row);
-    assert_eq(1, start.pT1.end_position.col);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(2, start.pT1.pA.end_position.row);
+    assert_eq(2, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(2, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(1, start.end_position.row);
-    assert_eq(1, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(2, start.end_position.row);
+    assert_eq(2, start.end_position.col);
 
     input = "a\nc\nc";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(1, start.pT1.pA.position.row);
-    assert_eq(0, start.pT1.pA.position.col);
-    assert_eq(2, start.pT1.pA.end_position.row);
-    assert_eq(0, start.pT1.pA.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(1, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(2, start.end_position.row);
-    assert_eq(0, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(1, start.end_position.col);
 
     input = "a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(!start.pT1.pA.position.valid);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(0, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(0, start.end_position.row);
-    assert_eq(0, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(1, start.end_position.col);
 }

spec/test_ast_token_positions.c

@@ -11,74 +11,74 @@ int main()
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(0, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->pT2->pToken->position.row);
-    assert_eq(1, start->pT2->pToken->position.col);
-    assert_eq(0, start->pT2->pToken->end_position.row);
-    assert_eq(2, start->pT2->pToken->end_position.col);
-    assert_eq(0, start->pT2->position.row);
-    assert_eq(1, start->pT2->position.col);
-    assert_eq(0, start->pT2->end_position.row);
-    assert_eq(2, start->pT2->end_position.col);
+    assert_eq(1, start->pT2->pToken->position.row);
+    assert_eq(2, start->pT2->pToken->position.col);
+    assert_eq(1, start->pT2->pToken->end_position.row);
+    assert_eq(3, start->pT2->pToken->end_position.col);
+    assert_eq(1, start->pT2->position.row);
+    assert_eq(2, start->pT2->position.col);
+    assert_eq(1, start->pT2->end_position.row);
+    assert_eq(3, start->pT2->end_position.col);
 
-    assert_eq(0, start->pT3->pToken->position.row);
-    assert_eq(3, start->pT3->pToken->position.col);
-    assert_eq(0, start->pT3->pToken->end_position.row);
-    assert_eq(5, start->pT3->pToken->end_position.col);
-    assert_eq(0, start->pT3->position.row);
-    assert_eq(3, start->pT3->position.col);
-    assert_eq(0, start->pT3->end_position.row);
-    assert_eq(5, start->pT3->end_position.col);
+    assert_eq(1, start->pT3->pToken->position.row);
+    assert_eq(4, start->pT3->pToken->position.col);
+    assert_eq(1, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(1, start->pT3->position.row);
+    assert_eq(4, start->pT3->position.col);
+    assert_eq(1, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(0, start->end_position.row);
-    assert_eq(5, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(6, start->end_position.col);
 
     input = "\n\n  bb\nc\ncc\n\n     a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(2, start->pT1->pToken->position.row);
-    assert_eq(2, start->pT1->pToken->position.col);
-    assert_eq(2, start->pT1->pToken->end_position.row);
-    assert_eq(3, start->pT1->pToken->end_position.col);
-    assert_eq(2, start->pT1->position.row);
-    assert_eq(2, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(3, start->pT1->end_position.col);
+    assert_eq(3, start->pT1->pToken->position.row);
+    assert_eq(3, start->pT1->pToken->position.col);
+    assert_eq(3, start->pT1->pToken->end_position.row);
+    assert_eq(4, start->pT1->pToken->end_position.col);
+    assert_eq(3, start->pT1->position.row);
+    assert_eq(3, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(4, start->pT1->end_position.col);
 
-    assert_eq(3, start->pT2->pToken->position.row);
-    assert_eq(0, start->pT2->pToken->position.col);
-    assert_eq(4, start->pT2->pToken->end_position.row);
-    assert_eq(1, start->pT2->pToken->end_position.col);
-    assert_eq(3, start->pT2->position.row);
-    assert_eq(0, start->pT2->position.col);
-    assert_eq(4, start->pT2->end_position.row);
-    assert_eq(1, start->pT2->end_position.col);
+    assert_eq(4, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(5, start->pT2->pToken->end_position.row);
+    assert_eq(2, start->pT2->pToken->end_position.col);
+    assert_eq(4, start->pT2->position.row);
+    assert_eq(1, start->pT2->position.col);
+    assert_eq(5, start->pT2->end_position.row);
+    assert_eq(2, start->pT2->end_position.col);
 
-    assert_eq(6, start->pT3->pToken->position.row);
-    assert_eq(5, start->pT3->pToken->position.col);
-    assert_eq(6, start->pT3->pToken->end_position.row);
-    assert_eq(5, start->pT3->pToken->end_position.col);
-    assert_eq(6, start->pT3->position.row);
-    assert_eq(5, start->pT3->position.col);
-    assert_eq(6, start->pT3->end_position.row);
-    assert_eq(5, start->pT3->end_position.col);
+    assert_eq(7, start->pT3->pToken->position.row);
+    assert_eq(6, start->pT3->pToken->position.col);
+    assert_eq(7, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(7, start->pT3->position.row);
+    assert_eq(6, start->pT3->position.col);
+    assert_eq(7, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
 
-    assert_eq(2, start->position.row);
-    assert_eq(2, start->position.col);
-    assert_eq(6, start->end_position.row);
-    assert_eq(5, start->end_position.col);
+    assert_eq(3, start->position.row);
+    assert_eq(3, start->position.col);
+    assert_eq(7, start->end_position.row);
+    assert_eq(6, start->end_position.col);
 
     return 0;
 }

spec/test_ast_token_positions.d

@@ -15,72 +15,72 @@ unittest
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(0, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.pT2.pToken.position.row);
-    assert_eq(1, start.pT2.pToken.position.col);
-    assert_eq(0, start.pT2.pToken.end_position.row);
-    assert_eq(2, start.pT2.pToken.end_position.col);
-    assert_eq(0, start.pT2.position.row);
-    assert_eq(1, start.pT2.position.col);
-    assert_eq(0, start.pT2.end_position.row);
-    assert_eq(2, start.pT2.end_position.col);
+    assert_eq(1, start.pT2.pToken.position.row);
+    assert_eq(2, start.pT2.pToken.position.col);
+    assert_eq(1, start.pT2.pToken.end_position.row);
+    assert_eq(3, start.pT2.pToken.end_position.col);
+    assert_eq(1, start.pT2.position.row);
+    assert_eq(2, start.pT2.position.col);
+    assert_eq(1, start.pT2.end_position.row);
+    assert_eq(3, start.pT2.end_position.col);
 
-    assert_eq(0, start.pT3.pToken.position.row);
-    assert_eq(3, start.pT3.pToken.position.col);
-    assert_eq(0, start.pT3.pToken.end_position.row);
-    assert_eq(5, start.pT3.pToken.end_position.col);
-    assert_eq(0, start.pT3.position.row);
-    assert_eq(3, start.pT3.position.col);
-    assert_eq(0, start.pT3.end_position.row);
-    assert_eq(5, start.pT3.end_position.col);
+    assert_eq(1, start.pT3.pToken.position.row);
+    assert_eq(4, start.pT3.pToken.position.col);
+    assert_eq(1, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(1, start.pT3.position.row);
+    assert_eq(4, start.pT3.position.col);
+    assert_eq(1, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(0, start.end_position.row);
-    assert_eq(5, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(6, start.end_position.col);
 
     input = "\n\n  bb\nc\ncc\n\n     a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(2, start.pT1.pToken.position.row);
-    assert_eq(2, start.pT1.pToken.position.col);
-    assert_eq(2, start.pT1.pToken.end_position.row);
-    assert_eq(3, start.pT1.pToken.end_position.col);
-    assert_eq(2, start.pT1.position.row);
-    assert_eq(2, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(3, start.pT1.end_position.col);
+    assert_eq(3, start.pT1.pToken.position.row);
+    assert_eq(3, start.pT1.pToken.position.col);
+    assert_eq(3, start.pT1.pToken.end_position.row);
+    assert_eq(4, start.pT1.pToken.end_position.col);
+    assert_eq(3, start.pT1.position.row);
+    assert_eq(3, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(4, start.pT1.end_position.col);
 
-    assert_eq(3, start.pT2.pToken.position.row);
-    assert_eq(0, start.pT2.pToken.position.col);
-    assert_eq(4, start.pT2.pToken.end_position.row);
-    assert_eq(1, start.pT2.pToken.end_position.col);
-    assert_eq(3, start.pT2.position.row);
-    assert_eq(0, start.pT2.position.col);
-    assert_eq(4, start.pT2.end_position.row);
-    assert_eq(1, start.pT2.end_position.col);
+    assert_eq(4, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(5, start.pT2.pToken.end_position.row);
+    assert_eq(2, start.pT2.pToken.end_position.col);
+    assert_eq(4, start.pT2.position.row);
+    assert_eq(1, start.pT2.position.col);
+    assert_eq(5, start.pT2.end_position.row);
+    assert_eq(2, start.pT2.end_position.col);
 
-    assert_eq(6, start.pT3.pToken.position.row);
-    assert_eq(5, start.pT3.pToken.position.col);
-    assert_eq(6, start.pT3.pToken.end_position.row);
-    assert_eq(5, start.pT3.pToken.end_position.col);
-    assert_eq(6, start.pT3.position.row);
-    assert_eq(5, start.pT3.position.col);
-    assert_eq(6, start.pT3.end_position.row);
-    assert_eq(5, start.pT3.end_position.col);
+    assert_eq(7, start.pT3.pToken.position.row);
+    assert_eq(6, start.pT3.pToken.position.col);
+    assert_eq(7, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(7, start.pT3.position.row);
+    assert_eq(6, start.pT3.position.col);
+    assert_eq(7, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
 
-    assert_eq(2, start.position.row);
-    assert_eq(2, start.position.col);
-    assert_eq(6, start.end_position.row);
-    assert_eq(5, start.end_position.col);
+    assert_eq(3, start.position.row);
+    assert_eq(3, start.position.col);
+    assert_eq(7, start.end_position.row);
+    assert_eq(6, start.end_position.col);
 }

spec/test_error_positions.c

@@ -12,28 +12,28 @@ int main()
     input = "a\n123\na  a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 3);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 4);
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 0);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 1);
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 2);
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 5);
-    assert(p_position(&context).col == 4);
+    assert(p_position(&context).row == 6);
+    assert(p_position(&context).col == 5);
 
     assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
     assert(strcmp(p_token_names[TOKEN_num], "num") == 0);

spec/test_error_positions.d

@@ -16,24 +16,24 @@ unittest
     input = "a\n123\na  a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
+    assert(p_position(&context) == p_position_t(3, 4));
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
+    assert(p_position(&context) == p_position_t(1, 1));
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
+    assert(p_position(&context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(5, 4));
+    assert(p_position(&context) == p_position_t(6, 5));
 
     assert(p_token_names[TOKEN_a] == "a");
     assert(p_token_names[TOKEN_num] == "num");

spec/test_lexer.c

@@ -41,68 +41,68 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 2u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 2u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 3u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 4u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 6u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 2u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 4u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 8u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 9u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 9u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 10u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 
     p_context_init(&context, (uint8_t const *)"", 0u);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 

spec/test_lexer.d

@@ -47,23 +47,23 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), p_position_t(0, 2), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), p_position_t(0, 4), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), p_position_t(0, 6), 1, TOKEN_times));
+    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), p_position_t(1, 2), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), p_position_t(1, 4), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), p_position_t(1, 8), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), p_position_t(1, 9), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
     p_context_init(&context, "");
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_named_optional_rule_component_ast.c

@@ -0,0 +1,45 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+    assert(start->r == NULL);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->r != NULL);
+    assert(start->pR == start->pR3);
+    assert(start->pR == start->r);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->r != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    return 0;
+}
+

spec/test_named_optional_rule_component_ast.d

@@ -0,0 +1,46 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+    assert(start.r is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.r != null);
+    assert(start.pR == start.pR3);
+    assert(start.pR == start.r);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_parser_rule_from_multiple_states.c

@@ -8,8 +8,8 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 2);
     assert(context.token == TOKEN___EOF);
 
     input = "a b";

spec/test_parser_rule_from_multiple_states.d

@@ -12,7 +12,7 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
+    assert(p_position(&context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";