v2.2.0

CHANGELOG.md

@@ -1,3 +1,38 @@
+## v2.2.0
+
+### Improvements
+
+- Allow multiple lexer modes to be specified for a lexer pattern (#35)
+- Document p_decode_code_point() API function (#34)
+
+## v2.1.1
+
+### Fixes
+
+- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
+
+## v2.1.0
+
+### Improvements
+
+- Report rule name and line number for conflicting AST node field positions errors (#32)
+
+## v2.0.0
+
+### Improvements
+
+- Log conflicting rules on reduce/reduce conflict (#31)
+- Use 1-based row and column values for position values (#30)
+
+### Fixes
+
+- Fix named optional rules (#29)
+
+### Upgrading
+
+- Adjust all uses of p_position_t row and col values to expect 1-based instead
+of 0-based values.
+
 ## v1.5.1
 
 ### Improvements

Gemfile

@@ -1,5 +1,6 @@
 source "https://rubygems.org"
 
+gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"

Gemfile.lock

@@ -1,40 +1,44 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.5.0)
-    docile (1.4.0)
-    psych (5.1.0)
+    base64 (0.2.0)
+    date (3.4.1)
+    diff-lcs (1.5.1)
+    docile (1.4.1)
+    psych (5.2.2)
+      date
       stringio
-    rake (13.0.6)
-    rdoc (6.5.0)
+    rake (13.2.1)
+    rdoc (6.10.0)
       psych (>= 4.0.0)
     redcarpet (3.6.0)
-    rspec (3.12.0)
-      rspec-core (~> 3.12.0)
-      rspec-expectations (~> 3.12.0)
-      rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
-      rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    stringio (3.0.7)
+    stringio (3.1.2)
     syntax (1.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  base64
   rake
   rdoc
   redcarpet

assets/parser.c.erb

@@ -60,6 +60,8 @@ void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * contex
     /* Lexer initialization. */
     newcontext.input = input;
     newcontext.input_length = input_length;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -365,7 +367,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {
@@ -567,7 +569,7 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  *************************************************************************/
 
 /** Invalid position value. */
-#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0xFFFFFFFFu, 0xFFFFFFFFu}
+#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
 
 /** Reduce ID type. */
 typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;

assets/parser.d.erb

@@ -65,12 +65,12 @@ public struct <%= @grammar.prefix %>position_t
     uint col;
 
     /** Invalid position value. */
-    enum INVALID = <%= @grammar.prefix %>position_t(0xFFFF_FFFF, 0xFFFF_FFFF);
+    enum INVALID = <%= @grammar.prefix %>position_t(0u, 0u);
 
     /** Return whether the position is valid. */
     public @property bool valid()
     {
-        return row != 0xFFFF_FFFFu;
+        return row != 0u;
     }
 }
 
@@ -235,6 +235,8 @@ public void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t *
 
     /* Lexer initialization. */
     newcontext.input = input;
+    newcontext.text_position.row = 1u;
+    newcontext.text_position.col = 1u;
     newcontext.mode = <%= @lexer.mode_id("default") %>;
 
     /* Copy to the user's context structure. */
@@ -534,7 +536,7 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 0u;
+                    attempt_match.delta_position.col = 1u;
                 }
                 else
                 {

assets/parser.h.erb

@@ -53,7 +53,7 @@ typedef struct
 } <%= @grammar.prefix %>position_t;
 
 /** Return whether the position is valid. */
-#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0xFFFFFFFFu)
+#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
 
 /** User header code blocks. */
 <%= @grammar.code_blocks.fetch("header", "") %>

doc/user_guide.md

@@ -536,6 +536,28 @@ It also returns the `str` token now that the token is complete.
 Note that the token name `str` above could have been `string` instead - the
 namespace for token names is distinct from the namespace for lexer modes.
 
+Multiple modes can be specified for a token or pattern or drop statement.
+For example, if the grammar wanted to only recognize an identifier following
+a `.` token and not other keywords, it could switch to an `identonly` mode
+when matching a `.`
+The `ident` token pattern will be matched in either the `default` or
+`identonly` mode.
+
+```
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+```
+
 ##> Specifying parser value types - the `ptype` statement
 
 The `ptype` statement is used to define parser value type(s).
@@ -641,13 +663,14 @@ This example uses the default start rule name of `Start`.
 
 A parser rule has zero or more fields on the right side of its definition.
 Each of these fields is either a token name or a rule name.
+A field can be immediately followed by a `?` character to signify that it is
+optional.
 A field can optionally be followed by a `:` and then a field alias name.
 If present, the field alias name is used to refer to the field value in user
 code blocks, or if AST mode is active, the field alias name is used as the
 field name in the generated AST node structure.
-A field can be immediately followed by a `?` character to signify that it is
-optional.
-Another example:
+An optional and named field must use the format `field?:name`.
+Example:
 
 ```
 token public;
@@ -655,7 +678,7 @@ token private;
 token int;
 token ident /[a-zA-Z_][a-zA-Z_0-9]*/;
 token semicolon /;/;
-IntegerDeclaration -> Visibility? int ident:name semicolon;
+IntegerDeclaration -> Visibility?:visibility int ident:name semicolon;
 Visibility -> public;
 Visibility -> private;
 ```
@@ -663,7 +686,7 @@ Visibility -> private;
 In a parser rule code block, parser values for the right side fields are
 accessible as `$1` for the first field's parser value, `$2` for the second
 field's parser value, etc...
-For the `IntegerDeclaration` rule, the third field value can also be referred
+For the `IntegerDeclaration` rule, the first field value can also be referred to as `${visibility}` and the third field value can also be referred
 to as `${name}`.
 The `$$` symbol accesses the output parser value for this rule.
 The above examples demonstrate how the parser values for the rule components
@@ -761,10 +784,16 @@ Propane generates the following result code constants:
 * `P_EOF`: The lexer reached the end of the input string.
 * `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
 
-Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
+Result codes are returned by the functions `p_decode_code_point()`, `p_lex()`, and `p_parse()`.
 
 ##> Types
 
+### `p_code_point_t`
+
+The `p_code_point_t` type is aliased to a 32-bit unsigned integer.
+It is used to store decoded code points from the input text and perform
+lexing based on the grammar's lexer patterns.
+
 ### `p_context_t`
 
 Propane defines a `p_context_t` structure type.
@@ -776,7 +805,7 @@ A pointer to this instance is passed to the generated functions.
 ### `p_position_t`
 
 The `p_position_t` structure contains two fields `row` and `col`.
-These fields contain the 0-based row and column describing a parser position.
+These fields contain the 1-based row and column describing a parser position.
 
 For D targets, the `p_position_t` structure can be checked for validity by
 querying the `valid` property.
@@ -998,6 +1027,26 @@ if (p_parse(&context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_decode_code_point`
+
+The `p_decode_code_point()` function can be used to decode code points from a
+UTF-8 string.
+It does not require a lexer/parser context structure and can be used as a
+standalone UTF-8 decoder or from within a lexer or parser user code block.
+
+D Example:
+
+```
+size_t result;
+p_code_point_t code_point;
+ubyte code_point_length;
+
+result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+assert(result == P_SUCCESS);
+assert(code_point == 0x1F9E1u);
+assert(code_point_length == 4u);
+```
+
 ##> Data
 
 ### `p_token_names`

lib/propane/generator.rb

@@ -43,8 +43,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.mode.nil?
-          pattern.mode = "default"
+        if pattern.modes.empty?
+          pattern.modes << "default"
           found_default = true
         end
         pattern.ptypename ||= "default"

lib/propane/grammar.rb

@@ -25,7 +25,7 @@ class Propane
       @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
-      @mode = nil
+      @modeline = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
@@ -58,7 +58,7 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @mode.nil? && parse_mode_label!
+      elsif @modeline.nil? && parse_mode_label!
       elsif parse_ast_statement!
       elsif parse_ast_prefix_statement!
       elsif parse_ast_suffix_statement!
@@ -81,8 +81,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
-        @mode = md[1]
+      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
+        @modeline = md[1]
       end
     end
 
@@ -117,7 +117,7 @@ class Propane
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -153,9 +153,9 @@ class Propane
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
         @patterns << pattern
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -173,7 +173,7 @@ class Propane
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -186,8 +186,8 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
-        @mode = nil
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
+        @modeline = nil
         true
       end
     end
@@ -198,7 +198,7 @@ class Propane
         if @ast && ptypename
           raise Error.new("Multiple ptypes are unsupported in AST mode")
         end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}(?::#{IDENTIFIER_REGEX})?\??\s*)*)\s*/, "expected rule component list")
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         if @ast
           consume!(/;/, "expected `;'")
@@ -208,7 +208,7 @@ class Propane
           end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -225,8 +225,8 @@ class Propane
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
-        @mode = nil
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
+        @modeline = nil
         true
       end
     end
@@ -247,7 +247,7 @@ class Propane
         else
           @code_blocks[name] = code
         end
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -315,6 +315,14 @@ class Propane
       end
     end
 
+    def get_modes_from_modeline
+      if @modeline
+        Set[*@modeline.split(",").map(&:strip)]
+      else
+        Set.new
+      end
+    end
+
   end
 
 end

lib/propane/lexer.rb

@@ -26,8 +26,14 @@ class Propane
     private
 
     def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
-        pattern.mode
+      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
+        result + pattern.modes
+      end
+      @modes = modenames.reduce({}) do |result, modename|
+        result[modename] = @grammar.patterns.select do |pattern|
+          pattern.modes.include?(modename)
+        end
+        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -14,6 +14,7 @@ class Propane
       @item_sets = []
       @item_sets_set = {}
       @warnings = Set.new
+      @errors = Set.new
       @options = options
       start_item = Item.new(grammar.rules.first, 0)
       eval_item_sets = Set[ItemSet.new([start_item])]
@@ -41,8 +42,18 @@ class Propane
       build_reduce_actions!
       build_tables!
       write_log!
+      errormessage = ""
+      if @errors.size > 0
+        errormessage += @errors.join("\n")
+      end
       if @warnings.size > 0 && @options[:warnings_as_errors]
-        raise Error.new("Fatal errors (-w):\n" + @warnings.join("\n"))
+        if errormessage != ""
+          errormessage += "\n"
+        end
+        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
+      end
+      if errormessage != ""
+        raise Error.new(errormessage)
       end
     end
 
@@ -171,7 +182,7 @@ class Propane
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            raise Error.new("Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number})")
+            @errors << "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
           end
           reduce_actions[lookahead_token] = reduce_rule
         end

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [String, nil]
-    #   Lexer mode for this pattern.
-    attr_accessor :mode
+    # @return [Set]
+    #   Lexer modes for this pattern.
+    attr_accessor :modes
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,14 +46,14 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :mode
-    #   Lexer mode for this pattern.
+    # @option options [String, nil] :modes
+    #   Lexer modes for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @mode = options[:mode]
+      @modes = options[:modes]
       @ptypename = options[:ptypename]
       regex = Regex.new(@pattern)
       regex.nfa.end_state.accepts = self

lib/propane/rule_set.rb

@@ -119,6 +119,8 @@ class Propane
     def build_ast_fields(grammar)
       field_ast_node_indexes = {}
       field_indexes_across_all_rules = {}
+      # Stores the index into @ast_fields by field alias name.
+      field_aliases = {}
       @ast_fields = []
       @rules.each do |rule|
         rule.components.each_with_index do |component, i|
@@ -136,6 +138,16 @@ class Propane
             field_ast_node_indexes[field_name] = @ast_fields.size
             @ast_fields << {field_name => struct_name}
           end
+          rule.aliases.each do |alias_name, index|
+            if index == i
+              alias_ast_fields_index = field_ast_node_indexes[field_name]
+              if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index
+                raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
+              end
+              field_aliases[alias_name] = alias_ast_fields_index
+              @ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1]
+            end
+          end
           field_indexes_across_all_rules[node_name] ||= Set.new
           field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
           rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
@@ -150,18 +162,6 @@ class Propane
             "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
         end
       end
-      # Now merge in the field aliases as given by the user in the
-      # grammar.
-      field_aliases = {}
-      @rules.each do |rule|
-        rule.aliases.each do |alias_name, index|
-          if field_aliases[alias_name] && field_aliases[alias_name] != index
-            raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}`")
-          end
-          field_aliases[alias_name] = index
-          @ast_fields[index][alias_name] = @ast_fields[index].first[1]
-        end
-      end
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.5.1"
+  VERSION = "2.2.0"
 end

spec/propane/grammar_spec.rb

@@ -151,30 +151,30 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m1"
+      expect(o.modes).to eq Set["m1"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m2"
+      expect(o.modes).to eq Set["m2"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m3"
+      expect(o.modes).to eq Set["m3"]
     end
 
     it "allows assigning ptypes to tokens and rules" do

spec/propane_spec.rb

@@ -621,6 +621,62 @@ EOF
         ])
       end
 
+      it "multiple lexer modes may apply to a pattern" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  printf("ident: %c\\n", $3);
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  writeln("ident: ", $3);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_multiple_modes.#{language}", language: language)
+        results = run_test
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "ident: d",
+          "pass1",
+          "ident: a",
+          "pass2",
+        ])
+      end
+
       it "executes user code associated with a parser rule" do
         case language
         when "c"
@@ -689,6 +745,7 @@ EOF
         results = run_propane(capture: true, language: language)
         expect(results.status).to_not eq 0
         expect(results.stderr).to match %r{Error: reduce/reduce conflict \(state \d+\) between rule E#\d+ \(defined on line 10\) and rule F#\d+ \(defined on line 11\)}
+        expect(File.binread("spec/run/testparser.log")).to match %r{Reduce.actions:}
       end
 
       it "provides matched text to user code blocks" do
@@ -1117,6 +1174,47 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows naming an optional rule component in AST generation mode" do
+        if language == "d"
+          write_grammar <<EOF
+ast;
+
+<<
+import std.stdio;
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        else
+          write_grammar <<EOF
+ast;
+
+<<
+#include <stdio.h>
+>>
+
+token a;
+token b;
+token c;
+token d;
+Start -> a?:a b R?:r;
+R -> c d;
+R -> d c;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_named_optional_rule_component_ast.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "stores token and rule positions in AST nodes" do
         write_grammar <<EOF
 ast;
@@ -1177,6 +1275,29 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "aliases the correct field when multiple rules are in a rule set in AST mode" do
+        write_grammar <<EOF
+ast;
+
+token a;
+token b;
+token c;
+drop /\\s+/;
+Start -> a;
+Start -> Foo;
+Start -> T:first T:second T:third;
+Foo -> b;
+T -> a;
+T -> b;
+T -> c;
+EOF
+        run_propane(language: language)
+        compile("spec/test_ast_field_aliases.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows specifying field aliases when AST mode is not enabled" do
         if language == "d"
           write_grammar <<EOF
@@ -1211,6 +1332,56 @@ Start -> id:first id:second <<
   printf("first is %s\\n", ${first});
   printf("second is %s\\n", ${second});
 >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_field_aliases.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        expect(results.stdout).to match /first is foo1.*second is bar2/m
+      end
+
+      it "aliases the correct field when multiple rules are in a rule set when AST mode is not enabled" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype string;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  $$ = match;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  writeln("first is ", ${first});
+  writeln("second is ", ${second});
+>>
+Foo -> ;
+EOF
+        else
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+#include <string.h>
+>>
+ptype char const *;
+token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
+  char * s = malloc(match_length + 1);
+  strncpy(s, (char const *)match, match_length);
+  s[match_length] = 0;
+  $$ = s;
+>>
+drop /\\s+/;
+Start -> id;
+Start -> Foo;
+Start -> id:first id:second <<
+  printf("first is %s\\n", ${first});
+  printf("second is %s\\n", ${second});
+>>
+Foo -> ;
 EOF
         end
         run_propane(language: language)

spec/test_ast_invalid_positions.c

@@ -11,92 +11,92 @@ int main()
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(2, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(2, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(2, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(7, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(7, start->pT1->end_position.col);
+    assert_eq(3, start->pT1->pA->position.row);
+    assert_eq(3, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(8, start->pT1->pA->end_position.col);
+    assert_eq(2, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(8, start->pT1->end_position.col);
 
-    assert_eq(1, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(7, start->end_position.col);
+    assert_eq(2, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(8, start->end_position.col);
 
     input = "a\nbb";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->pA->position.row);
-    assert_eq(0, start->pT1->pA->position.col);
-    assert_eq(1, start->pT1->pA->end_position.row);
-    assert_eq(1, start->pT1->pA->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(1, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(2, start->pT1->pA->end_position.row);
+    assert_eq(2, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(2, start->pT1->end_position.row);
+    assert_eq(2, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(1, start->end_position.row);
-    assert_eq(1, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(2, start->end_position.row);
+    assert_eq(2, start->end_position.col);
 
     input = "a\nc\nc";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->pA->position.row);
-    assert_eq(0, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(0, start->pT1->pA->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(2, start->pT1->pA->position.row);
+    assert_eq(1, start->pT1->pA->position.col);
+    assert_eq(3, start->pT1->pA->end_position.row);
+    assert_eq(1, start->pT1->pA->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(0, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(3, start->end_position.row);
+    assert_eq(1, start->end_position.col);
 
     input = "a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
     assert(!p_position_valid(start->pT1->pA->position));
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(0, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(0, start->end_position.row);
-    assert_eq(0, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(1, start->end_position.col);
 
     return 0;
 }

spec/test_ast_invalid_positions.d

@@ -15,90 +15,90 @@ unittest
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(2, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(2, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(2, start.pT1.pA.position.row);
-    assert_eq(2, start.pT1.pA.position.col);
-    assert_eq(2, start.pT1.pA.end_position.row);
-    assert_eq(7, start.pT1.pA.end_position.col);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(7, start.pT1.end_position.col);
+    assert_eq(3, start.pT1.pA.position.row);
+    assert_eq(3, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(8, start.pT1.pA.end_position.col);
+    assert_eq(2, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(8, start.pT1.end_position.col);
 
-    assert_eq(1, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(2, start.end_position.row);
-    assert_eq(7, start.end_position.col);
+    assert_eq(2, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(8, start.end_position.col);
 
     input = "a\nbb";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(1, start.pT1.pA.position.row);
-    assert_eq(0, start.pT1.pA.position.col);
-    assert_eq(1, start.pT1.pA.end_position.row);
-    assert_eq(1, start.pT1.pA.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(1, start.pT1.end_position.row);
-    assert_eq(1, start.pT1.end_position.col);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(2, start.pT1.pA.end_position.row);
+    assert_eq(2, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(2, start.pT1.end_position.row);
+    assert_eq(2, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(1, start.end_position.row);
-    assert_eq(1, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(2, start.end_position.row);
+    assert_eq(2, start.end_position.col);
 
     input = "a\nc\nc";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(start.pT1.pA.position.valid);
-    assert_eq(1, start.pT1.pA.position.row);
-    assert_eq(0, start.pT1.pA.position.col);
-    assert_eq(2, start.pT1.pA.end_position.row);
-    assert_eq(0, start.pT1.pA.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(2, start.pT1.pA.position.row);
+    assert_eq(1, start.pT1.pA.position.col);
+    assert_eq(3, start.pT1.pA.end_position.row);
+    assert_eq(1, start.pT1.pA.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(2, start.end_position.row);
-    assert_eq(0, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(3, start.end_position.row);
+    assert_eq(1, start.end_position.col);
 
     input = "a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
     assert(!start.pT1.pA.position.valid);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(0, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(0, start.end_position.row);
-    assert_eq(0, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(1, start.end_position.col);
 }

spec/test_ast_token_positions.c

@@ -11,74 +11,74 @@ int main()
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(0, start->pT1->pToken->position.row);
-    assert_eq(0, start->pT1->pToken->position.col);
-    assert_eq(0, start->pT1->pToken->end_position.row);
-    assert_eq(0, start->pT1->pToken->end_position.col);
-    assert_eq(0, start->pT1->position.row);
-    assert_eq(0, start->pT1->position.col);
-    assert_eq(0, start->pT1->end_position.row);
-    assert_eq(0, start->pT1->end_position.col);
+    assert_eq(1, start->pT1->pToken->position.row);
+    assert_eq(1, start->pT1->pToken->position.col);
+    assert_eq(1, start->pT1->pToken->end_position.row);
+    assert_eq(1, start->pT1->pToken->end_position.col);
+    assert_eq(1, start->pT1->position.row);
+    assert_eq(1, start->pT1->position.col);
+    assert_eq(1, start->pT1->end_position.row);
+    assert_eq(1, start->pT1->end_position.col);
 
-    assert_eq(0, start->pT2->pToken->position.row);
-    assert_eq(1, start->pT2->pToken->position.col);
-    assert_eq(0, start->pT2->pToken->end_position.row);
-    assert_eq(2, start->pT2->pToken->end_position.col);
-    assert_eq(0, start->pT2->position.row);
-    assert_eq(1, start->pT2->position.col);
-    assert_eq(0, start->pT2->end_position.row);
-    assert_eq(2, start->pT2->end_position.col);
+    assert_eq(1, start->pT2->pToken->position.row);
+    assert_eq(2, start->pT2->pToken->position.col);
+    assert_eq(1, start->pT2->pToken->end_position.row);
+    assert_eq(3, start->pT2->pToken->end_position.col);
+    assert_eq(1, start->pT2->position.row);
+    assert_eq(2, start->pT2->position.col);
+    assert_eq(1, start->pT2->end_position.row);
+    assert_eq(3, start->pT2->end_position.col);
 
-    assert_eq(0, start->pT3->pToken->position.row);
-    assert_eq(3, start->pT3->pToken->position.col);
-    assert_eq(0, start->pT3->pToken->end_position.row);
-    assert_eq(5, start->pT3->pToken->end_position.col);
-    assert_eq(0, start->pT3->position.row);
-    assert_eq(3, start->pT3->position.col);
-    assert_eq(0, start->pT3->end_position.row);
-    assert_eq(5, start->pT3->end_position.col);
+    assert_eq(1, start->pT3->pToken->position.row);
+    assert_eq(4, start->pT3->pToken->position.col);
+    assert_eq(1, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(1, start->pT3->position.row);
+    assert_eq(4, start->pT3->position.col);
+    assert_eq(1, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
 
-    assert_eq(0, start->position.row);
-    assert_eq(0, start->position.col);
-    assert_eq(0, start->end_position.row);
-    assert_eq(5, start->end_position.col);
+    assert_eq(1, start->position.row);
+    assert_eq(1, start->position.col);
+    assert_eq(1, start->end_position.row);
+    assert_eq(6, start->end_position.col);
 
     input = "\n\n  bb\nc\ncc\n\n     a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(2, start->pT1->pToken->position.row);
-    assert_eq(2, start->pT1->pToken->position.col);
-    assert_eq(2, start->pT1->pToken->end_position.row);
-    assert_eq(3, start->pT1->pToken->end_position.col);
-    assert_eq(2, start->pT1->position.row);
-    assert_eq(2, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(3, start->pT1->end_position.col);
+    assert_eq(3, start->pT1->pToken->position.row);
+    assert_eq(3, start->pT1->pToken->position.col);
+    assert_eq(3, start->pT1->pToken->end_position.row);
+    assert_eq(4, start->pT1->pToken->end_position.col);
+    assert_eq(3, start->pT1->position.row);
+    assert_eq(3, start->pT1->position.col);
+    assert_eq(3, start->pT1->end_position.row);
+    assert_eq(4, start->pT1->end_position.col);
 
-    assert_eq(3, start->pT2->pToken->position.row);
-    assert_eq(0, start->pT2->pToken->position.col);
-    assert_eq(4, start->pT2->pToken->end_position.row);
-    assert_eq(1, start->pT2->pToken->end_position.col);
-    assert_eq(3, start->pT2->position.row);
-    assert_eq(0, start->pT2->position.col);
-    assert_eq(4, start->pT2->end_position.row);
-    assert_eq(1, start->pT2->end_position.col);
+    assert_eq(4, start->pT2->pToken->position.row);
+    assert_eq(1, start->pT2->pToken->position.col);
+    assert_eq(5, start->pT2->pToken->end_position.row);
+    assert_eq(2, start->pT2->pToken->end_position.col);
+    assert_eq(4, start->pT2->position.row);
+    assert_eq(1, start->pT2->position.col);
+    assert_eq(5, start->pT2->end_position.row);
+    assert_eq(2, start->pT2->end_position.col);
 
-    assert_eq(6, start->pT3->pToken->position.row);
-    assert_eq(5, start->pT3->pToken->position.col);
-    assert_eq(6, start->pT3->pToken->end_position.row);
-    assert_eq(5, start->pT3->pToken->end_position.col);
-    assert_eq(6, start->pT3->position.row);
-    assert_eq(5, start->pT3->position.col);
-    assert_eq(6, start->pT3->end_position.row);
-    assert_eq(5, start->pT3->end_position.col);
+    assert_eq(7, start->pT3->pToken->position.row);
+    assert_eq(6, start->pT3->pToken->position.col);
+    assert_eq(7, start->pT3->pToken->end_position.row);
+    assert_eq(6, start->pT3->pToken->end_position.col);
+    assert_eq(7, start->pT3->position.row);
+    assert_eq(6, start->pT3->position.col);
+    assert_eq(7, start->pT3->end_position.row);
+    assert_eq(6, start->pT3->end_position.col);
 
-    assert_eq(2, start->position.row);
-    assert_eq(2, start->position.col);
-    assert_eq(6, start->end_position.row);
-    assert_eq(5, start->end_position.col);
+    assert_eq(3, start->position.row);
+    assert_eq(3, start->position.col);
+    assert_eq(7, start->end_position.row);
+    assert_eq(6, start->end_position.col);
 
     return 0;
 }

spec/test_ast_token_positions.d

@@ -15,72 +15,72 @@ unittest
     assert(p_parse(&context) == P_SUCCESS);
     Start * start = p_result(&context);
 
-    assert_eq(0, start.pT1.pToken.position.row);
-    assert_eq(0, start.pT1.pToken.position.col);
-    assert_eq(0, start.pT1.pToken.end_position.row);
-    assert_eq(0, start.pT1.pToken.end_position.col);
-    assert_eq(0, start.pT1.position.row);
-    assert_eq(0, start.pT1.position.col);
-    assert_eq(0, start.pT1.end_position.row);
-    assert_eq(0, start.pT1.end_position.col);
+    assert_eq(1, start.pT1.pToken.position.row);
+    assert_eq(1, start.pT1.pToken.position.col);
+    assert_eq(1, start.pT1.pToken.end_position.row);
+    assert_eq(1, start.pT1.pToken.end_position.col);
+    assert_eq(1, start.pT1.position.row);
+    assert_eq(1, start.pT1.position.col);
+    assert_eq(1, start.pT1.end_position.row);
+    assert_eq(1, start.pT1.end_position.col);
 
-    assert_eq(0, start.pT2.pToken.position.row);
-    assert_eq(1, start.pT2.pToken.position.col);
-    assert_eq(0, start.pT2.pToken.end_position.row);
-    assert_eq(2, start.pT2.pToken.end_position.col);
-    assert_eq(0, start.pT2.position.row);
-    assert_eq(1, start.pT2.position.col);
-    assert_eq(0, start.pT2.end_position.row);
-    assert_eq(2, start.pT2.end_position.col);
+    assert_eq(1, start.pT2.pToken.position.row);
+    assert_eq(2, start.pT2.pToken.position.col);
+    assert_eq(1, start.pT2.pToken.end_position.row);
+    assert_eq(3, start.pT2.pToken.end_position.col);
+    assert_eq(1, start.pT2.position.row);
+    assert_eq(2, start.pT2.position.col);
+    assert_eq(1, start.pT2.end_position.row);
+    assert_eq(3, start.pT2.end_position.col);
 
-    assert_eq(0, start.pT3.pToken.position.row);
-    assert_eq(3, start.pT3.pToken.position.col);
-    assert_eq(0, start.pT3.pToken.end_position.row);
-    assert_eq(5, start.pT3.pToken.end_position.col);
-    assert_eq(0, start.pT3.position.row);
-    assert_eq(3, start.pT3.position.col);
-    assert_eq(0, start.pT3.end_position.row);
-    assert_eq(5, start.pT3.end_position.col);
+    assert_eq(1, start.pT3.pToken.position.row);
+    assert_eq(4, start.pT3.pToken.position.col);
+    assert_eq(1, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(1, start.pT3.position.row);
+    assert_eq(4, start.pT3.position.col);
+    assert_eq(1, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
 
-    assert_eq(0, start.position.row);
-    assert_eq(0, start.position.col);
-    assert_eq(0, start.end_position.row);
-    assert_eq(5, start.end_position.col);
+    assert_eq(1, start.position.row);
+    assert_eq(1, start.position.col);
+    assert_eq(1, start.end_position.row);
+    assert_eq(6, start.end_position.col);
 
     input = "\n\n  bb\nc\ncc\n\n     a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_SUCCESS);
     start = p_result(&context);
 
-    assert_eq(2, start.pT1.pToken.position.row);
-    assert_eq(2, start.pT1.pToken.position.col);
-    assert_eq(2, start.pT1.pToken.end_position.row);
-    assert_eq(3, start.pT1.pToken.end_position.col);
-    assert_eq(2, start.pT1.position.row);
-    assert_eq(2, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(3, start.pT1.end_position.col);
+    assert_eq(3, start.pT1.pToken.position.row);
+    assert_eq(3, start.pT1.pToken.position.col);
+    assert_eq(3, start.pT1.pToken.end_position.row);
+    assert_eq(4, start.pT1.pToken.end_position.col);
+    assert_eq(3, start.pT1.position.row);
+    assert_eq(3, start.pT1.position.col);
+    assert_eq(3, start.pT1.end_position.row);
+    assert_eq(4, start.pT1.end_position.col);
 
-    assert_eq(3, start.pT2.pToken.position.row);
-    assert_eq(0, start.pT2.pToken.position.col);
-    assert_eq(4, start.pT2.pToken.end_position.row);
-    assert_eq(1, start.pT2.pToken.end_position.col);
-    assert_eq(3, start.pT2.position.row);
-    assert_eq(0, start.pT2.position.col);
-    assert_eq(4, start.pT2.end_position.row);
-    assert_eq(1, start.pT2.end_position.col);
+    assert_eq(4, start.pT2.pToken.position.row);
+    assert_eq(1, start.pT2.pToken.position.col);
+    assert_eq(5, start.pT2.pToken.end_position.row);
+    assert_eq(2, start.pT2.pToken.end_position.col);
+    assert_eq(4, start.pT2.position.row);
+    assert_eq(1, start.pT2.position.col);
+    assert_eq(5, start.pT2.end_position.row);
+    assert_eq(2, start.pT2.end_position.col);
 
-    assert_eq(6, start.pT3.pToken.position.row);
-    assert_eq(5, start.pT3.pToken.position.col);
-    assert_eq(6, start.pT3.pToken.end_position.row);
-    assert_eq(5, start.pT3.pToken.end_position.col);
-    assert_eq(6, start.pT3.position.row);
-    assert_eq(5, start.pT3.position.col);
-    assert_eq(6, start.pT3.end_position.row);
-    assert_eq(5, start.pT3.end_position.col);
+    assert_eq(7, start.pT3.pToken.position.row);
+    assert_eq(6, start.pT3.pToken.position.col);
+    assert_eq(7, start.pT3.pToken.end_position.row);
+    assert_eq(6, start.pT3.pToken.end_position.col);
+    assert_eq(7, start.pT3.position.row);
+    assert_eq(6, start.pT3.position.col);
+    assert_eq(7, start.pT3.end_position.row);
+    assert_eq(6, start.pT3.end_position.col);
 
-    assert_eq(2, start.position.row);
-    assert_eq(2, start.position.col);
-    assert_eq(6, start.end_position.row);
-    assert_eq(5, start.end_position.col);
+    assert_eq(3, start.position.row);
+    assert_eq(3, start.position.col);
+    assert_eq(7, start.end_position.row);
+    assert_eq(6, start.end_position.col);
 }

spec/test_error_positions.c

@@ -12,28 +12,28 @@ int main()
     input = "a\n123\na  a";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 3);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 4);
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 0);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 1);
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 2);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 3);
+    assert(p_position(&context).col == 2);
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 5);
-    assert(p_position(&context).col == 4);
+    assert(p_position(&context).row == 6);
+    assert(p_position(&context).col == 5);
 
     assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
     assert(strcmp(p_token_names[TOKEN_num], "num") == 0);

spec/test_error_positions.d

@@ -16,24 +16,24 @@ unittest
     input = "a\n123\na  a";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(2, 3));
+    assert(p_position(&context) == p_position_t(3, 4));
     assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 0));
+    assert(p_position(&context) == p_position_t(1, 1));
     assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(2, 1));
+    assert(p_position(&context) == p_position_t(3, 2));
 
     input = "a 12\n\na\n\n77\na   \xAA";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(5, 4));
+    assert(p_position(&context) == p_position_t(6, 5));
 
     assert(p_token_names[TOKEN_a] == "a");
     assert(p_token_names[TOKEN_num] == "num");

spec/test_lexer.c

@@ -41,68 +41,68 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 2u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 2u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 3u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 4u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 6u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 2u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 4u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 5u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 8u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 7u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 9u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 9u);
+    assert(token_info.position.row == 2u);
+    assert(token_info.position.col == 10u);
+    assert(token_info.end_position.row == 2u);
+    assert(token_info.end_position.col == 10u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 
     p_context_init(&context, (uint8_t const *)"", 0u);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 0u);
-    assert(token_info.end_position.col == 0u);
+    assert(token_info.position.row == 1u);
+    assert(token_info.position.col == 1u);
+    assert(token_info.end_position.row == 1u);
+    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
 

spec/test_lexer.d

@@ -47,23 +47,23 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 2), p_position_t(0, 2), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 4), p_position_t(0, 4), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 6), p_position_t(0, 6), 1, TOKEN_times));
+    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 0), p_position_t(1, 2), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 4), p_position_t(1, 4), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 6), p_position_t(1, 8), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 9), p_position_t(1, 9), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
 
     p_context_init(&context, "");
     assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(0, 0), p_position_t(0, 0), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
 }

spec/test_lexer_multiple_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc.def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc .  abc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_multiple_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc.def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc .  abc`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}

spec/test_named_optional_rule_component_ast.c

@@ -0,0 +1,45 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include "testutils.h"
+
+int main()
+{
+    char const * input = "b";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert_eq(TOKEN_b, start->pToken2->token);
+    assert(start->pR3 == NULL);
+    assert(start->pR == NULL);
+    assert(start->r == NULL);
+
+    input = "abcd";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a != NULL);
+    assert_eq(TOKEN_a, start->pToken1->token);
+    assert(start->pToken2 != NULL);
+    assert(start->pR3 != NULL);
+    assert(start->pR != NULL);
+    assert(start->r != NULL);
+    assert(start->pR == start->pR3);
+    assert(start->pR == start->r);
+    assert_eq(TOKEN_c, start->pR->pToken1->token);
+
+    input = "bdc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start->a == NULL);
+    assert(start->pToken2 != NULL);
+    assert(start->r != NULL);
+    assert_eq(TOKEN_d, start->pR->pToken1->token);
+
+    return 0;
+}
+

spec/test_named_optional_rule_component_ast.d

@@ -0,0 +1,46 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "b";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    Start * start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert_eq(TOKEN_b, start.pToken2.token);
+    assert(start.pR3 is null);
+    assert(start.pR is null);
+    assert(start.r is null);
+
+    input = "abcd";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 != null);
+    assert_eq(TOKEN_a, start.pToken1.token);
+    assert(start.pToken2 != null);
+    assert(start.pR3 != null);
+    assert(start.pR != null);
+    assert(start.r != null);
+    assert(start.pR == start.pR3);
+    assert(start.pR == start.r);
+    assert_eq(TOKEN_c, start.pR.pToken1.token);
+
+    input = "bdc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    start = p_result(&context);
+    assert(start.pToken1 is null);
+    assert(start.pToken2 !is null);
+    assert(start.pR !is null);
+    assert_eq(TOKEN_d, start.pR.pToken1.token);
+}

spec/test_parser_rule_from_multiple_states.c

@@ -8,8 +8,8 @@ int main()
     p_context_t context;
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 0);
-    assert(p_position(&context).col == 1);
+    assert(p_position(&context).row == 1);
+    assert(p_position(&context).col == 2);
     assert(context.token == TOKEN___EOF);
 
     input = "a b";

spec/test_parser_rule_from_multiple_states.d

@@ -12,7 +12,7 @@ unittest
     p_context_t context;
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(0, 1));
+    assert(p_position(&context) == p_position_t(1, 2));
     assert(context.token == TOKEN___EOF);
 
     input = "a b";