v2.2.1

CHANGELOG.md

@@ -1,3 +1,10 @@
+## v2.2.0
+
+### Improvements
+
+- Allow multiple lexer modes to be specified for a lexer pattern (#35)
+- Document p_decode_code_point() API function (#34)
+
 ## v2.1.1
 
 ### Fixes

Gemfile

@@ -1,5 +1,6 @@
 source "https://rubygems.org"
 
+gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"

Gemfile.lock

@@ -1,21 +1,22 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    base64 (0.2.0)
     date (3.4.1)
-    diff-lcs (1.5.1)
+    diff-lcs (1.6.1)
     docile (1.4.1)
-    psych (5.2.2)
+    psych (5.2.3)
       date
       stringio
     rake (13.2.1)
-    rdoc (6.10.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
-    redcarpet (3.6.0)
+    redcarpet (3.6.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.2)
+    rspec-core (3.13.3)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.3)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -30,13 +31,14 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.1)
     simplecov_json_formatter (0.1.4)
-    stringio (3.1.2)
+    stringio (3.1.6)
     syntax (1.2.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  base64
   rake
   rdoc
   redcarpet

assets/parser.c.erb

@@ -712,17 +712,22 @@ const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    {
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , NULL
+        NULL, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u},
 <%     end %>
-    },
 <%   end %>
 };
 

assets/parser.d.erb

@@ -8,6 +8,7 @@
 module <%= @grammar.modulename %>;
 <% end %>
 
+import core.memory;
 import core.stdc.stdlib : malloc;
 
 /**************************************************************************
@@ -875,17 +876,22 @@ immutable ushort[<%= rule.rule_set_node_field_index_map.size %>] r_<%= rule.name
 /** Parser reduce table. */
 private immutable reduce_t[] parser_reduce_table = [
 <%   @parser.reduce_table.each do |reduce| %>
-    reduce_t(<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u
+    reduce_t(
+        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
+        <%= reduce[:rule_id] %>u, /* Rule ID */
+        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
 <%     if @grammar.ast %>
+        <%= reduce[:n_states] %>u, /* Number of states */
 <%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-             , null
+        null, /* No rule set node field index map (flat map) */
 <%       else %>
-             , &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0]
+        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
 <%       end %>
-             , <%= reduce[:rule].rule_set.ast_fields.size %>
+        <%= reduce[:rule].rule_set.ast_fields.size %>, /* Number of AST fields */
-             , <%= reduce[:propagate_optional_target] %>
+        <%= reduce[:propagate_optional_target] %>), /* Propagate optional target? */
+<%     else %>
+        <%= reduce[:n_states] %>u), /* Number of states */
 <%     end %>
-            ),
 <%   end %>
 ];
 
@@ -1074,7 +1080,9 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
             else if (parser_reduce_table[reduce_index].n_states > 0)
             {
                 size_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                ASTNode * node = cast(ASTNode *)malloc(ASTNode.sizeof + n_fields * (void *).sizeof);
+                size_t node_size = ASTNode.sizeof + n_fields * (void *).sizeof;
+                ASTNode * node = cast(ASTNode *)malloc(node_size);
+                GC.addRange(node, node_size);
                 node.position = <%= @grammar.prefix %>position_t.INVALID;
                 node.end_position = <%= @grammar.prefix %>position_t.INVALID;
                 foreach (i; 0..n_fields)

doc/user_guide.md

@@ -536,6 +536,28 @@ It also returns the `str` token now that the token is complete.
 Note that the token name `str` above could have been `string` instead - the
 namespace for token names is distinct from the namespace for lexer modes.
 
+Multiple modes can be specified for a token or pattern or drop statement.
+For example, if the grammar wanted to only recognize an identifier following
+a `.` token and not other keywords, it could switch to an `identonly` mode
+when matching a `.`
+The `ident` token pattern will be matched in either the `default` or
+`identonly` mode.
+
+```
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+```
+
 ##> Specifying parser value types - the `ptype` statement
 
 The `ptype` statement is used to define parser value type(s).
@@ -762,10 +784,16 @@ Propane generates the following result code constants:
 * `P_EOF`: The lexer reached the end of the input string.
 * `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
 
-Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
+Result codes are returned by the functions `p_decode_code_point()`, `p_lex()`, and `p_parse()`.
 
 ##> Types
 
+### `p_code_point_t`
+
+The `p_code_point_t` type is aliased to a 32-bit unsigned integer.
+It is used to store decoded code points from the input text and perform
+lexing based on the grammar's lexer patterns.
+
 ### `p_context_t`
 
 Propane defines a `p_context_t` structure type.
@@ -999,6 +1027,26 @@ if (p_parse(&context) == P_UNEXPECTED_TOKEN)
 }
 ```
 
+### `p_decode_code_point`
+
+The `p_decode_code_point()` function can be used to decode code points from a
+UTF-8 string.
+It does not require a lexer/parser context structure and can be used as a
+standalone UTF-8 decoder or from within a lexer or parser user code block.
+
+D Example:
+
+```
+size_t result;
+p_code_point_t code_point;
+ubyte code_point_length;
+
+result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
+assert(result == P_SUCCESS);
+assert(code_point == 0x1F9E1u);
+assert(code_point_length == 4u);
+```
+
 ##> Data
 
 ### `p_token_names`

lib/propane/generator.rb

@@ -43,8 +43,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.mode.nil?
+        if pattern.modes.empty?
-          pattern.mode = "default"
+          pattern.modes << "default"
           found_default = true
         end
         pattern.ptypename ||= "default"

lib/propane/grammar.rb

@@ -25,7 +25,7 @@ class Propane
       @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
-      @mode = nil
+      @modeline = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
@@ -58,7 +58,7 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @mode.nil? && parse_mode_label!
+      elsif @modeline.nil? && parse_mode_label!
       elsif parse_ast_statement!
       elsif parse_ast_prefix_statement!
       elsif parse_ast_suffix_statement!
@@ -81,8 +81,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
+      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
-        @mode = md[1]
+        @modeline = md[1]
       end
     end
 
@@ -117,7 +117,7 @@ class Propane
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -153,9 +153,9 @@ class Propane
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
         @patterns << pattern
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -173,7 +173,7 @@ class Propane
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -186,8 +186,8 @@ class Propane
         end
         consume!(/\s+/)
         consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -208,7 +208,7 @@ class Propane
           end
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -225,8 +225,8 @@ class Propane
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -247,7 +247,7 @@ class Propane
         else
           @code_blocks[name] = code
         end
-        @mode = nil
+        @modeline = nil
         true
       end
     end
@@ -315,6 +315,14 @@ class Propane
       end
     end
 
+    def get_modes_from_modeline
+      if @modeline
+        Set[*@modeline.split(",").map(&:strip)]
+      else
+        Set.new
+      end
+    end
+
   end
 
 end

lib/propane/lexer.rb

@@ -26,8 +26,14 @@ class Propane
     private
 
     def build_tables!
-      @modes = @grammar.patterns.group_by do |pattern|
+      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
-        pattern.mode
+        result + pattern.modes
+      end
+      @modes = modenames.reduce({}) do |result, modename|
+        result[modename] = @grammar.patterns.select do |pattern|
+          pattern.modes.include?(modename)
+        end
+        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -92,7 +92,7 @@ class Propane
               propagate_optional_target: rule.optional? && rule.components.size == 1}]
           elsif reduce_actions = item_set.reduce_actions
             reduce_actions.map do |token, rule|
-              {token_id: token.id, rule_id: rule.id, rule: rule,
+              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
                rule_set_id: rule.rule_set.id, n_states: rule.components.size,
                propagate_optional_target: rule.optional? && rule.components.size == 1}
             end

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [String, nil]
+    # @return [Set]
-    #   Lexer mode for this pattern.
+    #   Lexer modes for this pattern.
-    attr_accessor :mode
+    attr_accessor :modes
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,14 +46,14 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :mode
+    # @option options [String, nil] :modes
-    #   Lexer mode for this pattern.
+    #   Lexer modes for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @mode = options[:mode]
+      @modes = options[:modes]
       @ptypename = options[:ptypename]
       regex = Regex.new(@pattern)
       regex.nfa.end_state.accepts = self

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "2.1.1"
+  VERSION = "2.2.1"
 end

spec/ast_node_memory_remains.c.propane

@@ -0,0 +1,151 @@
+ast;
+ast_prefix P;
+
+<<header
+#include <stdio.h>
+#include <stdint.h>
+
+typedef union
+{
+    uint64_t i64;
+    const uint8_t * s;
+    double dou;
+} TokenVal;
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.i64 = 64u;
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  $$.s = match;
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/ast_node_memory_remains.d.propane

@@ -0,0 +1,177 @@
+ast;
+ast_prefix P;
+
+<<
+import std.bigint;
+
+private string stringvalue;
+
+union TokenVal
+{
+    BigInt bi;
+    string s;
+    double dou;
+}
+>>
+
+ptype TokenVal;
+
+# Keywords.
+token byte;
+token def;
+token int;
+token long;
+token module;
+token return;
+token short;
+token size_t;
+token ssize_t;
+token ubyte;
+token uint;
+token ulong;
+token ushort;
+
+# Symbols.
+token arrow /->/;
+token comma /,/;
+token lbrace /\{/;
+token lparen /\(/;
+token rbrace /\}/;
+token rparen /\)/;
+token semicolon /;/;
+
+# Integer literals.
+token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
+  $$.bi = BigInt(match[0..3]);
+  foreach (c; match[3..$])
+  {
+    if (('0' <= c) && (c <= '9'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - '0');
+    }
+    if (('a' <= c) && (c <= 'f'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'a' + 10);
+    }
+    if (('A' <= c) && (c <= 'F'))
+    {
+      $$.bi *= 0x10;
+      $$.bi += (c - 'A' + 10);
+    }
+  }
+>>
+
+# Identifier.
+token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
+  if (match[0] == '$')
+  {
+    $$.s = match[1..$];
+  }
+  else
+  {
+    $$.s = match;
+  }
+  $mode(default);
+  return $token(ident);
+>>
+
+# Comments.
+drop /#.*/;
+
+# Whitespace.
+drop /[ \r\n]*/;
+
+start Module;
+
+# Assignment operators - right associative
+Expression -> Expression_Or:exp0;
+
+# Logical OR operator - left associative
+Expression_Or -> Expression_And:exp0;
+
+# Logical AND operator - left associative
+Expression_And -> Expression_Comp:exp0;
+
+# Equality operators - left associative
+Expression_Comp -> Expression_Relational:exp0;
+
+# Relational operators - left associative
+Expression_Relational -> Expression_REMatch:exp0;
+
+# Regular expression - left associative
+Expression_REMatch -> Expression_BinOr:exp0;
+
+# Binary OR operator - left associative
+Expression_BinOr -> Expression_Xor:exp0;
+
+# Binary XOR operator - left associative
+Expression_Xor -> Expression_BinAnd:exp0;
+
+# Binary AND operator - left associative
+Expression_BinAnd -> Expression_BitShift:exp0;
+
+# Bit shift operators - left associative
+Expression_BitShift -> Expression_Plus:exp0;
+
+# Add/subtract operators - left associative
+Expression_Plus -> Expression_Mul:exp0;
+
+# Multiplication/divide/modulus operators - left associative
+Expression_Mul -> Expression_Range:exp0;
+
+# Range construction operators - left associative
+Expression_Range -> Expression_UnaryPrefix:exp0;
+
+# Unary prefix operators
+Expression_UnaryPrefix -> Expression_Dot:exp0;
+
+# Postfix operators
+Expression_Dot -> Expression_Ident:exp0;
+Expression_Dot -> Expression_Dot:exp1 lparen rparen;
+
+# Literals, identifiers, and parenthesized expressions
+Expression_Ident -> Literal;
+Expression_Ident -> ident;
+
+FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
+
+FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
+FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
+
+FunctionReturnType -> arrow Type;
+
+Literal -> LiteralInteger;
+LiteralInteger -> hex_int_l;
+
+Module -> ModuleStatement? ModuleItems;
+
+ModuleItem -> FunctionDefinition;
+
+ModuleItems -> ;
+ModuleItems -> ModuleItems ModuleItem;
+
+ModulePath -> ident;
+
+ModuleStatement -> module ModulePath semicolon;
+
+ReturnStatement -> return Expression?:exp0 semicolon;
+
+Statements -> ;
+Statements -> Statements Statement;
+Statement -> Expression semicolon;
+Statement -> ReturnStatement;
+
+Type -> TypeBase;
+
+TypeBase -> byte;
+TypeBase -> ubyte;
+TypeBase -> short;
+TypeBase -> ushort;
+TypeBase -> int;
+TypeBase -> uint;
+TypeBase -> long;
+TypeBase -> ulong;
+TypeBase -> size_t;
+TypeBase -> ssize_t;

spec/propane/grammar_spec.rb

@@ -151,30 +151,30 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m1"
+      expect(o.modes).to eq Set["m1"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m2"
+      expect(o.modes).to eq Set["m2"]
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.mode).to be_nil
+      expect(o.modes).to be_empty
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.mode).to eq "m3"
+      expect(o.modes).to eq Set["m3"]
     end
 
     it "allows assigning ptypes to tokens and rules" do

spec/propane_spec.rb

@@ -74,7 +74,7 @@ EOF
     when "c"
       result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
     when "d"
-      result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
+      result = system(*%w[ldc2 -g --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
     end
     expect(result).to be_truthy
   end
@@ -621,6 +621,62 @@ EOF
         ])
       end
 
+      it "multiple lexer modes may apply to a pattern" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+#include <stdio.h>
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+  return $token(ident);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  printf("ident: %c\\n", $3);
+>>
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+import std.stdio;
+>>
+ptype char;
+token abc;
+token def;
+default, identonly: token ident /[a-z]+/ <<
+  $$ = match[0];
+  $mode(default);
+>>
+token dot /\\./ <<
+  $mode(identonly);
+>>
+default, identonly: drop /\\s+/;
+Start -> abc dot ident <<
+  writeln("ident: ", $3);
+>>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_lexer_multiple_modes.#{language}", language: language)
+        results = run_test
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "ident: d",
+          "pass1",
+          "ident: a",
+          "pass2",
+        ])
+      end
+
       it "executes user code associated with a parser rule" do
         case language
         when "c"
@@ -1335,6 +1391,15 @@ EOF
         expect(results.status).to eq 0
         expect(results.stdout).to match /first is foo1.*second is bar2/m
       end
+
+      it "does not free memory allocated for AST nodes" do
+        write_grammar(File.read("spec/ast_node_memory_remains.#{language}.propane"))
+        run_propane(language: language)
+        compile("spec/test_ast_node_memory_remains.#{language}", language: language)
+        results = run_test
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_ast_node_memory_remains.c

@@ -0,0 +1,415 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+int main(int argc, char * argv[])
+{
+    const char * input =
+        "# 0\n"
+        "def byte_val() -> byte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 1\n"
+        "def short_val() -> short\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 2\n"
+        "def int_val() -> int\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 3\n"
+        "def long_val() -> long\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 4\n"
+        "def ssize_t_val() -> ssize_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 5\n"
+        "def byte_to_short() -> short\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 6\n"
+        "def byte_to_int() -> int\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 7\n"
+        "def byte_to_long() -> long\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 8\n"
+        "def byte_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return byte_val();\n"
+        "}\n"
+        "\n"
+        "# 9\n"
+        "def short_to_byte() -> byte\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 10\n"
+        "def short_to_int() -> int\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 11\n"
+        "def short_to_long() -> long\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 12\n"
+        "def short_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return short_val();\n"
+        "}\n"
+        "\n"
+        "# 13\n"
+        "def int_to_byte() -> byte\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 14\n"
+        "def int_to_short() -> short\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 15\n"
+        "def int_to_long() -> long\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 16\n"
+        "def int_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n"
+        "\n"
+        "# 17\n"
+        "def long_to_byte() -> byte\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 18\n"
+        "def long_to_short() -> short\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 19\n"
+        "def long_to_int() -> int\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 20\n"
+        "def long_to_ssize_t() -> ssize_t\n"
+        "{\n"
+        "    return long_val();\n"
+        "}\n"
+        "\n"
+        "# 21\n"
+        "def ssize_t_to_byte() -> byte\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 22\n"
+        "def ssize_t_to_short() -> short\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 23\n"
+        "def ssize_t_to_int() -> int\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 24\n"
+        "def ssize_t_to_long() -> long\n"
+        "{\n"
+        "    return ssize_t_val();\n"
+        "}\n"
+        "\n"
+        "# 25\n"
+        "def ubyte_val() -> ubyte\n"
+        "{\n"
+        "    return 0x42;\n"
+        "}\n"
+        "\n"
+        "# 26\n"
+        "def ushort_val() -> ushort\n"
+        "{\n"
+        "    return 0x4242;\n"
+        "}\n"
+        "\n"
+        "# 27\n"
+        "def uint_val() -> uint\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 28\n"
+        "def ulong_val() -> ulong\n"
+        "{\n"
+        "    return 0x4242_4242_4242_4242;\n"
+        "}\n"
+        "\n"
+        "# 29\n"
+        "def size_t_val() -> size_t\n"
+        "{\n"
+        "    return 0x42424242;\n"
+        "}\n"
+        "\n"
+        "# 30\n"
+        "def ubyte_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 31\n"
+        "def ubyte_to_uint() -> uint\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 32\n"
+        "def ubyte_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 33\n"
+        "def ubyte_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ubyte_val();\n"
+        "}\n"
+        "\n"
+        "# 34\n"
+        "def ushort_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 35\n"
+        "def ushort_to_uint() -> uint\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 36\n"
+        "def ushort_to_ulong() -> ulong\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 37\n"
+        "def ushort_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ushort_val();\n"
+        "}\n"
+        "\n"
+        "# 38\n"
+        "def uint_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 39\n"
+        "def uint_to_ushort() -> ushort\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 40\n"
+        "def uint_to_ulong() -> ulong\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 41\n"
+        "def uint_to_size_t() -> size_t\n"
+        "{\n"
+        "    return uint_val();\n"
+        "}\n"
+        "\n"
+        "# 42\n"
+        "def ulong_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 43\n"
+        "def ulong_to_ushort() -> ushort\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 44\n"
+        "def ulong_to_uint() -> uint\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 45\n"
+        "def ulong_to_size_t() -> size_t\n"
+        "{\n"
+        "    return ulong_val();\n"
+        "}\n"
+        "\n"
+        "# 46\n"
+        "def size_t_to_ubyte() -> ubyte\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 47\n"
+        "def size_t_to_ushort() -> ushort\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 48\n"
+        "def size_t_to_int() -> int\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 49\n"
+        "def size_t_to_ulong() -> ulong\n"
+        "{\n"
+        "    return size_t_val();\n"
+        "}\n"
+        "\n"
+        "# 50\n"
+        "def main() -> int\n"
+        "{\n"
+        "    return int_val();\n"
+        "}\n";
+    struct
+    {
+        const char * name;
+        p_token_t token;
+    } expected[] = {
+        {"byte_val", TOKEN_byte},
+        {"short_val", TOKEN_short},
+        {"int_val", TOKEN_int},
+        {"long_val", TOKEN_long},
+        {"ssize_t_val", TOKEN_ssize_t},
+        {"byte_to_short", TOKEN_short},
+        {"byte_to_int", TOKEN_int},
+        {"byte_to_long", TOKEN_long},
+        {"byte_to_ssize_t", TOKEN_ssize_t},
+        {"short_to_byte", TOKEN_byte},
+        {"short_to_int", TOKEN_int},
+        {"short_to_long", TOKEN_long},
+        {"short_to_ssize_t", TOKEN_ssize_t},
+        {"int_to_byte", TOKEN_byte},
+        {"int_to_short", TOKEN_short},
+        {"int_to_long", TOKEN_long},
+        {"int_to_ssize_t", TOKEN_ssize_t},
+        {"long_to_byte", TOKEN_byte},
+        {"long_to_short", TOKEN_short},
+        {"long_to_int", TOKEN_int},
+        {"long_to_ssize_t", TOKEN_ssize_t},
+        {"ssize_t_to_byte", TOKEN_byte},
+        {"ssize_t_to_short", TOKEN_short},
+        {"ssize_t_to_int", TOKEN_int},
+        {"ssize_t_to_long", TOKEN_long},
+        {"ubyte_val", TOKEN_ubyte},
+        {"ushort_val", TOKEN_ushort},
+        {"uint_val", TOKEN_uint},
+        {"ulong_val", TOKEN_ulong},
+        {"size_t_val", TOKEN_size_t},
+        {"ubyte_to_ushort", TOKEN_ushort},
+        {"ubyte_to_uint", TOKEN_uint},
+        {"ubyte_to_ulong", TOKEN_ulong},
+        {"ubyte_to_size_t", TOKEN_size_t},
+        {"ushort_to_ubyte", TOKEN_ubyte},
+        {"ushort_to_uint", TOKEN_uint},
+        {"ushort_to_ulong", TOKEN_ulong},
+        {"ushort_to_size_t", TOKEN_size_t},
+        {"uint_to_ubyte", TOKEN_ubyte},
+        {"uint_to_ushort", TOKEN_ushort},
+        {"uint_to_ulong", TOKEN_ulong},
+        {"uint_to_size_t", TOKEN_size_t},
+        {"ulong_to_ubyte", TOKEN_ubyte},
+        {"ulong_to_ushort", TOKEN_ushort},
+        {"ulong_to_uint", TOKEN_uint},
+        {"ulong_to_size_t", TOKEN_size_t},
+        {"size_t_to_ubyte", TOKEN_ubyte},
+        {"size_t_to_ushort", TOKEN_ushort},
+        {"size_t_to_int", TOKEN_int},
+        {"size_t_to_ulong", TOKEN_ulong},
+        {"main", TOKEN_int},
+    };
+    p_context_t context;
+    p_context_init(&context, (const uint8_t *)input, strlen(input));
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod->pModuleItems;
+    PFunctionDefinition ** pfds;
+    size_t n_pfds = 0u;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        if (pmi->pFunctionDefinition != NULL)
+        {
+            n_pfds++;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    pfds = malloc(n_pfds * sizeof(PModuleItems *));
+    pmis = pmod->pModuleItems;
+    size_t pfd_i = n_pfds;
+    while (pmis != NULL)
+    {
+        PModuleItem * pmi = pmis->pModuleItem;
+        PFunctionDefinition * pfd = pmi->pFunctionDefinition;
+        if (pfd != NULL)
+        {
+            pfd_i--;
+            assert(pfd_i < n_pfds);
+            pfds[pfd_i] = pfd;
+        }
+        pmis = pmis->pModuleItems;
+    }
+    assert_eq(51, n_pfds);
+    for (size_t i = 0; i < n_pfds; i++)
+    {
+        if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
+            (expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
+        {
+            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
+        }
+    }
+
+    return 0;
+}

spec/test_ast_node_memory_remains.d

@@ -0,0 +1,408 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "
+# 0
+def byte_val() -> byte
+{
+    return 0x42;
+}
+
+# 1
+def short_val() -> short
+{
+    return 0x4242;
+}
+
+# 2
+def int_val() -> int
+{
+    return 0x42424242;
+}
+
+# 3
+def long_val() -> long
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 4
+def ssize_t_val() -> ssize_t
+{
+    return 0x42424242;
+}
+
+# 5
+def byte_to_short() -> short
+{
+    return byte_val();
+}
+
+# 6
+def byte_to_int() -> int
+{
+    return byte_val();
+}
+
+# 7
+def byte_to_long() -> long
+{
+    return byte_val();
+}
+
+# 8
+def byte_to_ssize_t() -> ssize_t
+{
+    return byte_val();
+}
+
+# 9
+def short_to_byte() -> byte
+{
+    return short_val();
+}
+
+# 10
+def short_to_int() -> int
+{
+    return short_val();
+}
+
+# 11
+def short_to_long() -> long
+{
+    return short_val();
+}
+
+# 12
+def short_to_ssize_t() -> ssize_t
+{
+    return short_val();
+}
+
+# 13
+def int_to_byte() -> byte
+{
+    return int_val();
+}
+
+# 14
+def int_to_short() -> short
+{
+    return int_val();
+}
+
+# 15
+def int_to_long() -> long
+{
+    return int_val();
+}
+
+# 16
+def int_to_ssize_t() -> ssize_t
+{
+    return int_val();
+}
+
+# 17
+def long_to_byte() -> byte
+{
+    return long_val();
+}
+
+# 18
+def long_to_short() -> short
+{
+    return long_val();
+}
+
+# 19
+def long_to_int() -> int
+{
+    return long_val();
+}
+
+# 20
+def long_to_ssize_t() -> ssize_t
+{
+    return long_val();
+}
+
+# 21
+def ssize_t_to_byte() -> byte
+{
+    return ssize_t_val();
+}
+
+# 22
+def ssize_t_to_short() -> short
+{
+    return ssize_t_val();
+}
+
+# 23
+def ssize_t_to_int() -> int
+{
+    return ssize_t_val();
+}
+
+# 24
+def ssize_t_to_long() -> long
+{
+    return ssize_t_val();
+}
+
+# 25
+def ubyte_val() -> ubyte
+{
+    return 0x42;
+}
+
+# 26
+def ushort_val() -> ushort
+{
+    return 0x4242;
+}
+
+# 27
+def uint_val() -> uint
+{
+    return 0x42424242;
+}
+
+# 28
+def ulong_val() -> ulong
+{
+    return 0x4242_4242_4242_4242;
+}
+
+# 29
+def size_t_val() -> size_t
+{
+    return 0x42424242;
+}
+
+# 30
+def ubyte_to_ushort() -> ushort
+{
+    return ubyte_val();
+}
+
+# 31
+def ubyte_to_uint() -> uint
+{
+    return ubyte_val();
+}
+
+# 32
+def ubyte_to_ulong() -> ulong
+{
+    return ubyte_val();
+}
+
+# 33
+def ubyte_to_size_t() -> size_t
+{
+    return ubyte_val();
+}
+
+# 34
+def ushort_to_ubyte() -> ubyte
+{
+    return ushort_val();
+}
+
+# 35
+def ushort_to_uint() -> uint
+{
+    return ushort_val();
+}
+
+# 36
+def ushort_to_ulong() -> ulong
+{
+    return ushort_val();
+}
+
+# 37
+def ushort_to_size_t() -> size_t
+{
+    return ushort_val();
+}
+
+# 38
+def uint_to_ubyte() -> ubyte
+{
+    return uint_val();
+}
+
+# 39
+def uint_to_ushort() -> ushort
+{
+    return uint_val();
+}
+
+# 40
+def uint_to_ulong() -> ulong
+{
+    return uint_val();
+}
+
+# 41
+def uint_to_size_t() -> size_t
+{
+    return uint_val();
+}
+
+# 42
+def ulong_to_ubyte() -> ubyte
+{
+    return ulong_val();
+}
+
+# 43
+def ulong_to_ushort() -> ushort
+{
+    return ulong_val();
+}
+
+# 44
+def ulong_to_uint() -> uint
+{
+    return ulong_val();
+}
+
+# 45
+def ulong_to_size_t() -> size_t
+{
+    return ulong_val();
+}
+
+# 46
+def size_t_to_ubyte() -> ubyte
+{
+    return size_t_val();
+}
+
+# 47
+def size_t_to_ushort() -> ushort
+{
+    return size_t_val();
+}
+
+# 48
+def size_t_to_int() -> int
+{
+    return size_t_val();
+}
+
+# 49
+def size_t_to_ulong() -> ulong
+{
+    return size_t_val();
+}
+
+# 50
+def main() -> int
+{
+    return int_val();
+}
+";
+    struct Expected
+    {
+        string name;
+        p_token_t token;
+    }
+    Expected[] expected = [
+        Expected("byte_val", TOKEN_byte),
+        Expected("short_val", TOKEN_short),
+        Expected("int_val", TOKEN_int),
+        Expected("long_val", TOKEN_long),
+        Expected("ssize_t_val", TOKEN_ssize_t),
+        Expected("byte_to_short", TOKEN_short),
+        Expected("byte_to_int", TOKEN_int),
+        Expected("byte_to_long", TOKEN_long),
+        Expected("byte_to_ssize_t", TOKEN_ssize_t),
+        Expected("short_to_byte", TOKEN_byte),
+        Expected("short_to_int", TOKEN_int),
+        Expected("short_to_long", TOKEN_long),
+        Expected("short_to_ssize_t", TOKEN_ssize_t),
+        Expected("int_to_byte", TOKEN_byte),
+        Expected("int_to_short", TOKEN_short),
+        Expected("int_to_long", TOKEN_long),
+        Expected("int_to_ssize_t", TOKEN_ssize_t),
+        Expected("long_to_byte", TOKEN_byte),
+        Expected("long_to_short", TOKEN_short),
+        Expected("long_to_int", TOKEN_int),
+        Expected("long_to_ssize_t", TOKEN_ssize_t),
+        Expected("ssize_t_to_byte", TOKEN_byte),
+        Expected("ssize_t_to_short", TOKEN_short),
+        Expected("ssize_t_to_int", TOKEN_int),
+        Expected("ssize_t_to_long", TOKEN_long),
+        Expected("ubyte_val", TOKEN_ubyte),
+        Expected("ushort_val", TOKEN_ushort),
+        Expected("uint_val", TOKEN_uint),
+        Expected("ulong_val", TOKEN_ulong),
+        Expected("size_t_val", TOKEN_size_t),
+        Expected("ubyte_to_ushort", TOKEN_ushort),
+        Expected("ubyte_to_uint", TOKEN_uint),
+        Expected("ubyte_to_ulong", TOKEN_ulong),
+        Expected("ubyte_to_size_t", TOKEN_size_t),
+        Expected("ushort_to_ubyte", TOKEN_ubyte),
+        Expected("ushort_to_uint", TOKEN_uint),
+        Expected("ushort_to_ulong", TOKEN_ulong),
+        Expected("ushort_to_size_t", TOKEN_size_t),
+        Expected("uint_to_ubyte", TOKEN_ubyte),
+        Expected("uint_to_ushort", TOKEN_ushort),
+        Expected("uint_to_ulong", TOKEN_ulong),
+        Expected("uint_to_size_t", TOKEN_size_t),
+        Expected("ulong_to_ubyte", TOKEN_ubyte),
+        Expected("ulong_to_ushort", TOKEN_ushort),
+        Expected("ulong_to_uint", TOKEN_uint),
+        Expected("ulong_to_size_t", TOKEN_size_t),
+        Expected("size_t_to_ubyte", TOKEN_ubyte),
+        Expected("size_t_to_ushort", TOKEN_ushort),
+        Expected("size_t_to_int", TOKEN_int),
+        Expected("size_t_to_ulong", TOKEN_ulong),
+        Expected("main", TOKEN_int),
+    ];
+    p_context_t context;
+    p_context_init(&context, input);
+    size_t result = p_parse(&context);
+    assert_eq(P_SUCCESS, result);
+    PModule * pmod = p_result(&context);
+    PModuleItems * pmis = pmod.pModuleItems;
+    PFunctionDefinition *[] pfds;
+    while (pmis !is null)
+    {
+        PModuleItem * pmi = pmis.pModuleItem;
+        if (pmi is null)
+        {
+            stderr.writeln("pmi is null!!!?");
+            assert(0);
+        }
+        PFunctionDefinition * pfd = pmi.pFunctionDefinition;
+        if (pfd !is null)
+        {
+            pfds = [pfd] ~ pfds;
+        }
+        pmis = pmis.pModuleItems;
+    }
+    assert_eq(51, pfds.length);
+    for (size_t i = 0; i < pfds.length; i++)
+    {
+        if ((expected[i].name != pfds[i].name.pvalue.s) ||
+            (expected[i].token != pfds[i].returntype.pType.pTypeBase.pToken1.token))
+        {
+            stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
+        }
+    }
+}

spec/test_lexer_multiple_modes.c

@@ -0,0 +1,20 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+
+int main()
+{
+    char const * input = "abc.def";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass1\n");
+
+    input = "abc .  abc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    printf("pass2\n");
+
+    return 0;
+}

spec/test_lexer_multiple_modes.d

@@ -0,0 +1,21 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = `abc.def`;
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass1");
+
+    input = `abc .  abc`;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+    writeln("pass2");
+}