v1.2.0

CHANGELOG.md

@@ -1,3 +1,25 @@
+## v1.2.0
+
+### New Features
+
+- Allow one line user code blocks (#21)
+- Add backslash escape codes (#19)
+- Add API to access unexpected token found (#18)
+- Add token_names API (#17)
+- Add D example to user guide for p_context_init() (#16)
+- Allow user termination from lexer code blocks (#15)
+
+### Fixes
+
+- Fix generator hang when state transition cycle is present (#20)
+
+## v1.1.0
+
+### New Features
+
+- Add user parser terminations (#13)
+- Document generated parser API in user guide (#14)
+
 ## v1.0.0
 
 - Initial release

assets/parser.c.erb

@@ -3,6 +3,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+const char * <%= @grammar.prefix %>token_names[] = {
+<% @grammar.tokens.each_with_index do |token, index| %>
+    "<%= token.name %>",
+<% end %>
+};
+
 /**************************************************************************
  * User code blocks
  *************************************************************************/
@@ -422,6 +433,8 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -441,6 +454,12 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state->code_id, match, match_info.length, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -511,6 +530,8 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -820,7 +841,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context->token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -953,3 +974,13 @@ size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t
 {
     return context->user_terminate_code;
 }
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context->token;
+}

assets/parser.d.erb

@@ -31,7 +31,7 @@ public enum : size_t
 }
 
 /** Token type. */
-public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
 
 /** Token IDs. */
 public enum : <%= @grammar.prefix %>token_t
@@ -43,6 +43,7 @@ public enum : <%= @grammar.prefix %>token_t
 <%   end %>
 <% end %>
     INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
+    TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
 }
 
 /** Code point type. */
@@ -538,6 +539,8 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -557,6 +560,12 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state.code_id, match, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -627,6 +636,8 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -848,7 +859,7 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context.token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -972,3 +983,13 @@ public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>co
 {
     return context.user_terminate_code;
 }
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context.token;
+}

assets/parser.h.erb

@@ -23,7 +23,7 @@
 #define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
 
 /** Token type. */
-typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
 
 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -33,6 +33,7 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
 
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
@@ -115,6 +116,13 @@ typedef struct
     size_t user_terminate_code;
 } <%= @grammar.prefix %>context_t;
 
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+extern const char * <%= @grammar.prefix %>token_names[];
+
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
@@ -129,3 +137,5 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
 
 size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -77,33 +77,15 @@ token rparen /\\)/;
 # Drop whitespace.
 drop /\\s+/;
 
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
 Grammar files can contain comment lines beginning with `#` which are ignored.
@@ -117,8 +99,8 @@ lowercase character and beginning a rule name with an uppercase character.
 
 ##> User Code Blocks
 
-User code blocks begin with the line following a "<<" token and end with the
-line preceding a grammar line consisting of solely the ">>" token.
+User code blocks begin following a "<<" token and end with a ">>" token found
+at the end of a line.
 All text lines in the code block are copied verbatim into the output file.
 
 ### Standalone Code Blocks
@@ -189,9 +171,7 @@ This parser value can then be used later in a parser rule.
 Example:
 
 ```
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
 ```
 
 Parser rule code blocks appear following a rule expression.
@@ -238,9 +218,7 @@ lexer.
 Example:
 
 ```
-token if <<
-  writeln("'if' keyword lexed");
->>
+token if << writeln("'if' keyword lexed"); >>
 ```
 
 The `token` statement is actually a shortcut statement for a combination of a
@@ -277,9 +255,7 @@ code but may not result in a matched token.
 Example:
 
 ```
-/foo+/ <<
-  writeln("saw a foo pattern");
->>
+/foo+/ << writeln("saw a foo pattern"); >>
 ```
 
 This can be especially useful with ${#Lexer modes}.
@@ -325,9 +301,16 @@ Regular expressions can include many special characters:
   * The `(` character begins a matching group.
   * The `{` character begins a count qualifier.
   * The `\` character escapes the following character and changes its meaning:
+    * The `\a` sequence matches an ASCII bell character (0x07).
+    * The `\b` sequence matches an ASCII backspace character (0x08).
     * The `\d` sequence matches any character `0` through `9`.
+    * The `\f` sequence matches an ASCII form feed character (0x0C).
+    * The `\n` sequence matches an ASCII new line character (0x0A).
+    * The `\r` sequence matches an ASCII carriage return character (0x0D).
     * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
     `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * The `\t` sequence matches an ASCII tab character (0x09).
+    * The `\v` sequence matches an ASCII vertical tab character (0x0B).
     * Any other character matches itself.
   * The `|` character creates an alternate match.
 
@@ -381,9 +364,7 @@ tokenid str;
   mystringvalue = "";
   $mode(string);
 >>
-string: /[^"]+/ <<
-  mystringvalue += match;
->>
+string: /[^"]+/ << mystringvalue += match; >>
 string: /"/ <<
   $mode(default);
   return $token(str);
@@ -440,20 +421,12 @@ ptype Value;
 ptype array = Value[];
 ptype dict = Value[string];
 
-Object -> lbrace rbrace <<
-  $$ = new Value();
->>
+Object -> lbrace rbrace << $$ = new Value(); >>
 
-Values (array) -> Value <<
-  $$ = [$1];
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>
+Values (array) -> Value << $$ = [$1]; >>
+Values -> Values comma Value << $$ = $1 ~ [$3]; >>
 
-KeyValue (dict) -> string colon Value <<
-  $$ = [$1: $3];
->>
+KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
 ```
 
 In this example, the default parser value type is `Value`.
@@ -486,12 +459,8 @@ Example:
 
 ```
 ptype ulong;
-token word /[a-z]+/ <<
-  $$ = match.length;
->>
-Start -> word <<
-  $$ = $1;
->>
+token word /[a-z]+/ << $$ = match.length; >>
+Start -> word << $$ = $1; >>
 ```
 
 In the above example the `Start` rule is defined to match a single `word`
@@ -500,33 +469,15 @@ token.
 Example:
 
 ```
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 ```
 
 A parser rule has zero or more terms on the right side of its definition.
@@ -574,23 +525,22 @@ default.
 It can also be used when generating multiple lexers/parsers to be used in the
 same program to avoid symbol collisions.
 
-##> User termination of the parser
+##> User termination of the lexer or parser
 
-Propane supports allowing parser user code blocks to terminate execution of the
-parser.
-One example use of this functionality is to detect and report an error before
-the parser continues parsing the remainder of the input.
-Another use of this features is to begin parsing input and determine whether a
-different parser should be used instead.
+Propane supports allowing lexer or parser user code blocks to terminate
+execution of the parser.
+Some example uses of this functionality could be to:
 
-To terminate parsing from a parser user code block, use the `$terminate(code)`
-function, passing an integer expression argument.
+  * Detect integer overflow when lexing an integer literal constant.
+  * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
+  * Determine whether parsing should stop and instead be performed using a different parser version.
+
+To terminate parsing from a lexer or parser user code block, use the
+`$terminate(code)` function, passing an integer expression argument.
 For example:
 
 ```
-NewExpression -> new Expression <<
-  $terminate(42);
->>
+NewExpression -> new Expression << $terminate(42); >>
 ```
 
 The value passed to the `$terminate()` function is known as the "user terminate
@@ -645,13 +595,20 @@ structure.
 The input to be used for lexing/parsing is passed in when initializing the
 context structure.
 
-Example:
+C example:
 
 ```
 p_context_t context;
 p_context_init(&context, input, input_length);
 ```
 
+D example:
+
+```
+p_context_t context;
+p_context_init(&context, input);
+```
+
 ### `p_parse`
 
 The `p_parse()` function is the main entry point to the parser.
@@ -718,6 +675,44 @@ if (p_parse(&context) == P_USER_TERMINATED)
 }
 ```
 
+### `p_token`
+
+The `p_token()` function can be used to retrieve the current parse token.
+This is useful after `p_parse()` returns a `P_UNEXPECTED_TOKEN` value.
+terminate code after `p_parse()` returns a `P_USER_TERMINATED` value to
+indicate what token the parser was not expecting.
+
+Example:
+
+```
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_token_t unexpected_token = p_token(&context);
+}
+```
+
+##> Data
+
+### `p_token_names`
+
+The `p_token_names` array contains the grammar-specified token names.
+It is indexed by the token ID.
+
+C example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_position_t error_position = p_position(&context);
+    fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
+        p_token_names[context.token],
+        error_position.row + 1, error_position.col + 1);
+}
+```
+
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane/generator.rb

@@ -198,6 +198,16 @@ class Propane
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
       end
+      code = code.gsub(/\$terminate\((.*)\);/) do |match|
+        user_terminate_code = $1
+        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
+        case @language
+        when "c"
+          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "d"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
           case @language
@@ -216,15 +226,6 @@ class Propane
             "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
           end
         end
-        code = code.gsub(/\$terminate\((.*)\);/) do |match|
-          user_terminate_code = $1
-          case @language
-          when "c"
-            "context->user_terminate_code = (#{user_terminate_code}); return P_USER_TERMINATED;"
-          when "d"
-            "context.user_terminate_code = (#{user_terminate_code}); return P_USER_TERMINATED;"
-          end
-        end
       else
         code = code.gsub(/\$\$/) do |match|
           case @language

lib/propane/grammar.rb

@@ -35,6 +35,10 @@ class Propane
       @tokens.size
     end
 
+    def terminate_token_id
+      @tokens.size + 1
+    end
+
     private
 
     def parse_grammar!
@@ -179,8 +183,10 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -218,8 +224,11 @@ class Propane
     end
 
     def parse_code_block!
-      if md = consume!(/<<\n(.*?)^>>\n/m)
-        md[1]
+      if md = consume!(/<<(.*?)>>\n/m)
+        code = md[1]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
+        code
       end
     end
 

lib/propane/parser.rb

@@ -134,7 +134,7 @@ class Propane
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
-      item_sets = item_set.leading_item_sets
+      item_sets = Set[item_set] + item_set.leading_item_sets
       reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|

lib/propane/parser/item_set.rb

@@ -87,14 +87,24 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
-      # This set includes this ItemSet.
-      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @in_sets.reduce(Set[self]) do |result, item_set|
-          result + item_set.leading_item_sets
+        result = Set.new
+        eval_sets = Set[self]
+        evaled = Set.new
+        while eval_sets.size > 0
+          eval_set = eval_sets.first
+          eval_sets.delete(eval_set)
+          evaled << eval_set
+          eval_set.in_sets.each do |in_set|
+            result << in_set
+            unless evaled.include?(in_set)
+              eval_sets << in_set
+            end
+          end
         end
+        result
       end
 
       # Represent the ItemSet as a String.

lib/propane/regex.rb

@@ -134,8 +134,18 @@ class Propane
       else
         c = @pattern.slice!(0)
         case c
+        when "a"
+          CharacterRangeUnit.new("\a", "\a")
+        when "b"
+          CharacterRangeUnit.new("\b", "\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "f"
+          CharacterRangeUnit.new("\f", "\f")
+        when "n"
+          CharacterRangeUnit.new("\n", "\n")
+        when "r"
+          CharacterRangeUnit.new("\r", "\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -145,6 +155,10 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "t"
+          CharacterRangeUnit.new("\t", "\t")
+        when "v"
+          CharacterRangeUnit.new("\v", "\v")
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

spec/propane_spec.rb

@@ -123,10 +123,8 @@ token plus /\\+/;
 token times /\\*/;
 drop /\\s+/;
 Start -> Foo;
-Foo -> int <<
->>
-Foo -> plus <<
->>
+Foo -> int <<>>
+Foo -> plus <<>>
 EOF
         build_parser(language: language)
         compile("spec/test_lexer.#{language}", language: language)
@@ -149,9 +147,7 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int <<
-  $$ = $1;
->>
+Start -> int << $$ = $1; >>
 EOF
         when "d"
           write_grammar <<EOF
@@ -165,9 +161,7 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int <<
-  $$ = $1;
->>
+Start -> int << $$ = $1; >>
 EOF
         end
         build_parser(language: language)
@@ -219,33 +213,15 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = (size_t)pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
         when "d"
           write_grammar <<EOF
@@ -271,33 +247,15 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 <<
-  $$ = $1;
->>
-E1 -> E2 <<
-  $$ = $1;
->>
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
+Start -> E1 << $$ = $1; >>
+E1 -> E2 << $$ = $1; >>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
+E2 -> E3 << $$ = $1; >>
+E2 -> E2 times E3 << $$ = $1 * $3; >>
+E3 -> E4 << $$ = $1; >>
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
+E4 -> integer << $$ = $1; >>
+E4 -> lparen E1 rparen << $$ = $2; >>
 EOF
         end
         build_parser(language: language)
@@ -408,9 +366,7 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
-  writeln("def!");
->>
+/def/ << writeln("def!"); >>
 Start -> abc;
 EOF
         end
@@ -435,9 +391,7 @@ EOF
 #include <stdio.h>
 >>
 token abc;
-/def/ <<
-  printf("def!\\n");
->>
+/def/ << printf("def!\\n"); >>
 /ghi/ <<
   printf("ghi!\\n");
   return $token(abc);
@@ -450,9 +404,7 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
-  writeln("def!");
->>
+/def/ << writeln("def!"); >>
 /ghi/ <<
   writeln("ghi!");
   return $token(abc);
@@ -541,15 +493,9 @@ EOF
 >>
 token a;
 token b;
-Start -> A B <<
-  printf("Start!\\n");
->>
-A -> a <<
-  printf("A!\\n");
->>
-B -> b <<
-  printf("B!\\n");
->>
+Start -> A B << printf("Start!\\n"); >>
+A -> a << printf("A!\\n"); >>
+B -> b << printf("B!\\n"); >>
 EOF
         when "d"
           write_grammar <<EOF
@@ -558,15 +504,9 @@ import std.stdio;
 >>
 token a;
 token b;
-Start -> A B <<
-  writeln("Start!");
->>
-A -> a <<
-  writeln("A!");
->>
-B -> b <<
-  writeln("B!");
->>
+Start -> A B << writeln("Start!"); >>
+A -> a << writeln("A!"); >>
+B -> b << writeln("B!"); >>
 EOF
         end
         build_parser(language: language)
@@ -584,15 +524,9 @@ EOF
         write_grammar <<EOF
 ptype #{language == "c" ? "uint32_t" : "uint"};
 token a;
-Start -> As <<
-  $$ = $1;
->>
-As -> <<
-  $$ = 0u;
->>
-As -> As a <<
-  $$ = $1 + 1u;
->>
+Start -> As << $$ = $1; >>
+As -> << $$ = 0u; >>
+As -> As a << $$ = $1 + 1u; >>
 EOF
         build_parser(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
@@ -730,6 +664,25 @@ EOF
         expect(results.status).to eq 0
       end
 
+      it "allows the user to terminate the lexer" do
+        write_grammar <<EOF
+token a;
+token b <<
+  $terminate(8675309);
+>>
+token c;
+Start -> Any;
+Any -> a;
+Any -> b;
+Any -> c;
+EOF
+        build_parser(language: language)
+        compile("spec/test_user_terminate_lexer.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
       it "allows the user to terminate the parser" do
         write_grammar <<EOF
 token a;
@@ -737,9 +690,7 @@ token b;
 token c;
 Start -> Any;
 Any -> a Any;
-Any -> b Any <<
-  $terminate(4200);
->>
+Any -> b Any << $terminate(4200); >>
 Any -> c Any;
 Any -> ;
 EOF
@@ -749,6 +700,73 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "matches backslash escape sequences" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+  #include <stdio.h>
+>>
+tokenid t;
+/\\a/ << printf("A\\n"); >>
+/\\b/ << printf("B\\n"); >>
+/\\t/ << printf("T\\n"); >>
+/\\n/ << printf("N\\n"); >>
+/\\v/ << printf("V\\n"); >>
+/\\f/ << printf("F\\n"); >>
+/\\r/ << printf("R\\n"); >>
+/t/ << return $token(t); >>
+Start -> t;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+  import std.stdio;
+>>
+tokenid t;
+/\\a/ << writeln("A"); >>
+/\\b/ << writeln("B"); >>
+/\\t/ << writeln("T"); >>
+/\\n/ << writeln("N"); >>
+/\\v/ << writeln("V"); >>
+/\\f/ << writeln("F"); >>
+/\\r/ << writeln("R"); >>
+/t/ <<
+  return $token(t);
+>>
+Start -> t;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_match_backslashes.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "A",
+          "B",
+          "T",
+          "N",
+          "V",
+          "F",
+          "R",
+        ])
+      end
+
+      it "handles when an item set leads to itself" do
+        write_grammar <<EOF
+token one;
+token two;
+
+Start -> Opt one Start;
+Start -> ;
+
+Opt -> two;
+Opt -> ;
+EOF
+        build_parser(language: language)
+      end
     end
   end
 end

spec/test_error_positions.c

@@ -14,14 +14,14 @@ int main()
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 2);
     assert(p_position(&context).col == 3);
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 0);
     assert(p_position(&context).col == 0);
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
@@ -35,5 +35,8 @@ int main()
     assert(p_position(&context).row == 5);
     assert(p_position(&context).col == 4);
 
+    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
+    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+
     return 0;
 }

spec/test_error_positions.d

@@ -17,13 +17,13 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
@@ -34,4 +34,7 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
     assert(p_position(&context) == p_position_t(5, 4));
+
+    assert(p_token_names[TOKEN_a] == "a");
+    assert(p_token_names[TOKEN_num] == "num");
 }

spec/test_match_backslashes.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_match_backslashes.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_user_terminate_lexer.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+
+    return 0;
+}

spec/test_user_terminate_lexer.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+}