v1.2.0

CHANGELOG.md

@@ -1,3 +1,25 @@
+## v1.2.0
+
+### New Features
+
+- Allow one line user code blocks (#21)
+- Add backslash escape codes (#19)
+- Add API to access unexpected token found (#18)
+- Add token_names API (#17)
+- Add D example to user guide for p_context_init() (#16)
+- Allow user termination from lexer code blocks (#15)
+
+### Fixes
+
+- Fix generator hang when state transition cycle is present (#20)
+
+## v1.1.0
+
+### New Features
+
+- Add user parser terminations (#13)
+- Document generated parser API in user guide (#14)
+
 ## v1.0.0
 
 - Initial release

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2023 Josh Holtrop
+Copyright (c) 2010-2024 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.c.erb

@@ -3,6 +3,17 @@
 #include <stdlib.h>
 #include <string.h>
 
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+const char * <%= @grammar.prefix %>token_names[] = {
+<% @grammar.tokens.each_with_index do |token, index| %>
+    "<%= token.name %>",
+<% end %>
+};
+
 /**************************************************************************
  * User code blocks
  *************************************************************************/
@@ -21,6 +32,7 @@
 #define P_UNEXPECTED_TOKEN 3u
 #define P_DROP 4u
 #define P_EOF 5u
+#define P_USER_TERMINATED 6u
 <% end %>
 
 /* An invalid ID value. */
@@ -390,7 +402,6 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
-            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -422,6 +433,8 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -441,6 +454,12 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state->code_id, match, match_info.length, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -511,6 +530,8 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -738,12 +759,13 @@ static void state_values_stack_free(state_values_stack_t * stack)
  *
  * @param rule The ID of the rule.
  *
- * @return Parse value.
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
-static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
+static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
 {
-    <%= @grammar.prefix %>value_t _pvalue = {0};
-
     switch (rule)
     {
 <% @grammar.rules.each do |rule| %>
@@ -756,7 +778,7 @@ static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_value
     default: break;
     }
 
-    return _pvalue;
+    return P_SUCCESS;
 }
 
 /**
@@ -819,7 +841,7 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context->token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -889,7 +911,12 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
+            <%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
+            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+            reduced_parser_value = reduced_parser_value2;
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
@@ -934,3 +961,26 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
 {
     return context->text_position;
 }
+
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
+size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
+{
+    return context->user_terminate_code;
+}
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context->token;
+}

assets/parser.d.erb

@@ -27,10 +27,11 @@ public enum : size_t
     <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN,
     <%= @grammar.prefix.upcase %>DROP,
     <%= @grammar.prefix.upcase %>EOF,
+    <%= @grammar.prefix.upcase %>USER_TERMINATED,
 }
 
 /** Token type. */
-public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.invalid_token_id) %>;
+public alias <%= @grammar.prefix %>token_t = <%= get_type_for(@grammar.terminate_token_id) %>;
 
 /** Token IDs. */
 public enum : <%= @grammar.prefix %>token_t
@@ -42,6 +43,7 @@ public enum : <%= @grammar.prefix %>token_t
 <%   end %>
 <% end %>
     INVALID_TOKEN_ID = <%= @grammar.invalid_token_id %>,
+    TERMINATE_TOKEN_ID = <%= @grammar.terminate_token_id %>,
 }
 
 /** Code point type. */
@@ -114,6 +116,9 @@ public struct <%= @grammar.prefix %>context_t
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
+
+    /** User terminate code. */
+    size_t user_terminate_code;
 }
 
 /**************************************************************************
@@ -141,6 +146,7 @@ private enum : size_t
     P_UNEXPECTED_TOKEN,
     P_DROP,
     P_EOF,
+    P_USER_TERMINATED,
 }
 <% end %>
 
@@ -502,7 +508,6 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
-            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -534,6 +539,8 @@ private size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -553,6 +560,12 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
             string match = context.input[context.input_index..(context.input_index + match_info.length)];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
                 match_info.accepting_state.code_id, match, &token_info);
+            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
+             * that the user code is requesting to terminate the lexer. */
+            if (user_code_token == TERMINATE_TOKEN_ID)
+            {
+                return P_USER_TERMINATED;
+            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -623,6 +636,8 @@ private size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%=
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
+ * @retval P_USER_TERMINATED
+ *   User code has requested to terminate the lexer.
  */
 public size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -762,12 +777,13 @@ private immutable parser_state_t[] parser_state_table = [
  *
  * @param rule The ID of the rule.
  *
- * @return Parse value.
+ * @retval P_SUCCESS
+ *   Continue parsing.
+ * @retval P_USER_TERMINATED
+ *   User requested to terminate parsing.
  */
-private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[] statevalues, uint n_states)
+private size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint rule, state_value_t[] statevalues, uint n_states, <%= @grammar.prefix %>context_t * context)
 {
-    <%= @grammar.prefix %>value_t _pvalue;
-
     switch (rule)
     {
 <% @grammar.rules.each do |rule| %>
@@ -780,7 +796,7 @@ private <%= @grammar.prefix %>value_t parser_user_code(uint rule, state_value_t[
     default: break;
     }
 
-    return _pvalue;
+    return P_SUCCESS;
 }
 
 /**
@@ -843,7 +859,7 @@ private size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The value context.token holds the unexpected token.
+ *   The function p_token(&context) can be used to get the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
@@ -907,7 +923,12 @@ public size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * cont
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states);
+            <%= @grammar.prefix %>value_t reduced_parser_value2;
+            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
+            {
+                return P_USER_TERMINATED;
+            }
+            reduced_parser_value = reduced_parser_value2;
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             statevalues.length -= parser_reduce_table[reduce_index].n_states;
             continue;
@@ -949,3 +970,26 @@ public <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @gram
 {
     return context.text_position;
 }
+
+/**
+ * Get the user terminate code.
+ *
+ * @param context
+ *   Lexer/parser context structure.
+ *
+ * @return User terminate code.
+ */
+public size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
+{
+    return context.user_terminate_code;
+}
+
+/**
+ * Get the parse token.
+ *
+ * @return Parse token.
+ */
+public <%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
+{
+    return context.token;
+}

assets/parser.h.erb

@@ -20,9 +20,10 @@
 #define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
 #define <%= @grammar.prefix.upcase %>DROP 4u
 #define <%= @grammar.prefix.upcase %>EOF 5u
+#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
 
 /** Token type. */
-typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
 
 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -32,6 +33,7 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
+#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
 
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
@@ -109,8 +111,18 @@ typedef struct
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
+
+    /** User terminate code. */
+    size_t user_terminate_code;
 } <%= @grammar.prefix %>context_t;
 
+/**************************************************************************
+ * Public data
+ *************************************************************************/
+
+/** Token names. */
+extern const char * <%= @grammar.prefix %>token_names[];
+
 void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
@@ -123,3 +135,7 @@ size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
+
+size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
+
+<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

doc/user_guide.md

@@ -13,7 +13,7 @@ Propane is a LALR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven shift/reduce parser to parse input in linear time
-  * target C or D language outputs
+  * targets C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
@@ -77,33 +77,15 @@ token rparen /\\)/;
 # Drop whitespace.
 drop /\\s+/;
 
-Start -> E1 <<
+Start -> E1 << $$ = $1; >>
-  $$ = $1;
+E1 -> E2 << $$ = $1; >>
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E1 -> E2 <<
+E2 -> E3 << $$ = $1; >>
-  $$ = $1;
+E2 -> E2 times E3 << $$ = $1 * $3; >>
->>
+E3 -> E4 << $$ = $1; >>
-E1 -> E1 plus E2 <<
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
-  $$ = $1 + $3;
+E4 -> integer << $$ = $1; >>
->>
+E4 -> lparen E1 rparen << $$ = $2; >>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
 ```
 
 Grammar files can contain comment lines beginning with `#` which are ignored.
@@ -117,8 +99,8 @@ lowercase character and beginning a rule name with an uppercase character.
 
 ##> User Code Blocks
 
-User code blocks begin with the line following a "<<" token and end with the
+User code blocks begin following a "<<" token and end with a ">>" token found
-line preceding a grammar line consisting of solely the ">>" token.
+at the end of a line.
 All text lines in the code block are copied verbatim into the output file.
 
 ### Standalone Code Blocks
@@ -189,9 +171,7 @@ This parser value can then be used later in a parser rule.
 Example:
 
 ```
-E1 -> E1 plus E2 <<
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
-  $$ = $1 + $3;
->>
 ```
 
 Parser rule code blocks appear following a rule expression.
@@ -238,9 +218,7 @@ lexer.
 Example:
 
 ```
-token if <<
+token if << writeln("'if' keyword lexed"); >>
-  writeln("'if' keyword lexed");
->>
 ```
 
 The `token` statement is actually a shortcut statement for a combination of a
@@ -277,9 +255,7 @@ code but may not result in a matched token.
 Example:
 
 ```
-/foo+/ <<
+/foo+/ << writeln("saw a foo pattern"); >>
-  writeln("saw a foo pattern");
->>
 ```
 
 This can be especially useful with ${#Lexer modes}.
@@ -325,9 +301,16 @@ Regular expressions can include many special characters:
   * The `(` character begins a matching group.
   * The `{` character begins a count qualifier.
   * The `\` character escapes the following character and changes its meaning:
+    * The `\a` sequence matches an ASCII bell character (0x07).
+    * The `\b` sequence matches an ASCII backspace character (0x08).
     * The `\d` sequence matches any character `0` through `9`.
+    * The `\f` sequence matches an ASCII form feed character (0x0C).
+    * The `\n` sequence matches an ASCII new line character (0x0A).
+    * The `\r` sequence matches an ASCII carriage return character (0x0D).
     * The `\s` sequence matches a space, horizontal tab `\t`, carriage return
     `\r`, a form feed `\f`, or a vertical tab `\v` character.
+    * The `\t` sequence matches an ASCII tab character (0x09).
+    * The `\v` sequence matches an ASCII vertical tab character (0x0B).
     * Any other character matches itself.
   * The `|` character creates an alternate match.
 
@@ -381,9 +364,7 @@ tokenid str;
   mystringvalue = "";
   $mode(string);
 >>
-string: /[^"]+/ <<
+string: /[^"]+/ << mystringvalue += match; >>
-  mystringvalue += match;
->>
 string: /"/ <<
   $mode(default);
   return $token(str);
@@ -440,20 +421,12 @@ ptype Value;
 ptype array = Value[];
 ptype dict = Value[string];
 
-Object -> lbrace rbrace <<
+Object -> lbrace rbrace << $$ = new Value(); >>
-  $$ = new Value();
->>
 
-Values (array) -> Value <<
+Values (array) -> Value << $$ = [$1]; >>
-  $$ = [$1];
+Values -> Values comma Value << $$ = $1 ~ [$3]; >>
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>
 
-KeyValue (dict) -> string colon Value <<
+KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
-  $$ = [$1: $3];
->>
 ```
 
 In this example, the default parser value type is `Value`.
@@ -486,12 +459,8 @@ Example:
 
 ```
 ptype ulong;
-token word /[a-z]+/ <<
+token word /[a-z]+/ << $$ = match.length; >>
-  $$ = match.length;
+Start -> word << $$ = $1; >>
->>
-Start -> word <<
-  $$ = $1;
->>
 ```
 
 In the above example the `Start` rule is defined to match a single `word`
@@ -500,33 +469,15 @@ token.
 Example:
 
 ```
-Start -> E1 <<
+Start -> E1 << $$ = $1; >>
-  $$ = $1;
+E1 -> E2 << $$ = $1; >>
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E1 -> E2 <<
+E2 -> E3 << $$ = $1; >>
-  $$ = $1;
+E2 -> E2 times E3 << $$ = $1 * $3; >>
->>
+E3 -> E4 << $$ = $1; >>
-E1 -> E1 plus E2 <<
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
-  $$ = $1 + $3;
+E4 -> integer << $$ = $1; >>
->>
+E4 -> lparen E1 rparen << $$ = $2; >>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
 ```
 
 A parser rule has zero or more terms on the right side of its definition.
@@ -574,6 +525,194 @@ default.
 It can also be used when generating multiple lexers/parsers to be used in the
 same program to avoid symbol collisions.
 
+##> User termination of the lexer or parser
+
+Propane supports allowing lexer or parser user code blocks to terminate
+execution of the parser.
+Some example uses of this functionality could be to:
+
+  * Detect integer overflow when lexing an integer literal constant.
+  * Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
+  * Determine whether parsing should stop and instead be performed using a different parser version.
+
+To terminate parsing from a lexer or parser user code block, use the
+`$terminate(code)` function, passing an integer expression argument.
+For example:
+
+```
+NewExpression -> new Expression << $terminate(42); >>
+```
+
+The value passed to the `$terminate()` function is known as the "user terminate
+code".
+If the parser returns a `P_USER_TERMINATED` result code, then the user
+terminate code can be accessed using the `p_user_terminate_code()` API
+function.
+
+#> Propane generated API
+
+By default, Propane uses a prefix of `p_` when generating a lexer/parser.
+This prefix is used for all publicly declared types and functions.
+The uppercase version of the prefix is used for all constant values.
+
+This section documents the generated API using the default `p_` or `P_` names.
+
+##> Constants
+
+Propane generates the following result code constants:
+
+* `P_SUCCESS`: A successful decode/lex/parse operation has taken place.
+* `P_DECODE_ERROR`: An error occurred when decoding UTF-8 input.
+* `P_UNEXPECTED_INPUT`: Input was received by the lexer that does not match any lexer pattern.
+* `P_UNEXPECTED_TOKEN`: A token was seen in a location that does not match any parser rule.
+* `P_DROP`: The lexer matched a drop pattern.
+* `P_EOF`: The lexer reached the end of the input string.
+* `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
+
+Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
+
+##> Types
+
+### `p_context_t`
+
+Propane defines a `p_context_t` structure type.
+The structure is intended to be used opaquely and stores information related to
+the state of the lexer and parser.
+Integrating code must define an instance of the `p_context_t` structure.
+A pointer to this instance is passed to the generated functions.
+
+### `p_position_t`
+
+The `p_position_t` structure contains two fields `row` and `col`.
+These fields contain the 0-based row and column describing a parser position.
+
+##> Functions
+
+### `p_context_init`
+
+The `p_context_init()` function must be called to initialize the context
+structure.
+The input to be used for lexing/parsing is passed in when initializing the
+context structure.
+
+C example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+```
+
+D example:
+
+```
+p_context_t context;
+p_context_init(&context, input);
+```
+
+### `p_parse`
+
+The `p_parse()` function is the main entry point to the parser.
+It must be passed a pointer to an initialized context structure.
+
+Example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+```
+
+### `p_result`
+
+The `p_result()` function can be used to retrieve the final parse value after
+`p_parse()` returns a `P_SUCCESS` value.
+
+Example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+if (p_parse(&context) == P_SUCCESS)
+{
+    result = p_result(&context);
+}
+```
+
+### `p_position`
+
+The `p_position()` function can be used to retrieve the parser position where
+an error occurred.
+
+Example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_position_t error_position = p_position(&context);
+    fprintf(stderr, "Error: unexpected token at row %u column %u\n",
+        error_position.row + 1, error_position.col + 1);
+}
+```
+
+### `p_user_terminate_code`
+
+The `p_user_terminate_code()` function can be used to retrieve the user
+terminate code after `p_parse()` returns a `P_USER_TERMINATED` value.
+User terminate codes are arbitrary values that can be defined by the user to
+be returned when the user requests to terminate parsing.
+They have no particular meaning to Propane.
+
+Example:
+
+```
+if (p_parse(&context) == P_USER_TERMINATED)
+{
+    size_t user_terminate_code = p_user_terminate_code(&context);
+}
+```
+
+### `p_token`
+
+The `p_token()` function can be used to retrieve the current parse token.
+This is useful after `p_parse()` returns a `P_UNEXPECTED_TOKEN` value.
+terminate code after `p_parse()` returns a `P_USER_TERMINATED` value to
+indicate what token the parser was not expecting.
+
+Example:
+
+```
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_token_t unexpected_token = p_token(&context);
+}
+```
+
+##> Data
+
+### `p_token_names`
+
+The `p_token_names` array contains the grammar-specified token names.
+It is indexed by the token ID.
+
+C example:
+
+```
+p_context_t context;
+p_context_init(&context, input, input_length);
+size_t result = p_parse(&context);
+if (p_parse(&context) == P_UNEXPECTED_TOKEN)
+{
+    p_position_t error_position = p_position(&context);
+    fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
+        p_token_names[context.token],
+        error_position.row + 1, error_position.col + 1);
+}
+```
+
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane/generator.rb

@@ -198,9 +198,24 @@ class Propane
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
       end
+      code = code.gsub(/\$terminate\((.*)\);/) do |match|
+        user_terminate_code = $1
+        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
+        case @language
+        when "c"
+          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        when "d"
+          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
+        end
+      end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          "_pvalue.v_#{rule.ptypename}"
+          case @language
+          when "c"
+            "_pvalue->v_#{rule.ptypename}"
+          when "d"
+            "_pvalue.v_#{rule.ptypename}"
+          end
         end
         code = code.gsub(/\$(\d+)/) do |match|
           index = $1.to_i

lib/propane/grammar.rb

@@ -35,6 +35,10 @@ class Propane
       @tokens.size
     end
 
+    def terminate_token_id
+      @tokens.size + 1
+    end
+
     private
 
     def parse_grammar!
@@ -179,8 +183,10 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -218,8 +224,11 @@ class Propane
     end
 
     def parse_code_block!
-      if md = consume!(/<<\n(.*?)^>>\n/m)
+      if md = consume!(/<<(.*?)>>\n/m)
-        md[1]
+        code = md[1]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
+        code
       end
     end
 

lib/propane/parser.rb

@@ -134,7 +134,7 @@ class Propane
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
-      item_sets = item_set.leading_item_sets
+      item_sets = Set[item_set] + item_set.leading_item_sets
       reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|

lib/propane/parser/item_set.rb

@@ -87,14 +87,24 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
-      # This set includes this ItemSet.
-      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @in_sets.reduce(Set[self]) do |result, item_set|
+        result = Set.new
-          result + item_set.leading_item_sets
+        eval_sets = Set[self]
+        evaled = Set.new
+        while eval_sets.size > 0
+          eval_set = eval_sets.first
+          eval_sets.delete(eval_set)
+          evaled << eval_set
+          eval_set.in_sets.each do |in_set|
+            result << in_set
+            unless evaled.include?(in_set)
+              eval_sets << in_set
+            end
+          end
         end
+        result
       end
 
       # Represent the ItemSet as a String.

lib/propane/regex.rb

@@ -134,8 +134,18 @@ class Propane
       else
         c = @pattern.slice!(0)
         case c
+        when "a"
+          CharacterRangeUnit.new("\a", "\a")
+        when "b"
+          CharacterRangeUnit.new("\b", "\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
+        when "f"
+          CharacterRangeUnit.new("\f", "\f")
+        when "n"
+          CharacterRangeUnit.new("\n", "\n")
+        when "r"
+          CharacterRangeUnit.new("\r", "\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -145,6 +155,10 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
+        when "t"
+          CharacterRangeUnit.new("\t", "\t")
+        when "v"
+          CharacterRangeUnit.new("\v", "\v")
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

spec/propane_spec.rb

@@ -123,10 +123,8 @@ token plus /\\+/;
 token times /\\*/;
 drop /\\s+/;
 Start -> Foo;
-Foo -> int <<
+Foo -> int <<>>
->>
+Foo -> plus <<>>
-Foo -> plus <<
->>
 EOF
         build_parser(language: language)
         compile("spec/test_lexer.#{language}", language: language)
@@ -149,9 +147,7 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int <<
+Start -> int << $$ = $1; >>
-  $$ = $1;
->>
 EOF
         when "d"
           write_grammar <<EOF
@@ -165,9 +161,7 @@ token int /\\d+/ <<
   }
   $$ = v;
 >>
-Start -> int <<
+Start -> int << $$ = $1; >>
-  $$ = $1;
->>
 EOF
         end
         build_parser(language: language)
@@ -219,33 +213,15 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 <<
+Start -> E1 << $$ = $1; >>
-  $$ = $1;
+E1 -> E2 << $$ = $1; >>
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E1 -> E2 <<
+E2 -> E3 << $$ = $1; >>
-  $$ = $1;
+E2 -> E2 times E3 << $$ = $1 * $3; >>
->>
+E3 -> E4 << $$ = $1; >>
-E1 -> E1 plus E2 <<
+E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
-  $$ = $1 + $3;
+E4 -> integer << $$ = $1; >>
->>
+E4 -> lparen E1 rparen << $$ = $2; >>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = (size_t)pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
 EOF
         when "d"
           write_grammar <<EOF
@@ -271,33 +247,15 @@ token lparen /\\(/;
 token rparen /\\)/;
 drop /\\s+/;
 
-Start -> E1 <<
+Start -> E1 << $$ = $1; >>
-  $$ = $1;
+E1 -> E2 << $$ = $1; >>
->>
+E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E1 -> E2 <<
+E2 -> E3 << $$ = $1; >>
-  $$ = $1;
+E2 -> E2 times E3 << $$ = $1 * $3; >>
->>
+E3 -> E4 << $$ = $1; >>
-E1 -> E1 plus E2 <<
+E3 -> E3 power E4 << $$ = pow($1, $3); >>
-  $$ = $1 + $3;
+E4 -> integer << $$ = $1; >>
->>
+E4 -> lparen E1 rparen << $$ = $2; >>
-E2 -> E3 <<
-  $$ = $1;
->>
-E2 -> E2 times E3 <<
-  $$ = $1 * $3;
->>
-E3 -> E4 <<
-  $$ = $1;
->>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer <<
-  $$ = $1;
->>
-E4 -> lparen E1 rparen <<
-  $$ = $2;
->>
 EOF
         end
         build_parser(language: language)
@@ -408,9 +366,7 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
+/def/ << writeln("def!"); >>
-  writeln("def!");
->>
 Start -> abc;
 EOF
         end
@@ -435,9 +391,7 @@ EOF
 #include <stdio.h>
 >>
 token abc;
-/def/ <<
+/def/ << printf("def!\\n"); >>
-  printf("def!\\n");
->>
 /ghi/ <<
   printf("ghi!\\n");
   return $token(abc);
@@ -450,9 +404,7 @@ EOF
 import std.stdio;
 >>
 token abc;
-/def/ <<
+/def/ << writeln("def!"); >>
-  writeln("def!");
->>
 /ghi/ <<
   writeln("ghi!");
   return $token(abc);
@@ -541,15 +493,9 @@ EOF
 >>
 token a;
 token b;
-Start -> A B <<
+Start -> A B << printf("Start!\\n"); >>
-  printf("Start!\\n");
+A -> a << printf("A!\\n"); >>
->>
+B -> b << printf("B!\\n"); >>
-A -> a <<
-  printf("A!\\n");
->>
-B -> b <<
-  printf("B!\\n");
->>
 EOF
         when "d"
           write_grammar <<EOF
@@ -558,15 +504,9 @@ import std.stdio;
 >>
 token a;
 token b;
-Start -> A B <<
+Start -> A B << writeln("Start!"); >>
-  writeln("Start!");
+A -> a << writeln("A!"); >>
->>
+B -> b << writeln("B!"); >>
-A -> a <<
-  writeln("A!");
->>
-B -> b <<
-  writeln("B!");
->>
 EOF
         end
         build_parser(language: language)
@@ -584,15 +524,9 @@ EOF
         write_grammar <<EOF
 ptype #{language == "c" ? "uint32_t" : "uint"};
 token a;
-Start -> As <<
+Start -> As << $$ = $1; >>
-  $$ = $1;
+As -> << $$ = 0u; >>
->>
+As -> As a << $$ = $1 + 1u; >>
-As -> <<
-  $$ = 0u;
->>
-As -> As a <<
-  $$ = $1 + 1u;
->>
 EOF
         build_parser(language: language)
         compile("spec/test_parsing_lists.#{language}", language: language)
@@ -729,6 +663,110 @@ EOF
         expect(results.stderr).to eq ""
         expect(results.status).to eq 0
       end
+
+      it "allows the user to terminate the lexer" do
+        write_grammar <<EOF
+token a;
+token b <<
+  $terminate(8675309);
+>>
+token c;
+Start -> Any;
+Any -> a;
+Any -> b;
+Any -> c;
+EOF
+        build_parser(language: language)
+        compile("spec/test_user_terminate_lexer.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "allows the user to terminate the parser" do
+        write_grammar <<EOF
+token a;
+token b;
+token c;
+Start -> Any;
+Any -> a Any;
+Any -> b Any << $terminate(4200); >>
+Any -> c Any;
+Any -> ;
+EOF
+        build_parser(language: language)
+        compile("spec/test_user_terminate.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
+
+      it "matches backslash escape sequences" do
+        case language
+        when "c"
+          write_grammar <<EOF
+<<
+  #include <stdio.h>
+>>
+tokenid t;
+/\\a/ << printf("A\\n"); >>
+/\\b/ << printf("B\\n"); >>
+/\\t/ << printf("T\\n"); >>
+/\\n/ << printf("N\\n"); >>
+/\\v/ << printf("V\\n"); >>
+/\\f/ << printf("F\\n"); >>
+/\\r/ << printf("R\\n"); >>
+/t/ << return $token(t); >>
+Start -> t;
+EOF
+        when "d"
+          write_grammar <<EOF
+<<
+  import std.stdio;
+>>
+tokenid t;
+/\\a/ << writeln("A"); >>
+/\\b/ << writeln("B"); >>
+/\\t/ << writeln("T"); >>
+/\\n/ << writeln("N"); >>
+/\\v/ << writeln("V"); >>
+/\\f/ << writeln("F"); >>
+/\\r/ << writeln("R"); >>
+/t/ <<
+  return $token(t);
+>>
+Start -> t;
+EOF
+        end
+        build_parser(language: language)
+        compile("spec/test_match_backslashes.#{language}", language: language)
+        results = run
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+        verify_lines(results.stdout, [
+          "A",
+          "B",
+          "T",
+          "N",
+          "V",
+          "F",
+          "R",
+        ])
+      end
+
+      it "handles when an item set leads to itself" do
+        write_grammar <<EOF
+token one;
+token two;
+
+Start -> Opt one Start;
+Start -> ;
+
+Opt -> two;
+Opt -> ;
+EOF
+        build_parser(language: language)
+      end
     end
   end
 end

spec/test_error_positions.c

@@ -14,14 +14,14 @@ int main()
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 2);
     assert(p_position(&context).col == 3);
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context).row == 0);
     assert(p_position(&context).col == 0);
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, (uint8_t const *)input, strlen(input));
@@ -35,5 +35,8 @@ int main()
     assert(p_position(&context).row == 5);
     assert(p_position(&context).col == 4);
 
+    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
+    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
+
     return 0;
 }

spec/test_error_positions.d

@@ -17,13 +17,13 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(2, 3));
-    assert(context.token == TOKEN_a);
+    assert(p_token(&context) == TOKEN_a);
 
     input = "12";
     p_context_init(&context, input);
     assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
     assert(p_position(&context) == p_position_t(0, 0));
-    assert(context.token == TOKEN_num);
+    assert(p_token(&context) == TOKEN_num);
 
     input = "a 12\n\nab";
     p_context_init(&context, input);
@@ -34,4 +34,7 @@ unittest
     p_context_init(&context, input);
     assert(p_parse(&context) == P_DECODE_ERROR);
     assert(p_position(&context) == p_position_t(5, 4));
+
+    assert(p_token_names[TOKEN_a] == "a");
+    assert(p_token_names[TOKEN_num] == "num");
 }

spec/test_match_backslashes.c

@@ -0,0 +1,13 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    return 0;
+}

spec/test_match_backslashes.d

@@ -0,0 +1,15 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "\a\b\t\n\v\f\rt";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+}

spec/test_user_terminate.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "aacc";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abc";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 4200);
+
+    return 0;
+}

spec/test_user_terminate.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "aacc";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "abc";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 4200);
+}

spec/test_user_terminate_lexer.c

@@ -0,0 +1,19 @@
+#include "testparser.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+int main()
+{
+    char const * input = "a";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+
+    return 0;
+}

spec/test_user_terminate_lexer.d

@@ -0,0 +1,20 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "a";
+    p_context_t context;
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "b";
+    p_context_init(&context, input);
+    assert(p_parse(&context) == P_USER_TERMINATED);
+    assert(p_user_terminate_code(&context) == 8675309);
+}