User guide: document user code blocks

assets/parser.c.erb

@@ -7,9 +7,7 @@
  * User code blocks
  *************************************************************************/
 
-<% @grammar.code_blocks.each do |code| %>
+<%= @grammar.code_blocks.fetch("", "") %>
-<%= code %>
-<% end %>
 
 /**************************************************************************
  * Private types

assets/parser.d.erb

@@ -12,9 +12,7 @@ module <%= @grammar.modulename %>;
  * User code blocks
  *************************************************************************/
 
-<% @grammar.code_blocks.each do |code| %>
+<%= @grammar.code_blocks.fetch("", "") %>
-<%= code %>
-<% end %>
 
 /**************************************************************************
  * Public types

assets/parser.h.erb

@@ -36,6 +36,9 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
+/** User header code blocks. */
+<%= @grammar.code_blocks.fetch("header", "") %>
+
 /** Parser values type(s). */
 typedef union
 {

doc/user_guide.md

@@ -13,9 +13,9 @@ Propane is an LR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven parser to parse input in linear time
+  * target C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
-  * supports D language
 
 #> Installation
 
@@ -103,6 +103,89 @@ E4 -> lparen E1 rparen <<
 >>
 ```
 
+##> User Code Blocks
+
+User code blocks begin with the line following a "<<" token and end with the
+line preceding a grammar line consisting of solely the ">>" token.
+All text lines in the code block are copied verbatim into the output file.
+
+### Standalone Code Blocks
+
+C example:
+
+```
+#include <stdio.h>
+```
+
+D example:
+
+```
+<<
+import std.stdio;
+>>
+```
+
+Standalone code blocks are emitted early in the output file as top-level code
+outside the context of any function.
+Standalone code blocks are a good place to include/import any other necessary
+supporting code modules.
+They can also define helper functions that can be reused by lexer or parser
+user code blocks.
+They are emitted in the order they are defined in the grammar file.
+
+For a C target, the word "header" may immediately follow the "<<" token to
+cause Propane to emit the code block in the generated header file rather than
+the generated implementation file.
+This allows including another header that may be necessary to define any types
+needed by a `ptype` directive, for example:
+
+    <<header
+    #include "mytypes.h"
+    >>
+
+### Lexer pattern code blocks
+
+Example:
+
+```
+ptype ulong;
+
+token integer /\\d+/ <<
+  ulong v;
+  foreach (c; match)
+  {
+    v *= 10;
+    v += (c - '0');
+  }
+  $$ = v;
+>>
+```
+
+Lexer code blocks appear following a `token` or `pattern` expression.
+User code in a lexer code block will be executed when the lexer matches the
+given pattern.
+Assignment to the `$$` symbol will associate a parser value with the lexed
+token.
+This parser value can then be used later in a parser rule.
+
+### Parser rule code blocks
+
+Example:
+
+```
+E1 -> E1 plus E2 <<
+  $$ = $1 + $3;
+>>
+```
+
+Parser rule code blocks appear following a rule expression.
+User code in a parser rule code block will be executed when the parser reduces
+the given rule.
+Assignment to the `$$` symbol will associate a parser value with the reduced
+rule.
+Parser values for the rules or tokens in the rule pattern can be accessed
+positionally with tokens `$1`, `$2`, `$3`, etc...
+
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane/grammar.rb

@@ -18,7 +18,7 @@ class Propane
       @patterns = []
       @tokens = []
       @rules = []
-      @code_blocks = []
+      @code_blocks = {}
       @line_number = 1
       @next_line_number = @line_number
       @mode = nil
@@ -191,8 +191,13 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if code = parse_code_block!
+      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
-        @code_blocks << code
+        name, code = md[1..2]
+        if @code_blocks[name]
+          @code_blocks[name] += code
+        else
+          @code_blocks[name] = code
+        end
         @mode = nil
         true
       end

spec/json_parser.c.propane

@@ -0,0 +1,183 @@
+<<header
+#include "json_types.h"
+#include "testutils.h"
+>>
+<<
+#include "math.h"
+#include <stdbool.h>
+static str_t string_value;
+>>
+
+ptype JSONValue *;
+
+drop /\s+/;
+token lbrace /\{/;
+token rbrace /\}/;
+token lbracket /\[/;
+token rbracket /\]/;
+token comma /,/;
+token colon /:/;
+token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
+    double n = 0.0;
+    bool negative = false;
+    size_t i = 0u;
+    if (match[i] == '-')
+    {
+        negative = true;
+        i++;
+    }
+    while ('0' <= match[i] && match[i] <= '9')
+    {
+        n *= 10.0;
+        n += (match[i] - '0');
+        i++;
+    }
+    if (match[i] == '.')
+    {
+        i++;
+        double mult = 0.1;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            n += mult * (match[i] - '0');
+            mult /= 10.0;
+            i++;
+        }
+    }
+    if (match[i] == 'e' || match[i] == 'E')
+    {
+        bool exp_negative = false;
+        i++;
+        if (match[i] == '-')
+        {
+            exp_negative = true;
+            i++;
+        }
+        else if (match[i] == '+')
+        {
+            i++;
+        }
+        long exp = 0.0;
+        while ('0' <= match[i] && match[i] <= '9')
+        {
+            exp *= 10;
+            exp += (match[i] - '0');
+            i++;
+        }
+        if (exp_negative)
+        {
+            exp = -exp;
+        }
+        n = pow(n, exp);
+    }
+    if (negative)
+    {
+        n = -n;
+    }
+    $$ = JSONValue_new(JSON_NUMBER);
+    $$->number = n;
+>>
+token true <<
+  $$ = JSONValue_new(JSON_TRUE);
+>>
+token false <<
+  $$ = JSONValue_new(JSON_FALSE);
+>>
+token null <<
+  $$ = JSONValue_new(JSON_NULL);
+>>
+/"/ <<
+  $mode(string);
+  str_init(&string_value, "");
+>>
+string: token string /"/ <<
+  $$ = JSONValue_new(JSON_STRING);
+  $$->string = string_value;
+  $mode(default);
+>>
+string: /\\"/ <<
+  str_append(&string_value, "\"");
+>>
+string: /\\\\/ <<
+  str_append(&string_value, "\\");
+>>
+string: /\\\// <<
+  str_append(&string_value, "/");
+>>
+string: /\\b/ <<
+  str_append(&string_value, "\b");
+>>
+string: /\\f/ <<
+  str_append(&string_value, "\f");
+>>
+string: /\\n/ <<
+  str_append(&string_value, "\n");
+>>
+string: /\\r/ <<
+  str_append(&string_value, "\r");
+>>
+string: /\\t/ <<
+  str_append(&string_value, "\t");
+>>
+string: /\\u[0-9a-fA-F]{4}/ <<
+  /* Not actually going to encode the code point for this example... */
+  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
+  str_append(&string_value, s);
+>>
+string: /[^\\]/ <<
+  char s[] = {match[0], 0};
+  str_append(&string_value, s);
+>>
+Start -> Value <<
+  $$ = $1;
+>>
+Value -> string <<
+  $$ = $1;
+>>
+Value -> number <<
+  $$ = $1;
+>>
+Value -> Object <<
+  $$ = $1;
+>>
+Value -> Array <<
+  $$ = $1;
+>>
+Value -> true <<
+  $$ = $1;
+>>
+Value -> false <<
+  $$ = $1;
+>>
+Value -> null <<
+  $$ = $1;
+>>
+Object -> lbrace rbrace <<
+  $$ = JSONObject_new();
+>>
+Object -> lbrace KeyValues rbrace <<
+  $$ = $2;
+>>
+KeyValues -> KeyValue <<
+  $$ = $1;
+>>
+KeyValues -> KeyValues comma KeyValue <<
+  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
+  $$ = $1;
+>>
+KeyValue -> string colon Value <<
+  $$ = JSONObject_new();
+  JSONObject_append($$, str_cstr(&$1->string), $3);
+>>
+Array -> lbracket rbracket <<
+  $$ = JSONArray_new();
+>>
+Array -> lbracket Values rbracket <<
+  $$ = $2;
+>>
+Values -> Value <<
+  $$ = $1;
+>>
+Values -> Values comma Value <<
+  JSONArray_append($1, $3);
+  $$ = $1;
+>>

spec/json_types.c

@@ -0,0 +1,64 @@
+#include "json_types.h"
+#include <string.h>
+#include <stdlib.h>
+#include "testutils.h"
+
+JSONValue * JSONValue_new(size_t id)
+{
+    JSONValue * jv = calloc(1, sizeof(JSONValue));
+    jv->id = id;
+    return jv;
+}
+
+JSONValue * JSONObject_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_OBJECT);
+    jv->object.size = 0u;
+    return jv;
+}
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
+{
+    size_t const size = object->object.size;
+    for (size_t i = 0u; i < size; i++)
+    {
+        if (strcmp(name, object->object.entries[i].name) == 0)
+        {
+            object->object.entries[i].value = value;
+            return;
+        }
+    }
+    size_t const new_size = size + 1;
+    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
+    if (size > 0)
+    {
+        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
+        free(object->object.entries);
+    }
+    object->object.entries = new_entries;
+    object->object.entries[size].name = name;
+    object->object.entries[size].value = value;
+    object->object.size = new_size;
+}
+
+JSONValue * JSONArray_new(void)
+{
+    JSONValue * jv = JSONValue_new(JSON_ARRAY);
+    jv->array.size = 0u;
+    return jv;
+}
+
+void JSONArray_append(JSONValue * array, JSONValue * value)
+{
+    size_t const size = array->array.size;
+    size_t const new_size = size + 1;
+    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
+    if (array->array.size > 0)
+    {
+        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
+        free(array->array.entries);
+    }
+    array->array.entries = new_entries;
+    array->array.entries[size] = value;
+    array->array.size = new_size;
+}

spec/json_types.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <stddef.h>
+#include "testutils.h"
+
+#define JSON_OBJECT 0u
+#define JSON_ARRAY 1u
+#define JSON_NUMBER 2u
+#define JSON_STRING 3u
+#define JSON_TRUE 4u
+#define JSON_FALSE 5u
+#define JSON_NULL 6u
+
+typedef struct JSONValue_s
+{
+    size_t id;
+    union
+    {
+        struct
+        {
+            size_t size;
+            struct
+            {
+                char const * name;
+                struct JSONValue_s * value;
+            } * entries;
+        } object;
+        struct
+        {
+            size_t size;
+            struct JSONValue_s ** entries;
+        } array;
+        double number;
+        str_t string;
+    };
+} JSONValue;
+
+JSONValue * JSONValue_new(size_t id);
+
+JSONValue * JSONObject_new(void);
+
+void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
+
+JSONValue * JSONArray_new(void);
+
+void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane_spec.rb

@@ -665,7 +665,7 @@ EOF
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.propane"))
+        write_grammar(File.read("spec/json_parser.#{language}.propane"))
         build_parser(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end

spec/test_parsing_json.c

@@ -0,0 +1,56 @@
+#include "testparser.h"
+#include "json_types.h"
+#include <string.h>
+#include <assert.h>
+
+int main()
+{
+    char const * input = "";
+    p_context_t context;
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+
+    input = "{}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_OBJECT);
+
+    input = "[]";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_ARRAY);
+
+    input = "-45.6";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == -45.6);
+
+    input = "2E-2";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    assert(p_result(&context)->id == JSON_NUMBER);
+    assert(p_result(&context)->number == 0.02);
+
+    input = "{\"hi\":true}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    JSONValue * o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(1, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "hi") == 0);
+    assert(o->object.entries[0].value->id == JSON_TRUE);
+
+    input = "{\"ff\": false, \"nn\": null}";
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
+    assert(p_parse(&context) == P_SUCCESS);
+    o = p_result(&context);
+    assert(o->id == JSON_OBJECT);
+    assert_eq(2, o->object.size);
+    assert(strcmp(o->object.entries[0].name, "ff") == 0);
+    assert(o->object.entries[0].value->id == JSON_FALSE);
+    assert(strcmp(o->object.entries[1].name, "nn") == 0);
+    assert(o->object.entries[1].value->id == JSON_NULL);
+
+    return 0;
+}

spec/testutils.c

@@ -1,6 +1,9 @@
 #include <stdio.h>
 #include <assert.h>
 #include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include "testutils.h"
 
 void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
 {
@@ -10,3 +13,26 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
         assert(false);
     }
 }
+
+void str_init(str_t * str, char const * cs)
+{
+    size_t length = strlen(cs);
+    str->cs = malloc(length + 1u);
+    strcpy(str->cs, cs);
+}
+
+void str_append(str_t * str, char const * cs)
+{
+    size_t length = strlen(str->cs);
+    size_t length2 = strlen(cs);
+    char * new_cs = malloc(length + length2 + 1u);
+    memcpy(new_cs, str->cs, length);
+    strcpy(&new_cs[length], cs);
+    free(str->cs);
+    str->cs = new_cs;
+}
+
+void str_free(str_t * str)
+{
+    free(str->cs);
+}

spec/testutils.h

@@ -5,3 +5,15 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 #define assert_eq(expected, actual) \
     assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
 
+typedef struct
+{
+    char * cs;
+} str_t;
+
+void str_init(str_t * str, char const * cs);
+void str_append(str_t * str, char const * cs);
+void str_free(str_t * str);
+static inline char * str_cstr(str_t * str)
+{
+    return str->cs;
+}