wip

assets/parser.c.erb

@@ -7,7 +7,9 @@
  * User code blocks
  *************************************************************************/
 
-<%= @grammar.code_blocks.fetch("", "") %>
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 /**************************************************************************
  * Private types

assets/parser.d.erb

@@ -12,7 +12,9 @@ module <%= @grammar.modulename %>;
  * User code blocks
  *************************************************************************/
 
-<%= @grammar.code_blocks.fetch("", "") %>
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 /**************************************************************************
  * Public types

assets/parser.h.erb

@@ -36,9 +36,6 @@ typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>tok
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
-/** User header code blocks. */
-<%= @grammar.code_blocks.fetch("header", "") %>
-
 /** Parser values type(s). */
 typedef union
 {

doc/user_guide.md

@@ -13,9 +13,9 @@ Propane is an LR Parser Generator (LPG) which:
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
   * generates a table-driven parser to parse input in linear time
-  * target C or D language outputs
   * is MIT-licensed
   * is distributable as a standalone Ruby script
+  * supports D language
 
 #> Installation
 
@@ -103,89 +103,6 @@ E4 -> lparen E1 rparen <<
 >>
 ```
 
-##> User Code Blocks
-
-User code blocks begin with the line following a "<<" token and end with the
-line preceding a grammar line consisting of solely the ">>" token.
-All text lines in the code block are copied verbatim into the output file.
-
-### Standalone Code Blocks
-
-C example:
-
-```
-#include <stdio.h>
-```
-
-D example:
-
-```
-<<
-import std.stdio;
->>
-```
-
-Standalone code blocks are emitted early in the output file as top-level code
-outside the context of any function.
-Standalone code blocks are a good place to include/import any other necessary
-supporting code modules.
-They can also define helper functions that can be reused by lexer or parser
-user code blocks.
-They are emitted in the order they are defined in the grammar file.
-
-For a C target, the word "header" may immediately follow the "<<" token to
-cause Propane to emit the code block in the generated header file rather than
-the generated implementation file.
-This allows including another header that may be necessary to define any types
-needed by a `ptype` directive, for example:
-
-    <<header
-    #include "mytypes.h"
-    >>
-
-### Lexer pattern code blocks
-
-Example:
-
-```
-ptype ulong;
-
-token integer /\\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-```
-
-Lexer code blocks appear following a `token` or `pattern` expression.
-User code in a lexer code block will be executed when the lexer matches the
-given pattern.
-Assignment to the `$$` symbol will associate a parser value with the lexed
-token.
-This parser value can then be used later in a parser rule.
-
-### Parser rule code blocks
-
-Example:
-
-```
-E1 -> E1 plus E2 <<
-  $$ = $1 + $3;
->>
-```
-
-Parser rule code blocks appear following a rule expression.
-User code in a parser rule code block will be executed when the parser reduces
-the given rule.
-Assignment to the `$$` symbol will associate a parser value with the reduced
-rule.
-Parser values for the rules or tokens in the rule pattern can be accessed
-positionally with tokens `$1`, `$2`, `$3`, etc...
-
 #> License
 
 Propane is licensed under the terms of the MIT License:

lib/propane/grammar.rb

@@ -18,7 +18,7 @@ class Propane
       @patterns = []
       @tokens = []
       @rules = []
-      @code_blocks = {}
+      @code_blocks = []
       @line_number = 1
       @next_line_number = @line_number
       @mode = nil
@@ -191,13 +191,8 @@ class Propane
     end
 
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)\n(.*?)^>>\n/m)
+      if code = parse_code_block!
-        name, code = md[1..2]
+        @code_blocks << code
-        if @code_blocks[name]
-          @code_blocks[name] += code
-        else
-          @code_blocks[name] = code
-        end
         @mode = nil
         true
       end

spec/json_parser.c.propane

@@ -1,183 +0,0 @@
-<<header
-#include "json_types.h"
-#include "testutils.h"
->>
-<<
-#include "math.h"
-#include <stdbool.h>
-static str_t string_value;
->>
-
-ptype JSONValue *;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n = 0.0;
-    bool negative = false;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative = false;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = JSONValue_new(JSON_NUMBER);
-    $$->number = n;
->>
-token true <<
-  $$ = JSONValue_new(JSON_TRUE);
->>
-token false <<
-  $$ = JSONValue_new(JSON_FALSE);
->>
-token null <<
-  $$ = JSONValue_new(JSON_NULL);
->>
-/"/ <<
-  $mode(string);
-  str_init(&string_value, "");
->>
-string: token string /"/ <<
-  $$ = JSONValue_new(JSON_STRING);
-  $$->string = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  str_append(&string_value, "\"");
->>
-string: /\\\\/ <<
-  str_append(&string_value, "\\");
->>
-string: /\\\// <<
-  str_append(&string_value, "/");
->>
-string: /\\b/ <<
-  str_append(&string_value, "\b");
->>
-string: /\\f/ <<
-  str_append(&string_value, "\f");
->>
-string: /\\n/ <<
-  str_append(&string_value, "\n");
->>
-string: /\\r/ <<
-  str_append(&string_value, "\r");
->>
-string: /\\t/ <<
-  str_append(&string_value, "\t");
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
-  str_append(&string_value, s);
->>
-string: /[^\\]/ <<
-  char s[] = {match[0], 0};
-  str_append(&string_value, s);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONObject_new();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
-  $$ = $1;
->>
-KeyValue -> string colon Value <<
-  $$ = JSONObject_new();
-  JSONObject_append($$, str_cstr(&$1->string), $3);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONArray_new();
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  JSONArray_append($1, $3);
-  $$ = $1;
->>

spec/json_types.c

@@ -1,64 +0,0 @@
-#include "json_types.h"
-#include <string.h>
-#include <stdlib.h>
-#include "testutils.h"
-
-JSONValue * JSONValue_new(size_t id)
-{
-    JSONValue * jv = calloc(1, sizeof(JSONValue));
-    jv->id = id;
-    return jv;
-}
-
-JSONValue * JSONObject_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_OBJECT);
-    jv->object.size = 0u;
-    return jv;
-}
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
-{
-    size_t const size = object->object.size;
-    for (size_t i = 0u; i < size; i++)
-    {
-        if (strcmp(name, object->object.entries[i].name) == 0)
-        {
-            object->object.entries[i].value = value;
-            return;
-        }
-    }
-    size_t const new_size = size + 1;
-    void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
-    if (size > 0)
-    {
-        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
-        free(object->object.entries);
-    }
-    object->object.entries = new_entries;
-    object->object.entries[size].name = name;
-    object->object.entries[size].value = value;
-    object->object.size = new_size;
-}
-
-JSONValue * JSONArray_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_ARRAY);
-    jv->array.size = 0u;
-    return jv;
-}
-
-void JSONArray_append(JSONValue * array, JSONValue * value)
-{
-    size_t const size = array->array.size;
-    size_t const new_size = size + 1;
-    JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
-    if (array->array.size > 0)
-    {
-        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
-        free(array->array.entries);
-    }
-    array->array.entries = new_entries;
-    array->array.entries[size] = value;
-    array->array.size = new_size;
-}

spec/json_types.h

@@ -1,46 +0,0 @@
-#pragma once
-
-#include <stddef.h>
-#include "testutils.h"
-
-#define JSON_OBJECT 0u
-#define JSON_ARRAY 1u
-#define JSON_NUMBER 2u
-#define JSON_STRING 3u
-#define JSON_TRUE 4u
-#define JSON_FALSE 5u
-#define JSON_NULL 6u
-
-typedef struct JSONValue_s
-{
-    size_t id;
-    union
-    {
-        struct
-        {
-            size_t size;
-            struct
-            {
-                char const * name;
-                struct JSONValue_s * value;
-            } * entries;
-        } object;
-        struct
-        {
-            size_t size;
-            struct JSONValue_s ** entries;
-        } array;
-        double number;
-        str_t string;
-    };
-} JSONValue;
-
-JSONValue * JSONValue_new(size_t id);
-
-JSONValue * JSONObject_new(void);
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
-
-JSONValue * JSONArray_new(void);
-
-void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane_spec.rb

@@ -665,7 +665,7 @@ EOF
       end
 
       it "allows creating a JSON parser" do
-        write_grammar(File.read("spec/json_parser.#{language}.propane"))
+        write_grammar(File.read("spec/json_parser.propane"))
         build_parser(language: language)
         compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
       end

spec/test_parsing_json.c

@@ -1,56 +0,0 @@
-#include "testparser.h"
-#include "json_types.h"
-#include <string.h>
-#include <assert.h>
-
-int main()
-{
-    char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
-
-    input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
-
-    input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
-
-    input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
-
-    input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(1, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "hi") == 0);
-    assert(o->object.entries[0].value->id == JSON_TRUE);
-
-    input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(2, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "ff") == 0);
-    assert(o->object.entries[0].value->id == JSON_FALSE);
-    assert(strcmp(o->object.entries[1].name, "nn") == 0);
-    assert(o->object.entries[1].value->id == JSON_NULL);
-
-    return 0;
-}

spec/testutils.c

@@ -1,9 +1,6 @@
 #include <stdio.h>
 #include <assert.h>
 #include <stdbool.h>
-#include <stdlib.h>
-#include <string.h>
-#include "testutils.h"
 
 void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
 {
@@ -13,26 +10,3 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
         assert(false);
     }
 }
-
-void str_init(str_t * str, char const * cs)
-{
-    size_t length = strlen(cs);
-    str->cs = malloc(length + 1u);
-    strcpy(str->cs, cs);
-}
-
-void str_append(str_t * str, char const * cs)
-{
-    size_t length = strlen(str->cs);
-    size_t length2 = strlen(cs);
-    char * new_cs = malloc(length + length2 + 1u);
-    memcpy(new_cs, str->cs, length);
-    strcpy(&new_cs[length], cs);
-    free(str->cs);
-    str->cs = new_cs;
-}
-
-void str_free(str_t * str)
-{
-    free(str->cs);
-}

spec/testutils.h

@@ -5,15 +5,3 @@ void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_
 #define assert_eq(expected, actual) \
     assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
 
-typedef struct
-{
-    char * cs;
-} str_t;
-
-void str_init(str_t * str, char const * cs);
-void str_append(str_t * str, char const * cs);
-void str_free(str_t * str);
-static inline char * str_cstr(str_t * str)
-{
-    return str->cs;
-}