Add RuleSet#start_token_set

.github/workflows/run-tests.yml

@@ -1,38 +0,0 @@
-name: Run Propane Tests
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        ruby-version: ['3.4']
-
-    steps:
-    - name: Install dependencies (Linux)
-      if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc
-
-    - name: Install dependencies (macOS)
-      if: runner.os == 'macOS'
-      run: brew install gcc ldc
-
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-
-    - name: Install dependencies
-      run: bundle install
-
-    - name: Run tests
-      run: rake all

.gitignore

@@ -2,8 +2,7 @@
 /.yardoc
 /_yardoc/
 /coverage/
-/dist/
+/doc/
-/gen/
 /pkg/
 /spec/reports/
 /tmp/

.rspec

@@ -1,2 +1,3 @@
+--format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,122 +0,0 @@
-## v2.3.0
-
-### New Features
-
-- Add \D, \S, \w, \W special character classes
-
-### Improvements
-
-- Include line numbers for pattern errors
-- Improve performance in a few places
-- Parallelize parser table generation on Linux hosts
-- Add github workflow to run unit tests
-
-### Fixes
-
-- Fix a couple clang warnings for C backend
-- Fix C backend not fully initializing pvalues when multiple ptypes are used with different sizes.
-- Fix some user guide examples
-
-## v2.2.1
-
-### Fixes
-
-- Fix GC issue for D backend when AST is enabled (#36)
-
-## v2.2.0
-
-### Improvements
-
-- Allow multiple lexer modes to be specified for a lexer pattern (#35)
-- Document p_decode_code_point() API function (#34)
-
-## v2.1.1
-
-### Fixes
-
-- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
-
-## v2.1.0
-
-### Improvements
-
-- Report rule name and line number for conflicting AST node field positions errors (#32)
-
-## v2.0.0
-
-### Improvements
-
-- Log conflicting rules on reduce/reduce conflict (#31)
-- Use 1-based row and column values for position values (#30)
-
-### Fixes
-
-- Fix named optional rules (#29)
-
-### Upgrading
-
-- Adjust all uses of p_position_t row and col values to expect 1-based instead
-of 0-based values.
-
-## v1.5.1
-
-### Improvements
-
-- Improve performance (#28)
-
-## v1.5.0
-
-### New Features
-
-- Track start and end text positions for tokens and rules in AST node structures (#27)
-- Add warnings for shift/reduce conflicts to log file (#25)
-- Add -w command line switch to treat warnings as errors and output to stderr (#26)
-- Add rule field aliases (#24)
-
-### Improvements
-
-- Show line numbers of rules on conflict (#23)
-
-## v1.4.0
-
-### New Features
-
-- Allow user to specify AST node name prefix or suffix
-- Allow specifying the start rule name
-- Allow rule terms to be marked as optional
-
-### Improvements
-
-- Give a better error message when a referenced ptype has not been declared
-
-## v1.3.0
-
-### New Features
-
-- Add AST generation (#22)
-
-## v1.2.0
-
-### New Features
-
-- Allow one line user code blocks (#21)
-- Add backslash escape codes (#19)
-- Add API to access unexpected token found (#18)
-- Add token_names API (#17)
-- Add D example to user guide for p_context_init() (#16)
-- Allow user termination from lexer code blocks (#15)
-
-### Fixes
-
-- Fix generator hang when state transition cycle is present (#20)
-
-## v1.1.0
-
-### New Features
-
-- Add user parser terminations (#13)
-- Document generated parser API in user guide (#14)
-
-## v1.0.0
-
-- Initial release

Gemfile

@@ -1,9 +1,4 @@
 source "https://rubygems.org"
 
-gem "base64"
 gem "rake"
 gem "rspec"
-gem "rdoc"
-gem "redcarpet"
-gem "syntax"
-gem "simplecov"

Gemfile.lock

@@ -1,54 +1,28 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    base64 (0.3.0)
+    diff-lcs (1.5.0)
-    date (3.5.1)
+    rake (13.0.6)
-    diff-lcs (1.6.2)
+    rspec (3.11.0)
-    docile (1.4.1)
+      rspec-core (~> 3.11.0)
-    erb (6.0.1)
+      rspec-expectations (~> 3.11.0)
-    psych (5.3.1)
+      rspec-mocks (~> 3.11.0)
-      date
+    rspec-core (3.11.0)
-      stringio
+      rspec-support (~> 3.11.0)
-    rake (13.3.1)
+    rspec-expectations (3.11.0)
-    rdoc (7.1.0)
-      erb
-      psych (>= 4.0.0)
-      tsort
-    redcarpet (3.6.1)
-    rspec (3.13.2)
-      rspec-core (~> 3.13.0)
-      rspec-expectations (~> 3.13.0)
-      rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.6)
-      rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
+      rspec-support (~> 3.11.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
+      rspec-support (~> 3.11.0)
-    rspec-support (3.13.6)
+    rspec-support (3.11.0)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.13.2)
-    simplecov_json_formatter (0.1.4)
-    stringio (3.2.0)
-    syntax (1.2.2)
-    tsort (0.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  base64
   rake
-  rdoc
-  redcarpet
   rspec
-  simplecov
-  syntax
 
 BUNDLED WITH
-   2.3.7
+   2.4.0.dev

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2022 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,104 +1,21 @@
 # The Propane Parser Generator
 
-Propane is a LALR Parser Generator (LPG) which:
+Propane is an LR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven shift/reduce parser to parse input in linear time
+  * generates a table-driven parser to parse input in linear time
-  * targets C, C++, or D language outputs
-  * optionally supports automatic full AST generation
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
 ## Installation
 
-Propane is designed to be distributed as a stand-alone single file script that
+TODO
-can be copied into and versioned in a project's source tree.
-The only requirement to run Propane is that the system has a Ruby interpreter
-installed.
-The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
-
-Simply copy the `propane` executable script into the desired location within
-the project to be built (typically the root of the repository) and mark it
-executable.
 
 ## Usage
 
-### Command Line Interface
+TODO: Write usage instructions here
-
-Propane is typically invoked from the command-line as `./propane`.
-
-    Usage: ./propane [options] <input-file> <output-file>
-    Options:
-      -h, --help  Show this usage and exit.
-      --log LOG   Write log file. This will show all parser states and their
-                  associated shifts and reduces. It can be helpful when
-                  debugging a grammar.
-      --version   Show program version and exit.
-      -w          Treat warnings as errors. This option will treat shift/reduce
-                  conflicts as fatal errors and will print them to stderr in
-                  addition to the log file.
-
-The user must specify the path to a Propane input grammar file and a path to an
-output file.
-The generated source code will be written to the output file.
-If a log file path is specified, Propane will write a log file containing
-detailed information about the parser states and transitions.
-
-### Propane Grammar File
-
-A Propane grammar file provides Propane with the patterns, tokens, grammar
-rules, and user code blocks from which to build the generated lexer and parser.
-
-Example grammar file:
-
-```
-<<
-import std.math;
->>
-
-# Parser values are unsigned integers.
-ptype ulong;
-
-# A few basic arithmetic operators.
-token plus /\+/;
-token times /\*/;
-token power /\*\*/;
-token integer /\d+/ <<
-  ulong v;
-  foreach (c; match)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\(/;
-token rparen /\)/;
-# Drop whitespace.
-drop /\s+/;
-
-Start -> E1 << $$ = $1; >>
-E1 -> E2 << $$ = $1; >>
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E2 -> E3 << $$ = $1; >>
-E2 -> E2 times E3 << $$ = $1 * $3; >>
-E3 -> E4 << $$ = $1; >>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer << $$ = $1; >>
-E4 -> lparen E1 rparen << $$ = $2; >>
-```
-
-Grammar files can contain comment lines beginning with `#` which are ignored.
-White space in the grammar file is also ignored.
-
-It is convention to use the extension `.propane` for the Propane grammar file,
-however any file name is accepted by Propane.
-
-See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
 
 ## Development
 

Rakefile

@@ -0,0 +1,9 @@
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
+  if args.example_pattern
+    task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  end
+end
+
+task :default => :spec

Rakefile.rb

@@ -1,46 +0,0 @@
-require "rake/clean"
-require "rspec/core/rake_task"
-require "simplecov"
-require "stringio"
-
-CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
-
-task :build_dist do
-  sh "ruby rb/build_dist.rb"
-end
-
-RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
-  if args.example_pattern
-    task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
-  end
-end
-task :spec do |task, args|
-  unless ENV["dist_specs"]
-    original_stdout = $stdout
-    sio = StringIO.new
-    $stdout = sio
-    SimpleCov.collate Dir["coverage/.resultset.json"]
-    $stdout = original_stdout
-    sio.string.lines.each do |line|
-      $stdout.write(line) unless line =~ /Coverage report generated for/
-    end
-  end
-end
-
-# dspec task is useful to test the distributable release script, but is not
-# useful for coverage information.
-desc "Dist Specs"
-task :dspec, [:example_string] => :build_dist do |task, args|
-  ENV["dist_specs"] = "1"
-  Rake::Task["spec"].execute(args)
-  ENV.delete("dist_specs")
-end
-
-task :default => :spec
-
-desc "Build user guide"
-task :user_guide do
-  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
-end
-
-task :all => [:spec, :dspec, :user_guide]

assets/parser.h.erb

@@ -1,219 +0,0 @@
-/**
- * @file
- *
- * This file is generated by Propane.
- */
-
-#pragma once
-
-#include <stdint.h>
-#include <stddef.h>
-
-/**************************************************************************
- * Public types
- *************************************************************************/
-
-/* Result codes. */
-#define <%= @grammar.prefix.upcase %>SUCCESS 0u
-#define <%= @grammar.prefix.upcase %>DECODE_ERROR 1u
-#define <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT 2u
-#define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
-#define <%= @grammar.prefix.upcase %>DROP 4u
-#define <%= @grammar.prefix.upcase %>EOF 5u
-#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
-
-/** Token type. */
-typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
-
-/** Token IDs. */
-<% @grammar.tokens.each_with_index do |token, index| %>
-#define TOKEN_<%= token.code_name %> <%= index %>u
-<%   unless token.id == index %>
-<%     raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
-<%   end %>
-<% end %>
-#define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
-#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
-
-/** Code point type. */
-typedef uint32_t <%= @grammar.prefix %>code_point_t;
-
-/**
- * A structure to keep track of input position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
-{
-    /** Input text row (0-based). */
-    uint32_t row;
-
-    /** Input text column (0-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
-
-/** Return whether the position is valid. */
-#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
-
-/** User header code blocks. */
-<%= @grammar.code_blocks.fetch("header", "") %>
-
-<% if @grammar.ast %>
-/** Parser values type. */
-typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
-<% else %>
-/** Parser values type(s). */
-typedef union
-{
-<%   @grammar.ptypes.each do |name, typestring| %>
-    <%= typestring %> v_<%= name %>;
-<%   end %>
-} <%= @grammar.prefix %>value_t;
-<% end %>
-
-<% if @grammar.ast %>
-/** AST node types. @{ */
-typedef struct <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>
-{
-    <% # ASTNode fields must be present in the same order here. # %>
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-    <%= @grammar.prefix %>token_t token;
-    <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.ast_prefix %>Token<%= @grammar.ast_suffix %>;
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-struct <%= name %>;
-<%   end %>
-
-<%   @parser.rule_sets.each do |name, rule_set| %>
-<%     next if name.start_with?("$") %>
-<%     next if rule_set.optional? %>
-typedef struct <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>
-{
-    <% # ASTNode fields must be present in the same order here. # %>
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
-    uint16_t n_fields;
-    uint8_t is_token;
-<%     rule_set.ast_fields.each do |fields| %>
-    union
-    {
-<%       fields.each do |field_name, type| %>
-        struct <%= type %> * <%= field_name %>;
-<%       end %>
-    };
-<%     end %>
-} <%= @grammar.ast_prefix %><%= name %><%= @grammar.ast_suffix %>;
-
-<%   end %>
-/** @} */
-<% end %>
-
-/** Lexed token information. */
-typedef struct
-{
-    /** Text position of first code point in token. */
-    <%= @grammar.prefix %>position_t position;
-
-    /** Text position of last code point in token. */
-    <%= @grammar.prefix %>position_t end_position;
-
-    /** Number of input bytes used by the token. */
-    size_t length;
-
-    /** Token that was lexed. */
-    <%= @grammar.prefix %>token_t token;
-
-    /** Parser value associated with the token. */
-    <%= @grammar.prefix %>value_t pvalue;
-} <%= @grammar.prefix %>token_info_t;
-
-/**
- * Lexer and parser context.
- *
- * The user must allocate an instance of this structure and pass it to any
- * public API function.
- */
-typedef struct
-{
-    /* Lexer context data. */
-
-    /** Input text. */
-    uint8_t const * input;
-
-    /** Input text length. */
-    size_t input_length;
-
-    /** Input text index (byte offset). */
-    size_t input_index;
-
-    /** Input text position (row/column). */
-    <%= @grammar.prefix %>position_t text_position;
-
-    /** Current lexer mode. */
-    size_t mode;
-
-    /* Parser context data. */
-
-    /** Parse result value. */
-<% if @grammar.ast %>
-    void * parse_result;
-<% else %>
-    <%= @grammar.prefix %>value_t parse_result;
-<% end %>
-
-    /** Unexpected token received. */
-    <%= @grammar.prefix %>token_t token;
-
-    /** User terminate code. */
-    size_t user_terminate_code;
-} <%= @grammar.prefix %>context_t;
-
-/**************************************************************************
- * Public data
- *************************************************************************/
-
-/** Token names. */
-extern const char * <%= @grammar.prefix %>token_names[];
-
-void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
-
-size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
-        <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
-
-size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
-
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-<%   end %>
-
-<% if @grammar.ast %>
-<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-<%   end %>
-<% else %>
-<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-<%   end %>
-<% end %>
-
-<% if @grammar.ast %>
-void <%= @grammar.prefix %>free_ast(<%= @grammar.ast_prefix %><%= @grammar.start_rules[0] %><%= @grammar.ast_suffix %> * ast);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-void <%= @grammar.prefix %>free_ast_<%= start_rule %>(<%= @grammar.ast_prefix %><%= start_rule %><%= @grammar.ast_suffix %> * ast);
-<%   end %>
-<% end %>
-
-<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
-
-size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
-
-<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

extra/vim/ftdetect/propane.vim

@@ -1 +0,0 @@
-au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -1,33 +0,0 @@
-" Vim syntax file for Propane
-" Language: propane
-" Maintainer: Josh Holtrop
-" URL: https://github.com/holtrop/propane
-
-if exists("b:current_syntax")
-  finish
-endif
-
-if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
-endif
-
-exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
-
-syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
-
-syn match propaneComment "#.*"
-syn match propaneOperator "->"
-syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
-syn match propaneFieldOperator ":" contained
-syn match propaneOperator "?"
-syn keyword propaneKeyword ast ast_prefix ast_suffix drop module prefix ptype start token tokenid
-
-syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
-
-hi def link propaneComment Comment
-hi def link propaneKeyword Keyword
-hi def link propaneRegex String
-hi def link propaneOperator Operator
-hi def link propaneFieldOperator Operator
-hi def link propaneDelimiter Delimiter
-hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -1,7 +1,5 @@
 require "erb"
 require "set"
-require "stringio"
-require_relative "propane/assets"
 require_relative "propane/cli"
 require_relative "propane/code_point_range"
 require_relative "propane/fa"
@@ -14,27 +12,37 @@ require_relative "propane/lexer/dfa"
 require_relative "propane/parser"
 require_relative "propane/parser/item"
 require_relative "propane/parser/item_set"
-require_relative "propane/pattern"
 require_relative "propane/regex"
 require_relative "propane/regex/nfa"
 require_relative "propane/regex/unit"
 require_relative "propane/rule_set"
 require_relative "propane/rule"
 require_relative "propane/token"
-require_relative "propane/util"
 require_relative "propane/version"
 
 class Propane
 
+  # EOF.
+  TOKEN_EOF = 0xFFFFFFFC
+
+  # Decoding error.
+  TOKEN_DECODE_ERROR = 0xFFFFFFFD
+
+  # Token ID for a "dropped" token.
+  TOKEN_DROP = 0xFFFFFFFE
+
+  # Invalid token ID.
+  TOKEN_NONE = 0xFFFFFFFF
+
   class Error < RuntimeError
   end
 
   class << self
 
-    def run(input_file, output_file, log_file, options)
+    def run(input_file, output_file, log_file)
       begin
         grammar = Grammar.new(File.read(input_file))
-        generator = Generator.new(grammar, output_file, log_file, options)
+        generator = Generator.new(grammar, output_file, log_file)
         generator.generate
       rescue Error => e
         $stderr.puts e.message

lib/propane/assets.rb

@@ -1,10 +0,0 @@
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
-        File.binread(path)
-      end
-    end
-  end
-end

lib/propane/cli.rb

@@ -4,21 +4,15 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  -h, --help  Show this usage and exit.
+  --log LOG   Write log file
-  --log LOG   Write log file. This will show all parser states and their
+  --version   Show program version and exit
-              associated shifts and reduces. It can be helpful when
+  -h, --help  Show this usage and exit
-              debugging a grammar.
-  --version   Show program version and exit.
-  -w          Treat warnings as errors. This option will treat shift/reduce
-              conflicts as fatal errors and will print them to stderr in
-              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
-        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -30,13 +24,11 @@ EOF
               log_file = args[i]
             end
           when "--version"
-            puts "propane version #{VERSION}"
+            puts "propane v#{VERSION}"
             return 0
           when "-h", "--help"
             puts USAGE
             return 0
-          when "-w"
-            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -53,7 +45,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file, options)
+        Propane.run(*params, log_file)
       end
 
     end

lib/propane/generator.rb

@@ -2,123 +2,43 @@ class Propane
 
   class Generator
 
-    def initialize(grammar, output_file, log_file, options)
+    def initialize(grammar, output_file, log_file)
       @grammar = grammar
       @output_file = output_file
-      if log_file
+      @log_file = log_file
-        @log = File.open(log_file, "wb")
+      @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
-      else
-        @log = StringIO.new
-      end
-      @language =
-        if output_file.end_with?(".d")
-          "d"
-        else
-          "c"
-        end
-      @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [nil]
+      erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.d.erb")), trim_mode: "<>")
-      if @language == "c"
-        extensions += %w[h]
-      end
-      extensions.each do |extension|
-        template = Assets.get("parser.#{extension || @language}.erb")
-        if extension
-          output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
-        else
-          output_file = @output_file
-        end
-        erb = ERB.new(template, trim_mode: "<>")
       result = erb.result(binding.clone)
-        File.open(output_file, "wb") do |fh|
+      File.open(@output_file, "wb") do |fh|
         fh.write(result)
       end
     end
-      @log.close
-    end
 
     private
 
     def process_grammar!
-      # Assign default pattern mode to patterns without a mode assigned.
-      found_default = false
-      @grammar.patterns.each do |pattern|
-        if pattern.modes.empty?
-          pattern.modes << "default"
-          found_default = true
-        end
-        pattern.ptypename ||= "default"
-      end
-      unless found_default
-        raise Error.new("No patterns found for default mode")
-      end
-      check_ptypes!
-      # Add EOF token.
-      @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
-      @grammar.tokens.each_with_index do |token, token_id|
+      @grammar.tokens.each do |token|
-        # Assign token ID.
-        token.id = token_id
-        # Set default ptypename if none given.
-        token.ptypename ||= "default"
-        # Check for token name conflicts.
         if tokens_by_name.include?(token.name)
           raise Error.new("Duplicate token name #{token.name.inspect}")
         end
         tokens_by_name[token.name] = token
       end
-      # Create real start rule(s).
-      real_start_rules = @grammar.start_rules.map do |start_rule|
-        unless @grammar.rules.find {|rule| rule.name == start_rule}
-          raise Error.new("Start rule `#{start_rule}` not found")
-        end
-        Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil)
-      end
-      # Add real start rules before user-given rules.
-      @grammar.rules = real_start_rules + @grammar.rules
-      # Generate and add rules for optional components.
-      generate_optional_component_rules!(tokens_by_name)
-      # Build rule sets.
       rule_sets = {}
-      rule_set_id = @grammar.tokens.size
+      @grammar.rules.each do |rule|
-      @grammar.rules.each_with_index do |rule, rule_id|
-        # Assign rule ID.
-        rule.id = rule_id
-        # Check for token/rule name conflict.
         if tokens_by_name.include?(rule.name)
           raise Error.new("Rule name collides with token name #{rule.name.inspect}")
         end
-        # Build rule sets of all rules with the same name.
+        rule_sets[rule.name] ||= RuleSet.new(rule.name)
-        unless rule_sets[rule.name]
+        rule_sets[rule.name] << rule
-          rule_sets[rule.name] = RuleSet.new(rule.name, rule_set_id)
-          rule_set_id += 1
       end
-        rule_set = rule_sets[rule.name]
+      unless rule_sets["Start"]
-        if rule_set.ptypename && rule.ptypename && rule_set.ptypename != rule.ptypename
+        raise Error.new("Start rule not found")
-          raise Error.new("Conflicting ptypes for rule #{rule.name}")
       end
-        rule_set.ptypename ||= rule.ptypename
-        rule.rule_set = rule_set
-        rule_set << rule
-      end
-      rule_sets.each do |name, rule_set|
-        rule_set.ptypename ||= "default"
-        # Assign rule set ptypenames back to rules.
-        rule_set.rules.each do |rule|
-          rule.ptypename = rule_set.ptypename
-        end
-      end
-      # Generate lexer user code IDs for lexer patterns with user code blocks.
-      @grammar.patterns.select do |pattern|
-        pattern.code
-      end.each_with_index do |pattern, code_id|
-        pattern.code_id = code_id
-      end
-      # Map rule components from names to Token/RuleSet objects.
       @grammar.rules.each do |rule|
         rule.components.map! do |component|
           if tokens_by_name[component]
@@ -131,55 +51,8 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
-      rule_sets.each do |name, rule_set|
+      @lexer = Lexer.new(@grammar.tokens, @grammar.drop_tokens)
-        rule_set.finalize(@grammar)
+      @parser = Parser.new(rule_sets["Start"])
-      end
-      # Generate the lexer.
-      @lexer = Lexer.new(@grammar)
-      # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, @log, @options)
-    end
-
-    # Check that any referenced ptypes have been defined.
-    def check_ptypes!
-      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
-        if potor.ptypename
-          unless @grammar.ptypes.include?(potor.ptypename)
-            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
-          end
-        end
-      end
-    end
-
-    # Generate and add rules for any optional components.
-    def generate_optional_component_rules!(tokens_by_name)
-      optional_rules_added = Set.new
-      @grammar.rules.each do |rule|
-        rule.components.each do |component|
-          if component =~ /^(.*)\?$/
-            c = $1
-            unless optional_rules_added.include?(component)
-              # Create two rules for the optional component: one empty and
-              # one just matching the component.
-              # We need to find the ptypename for the optional component in
-              # order to copy it to the generated rules.
-              if tokens_by_name[c]
-                # The optional component is a token.
-                ptypename = tokens_by_name[c].ptypename
-              else
-                # The optional component must be a rule, so find any instance
-                # of that rule that specifies a ptypename.
-                ptypename = @grammar.rules.reduce(nil) do |result, rule|
-                  rule.name == c && rule.ptypename ? rule.ptypename : result
-                end
-              end
-              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              @grammar.rules << Rule.new(component, [c], "$$ = $1;\n", ptypename, rule.line_number)
-              optional_rules_added << component
-            end
-          end
-        end
-      end
     end
 
     # Determine which grammar rules could expand to empty sequences.
@@ -238,142 +111,6 @@ class Propane
       end
     end
 
-    # Expand expansions in user code block.
-    #
-    # @param code [String]
-    #   User code block.
-    # @param parser [Boolean]
-    #   Whether the user code is for the parser or lexer.
-    # @param rule [Rule, nil]
-    #   The Rule associated with the user code if user code is for the parser.
-    # @param pattern [Pattern, nil]
-    #   The Pattern associated with the user code if user code is for the lexer.
-    #
-    # @return [String]
-    #   Expanded user code block.
-    def expand_code(code, parser, rule, pattern)
-      code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
-        "TOKEN_#{Token.code_name($1)}"
-      end
-      code = code.gsub(/\$terminate\((.*)\);/) do |match|
-        user_terminate_code = $1
-        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
-        case @language
-        when "c"
-          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        when "d"
-          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        end
-      end
-      if parser
-        code = code.gsub(/\$\$/) do |match|
-          case @language
-          when "c"
-            "_pvalue->v_#{rule.ptypename}"
-          when "d"
-            "_pvalue.v_#{rule.ptypename}"
-          end
-        end
-        code = code.gsub(/\$(\d+)/) do |match|
-          index = $1.to_i
-          case @language
-          when "c"
-            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          when "d"
-            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-          end
-        end
-        code = code.gsub(/\$\{(\w+)\}/) do |match|
-          aliasname = $1
-          if index = rule.aliases[aliasname]
-            case @language
-            when "c"
-              "state_values_stack_index(statevalues, -(int)n_states + #{index})->pvalue.v_#{rule.components[index].ptypename}"
-            when "d"
-              "statevalues[$-n_states+#{index}].pvalue.v_#{rule.components[index].ptypename}"
-            end
-          else
-            raise Error.new("Field alias '#{aliasname}' not found")
-          end
-        end
-      else
-        code = code.gsub(/\$\$/) do |match|
-          if @grammar.ast
-            case @language
-            when "c"
-              "out_token_info->pvalue"
-            when "d"
-              "out_token_info.pvalue"
-            end
-          else
-            case @language
-            when "c"
-              "out_token_info->pvalue.v_#{pattern.ptypename}"
-            when "d"
-              "out_token_info.pvalue.v_#{pattern.ptypename}"
-            end
-          end
-        end
-        code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
-          mode_name = $1
-          mode_id = @lexer.mode_id(mode_name)
-          unless mode_id
-            raise Error.new("Lexer mode '#{mode_name}' not found")
-          end
-          case @language
-          when "c"
-            "context->mode = #{mode_id}u"
-          when "d"
-            "context.mode = #{mode_id}u"
-          end
-        end
-      end
-      code
-    end
-
-    # Get the parser value type for the start rule.
-    #
-    # @return [Array<String>]
-    #   Start rule parser value type name and type string.
-    def start_rule_type(start_rule_index = 0)
-      start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rules[start_rule_index]
-      end
-      [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
-    end
-
-    # Get an unsigned integer type that can hold the given maximum value.
-    #
-    # @param max [Integer]
-    #   Maximum value to store.
-    #
-    # @return [String]
-    #   Type.
-    def get_type_for(max)
-      if max <= 0xFF
-        case @language
-        when "c"
-          "uint8_t"
-        when "d"
-          "ubyte"
-        end
-      elsif max <= 0xFFFF
-        case @language
-        when "c"
-          "uint16_t"
-        when "d"
-          "ushort"
-        end
-      else
-        case @language
-        when "c"
-          "uint32_t"
-        else
-          "uint"
-        end
-      end
-    end
-
   end
 
 end

lib/propane/grammar.rb

@@ -2,337 +2,60 @@ class Propane
 
   class Grammar
 
-    # Reserve identifiers beginning with a double-underscore for internal use.
+    attr_reader :classname
-    IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
+    attr_reader :drop_tokens
-
-    attr_reader :ast
-    attr_reader :ast_prefix
-    attr_reader :ast_suffix
-    attr_reader :free_token_node
     attr_reader :modulename
-    attr_reader :patterns
+    attr_reader :rules
-    attr_accessor :rules
-    attr_reader :start_rules
     attr_reader :tokens
-    attr_reader :code_blocks
-    attr_reader :ptypes
-    attr_reader :prefix
 
     def initialize(input)
-      @patterns = []
-      @start_rules = []
       @tokens = []
+      @drop_tokens = []
       @rules = []
-      @code_blocks = {}
+      input = input.gsub("\r\n", "\n")
-      @line_number = 1
+      parse_grammar(input)
-      @next_line_number = @line_number
-      @modeline = nil
-      @input = input.gsub("\r\n", "\n")
-      @ptypes = {"default" => "void *"}
-      @prefix = "p_"
-      @ast = false
-      @ast_prefix = ""
-      @ast_suffix = ""
-      @free_token_node = nil
-      parse_grammar!
-      @start_rules << "Start" if @start_rules.empty?
-    end
-
-    def ptype
-      @ptypes["default"]
-    end
-
-    def invalid_token_id
-      @tokens.size
-    end
-
-    def terminate_token_id
-      @tokens.size + 1
     end
 
     private
 
-    def parse_grammar!
+    def parse_grammar(input)
-      while @input.size > 0
+      line_number = 1
-        parse_statement!
+      while !input.empty?
+        if sliced = input.slice!(/\A\s+/)
+          # Skip white space.
+        elsif sliced = input.slice!(/\A#.*\n/)
+          # Skip comment lines.
+        elsif sliced = input.slice!(/\Amodule\s+(\S+)\n/)
+          @modulename = $1
+        elsif sliced = input.slice!(/\Aclass\s+(\S+)\n/)
+          @classname = $1
+        elsif sliced = input.slice!(/\Atoken\s+(\S+)(?:\s+(\S+))?\n/)
+          name, pattern = $1, $2
+          if pattern.nil?
+            pattern = name
           end
+          unless name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
+            raise Error.new("Invalid token name #{name.inspect}")
           end
-
+          @tokens << Token.new(name: name, pattern: pattern, id: @tokens.size, line_number: line_number)
-    def parse_statement!
+        elsif sliced = input.slice!(/\Adrop\s+(\S+)\n/)
-      if parse_white_space!
+          pattern = $1
-      elsif parse_comment_line!
+          @drop_tokens << Token.new(pattern: pattern, line_number: line_number)
-      elsif @modeline.nil? && parse_mode_label!
+        elsif sliced = input.slice!(/\A(\S+)\s*->\s*(.*?)(?:;|<<\n(.*?)^>>\n)/m)
-      elsif parse_ast_statement!
+          rule_name, components, code = $1, $2, $3
-      elsif parse_ast_prefix_statement!
+          unless rule_name =~ /^[a-zA-Z_][a-zA-Z_0-9]*$/
-      elsif parse_ast_suffix_statement!
+            raise Error.new("Invalid rule name #{name.inspect}")
-      elsif parse_free_token_node_statement!
+          end
-      elsif parse_module_statement!
+          components = components.strip.split(/\s+/)
-      elsif parse_ptype_statement!
+          # Reserve rule ID 0 for the "real" start rule.
-      elsif parse_pattern_statement!
+          @rules << Rule.new(rule_name, components, code, line_number, @rules.size + 1)
-      elsif parse_start_statement!
-      elsif parse_token_statement!
-      elsif parse_tokenid_statement!
-      elsif parse_drop_statement!
-      elsif parse_rule_statement!
-      elsif parse_code_block_statement!
-      elsif parse_prefix_statement!
         else
-        if @input.size > 25
+          if input.size > 25
-          @input = @input.slice(0..20) + "..."
+            input = input.slice(0..20) + "..."
           end
-        raise Error.new("Unexpected grammar input at line #{@line_number}: #{@input.chomp}")
+          raise Error.new("Unexpected grammar input at line #{line_number}: #{input.chomp}")
         end
-    end
+        line_number += sliced.count("\n")
-
-    def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
-        @modeline = md[1]
-      end
-    end
-
-    def parse_white_space!
-      consume!(/\s+/)
-    end
-
-    def parse_comment_line!
-      consume!(/#.*\n/)
-    end
-
-    def parse_ast_statement!
-      if consume!(/ast\s*;/)
-        @ast = true
-      end
-    end
-
-    def parse_ast_prefix_statement!
-      if md = consume!(/ast_prefix\s+(\w+)\s*;/)
-        @ast_prefix = md[1]
-      end
-    end
-
-    def parse_ast_suffix_statement!
-      if md = consume!(/ast_suffix\s+(\w+)\s*;/)
-        @ast_suffix = md[1]
-      end
-    end
-
-    def parse_free_token_node_statement!
-      if md = consume!(/free_token_node\s+(\w+)\s*;/)
-        @free_token_node = md[1]
-      end
-    end
-
-    def parse_module_statement!
-      if consume!(/module\s+/)
-        md = consume!(/([\w.]+)\s*/, "expected module name")
-        @modulename = md[1]
-        consume!(/;/, "expected `;'")
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_ptype_statement!
-      if consume!(/ptype\s+/)
-        name = "default"
-        if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
-          end
-          name = md[1]
-        end
-        md = consume!(/([^;]+);/, "expected parser result type expression")
-        @ptypes[name] = md[1].strip
-      end
-    end
-
-    def parse_token_statement!
-      if consume!(/token\s+/)
-        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
-        name = md[1]
-        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
-          end
-          ptypename = md[1]
-        end
-        pattern = parse_pattern! || name
-        consume!(/\s+/)
-        unless code = parse_code_block!
-          consume!(/;/, "expected `;' or code block")
-        end
-        token = Token.new(name, ptypename, @line_number)
-        @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
-        @patterns << pattern
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_tokenid_statement!
-      if md = consume!(/tokenid\s+/)
-        md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
-        name = md[1]
-        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
-          end
-          ptypename = md[1]
-        end
-        consume!(/;/, "expected `;'");
-        token = Token.new(name, ptypename, @line_number)
-        @tokens << token
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_drop_statement!
-      if md = consume!(/drop\s+/)
-        pattern = parse_pattern!
-        unless pattern
-          raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
-        end
-        consume!(/\s+/)
-        consume!(/;/, "expected `;'")
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, modes: get_modes_from_modeline)
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_rule_statement!
-      if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
-        rule_name, ptypename = *md[1, 2]
-        if @ast && ptypename
-          raise Error.new("Multiple ptypes are unsupported in AST mode")
-        end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
-        components = md[1].strip.split(/\s+/)
-        if @ast
-          consume!(/;/, "expected `;'")
-        else
-          unless code = parse_code_block!
-            consume!(/;/, "expected `;' or code block")
-          end
-        end
-        @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_pattern_statement!
-      if pattern = parse_pattern!
-        consume!(/\s+/)
-        if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @ast
-            raise Error.new("Multiple ptypes are unsupported in AST mode")
-          end
-          ptypename = md[1]
-        end
-        unless code = parse_code_block!
-          raise Error.new("Line #{@line_number}: expected code block to follow pattern")
-        end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_start_statement!
-      if md = consume!(/start\s+([\w\s]*);/)
-        start_rules = md[1].split(/\s+/).map(&:strip)
-        start_rules.each do |start_rule|
-          @start_rules << start_rule unless @start_rules.include?(start_rule)
-        end
-      end
-    end
-
-    def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
-        name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
-        if @code_blocks[name]
-          @code_blocks[name] += code
-        else
-          @code_blocks[name] = code
-        end
-        @modeline = nil
-        true
-      end
-    end
-
-    def parse_prefix_statement!
-      if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
-        @prefix = md[1]
-        true
-      end
-    end
-
-    def parse_pattern!
-      if md = consume!(%r{/})
-        pattern = ""
-        while !consume!(%r{/})
-          if consume!(%r{\\})
-            pattern += "\\"
-            if md = consume!(%r{(.)})
-              pattern += md[1]
-            else
-              raise Error.new("Line #{@line_number}: unterminated escape sequence")
-            end
-          elsif md = consume!(%r{(.)})
-            pattern += md[1]
-          end
-        end
-        pattern
-      end
-    end
-
-    def parse_code_block!
-      if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
-        code
-      end
-    end
-
-    # Check if the input string matches the given regex.
-    #
-    # If so, remove the match from the input string, and update the line
-    # number. If the regex is not matched and an error message is provided,
-    # the error is raised.
-    #
-    # @param regex [Regexp]
-    #   Regex to attempt to match.
-    # @param error_message [String, nil]
-    #   Error message to display if the regex is not matched. If nil and the
-    #   regex is not matched, an error is not raised.
-    #
-    # @return [MatchData, nil]
-    #   MatchData for the given regex if it was matched and removed from the
-    #   input.
-    def consume!(regex, error_message = nil)
-      @line_number = @next_line_number
-      if md = @input.match(/\A#{regex}/)
-        @input.slice!(0, md[0].size)
-        @next_line_number += md[0].count("\n")
-        md
-      elsif error_message
-        raise Error.new("Line #{@line_number}: Error: #{error_message}")
-      else
-        false
-      end
-    end
-
-    def get_modes_from_modeline
-      if @modeline
-        Set[*@modeline.split(",").map(&:strip)]
-      else
-        Set.new
       end
     end
 

lib/propane/lexer.rb

@@ -1,73 +1,41 @@
 class Propane
   class Lexer
 
-    attr_reader :state_table
+    # @return [DFA]
-    attr_reader :transition_table
+    #   Lexer DFA.
-    attr_reader :mode_table
+    attr_accessor :dfa
 
-    def initialize(grammar)
+    def initialize(tokens, drop_tokens)
-      @grammar = grammar
+      @dfa = DFA.new(tokens, drop_tokens)
-      build_tables!
     end
 
-    # Get ID for a mode.
+    def build_tables
-    #
+      transition_table = []
-    # @param mode_name [String]
+      state_table = []
-    #   Mode name.
+      states = @dfa.enumerate
-    #
-    # @return [Integer, nil]
-    #   Mode ID.
-    def mode_id(mode_name)
-      if mode_info = @modes[mode_name]
-        mode_info[:id]
-      end
-    end
-
-    private
-
-    def build_tables!
-      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
-        result + pattern.modes
-      end
-      @modes = modenames.reduce({}) do |result, modename|
-        result[modename] = @grammar.patterns.select do |pattern|
-          pattern.modes.include?(modename)
-        end
-        result
-      end.transform_values do |patterns|
-        {dfa: DFA.new(patterns)}
-      end
-      @modes.each_with_index do |(mode_name, mode_info), index|
-        mode_info[:id] = index
-      end
-      @state_table = []
-      @transition_table = []
-      @mode_table = []
-      @modes.each do |mode_name, mode_info|
-        state_table_offset = @state_table.size
-        @mode_table << {
-          state_table_offset: state_table_offset,
-        }
-        states = mode_info[:dfa].enumerate
       states.each do |state, id|
-          token = state.accepts && state.accepts.token && state.accepts.token.id
+        accepts =
-          code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
+          if state.accepts.nil?
-          @state_table << {
+            TOKEN_NONE
-            transition_table_index: @transition_table.size,
+          elsif state.accepts.drop?
+            TOKEN_DROP
+          else
+            state.accepts.id
+          end
+        state_table << {
+          transition_table_index: transition_table.size,
           n_transitions: state.transitions.size,
-            accepts: !!state.accepts,
+          accepts: accepts,
-            token: token,
-            code_id: code_id,
         }
         state.transitions.each do |transition|
-            @transition_table << {
+          transition_table << {
             first: transition.code_point_range.first,
             last: transition.code_point_range.last,
-              destination: states[transition.destination] + state_table_offset,
+            destination: states[transition.destination],
           }
         end
       end
-      end
+      [transition_table, state_table]
     end
 
   end

lib/propane/lexer/dfa.rb

@@ -3,11 +3,11 @@ class Propane
 
     class DFA < FA
 
-      def initialize(patterns)
+      def initialize(tokens, drop_tokens)
         super()
         start_nfa = Regex::NFA.new
-        patterns.each do |pattern|
+        (tokens + drop_tokens).each do |token|
-          start_nfa.start_state.add_transition(nil, pattern.nfa.start_state)
+          start_nfa.start_state.add_transition(nil, token.nfa.start_state)
         end
         @nfa_state_sets = {}
         @states = []
@@ -40,7 +40,7 @@ class Propane
           nfa_state_set.each do |nfa_state|
             if nfa_state.accepts
               if state.accepts
-                if nfa_state.accepts.line_number < state.accepts.line_number
+                if nfa_state.accepts.id < state.accepts.id
                   state.accepts = nfa_state.accepts
                 end
               else

lib/propane/parser.rb

@@ -2,43 +2,25 @@ class Propane
 
   class Parser
 
-    attr_reader :state_table
+    def initialize(start_rule_set)
-    attr_reader :shift_table
+      @eof_token = Token.new(name: "$", id: TOKEN_EOF)
-    attr_reader :reduce_table
+      start_rule = Rule.new("$$", [start_rule_set, @eof_token], nil, nil, 0)
-    attr_reader :rule_sets
-
-    def initialize(grammar, rule_sets, log, options)
-      @grammar = grammar
-      @rule_sets = rule_sets
-      @log = log
       @item_sets = []
       @item_sets_set = {}
-      @warnings = Set.new
+      start_item = Item.new(start_rule, 0)
-      @errors = Set.new
+      eval_item_sets = Set[ItemSet.new([start_item])]
-      @options = options
-      start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule|
-        Item.new(start_rule, 0)
-      end
-      start_item_sets = start_items.map {|item| ItemSet.new([item])}
-      eval_item_sets = Set[*start_item_sets]
 
       while eval_item_sets.size > 0
-        item_set =
+        item_set = eval_item_sets.first
-          if start_item_sets.size > 0
-            # Ensure we evaluate start_item_sets first in order
-            start_item_sets.slice!(0)
-          else
-            eval_item_sets.first
-          end
         eval_item_sets.delete(item_set)
         unless @item_sets_set.include?(item_set)
           item_set.id = @item_sets.size
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
-          item_set.next_symbols.each do |next_symbol|
+          item_set.following_symbols.each do |following_symbol|
-            unless next_symbol.name == "$EOF"
+            unless following_symbol == @eof_token
-              next_item_set = item_set.build_next_item_set(next_symbol)
+              following_set = item_set.build_following_item_set(following_symbol)
-              eval_item_sets << next_item_set
+              eval_item_sets << following_set
             end
           end
         end
@@ -46,350 +28,48 @@ class Propane
 
       @item_sets.each do |item_set|
         process_item_set(item_set)
+        puts "Item set #{item_set.id}:"
+        ids = item_set.in_sets.map(&:id)
+        if ids.size > 0
+          puts "    (in from #{ids.join(", ")})"
+        end
+        puts item_set
+        item_set.following_item_set.each do |following_symbol, following_item_set|
+          puts " #{following_symbol.name} => #{following_item_set.id}"
+        end
+        puts
+      end
     end
 
-      build_reduce_actions!
+    def build_tables
-      build_tables!
+      shift_table = []
-      write_log!
+      state_table = []
-      errormessage = ""
+      @item_sets.each do |item_set|
-      if @errors.size > 0
+        shift_entries = item_set.following_symbols.select do |following_symbol|
-        errormessage += @errors.join("\n")
+          following_symbol.is_a?(Token)
+        end.map do |following_symbol|
+          {
+            token_id: following_symbol.id,
+            state_id: item_set.following_item_set[following_symbol].id,
+          }
         end
-      if @warnings.size > 0 && @options[:warnings_as_errors]
+        state_table << {
-        if errormessage != ""
+          shift_index: shift_table.size,
-          errormessage += "\n"
+          n_shifts: shift_entries.size,
-        end
+        }
-        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
+        shift_table += shift_entries
-      end
-      if errormessage != ""
-        raise Error.new(errormessage)
       end
+      [state_table, shift_table]
     end
 
     private
 
-    def build_tables!
-      @state_table = []
-      @shift_table = []
-      @reduce_table = []
-      @item_sets.each do |item_set|
-        unless item_set.reduce_rules.empty?
-          item_set.shift_entries.each do |shift_entry|
-            token = shift_entry[:symbol]
-            if item_set.reduce_actions
-              if rule = item_set.reduce_actions[token]
-                @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
-              end
-            end
-          end
-        end
-        reduce_entries =
-          if rule = item_set.reduce_rule
-            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
-              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
-              propagate_optional_target: rule.optional? && rule.components.size == 1}]
-          elsif reduce_actions = item_set.reduce_actions
-            reduce_actions.map do |token, rule|
-              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
-               propagate_optional_target: rule.optional? && rule.components.size == 1}
-            end
-          else
-            []
-          end
-        @state_table << {
-          shift_index: @shift_table.size,
-          n_shifts: item_set.shift_entries.size,
-          reduce_index: @reduce_table.size,
-          n_reduces: reduce_entries.size,
-        }
-        @shift_table += item_set.shift_entries
-        @reduce_table += reduce_entries
-      end
-    end
-
     def process_item_set(item_set)
-      item_set.next_symbols.each do |next_symbol|
+      item_set.following_symbols.each do |following_symbol|
-        unless next_symbol.name == "$EOF"
+        unless following_symbol == @eof_token
-          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
+          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
-          item_set.next_item_set[next_symbol] = next_item_set
+          item_set.following_item_set[following_symbol] = following_set
-          next_item_set.in_sets << item_set
+          following_set.in_sets << item_set
-        end
-      end
-    end
-
-    # Build the reduce actions for each ItemSet.
-    #
-    # @return [void]
-    def build_reduce_actions!
-      @item_sets.each do |item_set|
-        build_shift_entries(item_set)
-        build_reduce_actions_for_item_set(item_set)
-      end
-      item_sets_to_process = @item_sets.select do |item_set|
-        # We need lookahead reduce actions if:
-        # 1) There is more than one possible rule to reduce. In this case the
-        #    lookahead token can help choose which rule to reduce.
-        # 2) There is at least one shift action and one reduce action for
-        #    this item set. In this case the lookahead reduce actions are
-        #    needed to test for a Shift/Reduce conflict.
-        item_set.reduce_rules.size > 1 ||
-          (item_set.reduce_rules.size > 0 && item_set.shift_entries.size > 0)
-      end
-      if RbConfig::CONFIG["host_os"] =~ /linux/
-        item_sets_by_id = {}
-        item_sets_to_process.each do |item_set|
-          item_sets_by_id[item_set.object_id] = item_set
-        end
-        tokens_by_id = {}
-        @grammar.tokens.each do |token|
-          tokens_by_id[token.object_id] = token
-        end
-        rules_by_id = {}
-        @grammar.rules.each do |rule|
-          rules_by_id[rule.object_id] = rule
-        end
-        n_threads = Util.determine_n_threads
-        semaphore = Mutex.new
-        queue = Queue.new
-        threads = {}
-        n_threads.times do
-          piper, pipew = IO.pipe
-          thread = Thread.new do
-            loop do
-              item_set = nil
-              semaphore.synchronize do
-                item_set = item_sets_to_process.slice!(0)
-              end
-              break if item_set.nil?
-              fork do
-                piper.close
-                build_lookahead_reduce_actions_for_item_set(item_set, pipew)
-              end
-            end
-            queue.push(Thread.current)
-          end
-          threads[thread] = [piper, pipew]
-        end
-        until threads.empty?
-          thread = queue.pop
-          piper, pipew = threads[thread]
-          pipew.close
-          thread_txt = piper.read
-          thread_txt.each_line do |line|
-            if line.start_with?("RA,")
-              parts = line.split(",")
-              item_set_id, token_id, rule_id = parts[1..3].map(&:to_i)
-              item_set = item_sets_by_id[item_set_id]
-              unless item_set
-                raise "Internal error: could not find item set from thread"
-              end
-              token = tokens_by_id[token_id]
-              unless item_set
-                raise "Internal error: could not find token from thread"
-              end
-              rule = rules_by_id[rule_id]
-              unless item_set
-                raise "Internal error: could not find rule from thread"
-              end
-              item_set.reduce_actions ||= {}
-              item_set.reduce_actions[token] = rule
-            elsif line.start_with?("Error: ")
-              @errors << line.chomp
-            else
-              raise "Internal error: unhandled thread line #{line}"
-            end
-          end
-          thread.join
-          threads.delete(thread)
-        end
-      else
-        # Fall back to single threaded algorithm.
-        item_sets_to_process.each do |item_set|
-          item_set.reduce_actions = build_lookahead_reduce_actions_for_item_set(item_set)
-        end
-      end
-    end
-
-    # Build the shift entries for a single item set.
-    #
-    # @return [void]
-    def build_shift_entries(item_set)
-      item_set.shift_entries = item_set.next_symbols.map do |next_symbol|
-        state_id =
-          if next_symbol.name == "$EOF"
-            0
-          else
-            item_set.next_item_set[next_symbol].id
-          end
-        {
-          symbol: next_symbol,
-          state_id: state_id,
-        }
-      end
-    end
-
-    # Build the reduce actions for a single item set (parser state).
-    #
-    # @param item_set [ItemSet]
-    #   ItemSet (parser state)
-    #
-    # @return [void]
-    def build_reduce_actions_for_item_set(item_set)
-      # To build the reduce actions, we start by looking at any
-      # "complete" items, i.e., items where the parse position is at the
-      # end of a rule. These are the only rules that are candidates for
-      # reduction in the current ItemSet.
-      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
-
-      if item_set.reduce_rules.size == 1
-        item_set.reduce_rule = item_set.reduce_rules.first
-      end
-    end
-
-    # Build the reduce actions for a single item set (parser state).
-    #
-    # @param item_set [ItemSet]
-    #   ItemSet (parser state)
-    # @param fh [File]
-    #   Output file handle for multiprocessing mode.
-    #
-    # @return [Hash]
-    #   Mapping of lookahead Tokens to the Rules to reduce.
-    def build_lookahead_reduce_actions_for_item_set(item_set, fh = nil)
-      # We will be looking for all possible tokens that can follow instances of
-      # these rules. Rather than looking through the entire grammar for the
-      # possible following tokens, we will only look in the item sets leading
-      # up to this one. This restriction gives us a more precise lookahead set,
-      # and allows us to parse LALR grammars.
-      item_sets = Set[item_set] + item_set.leading_item_sets
-      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
-        lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
-        lookahead_tokens_for_rule.each do |lookahead_token|
-          if existing_reduce_rule = reduce_actions[lookahead_token]
-            error = "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
-            @errors << error
-            fh.puts(error) if fh
-          end
-          reduce_actions[lookahead_token] = reduce_rule
-          fh.puts "RA,#{item_set.object_id},#{lookahead_token.object_id},#{reduce_rule.object_id}" if fh
-        end
-        reduce_actions
-      end
-    end
-
-    # Build the set of lookahead Tokens that should cause the given Rule to be
-    # reduced in the given context of ItemSets.
-    #
-    # @param rule [Rule]
-    #   Rule to reduce.
-    # @param item_sets [Set<ItemSet>]
-    #   ItemSets to consider for the context in which to reduce this Rule.
-    #
-    # @return [Set<Token>]
-    #   Possible lookahead Tokens for the given Rule within the context of the
-    #   given ItemSets.
-    def build_lookahead_tokens_to_reduce(rule, item_sets)
-      # We need to look for possible following tokens for this reduce rule. We
-      # do this by looking for tokens that follow the reduce rule, or the
-      # start token set for any other rule that follows the reduce rule.
-      # While doing this, the following situations could arise:
-      # 1. We may come across a following rule that could be empty. In this
-      #    case, in addition to the start token set for that rule, we must also
-      #    continue to the next following symbol after the potentially empty
-      #    rule and continue the search for potential following tokens.
-      # 2. We may reach the end of a rule that was not one of the original
-      #    reduce rules. In this case, we must also search for all potential
-      #    following tokens for this rule as well.
-      lookahead_tokens = Set.new
-      rule_sets_to_check_after = [rule.rule_set]
-      checked_rule_sets = Set.new
-      while !rule_sets_to_check_after.empty?
-        rule_set = rule_sets_to_check_after.slice!(0)
-        checked_rule_sets << rule_set
-        # For each RuleSet we're checking, we're going to look through all
-        # items in the item sets of interest and gather all possible following
-        # tokens to form the lookahead token set.
-        item_sets.each do |item_set|
-          item_set.items.each do |item|
-            if item.next_symbol == rule_set
-              (1..).each do |offset|
-                case symbol = item.next_symbol(offset)
-                when nil
-                  rule_set = item.rule.rule_set
-                  unless checked_rule_sets.include?(rule_set)
-                    rule_sets_to_check_after << rule_set
-                    checked_rule_sets << rule_set
-                  end
-                  break
-                when Token
-                  lookahead_tokens << symbol
-                  break
-                when RuleSet
-                  lookahead_tokens += symbol.start_token_set
-                  unless symbol.could_be_empty?
-                    break
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-      lookahead_tokens
-    end
-
-    def write_log!
-      @log.puts Util.banner("Parser Rules")
-      @grammar.rules.each do |rule|
-        @log.puts
-        @log.puts "Rule #{rule.id}:"
-        @log.puts "  #{rule}"
-      end
-
-      @log.puts
-      @log.puts Util.banner("Parser Tokens")
-      @log.puts
-      @grammar.tokens.each do |token|
-        @log.puts "Token #{token.id}: #{token.name}"
-      end
-
-      @log.puts
-      @log.puts Util.banner("Parser Rule Sets")
-      @rule_sets.each do |rule_set_name, rule_set|
-        @log.puts
-        @log.puts "Rule Set #{rule_set.id}: #{rule_set_name}"
-        @log.puts "  Start token set: #{rule_set.start_token_set.map(&:name).join(", ")}"
-      end
-
-      @log.puts
-      @log.puts Util.banner("Parser States")
-      @item_sets.each do |item_set|
-        @log.puts
-        @log.puts "State #{item_set.id}:"
-        @log.puts item_set.to_s.gsub(/^/, "  ")
-        incoming_ids = item_set.in_sets.map(&:id)
-        @log.puts
-        @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
-        @log.puts "  Outgoing states:"
-        item_set.next_item_set.each do |next_symbol, next_item_set|
-          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
-        end
-        @log.puts
-        @log.puts "  Reduce actions:"
-        if item_set.reduce_rule
-          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
-        elsif item_set.reduce_actions
-          item_set.reduce_actions.each do |token, rule|
-            @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
-          end
-        end
-      end
-      if @warnings.size > 0
-        @log.puts
-        @log.puts "Warnings:"
-        @warnings.each do |warning|
-          @log.puts "  #{warning}"
         end
       end
     end

lib/propane/parser/item.rb

@@ -22,7 +22,6 @@ class Propane
       def initialize(rule, position)
         @rule = rule
         @position = position
-        @_hash = [@rule, @position].hash
       end
 
       # Hash function.
@@ -30,7 +29,7 @@ class Propane
       # @return [Integer]
       #   Hash code.
       def hash
-        @_hash
+        [@rule, @position].hash
       end
 
       # Compare Item objects.
@@ -57,7 +56,7 @@ class Propane
 
       # Return the set of Items obtained by "closing" the current item.
       #
-      # If the next symbol for the current item is another Rule name, then
+      # If the following symbol for the current item is another Rule name, then
       # this method will return all Items for that Rule with a position of 0.
       # Otherwise, an empty Array is returned.
       #
@@ -82,52 +81,36 @@ class Propane
         @position == @rule.components.size
       end
 
-      # Get the next symbol for the Item.
+      # Get the following symbol for the Item.
       #
-      # That is, the symbol which is after the parse position marker in the
+      # That is, the symbol which follows the parse position marker in the
-      # current Item.
-      #
-      # @param offset [Integer]
-      #   Offset from current parse position to examine.
-      #
-      # @return [Token, RuleSet, nil]
-      #   Next symbol for the Item.
-      def next_symbol(offset = 0)
-        @rule.components[@position + offset]
-      end
-
-      # Get the previous symbol for the Item.
-      #
-      # That is, the symbol which precedes the parse position marker in the
       # current Item.
       #
       # @return [Token, RuleSet, nil]
-      #   Previous symbol for this Item.
+      #   Following symbol for the Item.
-      def previous_symbol
+      def following_symbol
-        if @position > 0
+        @rule.components[@position]
-          @rule.components[@position - 1]
-        end
       end
 
-      # Get whether this Item's next symbol is the given symbol.
+      # Get whether this Item is followed by the provided symbol.
       #
       # @param symbol [Token, RuleSet]
       #   Symbol to query.
       #
       # @return [Boolean]
-      #   Whether this Item's next symbol is the given symbol.
+      #   Whether this Item is followed by the provided symbol.
-      def next_symbol?(symbol)
+      def followed_by?(symbol)
-        next_symbol == symbol
+        following_symbol == symbol
       end
 
-      # Get the next item for this Item.
+      # Get the following item for this Item.
       #
       # That is, the Item formed by moving the parse position marker one place
       # forward from its position in this Item.
       #
       # @return [Item]
-      #   The next item for this Item.
+      #   The following item for this Item.
-      def next_item
+      def following_item
         Item.new(@rule, @position + 1)
       end
 

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing. This is equivalent to a parser state.
+    # parser could currently be parsing.
     class ItemSet
 
       # @return [Set<Item>]
@@ -14,58 +14,41 @@ class Propane
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a next symbol to its ItemSet.
+      #   Maps a following symbol to its ItemSet.
-      attr_reader :next_item_set
+      attr_reader :following_item_set
 
       # @return [Set<ItemSet>]
       #   ItemSets leading to this item set.
       attr_reader :in_sets
 
-      # @return [nil, Rule]
-      #   Rule to reduce if there is only one possibility.
-      attr_accessor :reduce_rule
-
-      # @return [Set<Rule>]
-      #   Set of rules that could be reduced in this parser state.
-      attr_accessor :reduce_rules
-
-      # @return [nil, Hash]
-      #   Reduce actions, mapping lookahead tokens to rules, if there is
-      #   more than one rule that could be reduced.
-      attr_accessor :reduce_actions
-
-      # @return [Array<Hash>]
-      #   Shift table entries.
-      attr_accessor :shift_entries
-
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
       #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @next_item_set = {}
+        @following_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      # Get the set of next symbols for all Items in this ItemSet.
+      # Get the set of following symbols for all Items in this ItemSet.
       #
       # @return [Set<Token, RuleSet>]
-      #   Set of next symbols for all Items in this ItemSet.
+      #   Set of following symbols for all Items in this ItemSet.
-      def next_symbols
+      def following_symbols
-        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
+        Set.new(@items.map(&:following_symbol).compact)
       end
 
-      # Build a next ItemSet for the given next symbol.
+      # Build a following ItemSet for the given following symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Next symbol to build the next ItemSet for.
+      #   Following symbol to build the following ItemSet for.
       #
       # @return [ItemSet]
-      #   Next ItemSet for the given next symbol.
+      #   Following ItemSet for the given following symbol.
-      def build_next_item_set(symbol)
+      def build_following_item_set(symbol)
-        ItemSet.new(items_with_next(symbol).map(&:next_item))
+        ItemSet.new(items_followed_by(symbol).map(&:following_item))
       end
 
       # Hash function.
@@ -100,26 +83,13 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
+      # This set includes this ItemSet.
+      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @_leading_item_sets ||=
+        @in_sets.reduce(Set[self]) do |result, item_set|
-          begin
+          result + item_set.leading_item_sets
-            result = Set.new
-            eval_sets = Set[self]
-            evaled = Set.new
-            while eval_sets.size > 0
-              eval_set = eval_sets.first
-              eval_sets.delete(eval_set)
-              evaled << eval_set
-              eval_set.in_sets.each do |in_set|
-                result << in_set
-                unless evaled.include?(in_set)
-                  eval_sets << in_set
-                end
-              end
-            end
-            result
         end
       end
 
@@ -140,7 +110,7 @@ class Propane
       #
       # @return [void]
       def close!
-        eval_items = @items.dup
+        eval_items = @items
         while eval_items.size > 0
           item = eval_items.first
           eval_items.delete(item)
@@ -153,16 +123,16 @@ class Propane
         end
       end
 
-      # Get the Items with the given next symbol.
+      # Get the Items followed by the given following symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Next symbol.
+      #   Following symbol.
       #
       # @return [Array<Item>]
-      #   Items with the given next symbol.
+      #   Items followed by the given following symbol.
-      def items_with_next(symbol)
+      def items_followed_by(symbol)
         @items.select do |item|
-          item.next_symbol?(symbol)
+          item.followed_by?(symbol)
         end
       end
 

lib/propane/pattern.rb

@@ -1,65 +0,0 @@
-class Propane
-
-  class Pattern
-
-    # @return [String, nil]
-    #   Code block to execute when the pattern is matched.
-    attr_reader :code
-
-    # @option options [Integer, nil] :code_id
-    #   Code block ID.
-    attr_accessor :code_id
-
-    # @return [String, nil]
-    #   Pattern.
-    attr_reader :pattern
-
-    # @return [Token, nil]
-    #   Token to be returned by this pattern.
-    attr_reader :token
-
-    # @return [Integer, nil]
-    #   Line number where the pattern was defined in the input grammar.
-    attr_reader :line_number
-
-    # @return [Regex::NFA, nil]
-    #   Regex NFA for matching the pattern.
-    attr_reader :nfa
-
-    # @return [Set]
-    #   Lexer modes for this pattern.
-    attr_accessor :modes
-
-    # @return [String, nil]
-    #   Parser value type name.
-    attr_accessor :ptypename
-
-    # Construct a Pattern.
-    #
-    # @param options [Hash]
-    #   Optional parameters.
-    # @option options [String, nil] :code
-    #   Code block to execute when the pattern is matched.
-    # @option options [String, nil] :pattern
-    #   Pattern.
-    # @option options [Token, nil] :token
-    #   Token to be returned by this pattern.
-    # @option options [Integer, nil] :line_number
-    #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :modes
-    #   Lexer modes for this pattern.
-    def initialize(options)
-      @code = options[:code]
-      @pattern = options[:pattern]
-      @token = options[:token]
-      @line_number = options[:line_number]
-      @modes = options[:modes]
-      @ptypename = options[:ptypename]
-      regex = Regex.new(@pattern, @line_number)
-      regex.nfa.end_state.accepts = self
-      @nfa = regex.nfa
-    end
-
-  end
-
-end

lib/propane/regex.rb

@@ -4,13 +4,12 @@ class Propane
     attr_reader :unit
     attr_reader :nfa
 
-    def initialize(pattern, line_number)
+    def initialize(pattern)
       @pattern = pattern.dup
-      @line_number = line_number
       @unit = parse_alternates
       @nfa = @unit.to_nfa
       if @pattern != ""
-        raise Error.new(%[Line #{@line_number}: unexpected "#{@pattern}" in pattern])
+        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
       end
     end
 
@@ -42,7 +41,7 @@ class Propane
             mu = MultiplicityUnit.new(last_unit, min_count, max_count)
             au.replace_last!(mu)
           else
-            raise Error.new("Line #{@line_number}: #{c} follows nothing")
+            raise Error.new("#{c} follows nothing")
           end
         when "|"
           au.new_alternate!
@@ -60,7 +59,7 @@ class Propane
     def parse_group
       au = parse_alternates
       if @pattern[0] != ")"
-        raise Error.new("Line #{@line_number}: unterminated group in pattern")
+        raise Error.new("Unterminated group in pattern")
       end
       @pattern.slice!(0)
       au
@@ -71,7 +70,7 @@ class Propane
       index = 0
       loop do
         if @pattern == ""
-          raise Error.new("Line #{@line_number}: unterminated character class")
+          raise Error.new("Unterminated character class")
         end
         c = @pattern.slice!(0)
         if c == "]"
@@ -85,13 +84,13 @@ class Propane
         elsif c == "-" && @pattern[0] != "]"
           begin_cu = ccu.last_unit
           unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
-            raise Error.new("Line #{@line_number}: character range must be between single characters")
+            raise Error.new("Character range must be between single characters")
           end
           if @pattern[0] == "\\"
             @pattern.slice!(0)
             end_cu = parse_backslash
             unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
-              raise Error.new("Line #{@line_number}: character range must be between single characters")
+              raise Error.new("Character range must be between single characters")
             end
             max_code_point = end_cu.code_point
           else
@@ -117,7 +116,7 @@ class Propane
         elsif max_count.to_s != ""
           max_count = max_count.to_i
           if max_count < min_count
-            raise Error.new("Line #{@line_number}: maximum repetition count cannot be less than minimum repetition count")
+            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
           end
         else
           max_count = nil
@@ -125,33 +124,18 @@ class Propane
         @pattern = pattern
         [min_count, max_count]
       else
-        raise Error.new("Line #{@line_number}: unexpected match count following {")
+        raise Error.new("Unexpected match count at #{@pattern}")
       end
     end
 
     def parse_backslash
       if @pattern == ""
-        raise Error.new("Line #{@line_number}: error: unfollowed \\")
+        raise Error.new("Error: unfollowed \\")
       else
         c = @pattern.slice!(0)
         case c
-        when "a"
-          CharacterRangeUnit.new("\a")
-        when "b"
-          CharacterRangeUnit.new("\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
-        when "D"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu.negate = true
-          ccu
-        when "f"
-          CharacterRangeUnit.new("\f")
-        when "n"
-          CharacterRangeUnit.new("\n")
-        when "r"
-          CharacterRangeUnit.new("\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -161,35 +145,6 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
-        when "S"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new(" ")
-          ccu << CharacterRangeUnit.new("\t")
-          ccu << CharacterRangeUnit.new("\r")
-          ccu << CharacterRangeUnit.new("\n")
-          ccu << CharacterRangeUnit.new("\f")
-          ccu << CharacterRangeUnit.new("\v")
-          ccu.negate = true
-          ccu
-        when "t"
-          CharacterRangeUnit.new("\t")
-        when "v"
-          CharacterRangeUnit.new("\v")
-        when "w"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("_")
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu << CharacterRangeUnit.new("a", "z")
-          ccu << CharacterRangeUnit.new("A", "Z")
-          ccu
-        when "W"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("_")
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu << CharacterRangeUnit.new("a", "z")
-          ccu << CharacterRangeUnit.new("A", "Z")
-          ccu.negate = true
-          ccu
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/regex/unit.rb

@@ -92,20 +92,17 @@ class Propane
         @units = []
         @negate = false
       end
-      def method_missing(*args, &block)
+      def initialize
-        @units.__send__(*args, &block)
+        @units = []
+      end
+      def method_missing(*args)
+        @units.__send__(*args)
       end
       def <<(thing)
         if thing.is_a?(CharacterClassUnit)
-          if thing.negate
-            CodePointRange.invert_ranges(thing.map(&:code_point_range)).each do |cpr|
-              CharacterRangeUnit.new(cpr.first, cpr.last)
-            end
-          else
           thing.each do |ccu_unit|
             @units << ccu_unit
           end
-          end
         else
           @units << thing
         end

lib/propane/rule.rb

@@ -6,21 +6,13 @@ class Propane
     #   Rule components.
     attr_reader :components
 
-    # @return [Hash]
-    #   Field aliases.
-    attr_reader :aliases
-
     # @return [String]
     #   User code associated with the rule.
     attr_reader :code
 
     # @return [Integer]
     #   Rule ID.
-    attr_accessor :id
+    attr_reader :id
-
-    # @return [String, nil]
-    #   Parser type name.
-    attr_accessor :ptypename
 
     # @return [Integer]
     #   Line number where the rule was defined in the input grammar.
@@ -30,15 +22,6 @@ class Propane
     #   Rule name.
     attr_reader :name
 
-    # @return [RuleSet]
-    #   The RuleSet that this Rule is a part of.
-    attr_accessor :rule_set
-
-    # @return [Array<Integer>]
-    #   Map this rule's components to their positions in the parent RuleSet's
-    #   node field pointer array. This is used for AST construction.
-    attr_accessor :rule_set_node_field_index_map
-
     # Construct a Rule.
     #
     # @param name [String]
@@ -47,28 +30,15 @@ class Propane
     #   Rule components.
     # @param code [String]
     #   User code associated with the rule.
-    # @param ptypename [String, nil]
-    #   Parser type name.
     # @param line_number [Integer]
     #   Line number where the rule was defined in the input grammar.
-    def initialize(name, components, code, ptypename, line_number)
+    # @param id [Integer]
+    #   Rule ID.
+    def initialize(name, components, code, line_number, id)
       @name = name
-      @aliases = {}
+      @components = components
-      @components = components.each_with_index.map do |component, i|
-        if component =~ /(\S+):(\S+)/
-          c, aliasname = $1, $2
-          if @aliases[aliasname]
-            raise Error.new("Error: duplicate field alias `#{aliasname}` for rule #{name} defined on line #{line_number}")
-          end
-          @aliases[aliasname] = i
-          c
-        else
-          component
-        end
-      end
-      @rule_set_node_field_index_map = components.map {0}
       @code = code
-      @ptypename = ptypename
+      @id = id
       @line_number = line_number
     end
 
@@ -82,33 +52,6 @@ class Propane
       @components.empty?
     end
 
-    # Return whether this is an optional Rule.
-    #
-    # @return [Boolean]
-    #   Whether this is an optional Rule.
-    def optional?
-      @name.end_with?("?")
-    end
-
-    # Represent the Rule as a String.
-    #
-    # @return [String]
-    #   Rule represented as a String.
-    def to_s
-      "#{@name} -> #{@components.map(&:name).join(" ")}"
-    end
-
-    # Check whether the rule set node field index map is just a 1:1 mapping.
-    #
-    # @return [Boolean]
-    #   Boolean indicating whether the rule set node field index map is just a
-    #   1:1 mapping.
-    def flat_rule_set_node_field_index_map?
-      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
-        v == i
-      end
-    end
-
   end
 
 end

lib/propane/rule_set.rb

@@ -1,24 +1,11 @@
 class Propane
 
-  # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
-    # @return [Array<Hash>]
-    #   AST fields.
-    attr_reader :ast_fields
-
-    # @return [Integer]
-    #   ID of the RuleSet.
-    attr_reader :id
-
     # @return [String]
     #   Name of the RuleSet.
     attr_reader :name
 
-    # @return [String, nil]
-    #   Parser type name.
-    attr_accessor :ptypename
-
     # @return [Array<Rule>]
     #   Rules in the RuleSet.
     attr_reader :rules
@@ -31,10 +18,7 @@ class Propane
     #
     # @param name [String]
     #   Name of the RuleSet.
-    # @param id [Integer]
+    def initialize(name)
-    #   ID of the RuleSet.
-    def initialize(name, id)
-      @id = id
       @name = name
       @rules = []
       @could_be_empty = false
@@ -56,24 +40,6 @@ class Propane
       @could_be_empty
     end
 
-    # Return whether this is an optional RuleSet.
-    #
-    # @return [Boolean]
-    #   Whether this is an optional RuleSet.
-    def optional?
-      @name.end_with?("?")
-    end
-
-    # For optional rule sets, return the underlying component that is optional.
-    def option_target
-      @rules.each do |rule|
-        if rule.components.size > 0
-          return rule.components[0]
-        end
-      end
-      raise "Optional rule target not found"
-    end
-
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -98,72 +64,6 @@ class Propane
       @_start_token_set
     end
 
-    # Finalize a RuleSet after adding all Rules to it.
-    def finalize(grammar)
-      if grammar.ast
-        build_ast_fields(grammar)
-      end
-    end
-
-    private
-
-    # Build the set of AST fields for this RuleSet.
-    #
-    # This is an Array of Hashes. Each entry in the Array corresponds to a
-    # field location in the AST node. The entry is a Hash. It could have one or
-    # two keys. It will always have the field name with a positional suffix as
-    # a key. It may also have the field name without the positional suffix if
-    # that field only exists in one position across all Rules in the RuleSet.
-    #
-    # @return [void]
-    def build_ast_fields(grammar)
-      field_ast_node_indexes = {}
-      field_indexes_across_all_rules = {}
-      # Stores the index into @ast_fields by field alias name.
-      field_aliases = {}
-      @ast_fields = []
-      @rules.each do |rule|
-        rule.components.each_with_index do |component, i|
-          if component.is_a?(RuleSet) && component.optional?
-            component = component.option_target
-          end
-          if component.is_a?(Token)
-            node_name = "Token"
-          else
-            node_name = component.name
-          end
-          struct_name = "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
-          field_name = "p#{node_name}#{i + 1}"
-          unless field_ast_node_indexes[field_name]
-            field_ast_node_indexes[field_name] = @ast_fields.size
-            @ast_fields << {field_name => struct_name}
-          end
-          rule.aliases.each do |alias_name, index|
-            if index == i
-              alias_ast_fields_index = field_ast_node_indexes[field_name]
-              if field_aliases[alias_name] && field_aliases[alias_name] != alias_ast_fields_index
-                raise Error.new("Error: conflicting AST node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
-              end
-              field_aliases[alias_name] = alias_ast_fields_index
-              @ast_fields[alias_ast_fields_index][alias_name] = @ast_fields[alias_ast_fields_index].first[1]
-            end
-          end
-          field_indexes_across_all_rules[node_name] ||= Set.new
-          field_indexes_across_all_rules[node_name] << field_ast_node_indexes[field_name]
-          rule.rule_set_node_field_index_map[i] = field_ast_node_indexes[field_name]
-        end
-      end
-      field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
-        if indexes_across_all_rules.size == 1
-          # If this field was only seen in one position across all rules,
-          # then add an alias to the positional field name that does not
-          # include the position.
-          @ast_fields[indexes_across_all_rules.first]["p#{node_name}"] =
-            "#{grammar.ast_prefix}#{node_name}#{grammar.ast_suffix}"
-        end
-      end
-    end
-
   end
 
 end

lib/propane/token.rb

@@ -2,54 +2,60 @@ class Propane
 
   class Token
 
-    class << self
-
-      # Name of the token to use in code (special characters replaced).
-      #
-      # @return [String]
-      #   Name of the token to use in code (special characters replaced).
-      def code_name(name)
-        name.sub(/^\$/, "__")
-      end
-
-    end
-
     # @return [String, nil]
     #   Token name.
     attr_reader :name
 
     # @return [String, nil]
-    #   Parser value type name.
+    #   Token pattern.
-    attr_accessor :ptypename
+    attr_reader :pattern
 
     # @return [Integer, nil]
     #   Token ID.
-    attr_accessor :id
+    attr_reader :id
 
     # @return [Integer, nil]
     #   Line number where the token was defined in the input grammar.
     attr_reader :line_number
 
+    # @return [Regex::NFA, nil]
+    #   Regex NFA for matching the token.
+    attr_reader :nfa
+
     # Construct a Token.
     #
-    # @param name [String, nil]
+    # @param options [Hash]
+    #   Optional parameters.
+    # @option options [String, nil] :name
     #   Token name.
-    # @param ptypename [String, nil]
+    # @option options [String, nil] :pattern
-    #   Parser value type for this token.
+    #   Token pattern.
-    # @param line_number [Integer, nil]
+    # @option options [Integer, nil] :id
+    #   Token ID.
+    # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    def initialize(name, ptypename, line_number)
+    def initialize(options)
-      @name = name
+      @name = options[:name]
-      @ptypename = ptypename
+      @pattern = options[:pattern]
-      @line_number = line_number
+      @id = options[:id]
+      @line_number = options[:line_number]
+      unless @pattern.nil?
+        regex = Regex.new(@pattern)
+        regex.nfa.end_state.accepts = self
+        @nfa = regex.nfa
+      end
     end
 
-    # Name of the token to use in code (special characters replaced).
+    def c_name
+      @name.upcase
+    end
+
+    # Whether the token is a drop token.
     #
-    # @return [String]
+    # @return [Boolean]
-    #   Name of the token to use in code (special characters replaced).
+    #   Whether the token is a drop token.
-    def code_name
+    def drop?
-      self.class.code_name(@name)
+      @name.nil?
     end
 
     def to_s

lib/propane/util.rb

@@ -1,43 +0,0 @@
-class Propane
-
-  # Utility methods.
-  module Util
-
-    class << self
-
-      def banner(message)
-        s = "*" * (message.size + 4)
-        "#{s}\n* #{message} *\n#{s}\n"
-      end
-
-      # Determine the number of threads to use.
-      #
-      # @return [Integer]
-      #   The number of threads to use.
-      def determine_n_threads
-        # Try to figure out how many threads are available on the host hardware.
-        begin
-          case RbConfig::CONFIG["host_os"]
-          when /linux/
-            return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
-          when /mswin|mingw|msys/
-            if `wmic cpu get NumberOfLogicalProcessors -value` =~ /NumberOfLogicalProcessors=(\d+)/
-              return $1.to_i
-            end
-          when /darwin/
-            if `sysctl -n hw.ncpu` =~ /(\d+)/
-              return $1.to_i
-            end
-          end
-        rescue
-        end
-
-        # If we can't figure it out, default to 4.
-        4
-      end
-
-    end
-
-  end
-
-end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "2.3.0"
+  VERSION = "0.1.0"
 end

propane.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec bundle exec ruby -Ilib bin/propane "$@"

rb/assets/user_guide.html.erb

@@ -1,68 +0,0 @@
-<html>
-  <head>
-    <title>Propane User Guide<%= subpage_title %> - Version <%= Propane::VERSION %></title>
-    <style>
-      body {
-        background-color: #CCC;
-        margin: 0px;
-        padding: 0px;
-      }
-      #body-content {
-        margin-left: auto;
-        margin-right: auto;
-        margin-top: 1em;
-        margin-bottom: 1em;
-        border: 1px solid black;
-        background-color: #FFF;
-        width: 120ex;
-        padding: 2ex;
-      }
-      a {
-        text-decoration: none;
-      }
-      .separator {
-        height: 2em;
-      }
-      .img_block_center {
-        display: block;
-        margin-left: auto;
-        margin-right: auto;
-      }
-      .page_nav {
-        width: 100%;
-        table-layout: fixed;
-      }
-      .page_nav_toc {
-        text-align: center;
-      }
-      .page_nav_next {
-        text-align: right;
-      }
-      .code {
-        padding-left: 2ex;
-        width: 116ex;
-        overflow-x: auto;
-      }
-      .ruby_code .normal {}
-      .ruby_code .comment { color: #005; font-style: italic; }
-      .ruby_code .keyword { color: #A00; font-weight: bold; }
-      .ruby_code .method { color: #077; }
-      .ruby_code .class { color: #074; }
-      .ruby_code .module { color: #050; }
-      .ruby_code .punct { color: #447; font-weight: bold; }
-      .ruby_code .symbol { color: #099; }
-      .ruby_code .string { color: #090; }
-      .ruby_code .char { color: #F07; }
-      .ruby_code .ident { color: #004; }
-      .ruby_code .constant { color: #07F; }
-      .ruby_code .regex { color: #B66; }
-      .ruby_code .number { color: #D55; }
-      .ruby_code .attribute { color: #377; }
-      .ruby_code .global { color: #3B7; }
-      .ruby_code .expr { color: #227; }
-    </style>
-  </head>
-  <body>
-    <div id="body-content"><%= content %></div>
-  </body>
-</html>

rb/build_dist.rb

@@ -1,129 +0,0 @@
-#!/usr/bin/env ruby
-
-require "erb"
-require "fileutils"
-require "digest/md5"
-
-if File.read("lib/propane/version.rb") =~ /VERSION = "(.+)"/
-  VERSION = $1
-else
-  raise "Could not determine version."
-end
-PROG_NAME = "propane"
-START_FILE = "bin/#{PROG_NAME}"
-LIB_DIR = "lib"
-DIST = "dist"
-
-ASSETS_TEMPLATE = <<EOF
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        case name
-<% Dir.glob("assets/*").each  do |asset_file| %>
-        when <%= File.basename(asset_file).inspect %>
-          <%= File.binread(asset_file).inspect %>
-<% end %>
-        end
-      end
-    end
-  end
-end
-EOF
-
-assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
-files_processed = {}
-combined_file = []
-
-combine_files = lambda do |file|
-  File.read(file, mode: "rb").each_line do |line|
-    if line =~ /^\s*require(?:_relative)?\s*"(.*)"$/
-      require_name = $1
-      if require_name =~ %r{^#{PROG_NAME}(?:/.*)?$}
-        path = "#{LIB_DIR}/#{require_name}.rb"
-        if File.exist?(path)
-          unless files_processed[path]
-            files_processed[path] = true
-            if require_name == "propane/assets"
-              combined_file << assets_module
-            else
-              combine_files[path]
-            end
-          end
-        else
-          raise "require path #{path.inspect} not found"
-        end
-      else
-        combined_file << line
-      end
-    else
-      combined_file << line
-    end
-  end
-end
-
-combine_files[START_FILE]
-
-# Strip Ruby comment lines and empty lines to save some space, but do not
-# remove lines that are in heredoc sections. This isn't terribly robust to be
-# used in the wild, but works for the heredoc instances for this project.
-stripped = []
-heredoc_end = nil
-combined_file.each do |line|
-  if line =~ /<<-?([A-Z]+)/
-    heredoc_end = $1
-  end
-  if heredoc_end and line =~ /^\s*#{heredoc_end}/
-    heredoc_end = nil
-  end
-  if line !~ /#\sspecs/
-    if heredoc_end or line !~ /^\s*(#[^!].*)?$/
-      stripped << line
-    end
-  end
-end
-
-license = File.read("LICENSE.txt").gsub(/^(.*?)$/) do |line|
-  if line.size > 0
-    "# #{line}"
-  else
-    "#"
-  end
-end
-
-require "zlib"
-require "base64"
-compressed_script = Zlib::Deflate.deflate(stripped.join)
-encoded_compressed_script = Base64.encode64(compressed_script).gsub("\n", "")
-hash = Digest::MD5.hexdigest(encoded_compressed_script)
-
-FileUtils.rm_rf(DIST)
-FileUtils.mkdir_p(DIST)
-File.open("#{DIST}/#{PROG_NAME}", "wb", 0755) do |fh|
-  fh.write(<<EOF)
-#!/usr/bin/env ruby
-
-#{license}
-
-script = File.join(File.dirname(__FILE__), ".propane-#{VERSION}-#{hash}.rb")
-unless File.exist?(script)
-  if File.read(__FILE__, mode: "rb") =~ /^#==>(.*)/
-    require "zlib"
-    require "base64"
-    encoded_compressed = $1
-    unescaped_compressed = Base64.decode64(encoded_compressed)
-    inflated = Zlib::Inflate.inflate(unescaped_compressed)
-    File.open(script, "wb") do |fh|
-      fh.write(inflated)
-    end
-  else
-    raise "Could not decompress."
-  end
-end
-load script
-if __FILE__ == $0
-  Propane::CLI.new.run(ARGV)
-end
-#==>#{encoded_compressed_script}
-EOF
-end

rb/gen_user_guide.rb

@@ -1,227 +0,0 @@
-#!/usr/bin/env ruby
-
-require "erb"
-require "fileutils"
-require "redcarpet"
-require "syntax"
-require "syntax/convertors/html"
-require "propane/version"
-
-def load_file(file_name)
-  contents = File.read(file_name)
-  contents.gsub(/\$\{include (.*?)\}/) do |match|
-    include_file_name = $1
-    File.read(include_file_name)
-  end
-end
-
-class Generator
-  class Section
-    attr_reader :number
-    attr_reader :title
-    attr_reader :page
-    attr_reader :contents
-    attr_reader :anchor
-    def initialize(number, title, page, anchor)
-      @number = number
-      @title = title.gsub(/[`]/, "")
-      @page = page
-      @anchor = anchor
-      @contents = ""
-    end
-    def append(contents)
-      @contents += contents
-    end
-  end
-
-  class Page
-    attr_reader :name
-    attr_reader :title
-    attr_accessor :contents
-    def initialize(name, title, contents = "")
-      @name = name
-      @title = title
-      @contents = contents
-    end
-  end
-
-  def initialize(input, output_file, multi_page)
-    current_page =
-      if multi_page
-        nil
-      else
-        File.basename(output_file).sub(/\.html$/, "")
-      end
-    @sections = []
-    @current_section_number = [0]
-    @lines = input.lines
-    while @lines.size > 0
-      line = @lines.slice!(0)
-      if line =~ /^```(.*)$/
-        @sections.last.append(gather_code_section($1))
-      elsif line.chomp == "${remove}"
-        remove_section
-      elsif line =~ /^(#+)(>)?\s*(.*)$/
-        level_text, new_page_text, title_text = $1, $2, $3
-        level = $1.size
-        new_page = !new_page_text.nil?
-        section_number = get_next_section_number(level)
-        anchor = make_anchor(section_number, title_text)
-        if multi_page and (new_page or current_page.nil?)
-          current_page = anchor
-        end
-        @sections << Section.new(section_number, title_text, current_page, anchor)
-        @sections.last.append("#{level_text} #{section_number} #{title_text}")
-      elsif @sections.size > 0
-        @sections.last.append(line)
-      end
-    end
-
-    renderer = Redcarpet::Render::HTML.new
-    @markdown_renderer = Redcarpet::Markdown.new(renderer)
-    changelog = @markdown_renderer.render(File.read("CHANGELOG.md"))
-
-    @pages = [Page.new("toc", "Table of Contents", render_toc)]
-    @sections.each do |section|
-      unless @pages.last.name == section.page
-        @pages << Page.new(section.page, "#{section.number} #{section.title}")
-      end
-      @pages.last.contents += render_section(section)
-    end
-
-    @pages.each do |page|
-      page.contents.gsub!("${changelog}", changelog)
-      page.contents.gsub!(%r{\$\{#(.+?)\}}) do |match|
-        section_name = $1
-        href = get_link_to_section(section_name)
-        %[<a href="#{href}">#{section_name}</a>]
-      end
-    end
-
-    template = File.read("rb/assets/user_guide.html.erb")
-    erb = ERB.new(template, trim_mode: "<>")
-
-    if multi_page
-      @pages.each_with_index do |page, page_index|
-        subpage_title = " - #{page.title}"
-        page_nav_bar = render_page_nav_bar(page_index)
-        content = page_nav_bar + separator + page.contents + separator + page_nav_bar
-        html_result = erb.result(binding.clone)
-        File.open(File.join(output_file, "#{page.name}.html"), "w") do |fh|
-          fh.write(html_result)
-        end
-      end
-    else
-      subpage_title = ""
-      content = @pages.reduce("") do |result, page|
-        result + page.contents
-      end
-      html_result = erb.result(binding.clone)
-      File.open(output_file, "w") do |fh|
-        fh.write(html_result)
-      end
-    end
-  end
-
-  def separator
-    %[<div class="separator"></div>]
-  end
-
-  def render_page_nav_bar(page_index)
-    page_nav_prev =
-      if page_index > 1
-        %[<a href="#{@pages[page_index - 1].name}.html">&laquo; Prev<br/>#{@pages[page_index - 1].title}</a>]
-      else
-        ""
-      end
-    page_nav_toc =
-      if page_index > 0
-        %[<a href="toc.html">Table of Contents</a>]
-      else
-        ""
-      end
-    page_nav_next =
-      if page_index < @pages.size - 1
-        %[<a href="#{@pages[page_index + 1].name}.html">Next &raquo<br/>#{@pages[page_index + 1].title}</a>]
-      else
-        ""
-      end
-    %[<table class="page_nav"><tr>] + \
-    %[<td class="page_nav_prev">#{page_nav_prev}</td>] + \
-    %[<td class="page_nav_toc">#{page_nav_toc}</td>] + \
-    %[<td class="page_nav_next">#{page_nav_next}</td>] + \
-    %[</tr></table>]
-  end
-
-  def render_toc
-    toc_content = %[<h1>Table of Contents</h1>\n]
-    @sections.each do |section|
-      indent = section.number.split(".").size - 1
-      toc_content += %[<span style="padding-left: #{4 * indent}ex;">]
-      toc_content += %[<a href="#{section.page}.html##{section.anchor}">#{section.number} #{section.title}</a><br/>\n]
-      toc_content += %[</span>]
-    end
-    toc_content
-  end
-
-  def render_section(section)
-    %[<a name="#{section.anchor}" />] + \
-      @markdown_renderer.render(section.contents)
-  end
-
-  def make_anchor(section_number, section_title)
-    "s" + ("#{section_number} #{section_title}").gsub(/[^a-zA-Z0-9]/, "_")
-  end
-
-  def get_link_to_section(section_name)
-    section = @sections.find do |section|
-      section.title == section_name
-    end
-    raise "Could not find section #{section_name}" unless section
-    "#{section.page}.html##{section.anchor}"
-  end
-
-  def gather_code_section(syntax)
-    code = ""
-    loop do
-      line = @lines.slice!(0)
-      if line =~ /^```/
-        break
-      end
-      code += line
-    end
-    if syntax != ""
-      convertor = Syntax::Convertors::HTML.for_syntax(syntax)
-      %[<div class="code #{syntax}_code">\n#{convertor.convert(code)}\n</div>\n]
-    else
-      %[<div class="code">\n<pre>#{code}</pre>\n</div>\n]
-    end
-  end
-
-  def remove_section
-    loop do
-      line = @lines.slice!(0)
-      if line.chomp == "${/remove}"
-        break
-      end
-    end
-  end
-
-  def get_next_section_number(level)
-    if @current_section_number.size == level - 1
-      @current_section_number << 1
-    elsif @current_section_number.size >= level
-      @current_section_number[level - 1] += 1
-      @current_section_number.slice!(level, @current_section_number.size)
-    else
-      raise "Section level change from #{@current_section_number.size} to #{level}"
-    end
-    @current_section_number.join(".")
-  end
-end
-
-input = load_file("doc/user_guide.md")
-FileUtils.rm_rf("gen/user_guide")
-FileUtils.mkdir_p("gen/user_guide")
-Generator.new(input, "gen/user_guide/user_guide.html", false)
-Generator.new(input, "gen/user_guide", true)

spec/ast_node_memory_remains.c.propane

@@ -1,151 +0,0 @@
-ast;
-ast_prefix P;
-
-<<header
-#include <stdio.h>
-#include <stdint.h>
-
-typedef union
-{
-    uint64_t i64;
-    const uint8_t * s;
-    double dou;
-} TokenVal;
->>
-
-ptype TokenVal;
-
-# Keywords.
-token byte;
-token def;
-token int;
-token long;
-token module;
-token return;
-token short;
-token size_t;
-token ssize_t;
-token ubyte;
-token uint;
-token ulong;
-token ushort;
-
-# Symbols.
-token arrow /->/;
-token comma /,/;
-token lbrace /\{/;
-token lparen /\(/;
-token rbrace /\}/;
-token rparen /\)/;
-token semicolon /;/;
-
-# Integer literals.
-token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
-  $$.i64 = 64u;
->>
-
-# Identifier.
-token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  $$.s = match;
-  $mode(default);
-  return $token(ident);
->>
-
-# Comments.
-drop /#.*/;
-
-# Whitespace.
-drop /[ \r\n]*/;
-
-start Module;
-
-# Assignment operators - right associative
-Expression -> Expression_Or:exp0;
-
-# Logical OR operator - left associative
-Expression_Or -> Expression_And:exp0;
-
-# Logical AND operator - left associative
-Expression_And -> Expression_Comp:exp0;
-
-# Equality operators - left associative
-Expression_Comp -> Expression_Relational:exp0;
-
-# Relational operators - left associative
-Expression_Relational -> Expression_REMatch:exp0;
-
-# Regular expression - left associative
-Expression_REMatch -> Expression_BinOr:exp0;
-
-# Binary OR operator - left associative
-Expression_BinOr -> Expression_Xor:exp0;
-
-# Binary XOR operator - left associative
-Expression_Xor -> Expression_BinAnd:exp0;
-
-# Binary AND operator - left associative
-Expression_BinAnd -> Expression_BitShift:exp0;
-
-# Bit shift operators - left associative
-Expression_BitShift -> Expression_Plus:exp0;
-
-# Add/subtract operators - left associative
-Expression_Plus -> Expression_Mul:exp0;
-
-# Multiplication/divide/modulus operators - left associative
-Expression_Mul -> Expression_Range:exp0;
-
-# Range construction operators - left associative
-Expression_Range -> Expression_UnaryPrefix:exp0;
-
-# Unary prefix operators
-Expression_UnaryPrefix -> Expression_Dot:exp0;
-
-# Postfix operators
-Expression_Dot -> Expression_Ident:exp0;
-Expression_Dot -> Expression_Dot:exp1 lparen rparen;
-
-# Literals, identifiers, and parenthesized expressions
-Expression_Ident -> Literal;
-Expression_Ident -> ident;
-
-FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
-
-FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
-FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
-
-FunctionReturnType -> arrow Type;
-
-Literal -> LiteralInteger;
-LiteralInteger -> hex_int_l;
-
-Module -> ModuleStatement? ModuleItems;
-
-ModuleItem -> FunctionDefinition;
-
-ModuleItems -> ;
-ModuleItems -> ModuleItems ModuleItem;
-
-ModulePath -> ident;
-
-ModuleStatement -> module ModulePath semicolon;
-
-ReturnStatement -> return Expression?:exp0 semicolon;
-
-Statements -> ;
-Statements -> Statements Statement;
-Statement -> Expression semicolon;
-Statement -> ReturnStatement;
-
-Type -> TypeBase;
-
-TypeBase -> byte;
-TypeBase -> ubyte;
-TypeBase -> short;
-TypeBase -> ushort;
-TypeBase -> int;
-TypeBase -> uint;
-TypeBase -> long;
-TypeBase -> ulong;
-TypeBase -> size_t;
-TypeBase -> ssize_t;

spec/ast_node_memory_remains.d.propane

@@ -1,177 +0,0 @@
-ast;
-ast_prefix P;
-
-<<
-import std.bigint;
-
-private string stringvalue;
-
-union TokenVal
-{
-    BigInt bi;
-    string s;
-    double dou;
-}
->>
-
-ptype TokenVal;
-
-# Keywords.
-token byte;
-token def;
-token int;
-token long;
-token module;
-token return;
-token short;
-token size_t;
-token ssize_t;
-token ubyte;
-token uint;
-token ulong;
-token ushort;
-
-# Symbols.
-token arrow /->/;
-token comma /,/;
-token lbrace /\{/;
-token lparen /\(/;
-token rbrace /\}/;
-token rparen /\)/;
-token semicolon /;/;
-
-# Integer literals.
-token hex_int_l /0[xX][0-9a-fA-F][0-9a-fA-F_]*/ <<
-  $$.bi = BigInt(match[0..3]);
-  foreach (c; match[3..$])
-  {
-    if (('0' <= c) && (c <= '9'))
-    {
-      $$.bi *= 0x10;
-      $$.bi += (c - '0');
-    }
-    if (('a' <= c) && (c <= 'f'))
-    {
-      $$.bi *= 0x10;
-      $$.bi += (c - 'a' + 10);
-    }
-    if (('A' <= c) && (c <= 'F'))
-    {
-      $$.bi *= 0x10;
-      $$.bi += (c - 'A' + 10);
-    }
-  }
->>
-
-# Identifier.
-token ident /\$?[a-zA-Z_][a-zA-Z_0-9]*\??/ <<
-  if (match[0] == '$')
-  {
-    $$.s = match[1..$];
-  }
-  else
-  {
-    $$.s = match;
-  }
-  $mode(default);
-  return $token(ident);
->>
-
-# Comments.
-drop /#.*/;
-
-# Whitespace.
-drop /[ \r\n]*/;
-
-start Module;
-
-# Assignment operators - right associative
-Expression -> Expression_Or:exp0;
-
-# Logical OR operator - left associative
-Expression_Or -> Expression_And:exp0;
-
-# Logical AND operator - left associative
-Expression_And -> Expression_Comp:exp0;
-
-# Equality operators - left associative
-Expression_Comp -> Expression_Relational:exp0;
-
-# Relational operators - left associative
-Expression_Relational -> Expression_REMatch:exp0;
-
-# Regular expression - left associative
-Expression_REMatch -> Expression_BinOr:exp0;
-
-# Binary OR operator - left associative
-Expression_BinOr -> Expression_Xor:exp0;
-
-# Binary XOR operator - left associative
-Expression_Xor -> Expression_BinAnd:exp0;
-
-# Binary AND operator - left associative
-Expression_BinAnd -> Expression_BitShift:exp0;
-
-# Bit shift operators - left associative
-Expression_BitShift -> Expression_Plus:exp0;
-
-# Add/subtract operators - left associative
-Expression_Plus -> Expression_Mul:exp0;
-
-# Multiplication/divide/modulus operators - left associative
-Expression_Mul -> Expression_Range:exp0;
-
-# Range construction operators - left associative
-Expression_Range -> Expression_UnaryPrefix:exp0;
-
-# Unary prefix operators
-Expression_UnaryPrefix -> Expression_Dot:exp0;
-
-# Postfix operators
-Expression_Dot -> Expression_Ident:exp0;
-Expression_Dot -> Expression_Dot:exp1 lparen rparen;
-
-# Literals, identifiers, and parenthesized expressions
-Expression_Ident -> Literal;
-Expression_Ident -> ident;
-
-FunctionDefinition -> def ident:name lparen FunctionParameterList?:parameters rparen FunctionReturnType?:returntype lbrace Statements rbrace;
-
-FunctionParameterList -> ident:name Type:type FunctionParameterListMore?:more;
-FunctionParameterListMore -> comma ident:name Type:type FunctionParameterListMore?:more;
-
-FunctionReturnType -> arrow Type;
-
-Literal -> LiteralInteger;
-LiteralInteger -> hex_int_l;
-
-Module -> ModuleStatement? ModuleItems;
-
-ModuleItem -> FunctionDefinition;
-
-ModuleItems -> ;
-ModuleItems -> ModuleItems ModuleItem;
-
-ModulePath -> ident;
-
-ModuleStatement -> module ModulePath semicolon;
-
-ReturnStatement -> return Expression?:exp0 semicolon;
-
-Statements -> ;
-Statements -> Statements Statement;
-Statement -> Expression semicolon;
-Statement -> ReturnStatement;
-
-Type -> TypeBase;
-
-TypeBase -> byte;
-TypeBase -> ubyte;
-TypeBase -> short;
-TypeBase -> ushort;
-TypeBase -> int;
-TypeBase -> uint;
-TypeBase -> long;
-TypeBase -> ulong;
-TypeBase -> size_t;
-TypeBase -> ssize_t;

spec/json_parser.c.propane

@@ -1,183 +0,0 @@
-<<header
-#include "json_types.h"
-#include "testutils.h"
->>
-<<
-#include "math.h"
-#include <stdbool.h>
-static str_t string_value;
->>
-
-ptype JSONValue *;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n = 0.0;
-    bool negative = false;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative = false;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp = 0.0;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = JSONValue_new(JSON_NUMBER);
-    $$->number = n;
->>
-token true <<
-  $$ = JSONValue_new(JSON_TRUE);
->>
-token false <<
-  $$ = JSONValue_new(JSON_FALSE);
->>
-token null <<
-  $$ = JSONValue_new(JSON_NULL);
->>
-/"/ <<
-  $mode(string);
-  str_init(&string_value, "");
->>
-string: token string /"/ <<
-  $$ = JSONValue_new(JSON_STRING);
-  $$->string = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  str_append(&string_value, "\"");
->>
-string: /\\\\/ <<
-  str_append(&string_value, "\\");
->>
-string: /\\\// <<
-  str_append(&string_value, "/");
->>
-string: /\\b/ <<
-  str_append(&string_value, "\b");
->>
-string: /\\f/ <<
-  str_append(&string_value, "\f");
->>
-string: /\\n/ <<
-  str_append(&string_value, "\n");
->>
-string: /\\r/ <<
-  str_append(&string_value, "\r");
->>
-string: /\\t/ <<
-  str_append(&string_value, "\t");
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match[2], (char)match[3], (char)match[4], (char)match[5], '}', 0};
-  str_append(&string_value, s);
->>
-string: /[^\\]/ <<
-  char s[] = {(char)match[0], 0};
-  str_append(&string_value, s);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONObject_new();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
-  $$ = $1;
->>
-KeyValue -> string colon Value <<
-  $$ = JSONObject_new();
-  JSONObject_append($$, str_cstr(&$1->string), $3);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONArray_new();
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  JSONArray_append($1, $3);
-  $$ = $1;
->>

spec/json_parser.d.propane

@@ -1,179 +0,0 @@
-<<
-import std.math;
-import json_types;
-private string string_value;
->>
-
-ptype JSONValue;
-ptype array = JSONValue[];
-ptype dict = JSONValue[string];
-ptype string = string;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n;
-    bool negative;
-    size_t i = 0u;
-    if (match[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match[i] && match[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match[i] - '0');
-        i++;
-    }
-    if (match[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            n += mult * (match[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match[i] == 'e' || match[i] == 'E')
-    {
-        bool exp_negative;
-        i++;
-        if (match[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match[i] == '+')
-        {
-            i++;
-        }
-        long exp;
-        while ('0' <= match[i] && match[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = new JSONNumber(n);
->>
-token true <<
-  $$ = new JSONTrue();
->>
-token false <<
-  $$ = new JSONFalse();
->>
-token null <<
-  $$ = new JSONNull();
->>
-/"/ <<
-  $mode(string);
-  string_value = "";
->>
-string: token string (string) /"/ <<
-  $$ = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  string_value ~= "\"";
->>
-string: /\\\\/ <<
-  string_value ~= "\\";
->>
-string: /\\\// <<
-  string_value ~= "/";
->>
-string: /\\b/ <<
-  string_value ~= "\b";
->>
-string: /\\f/ <<
-  string_value ~= "\f";
->>
-string: /\\n/ <<
-  string_value ~= "\n";
->>
-string: /\\r/ <<
-  string_value ~= "\r";
->>
-string: /\\t/ <<
-  string_value ~= "\t";
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match[2..6] ~ "}";
->>
-string: /[^\\]/ <<
-  string_value ~= match;
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = new JSONString($1);
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = new JSONObject();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = new JSONObject($2);
->>
-KeyValues (dict) -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  foreach (key, value; $3)
-  {
-    $1[key] = value;
-  }
-  $$ = $1;
->>
-KeyValue (dict) -> string colon Value <<
-  $$ = [$1: $3];
->>
-Array -> lbracket rbracket <<
-  $$ = new JSONArray();
->>
-Array -> lbracket Values rbracket <<
-  $$ = new JSONArray($2);
->>
-Values (array) -> Value <<
-  $$ = [$1];
->>
-Values -> Values comma Value <<
-  $$ = $1 ~ [$3];
->>

spec/json_types.c

@@ -1,64 +0,0 @@
-#include "json_types.h"
-#include <string.h>
-#include <stdlib.h>
-#include "testutils.h"
-
-JSONValue * JSONValue_new(size_t id)
-{
-    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
-    jv->id = id;
-    return jv;
-}
-
-JSONValue * JSONObject_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_OBJECT);
-    jv->object.size = 0u;
-    return jv;
-}
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
-{
-    size_t const size = object->object.size;
-    for (size_t i = 0u; i < size; i++)
-    {
-        if (strcmp(name, object->object.entries[i].name) == 0)
-        {
-            object->object.entries[i].value = value;
-            return;
-        }
-    }
-    size_t const new_size = size + 1;
-    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
-    if (size > 0)
-    {
-        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
-        free(object->object.entries);
-    }
-    object->object.entries = new_entries;
-    object->object.entries[size].name = name;
-    object->object.entries[size].value = value;
-    object->object.size = new_size;
-}
-
-JSONValue * JSONArray_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_ARRAY);
-    jv->array.size = 0u;
-    return jv;
-}
-
-void JSONArray_append(JSONValue * array, JSONValue * value)
-{
-    size_t const size = array->array.size;
-    size_t const new_size = size + 1;
-    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
-    if (array->array.size > 0)
-    {
-        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
-        free(array->array.entries);
-    }
-    array->array.entries = new_entries;
-    array->array.entries[size] = value;
-    array->array.size = new_size;
-}

spec/json_types.d

@@ -1,63 +0,0 @@
-class JSONValue
-{
-}
-
-class JSONObject : JSONValue
-{
-    JSONValue[string] value;
-
-    this()
-    {
-    }
-
-    this(JSONValue[string] value)
-    {
-        this.value = value;
-    }
-}
-
-class JSONArray : JSONValue
-{
-    JSONValue[] value;
-
-    this()
-    {
-    }
-
-    this(JSONValue[] value)
-    {
-        this.value = value;
-    }
-}
-
-class JSONNumber : JSONValue
-{
-    double value;
-
-    this(double value)
-    {
-        this.value = value;
-    }
-}
-
-class JSONString : JSONValue
-{
-    string value;
-
-    this(string value)
-    {
-        this.value = value;
-    }
-}
-
-class JSONTrue : JSONValue
-{
-}
-
-class JSONFalse : JSONValue
-{
-}
-
-class JSONNull : JSONValue
-{
-}

spec/json_types.h

@@ -1,48 +0,0 @@
-#pragma once
-
-#include <stddef.h>
-#include "testutils.h"
-
-#define JSON_OBJECT 0u
-#define JSON_ARRAY 1u
-#define JSON_NUMBER 2u
-#define JSON_STRING 3u
-#define JSON_TRUE 4u
-#define JSON_FALSE 5u
-#define JSON_NULL 6u
-
-typedef struct JSONObjectEntry_s
-{
-    char const * name;
-    struct JSONValue_s * value;
-} JSONObjectEntry;
-
-typedef struct JSONValue_s
-{
-    size_t id;
-    union
-    {
-        struct
-        {
-            size_t size;
-            JSONObjectEntry * entries;
-        } object;
-        struct
-        {
-            size_t size;
-            struct JSONValue_s ** entries;
-        } array;
-        double number;
-        str_t string;
-    };
-} JSONValue;
-
-JSONValue * JSONValue_new(size_t id);
-
-JSONValue * JSONObject_new(void);
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
-
-JSONValue * JSONArray_new(void);
-
-void JSONArray_append(JSONValue * array, JSONValue * value);

spec/propane/grammar_spec.rb

@@ -1,231 +0,0 @@
-class Propane
-  describe Grammar do
-    it "parses a user grammar" do
-      input = <<EOF
-# Comment line
-
-module a.b;
-ptype   XYZ  *  ;
-
-token while;
-
-token id
-  /[a-zA-Z_][a-zA-Z_0-9]*/;
-
-token token_with_code <<
-Code for the token
->>
-
-tokenid token_with_no_pattern;
-
-drop /\\s+/;
-
-A -> B <<
-  a = 42;
->>
-B -> C while id;
-B -> <<
-  b = 0;
->>
-EOF
-      grammar = Grammar.new(input)
-      expect(grammar.modulename).to eq "a.b"
-      expect(grammar.ptype).to eq "XYZ  *"
-      expect(grammar.ptypes).to eq("default" => "XYZ  *")
-      expect(grammar.prefix).to eq "p_"
-
-      o = grammar.tokens.find {|token| token.name == "while"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 6
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.pattern).to eq "while"
-      expect(o.line_number).to eq 6
-      expect(o.code).to be_nil
-
-      o = grammar.tokens.find {|token| token.name == "id"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 9
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
-      expect(o.line_number).to eq 9
-      expect(o.code).to be_nil
-
-      o = grammar.tokens.find {|token| token.name == "token_with_code"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 11
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.pattern).to eq "token_with_code"
-      expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
-
-      o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 15
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 17
-      expect(o.token).to be_nil
-      expect(o.code).to be_nil
-
-      expect(grammar.rules.size).to eq 3
-
-      o = grammar.rules[0]
-      expect(o.name).to eq "A"
-      expect(o.components).to eq %w[B]
-      expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
-
-      o = grammar.rules[1]
-      expect(o.name).to eq "B"
-      expect(o.components).to eq %w[C while id]
-      expect(o.line_number).to eq 22
-      expect(o.code).to be_nil
-
-      o = grammar.rules[2]
-      expect(o.name).to eq "B"
-      expect(o.components).to eq []
-      expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
-    end
-
-    it "parses code segments with semicolons" do
-      input = <<EOF
-token code1 <<
-  a = b;
-  return c;
->>
-
-token code2 <<
-  writeln("Hello there");
->>
-
-tokenid token_with_no_pattern;
-
-prefix myparser_;
-EOF
-      grammar = Grammar.new(input)
-      expect(grammar.prefix).to eq "myparser_"
-
-      o = grammar.tokens.find {|token| token.name == "code1"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 1
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
-
-      o = grammar.tokens.find {|token| token.name == "code2"}
-      expect(o).to_not be_nil
-      expect(o.line_number).to eq 6
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
-    end
-
-    it "supports mode labels" do
-      input = <<EOF
-token a;
-m1: token b;
-/foo/ <<
->>
-m2: /bar/ <<
->>
-drop /q/;
-m3: drop /r/;
-EOF
-      grammar = Grammar.new(input)
-
-      o = grammar.tokens.find {|token| token.name == "a"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.modes).to be_empty
-
-      o = grammar.tokens.find {|token| token.name == "b"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m1"]
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
-      expect(o).to_not be_nil
-      expect(o.modes).to be_empty
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
-      expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m2"]
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
-      expect(o).to_not be_nil
-      expect(o.modes).to be_empty
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
-      expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m3"]
-    end
-
-    it "allows assigning ptypes to tokens and rules" do
-      input = <<EOF
-ptype Subnode *;
-ptype string = char *;
-ptype integer = int;
-ptype node = Node *;
-
-token abc(string);
-token bar;
-tokenid int(integer);
-
-/xyz/ (string) <<
->>
-
-/z28/ <<
->>
-
-Start (node) -> R;
-R -> abc int;
-EOF
-      grammar = Grammar.new(input)
-
-      o = grammar.tokens.find {|token| token.name == "abc"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to eq "string"
-
-      o = grammar.tokens.find {|token| token.name == "bar"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to be_nil
-
-      o = grammar.tokens.find {|token| token.name == "int"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to eq "integer"
-
-      o = grammar.rules.find {|rule| rule.name == "Start"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to eq "node"
-
-      o = grammar.rules.find {|rule| rule.name == "R"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "xyz"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to eq "string"
-
-      o = grammar.patterns.find {|pattern| pattern.pattern == "z28"}
-      expect(o).to_not be_nil
-      expect(o.ptypename).to be_nil
-    end
-  end
-end

spec/propane/lexer/dfa_spec.rb

@@ -35,8 +35,7 @@ class TestLexer
       end
     end
     if last_accepts
-      name = last_accepts.token ? last_accepts.token.name : nil
+      [last_accepts.name, last_s]
-      [name, last_s]
     end
   end
 
@@ -52,7 +51,7 @@ end
 
 def run(grammar, input)
   grammar = Propane::Grammar.new(grammar)
-  token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
+  token_dfa = Propane::Lexer::DFA.new(grammar.tokens, grammar.drop_tokens)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)
 end
@@ -60,7 +59,7 @@ end
 describe Propane::Lexer::DFA do
   it "lexes a simple token" do
     expect(run(<<EOF, "foo")).to eq [["foo", "foo"]]
-token foo;
+token foo
 EOF
   end
 
@@ -70,8 +69,8 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo;
+token foo
-token bar;
+token bar
 EOF
   end
 
@@ -80,17 +79,17 @@ EOF
       ["identifier", "foobar"],
     ]
     expect(run(<<EOF, "foobar")).to eq expected
-token foo;
+token foo
-token bar;
+token bar
-token identifier /[a-z]+/;
+token identifier [a-z]+
 EOF
     expected = [
       ["plusplus", "++"],
       ["plus", "+"],
     ]
     expect(run(<<EOF, "+++")).to eq expected
-token plus /\\+/;
+token plus \\+
-token plusplus /\\+\\+/;
+token plusplus \\+\\+
 EOF
   end
 
@@ -101,9 +100,9 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo;
+token foo
-token bar;
+token bar
-token WS /\\s+/;
+token WS \\s+
 EOF
   end
 
@@ -114,86 +113,9 @@ EOF
       ["bar", "bar"],
     ]
     expect(run(<<EOF, "foo \tbar")).to eq expected
-token foo;
+token foo
-token bar;
+token bar
-drop /\\s+/;
+drop \\s+
-EOF
-  end
-
-  it "matches a semicolon" do
-    expected = [
-      ["semicolon", ";"],
-    ]
-    expect(run(<<EOF, ";")).to eq expected
-token semicolon /;/;
-EOF
-  end
-
-  it "matches a negated character class" do
-    expected = [
-      ["pattern", "/abc/"],
-    ]
-    expect(run(<<EOF, "/abc/")).to eq expected
-token pattern /\\/[^\\s]*\\//;
-EOF
-  end
-
-  it "matches special character classes " do
-    expected = [
-      ["a", "abc123_FOO"],
-    ]
-    expect(run(<<EOF, "abc123_FOO")).to eq expected
-token a /\\w+/;
-EOF
-    expected = [
-      ["b", "FROG*%$#"],
-    ]
-    expect(run(<<EOF, "FROG*%$#")).to eq expected
-token b /FROG\\D{1,4}/;
-EOF
-    expected = [
-      ["c", "$883366"],
-    ]
-    expect(run(<<EOF, "$883366")).to eq expected
-token c /$\\d+/;
-EOF
-    expected = [
-      ["d", "^&$@"],
-    ]
-    expect(run(<<EOF, "^&$@")).to eq expected
-token d /^\\W+/;
-EOF
-    expected = [
-      ["a", "abc123_FOO"],
-      [nil, " "],
-      ["b", "FROG*%$#"],
-      [nil, " "],
-      ["c", "$883366"],
-      [nil, " "],
-      ["d", "^&$@"],
-    ]
-    expect(run(<<EOF, "abc123_FOO FROG*%$# $883366 ^&$@")).to eq expected
-token a /\\w+/;
-token b /FROG\\D{1,4}/;
-token c /$\\d+/;
-token d /^\\W+/;
-drop /\\s+/;
-EOF
-  end
-
-  it "matches a negated character class with a nested inner negated character class" do
-    expected = [
-      ["t", "$&*"],
-    ]
-    expect(run(<<EOF, "$&*")).to eq expected
-token t /[^%\\W]+/;
-EOF
-  end
-
-  it "\\s matches a newline" do
-    expected = [["s", "\n"]]
-    expect(run(<<EOF, "\n")).to eq expected
-token s /\\s/;
 EOF
   end
 end

spec/propane/regex_spec.rb

@@ -2,14 +2,14 @@ class Propane
   RSpec.describe Regex do
 
     it "parses an empty expression" do
-      regex = Regex.new("", 1)
+      regex = Regex.new("")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0].size).to eq 0
     end
 
     it "parses a single character unit expression" do
-      regex = Regex.new("a", 1)
+      regex = Regex.new("a")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -19,7 +19,7 @@ class Propane
     end
 
     it "parses a group with a single character unit expression" do
-      regex = Regex.new("(a)", 1)
+      regex = Regex.new("(a)")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -33,7 +33,7 @@ class Propane
     end
 
     it "parses a *" do
-      regex = Regex.new("a*", 1)
+      regex = Regex.new("a*")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -47,7 +47,7 @@ class Propane
     end
 
     it "parses a +" do
-      regex = Regex.new("a+", 1)
+      regex = Regex.new("a+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -61,7 +61,7 @@ class Propane
     end
 
     it "parses a ?" do
-      regex = Regex.new("a?", 1)
+      regex = Regex.new("a?")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -75,7 +75,7 @@ class Propane
     end
 
     it "parses a multiplicity count" do
-      regex = Regex.new("a{5}", 1)
+      regex = Regex.new("a{5}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -89,7 +89,7 @@ class Propane
     end
 
     it "parses a minimum-only multiplicity count" do
-      regex = Regex.new("a{5,}", 1)
+      regex = Regex.new("a{5,}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -103,7 +103,7 @@ class Propane
     end
 
     it "parses a minimum and maximum multiplicity count" do
-      regex = Regex.new("a{5,8}", 1)
+      regex = Regex.new("a{5,8}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -118,7 +118,7 @@ class Propane
     end
 
     it "parses an escaped *" do
-      regex = Regex.new("a\\*", 1)
+      regex = Regex.new("a\\*")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -131,7 +131,7 @@ class Propane
     end
 
     it "parses an escaped +" do
-      regex = Regex.new("a\\+", 1)
+      regex = Regex.new("a\\+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -144,7 +144,7 @@ class Propane
     end
 
     it "parses an escaped \\" do
-      regex = Regex.new("\\\\d", 1)
+      regex = Regex.new("\\\\d")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -157,7 +157,7 @@ class Propane
     end
 
     it "parses a character class" do
-      regex = Regex.new("[a-z_]", 1)
+      regex = Regex.new("[a-z_]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -175,7 +175,7 @@ class Propane
     end
 
     it "parses a negated character class" do
-      regex = Regex.new("[^xyz]", 1)
+      regex = Regex.new("[^xyz]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -189,25 +189,8 @@ class Propane
       expect(ccu[0].first).to eq "x".ord
     end
 
-    it "parses a negated character class with inner character classes" do
-      regex = Regex.new("[^x\\sz]", 1)
-      expect(regex.unit).to be_a Regex::AlternatesUnit
-      expect(regex.unit.alternates.size).to eq 1
-      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
-      seq_unit = regex.unit.alternates[0]
-      expect(seq_unit.size).to eq 1
-      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
-      ccu = seq_unit[0]
-      expect(ccu.negate).to be_truthy
-      expect(ccu.size).to eq 8
-      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
-      expect(ccu[0].first).to eq "x".ord
-      expect(ccu[1].first).to eq " ".ord
-      expect(ccu[7].first).to eq "z".ord
-    end
-
     it "parses - as a plain character at beginning of a character class" do
-      regex = Regex.new("[-9]", 1)
+      regex = Regex.new("[-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -221,7 +204,7 @@ class Propane
     end
 
     it "parses - as a plain character at end of a character class" do
-      regex = Regex.new("[0-]", 1)
+      regex = Regex.new("[0-]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -237,7 +220,7 @@ class Propane
     end
 
     it "parses - as a plain character at beginning of a negated character class" do
-      regex = Regex.new("[^-9]", 1)
+      regex = Regex.new("[^-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -252,7 +235,7 @@ class Propane
     end
 
     it "parses . as a plain character in a character class" do
-      regex = Regex.new("[.]", 1)
+      regex = Regex.new("[.]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -267,7 +250,7 @@ class Propane
     end
 
     it "parses - as a plain character when escaped in middle of character class" do
-      regex = Regex.new("[0\\-9]", 1)
+      regex = Regex.new("[0\\-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -286,7 +269,7 @@ class Propane
     end
 
     it "parses alternates" do
-      regex = Regex.new("ab|c", 1)
+      regex = Regex.new("ab|c")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 2
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -296,7 +279,7 @@ class Propane
     end
 
     it "parses a ." do
-      regex = Regex.new("a.b", 1)
+      regex = Regex.new("a.b")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -307,7 +290,7 @@ class Propane
     end
 
     it "parses something complex" do
-      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+", 1)
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 4
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit

spec/spec_helper.rb

@@ -1,23 +1,5 @@
-unless ENV["dist_specs"]
 require "bundler/setup"
-  require "simplecov"
+require "propane"
-
-  class MyFormatter
-    def format(*args)
-    end
-  end
-  SimpleCov.start do
-    add_filter "/spec/"
-    add_filter "/.bundle/"
-    if ENV["partial_specs"]
-      command_name "RSpec-partial"
-    else
-      command_name "RSpec"
-    end
-    project_name "Propane"
-    merge_timeout 3600
-    formatter(MyFormatter)
-  end
 
 RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
@@ -27,6 +9,3 @@ unless ENV["dist_specs"]
     c.syntax = :expect
   end
 end
-end
-
-require "propane"

spec/test_ast.c

@@ -1,61 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
-    assert(start->pItems1 != NULL);
-    assert(start->pItems != NULL);
-    Items * items = start->pItems;
-    assert(items->pItem != NULL);
-    assert(items->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_a, items->pItem->pToken1->token);
-    assert_eq(11, items->pItem->pToken1->pvalue);
-    assert(items->pItemsMore != NULL);
-    ItemsMore * itemsmore = items->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore != NULL);
-    itemsmore = itemsmore->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore == NULL);
-
-    p_free_ast(start);
-
-    input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start->pItems == NULL);
-
-    p_free_ast(start);
-
-    input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start->pItems != NULL);
-    assert(start->pItems->pItem != NULL);
-    assert(start->pItems->pItem->pDual != NULL);
-    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
-    assert(start->pItems->pItem->pDual->pOne2 != NULL);
-    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
-    assert(start->pItems->pItem->pDual->pOne1 == NULL);
-
-    p_free_ast(start);
-
-    return 0;
-}

spec/test_ast.d

@@ -1,57 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
-    assert(start.pItems1 !is null);
-    assert(start.pItems !is null);
-    Items * items = start.pItems;
-    assert(items.pItem !is null);
-    assert(items.pItem.pToken1 !is null);
-    assert_eq(TOKEN_a, items.pItem.pToken1.token);
-    assert_eq(11, items.pItem.pToken1.pvalue);
-    assert(items.pItemsMore !is null);
-    ItemsMore * itemsmore = items.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
-    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
-    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore !is null);
-    itemsmore = itemsmore.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pToken1 !is null);
-    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
-    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore is null);
-
-    input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start.pItems is null);
-
-    input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start.pItems !is null);
-    assert(start.pItems.pItem !is null);
-    assert(start.pItems.pItem.pDual !is null);
-    assert(start.pItems.pItem.pDual.pTwo1 !is null);
-    assert(start.pItems.pItem.pDual.pOne2 !is null);
-    assert(start.pItems.pItem.pDual.pTwo2 is null);
-    assert(start.pItems.pItem.pDual.pOne1 is null);
-}

spec/test_ast_field_aliases.c

@@ -1,21 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(TOKEN_a, start->first->pToken->token);
-    assert_eq(TOKEN_b, start->second->pToken->token);
-    assert_eq(TOKEN_c, start->third->pToken->token);
-
-    p_free_ast(start);
-
-    return 0;
-}

spec/test_ast_field_aliases.d

@@ -1,21 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "\na\nb\nc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(TOKEN_a, start.first.pToken.token);
-    assert_eq(TOKEN_b, start.second.pToken.token);
-    assert_eq(TOKEN_c, start.third.pToken.token);
-}

spec/test_ast_invalid_positions.c

@@ -1,110 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(2, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(2, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(3, start->pT1->pA->position.row);
-    assert_eq(3, start->pT1->pA->position.col);
-    assert_eq(3, start->pT1->pA->end_position.row);
-    assert_eq(8, start->pT1->pA->end_position.col);
-    assert_eq(2, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(3, start->pT1->end_position.row);
-    assert_eq(8, start->pT1->end_position.col);
-
-    assert_eq(2, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(3, start->end_position.row);
-    assert_eq(8, start->end_position.col);
-
-    p_free_ast(start);
-
-    input = "a\nbb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(1, start->pT1->pA->position.col);
-    assert_eq(2, start->pT1->pA->end_position.row);
-    assert_eq(2, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(2, start->pT1->end_position.row);
-    assert_eq(2, start->pT1->end_position.col);
-
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(2, start->end_position.row);
-    assert_eq(2, start->end_position.col);
-
-    p_free_ast(start);
-
-    input = "a\nc\nc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(p_position_valid(start->pT1->pA->position));
-    assert_eq(2, start->pT1->pA->position.row);
-    assert_eq(1, start->pT1->pA->position.col);
-    assert_eq(3, start->pT1->pA->end_position.row);
-    assert_eq(1, start->pT1->pA->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(3, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
-
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(3, start->end_position.row);
-    assert_eq(1, start->end_position.col);
-
-    p_free_ast(start);
-
-    input = "a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert(!p_position_valid(start->pT1->pA->position));
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(1, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
-
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(1, start->end_position.row);
-    assert_eq(1, start->end_position.col);
-
-    p_free_ast(start);
-
-    return 0;
-}

spec/test_ast_invalid_positions.d

@@ -1,104 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "\na\n  bb ccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(2, start.pT1.pToken.position.row);
-    assert_eq(1, start.pT1.pToken.position.col);
-    assert_eq(2, start.pT1.pToken.end_position.row);
-    assert_eq(1, start.pT1.pToken.end_position.col);
-    assert(start.pT1.pA.position.valid);
-    assert_eq(3, start.pT1.pA.position.row);
-    assert_eq(3, start.pT1.pA.position.col);
-    assert_eq(3, start.pT1.pA.end_position.row);
-    assert_eq(8, start.pT1.pA.end_position.col);
-    assert_eq(2, start.pT1.position.row);
-    assert_eq(1, start.pT1.position.col);
-    assert_eq(3, start.pT1.end_position.row);
-    assert_eq(8, start.pT1.end_position.col);
-
-    assert_eq(2, start.position.row);
-    assert_eq(1, start.position.col);
-    assert_eq(3, start.end_position.row);
-    assert_eq(8, start.end_position.col);
-
-    input = "a\nbb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(1, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(1, start.pT1.pToken.end_position.col);
-    assert(start.pT1.pA.position.valid);
-    assert_eq(2, start.pT1.pA.position.row);
-    assert_eq(1, start.pT1.pA.position.col);
-    assert_eq(2, start.pT1.pA.end_position.row);
-    assert_eq(2, start.pT1.pA.end_position.col);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(1, start.pT1.position.col);
-    assert_eq(2, start.pT1.end_position.row);
-    assert_eq(2, start.pT1.end_position.col);
-
-    assert_eq(1, start.position.row);
-    assert_eq(1, start.position.col);
-    assert_eq(2, start.end_position.row);
-    assert_eq(2, start.end_position.col);
-
-    input = "a\nc\nc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(1, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(1, start.pT1.pToken.end_position.col);
-    assert(start.pT1.pA.position.valid);
-    assert_eq(2, start.pT1.pA.position.row);
-    assert_eq(1, start.pT1.pA.position.col);
-    assert_eq(3, start.pT1.pA.end_position.row);
-    assert_eq(1, start.pT1.pA.end_position.col);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(1, start.pT1.position.col);
-    assert_eq(3, start.pT1.end_position.row);
-    assert_eq(1, start.pT1.end_position.col);
-
-    assert_eq(1, start.position.row);
-    assert_eq(1, start.position.col);
-    assert_eq(3, start.end_position.row);
-    assert_eq(1, start.end_position.col);
-
-    input = "a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(1, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(1, start.pT1.pToken.end_position.col);
-    assert(!start.pT1.pA.position.valid);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(1, start.pT1.position.col);
-    assert_eq(1, start.pT1.end_position.row);
-    assert_eq(1, start.pT1.end_position.col);
-
-    assert_eq(1, start.position.row);
-    assert_eq(1, start.position.col);
-    assert_eq(1, start.end_position.row);
-    assert_eq(1, start.end_position.col);
-}

spec/test_ast_node_memory_remains.c

@@ -1,418 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdlib.h>
-#include "testutils.h"
-
-int main(int argc, char * argv[])
-{
-    const char * input =
-        "# 0\n"
-        "def byte_val() -> byte\n"
-        "{\n"
-        "    return 0x42;\n"
-        "}\n"
-        "\n"
-        "# 1\n"
-        "def short_val() -> short\n"
-        "{\n"
-        "    return 0x4242;\n"
-        "}\n"
-        "\n"
-        "# 2\n"
-        "def int_val() -> int\n"
-        "{\n"
-        "    return 0x42424242;\n"
-        "}\n"
-        "\n"
-        "# 3\n"
-        "def long_val() -> long\n"
-        "{\n"
-        "    return 0x4242_4242_4242_4242;\n"
-        "}\n"
-        "\n"
-        "# 4\n"
-        "def ssize_t_val() -> ssize_t\n"
-        "{\n"
-        "    return 0x42424242;\n"
-        "}\n"
-        "\n"
-        "# 5\n"
-        "def byte_to_short() -> short\n"
-        "{\n"
-        "    return byte_val();\n"
-        "}\n"
-        "\n"
-        "# 6\n"
-        "def byte_to_int() -> int\n"
-        "{\n"
-        "    return byte_val();\n"
-        "}\n"
-        "\n"
-        "# 7\n"
-        "def byte_to_long() -> long\n"
-        "{\n"
-        "    return byte_val();\n"
-        "}\n"
-        "\n"
-        "# 8\n"
-        "def byte_to_ssize_t() -> ssize_t\n"
-        "{\n"
-        "    return byte_val();\n"
-        "}\n"
-        "\n"
-        "# 9\n"
-        "def short_to_byte() -> byte\n"
-        "{\n"
-        "    return short_val();\n"
-        "}\n"
-        "\n"
-        "# 10\n"
-        "def short_to_int() -> int\n"
-        "{\n"
-        "    return short_val();\n"
-        "}\n"
-        "\n"
-        "# 11\n"
-        "def short_to_long() -> long\n"
-        "{\n"
-        "    return short_val();\n"
-        "}\n"
-        "\n"
-        "# 12\n"
-        "def short_to_ssize_t() -> ssize_t\n"
-        "{\n"
-        "    return short_val();\n"
-        "}\n"
-        "\n"
-        "# 13\n"
-        "def int_to_byte() -> byte\n"
-        "{\n"
-        "    return int_val();\n"
-        "}\n"
-        "\n"
-        "# 14\n"
-        "def int_to_short() -> short\n"
-        "{\n"
-        "    return int_val();\n"
-        "}\n"
-        "\n"
-        "# 15\n"
-        "def int_to_long() -> long\n"
-        "{\n"
-        "    return int_val();\n"
-        "}\n"
-        "\n"
-        "# 16\n"
-        "def int_to_ssize_t() -> ssize_t\n"
-        "{\n"
-        "    return int_val();\n"
-        "}\n"
-        "\n"
-        "# 17\n"
-        "def long_to_byte() -> byte\n"
-        "{\n"
-        "    return long_val();\n"
-        "}\n"
-        "\n"
-        "# 18\n"
-        "def long_to_short() -> short\n"
-        "{\n"
-        "    return long_val();\n"
-        "}\n"
-        "\n"
-        "# 19\n"
-        "def long_to_int() -> int\n"
-        "{\n"
-        "    return long_val();\n"
-        "}\n"
-        "\n"
-        "# 20\n"
-        "def long_to_ssize_t() -> ssize_t\n"
-        "{\n"
-        "    return long_val();\n"
-        "}\n"
-        "\n"
-        "# 21\n"
-        "def ssize_t_to_byte() -> byte\n"
-        "{\n"
-        "    return ssize_t_val();\n"
-        "}\n"
-        "\n"
-        "# 22\n"
-        "def ssize_t_to_short() -> short\n"
-        "{\n"
-        "    return ssize_t_val();\n"
-        "}\n"
-        "\n"
-        "# 23\n"
-        "def ssize_t_to_int() -> int\n"
-        "{\n"
-        "    return ssize_t_val();\n"
-        "}\n"
-        "\n"
-        "# 24\n"
-        "def ssize_t_to_long() -> long\n"
-        "{\n"
-        "    return ssize_t_val();\n"
-        "}\n"
-        "\n"
-        "# 25\n"
-        "def ubyte_val() -> ubyte\n"
-        "{\n"
-        "    return 0x42;\n"
-        "}\n"
-        "\n"
-        "# 26\n"
-        "def ushort_val() -> ushort\n"
-        "{\n"
-        "    return 0x4242;\n"
-        "}\n"
-        "\n"
-        "# 27\n"
-        "def uint_val() -> uint\n"
-        "{\n"
-        "    return 0x42424242;\n"
-        "}\n"
-        "\n"
-        "# 28\n"
-        "def ulong_val() -> ulong\n"
-        "{\n"
-        "    return 0x4242_4242_4242_4242;\n"
-        "}\n"
-        "\n"
-        "# 29\n"
-        "def size_t_val() -> size_t\n"
-        "{\n"
-        "    return 0x42424242;\n"
-        "}\n"
-        "\n"
-        "# 30\n"
-        "def ubyte_to_ushort() -> ushort\n"
-        "{\n"
-        "    return ubyte_val();\n"
-        "}\n"
-        "\n"
-        "# 31\n"
-        "def ubyte_to_uint() -> uint\n"
-        "{\n"
-        "    return ubyte_val();\n"
-        "}\n"
-        "\n"
-        "# 32\n"
-        "def ubyte_to_ulong() -> ulong\n"
-        "{\n"
-        "    return ubyte_val();\n"
-        "}\n"
-        "\n"
-        "# 33\n"
-        "def ubyte_to_size_t() -> size_t\n"
-        "{\n"
-        "    return ubyte_val();\n"
-        "}\n"
-        "\n"
-        "# 34\n"
-        "def ushort_to_ubyte() -> ubyte\n"
-        "{\n"
-        "    return ushort_val();\n"
-        "}\n"
-        "\n"
-        "# 35\n"
-        "def ushort_to_uint() -> uint\n"
-        "{\n"
-        "    return ushort_val();\n"
-        "}\n"
-        "\n"
-        "# 36\n"
-        "def ushort_to_ulong() -> ulong\n"
-        "{\n"
-        "    return ushort_val();\n"
-        "}\n"
-        "\n"
-        "# 37\n"
-        "def ushort_to_size_t() -> size_t\n"
-        "{\n"
-        "    return ushort_val();\n"
-        "}\n"
-        "\n"
-        "# 38\n"
-        "def uint_to_ubyte() -> ubyte\n"
-        "{\n"
-        "    return uint_val();\n"
-        "}\n"
-        "\n"
-        "# 39\n"
-        "def uint_to_ushort() -> ushort\n"
-        "{\n"
-        "    return uint_val();\n"
-        "}\n"
-        "\n"
-        "# 40\n"
-        "def uint_to_ulong() -> ulong\n"
-        "{\n"
-        "    return uint_val();\n"
-        "}\n"
-        "\n"
-        "# 41\n"
-        "def uint_to_size_t() -> size_t\n"
-        "{\n"
-        "    return uint_val();\n"
-        "}\n"
-        "\n"
-        "# 42\n"
-        "def ulong_to_ubyte() -> ubyte\n"
-        "{\n"
-        "    return ulong_val();\n"
-        "}\n"
-        "\n"
-        "# 43\n"
-        "def ulong_to_ushort() -> ushort\n"
-        "{\n"
-        "    return ulong_val();\n"
-        "}\n"
-        "\n"
-        "# 44\n"
-        "def ulong_to_uint() -> uint\n"
-        "{\n"
-        "    return ulong_val();\n"
-        "}\n"
-        "\n"
-        "# 45\n"
-        "def ulong_to_size_t() -> size_t\n"
-        "{\n"
-        "    return ulong_val();\n"
-        "}\n"
-        "\n"
-        "# 46\n"
-        "def size_t_to_ubyte() -> ubyte\n"
-        "{\n"
-        "    return size_t_val();\n"
-        "}\n"
-        "\n"
-        "# 47\n"
-        "def size_t_to_ushort() -> ushort\n"
-        "{\n"
-        "    return size_t_val();\n"
-        "}\n"
-        "\n"
-        "# 48\n"
-        "def size_t_to_int() -> int\n"
-        "{\n"
-        "    return size_t_val();\n"
-        "}\n"
-        "\n"
-        "# 49\n"
-        "def size_t_to_ulong() -> ulong\n"
-        "{\n"
-        "    return size_t_val();\n"
-        "}\n"
-        "\n"
-        "# 50\n"
-        "def main() -> int\n"
-        "{\n"
-        "    return int_val();\n"
-        "}\n";
-    struct
-    {
-        const char * name;
-        p_token_t token;
-    } expected[] = {
-        {"byte_val", TOKEN_byte},
-        {"short_val", TOKEN_short},
-        {"int_val", TOKEN_int},
-        {"long_val", TOKEN_long},
-        {"ssize_t_val", TOKEN_ssize_t},
-        {"byte_to_short", TOKEN_short},
-        {"byte_to_int", TOKEN_int},
-        {"byte_to_long", TOKEN_long},
-        {"byte_to_ssize_t", TOKEN_ssize_t},
-        {"short_to_byte", TOKEN_byte},
-        {"short_to_int", TOKEN_int},
-        {"short_to_long", TOKEN_long},
-        {"short_to_ssize_t", TOKEN_ssize_t},
-        {"int_to_byte", TOKEN_byte},
-        {"int_to_short", TOKEN_short},
-        {"int_to_long", TOKEN_long},
-        {"int_to_ssize_t", TOKEN_ssize_t},
-        {"long_to_byte", TOKEN_byte},
-        {"long_to_short", TOKEN_short},
-        {"long_to_int", TOKEN_int},
-        {"long_to_ssize_t", TOKEN_ssize_t},
-        {"ssize_t_to_byte", TOKEN_byte},
-        {"ssize_t_to_short", TOKEN_short},
-        {"ssize_t_to_int", TOKEN_int},
-        {"ssize_t_to_long", TOKEN_long},
-        {"ubyte_val", TOKEN_ubyte},
-        {"ushort_val", TOKEN_ushort},
-        {"uint_val", TOKEN_uint},
-        {"ulong_val", TOKEN_ulong},
-        {"size_t_val", TOKEN_size_t},
-        {"ubyte_to_ushort", TOKEN_ushort},
-        {"ubyte_to_uint", TOKEN_uint},
-        {"ubyte_to_ulong", TOKEN_ulong},
-        {"ubyte_to_size_t", TOKEN_size_t},
-        {"ushort_to_ubyte", TOKEN_ubyte},
-        {"ushort_to_uint", TOKEN_uint},
-        {"ushort_to_ulong", TOKEN_ulong},
-        {"ushort_to_size_t", TOKEN_size_t},
-        {"uint_to_ubyte", TOKEN_ubyte},
-        {"uint_to_ushort", TOKEN_ushort},
-        {"uint_to_ulong", TOKEN_ulong},
-        {"uint_to_size_t", TOKEN_size_t},
-        {"ulong_to_ubyte", TOKEN_ubyte},
-        {"ulong_to_ushort", TOKEN_ushort},
-        {"ulong_to_uint", TOKEN_uint},
-        {"ulong_to_size_t", TOKEN_size_t},
-        {"size_t_to_ubyte", TOKEN_ubyte},
-        {"size_t_to_ushort", TOKEN_ushort},
-        {"size_t_to_int", TOKEN_int},
-        {"size_t_to_ulong", TOKEN_ulong},
-        {"main", TOKEN_int},
-    };
-    p_context_t context;
-    p_context_init(&context, (const uint8_t *)input, strlen(input));
-    size_t result = p_parse(&context);
-    assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
-    PModuleItems * pmis = pmod->pModuleItems;
-    PFunctionDefinition ** pfds;
-    size_t n_pfds = 0u;
-    while (pmis != NULL)
-    {
-        PModuleItem * pmi = pmis->pModuleItem;
-        if (pmi->pFunctionDefinition != NULL)
-        {
-            n_pfds++;
-        }
-        pmis = pmis->pModuleItems;
-    }
-    pfds = (PFunctionDefinition **)malloc(n_pfds * sizeof(PModuleItems *));
-    pmis = pmod->pModuleItems;
-    size_t pfd_i = n_pfds;
-    while (pmis != NULL)
-    {
-        PModuleItem * pmi = pmis->pModuleItem;
-        PFunctionDefinition * pfd = pmi->pFunctionDefinition;
-        if (pfd != NULL)
-        {
-            pfd_i--;
-            assert(pfd_i < n_pfds);
-            pfds[pfd_i] = pfd;
-        }
-        pmis = pmis->pModuleItems;
-    }
-    assert_eq(51, n_pfds);
-    for (size_t i = 0; i < n_pfds; i++)
-    {
-        if (strncmp(expected[i].name, (const char *)pfds[i]->name->pvalue.s, strlen(expected[i].name)) != 0 ||
-            (expected[i].token != pfds[i]->returntype->pType->pTypeBase->pToken1->token))
-        {
-            fprintf(stderr, "Index %lu: expected %s/%u, got %u\n", i, expected[i].name, expected[i].token, pfds[i]->returntype->pType->pTypeBase->pToken1->token);
-        }
-    }
-
-    free(pfds);
-    p_free_ast(pmod);
-
-    return 0;
-}

spec/test_ast_node_memory_remains.d

@@ -1,408 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "
-# 0
-def byte_val() -> byte
-{
-    return 0x42;
-}
-
-# 1
-def short_val() -> short
-{
-    return 0x4242;
-}
-
-# 2
-def int_val() -> int
-{
-    return 0x42424242;
-}
-
-# 3
-def long_val() -> long
-{
-    return 0x4242_4242_4242_4242;
-}
-
-# 4
-def ssize_t_val() -> ssize_t
-{
-    return 0x42424242;
-}
-
-# 5
-def byte_to_short() -> short
-{
-    return byte_val();
-}
-
-# 6
-def byte_to_int() -> int
-{
-    return byte_val();
-}
-
-# 7
-def byte_to_long() -> long
-{
-    return byte_val();
-}
-
-# 8
-def byte_to_ssize_t() -> ssize_t
-{
-    return byte_val();
-}
-
-# 9
-def short_to_byte() -> byte
-{
-    return short_val();
-}
-
-# 10
-def short_to_int() -> int
-{
-    return short_val();
-}
-
-# 11
-def short_to_long() -> long
-{
-    return short_val();
-}
-
-# 12
-def short_to_ssize_t() -> ssize_t
-{
-    return short_val();
-}
-
-# 13
-def int_to_byte() -> byte
-{
-    return int_val();
-}
-
-# 14
-def int_to_short() -> short
-{
-    return int_val();
-}
-
-# 15
-def int_to_long() -> long
-{
-    return int_val();
-}
-
-# 16
-def int_to_ssize_t() -> ssize_t
-{
-    return int_val();
-}
-
-# 17
-def long_to_byte() -> byte
-{
-    return long_val();
-}
-
-# 18
-def long_to_short() -> short
-{
-    return long_val();
-}
-
-# 19
-def long_to_int() -> int
-{
-    return long_val();
-}
-
-# 20
-def long_to_ssize_t() -> ssize_t
-{
-    return long_val();
-}
-
-# 21
-def ssize_t_to_byte() -> byte
-{
-    return ssize_t_val();
-}
-
-# 22
-def ssize_t_to_short() -> short
-{
-    return ssize_t_val();
-}
-
-# 23
-def ssize_t_to_int() -> int
-{
-    return ssize_t_val();
-}
-
-# 24
-def ssize_t_to_long() -> long
-{
-    return ssize_t_val();
-}
-
-# 25
-def ubyte_val() -> ubyte
-{
-    return 0x42;
-}
-
-# 26
-def ushort_val() -> ushort
-{
-    return 0x4242;
-}
-
-# 27
-def uint_val() -> uint
-{
-    return 0x42424242;
-}
-
-# 28
-def ulong_val() -> ulong
-{
-    return 0x4242_4242_4242_4242;
-}
-
-# 29
-def size_t_val() -> size_t
-{
-    return 0x42424242;
-}
-
-# 30
-def ubyte_to_ushort() -> ushort
-{
-    return ubyte_val();
-}
-
-# 31
-def ubyte_to_uint() -> uint
-{
-    return ubyte_val();
-}
-
-# 32
-def ubyte_to_ulong() -> ulong
-{
-    return ubyte_val();
-}
-
-# 33
-def ubyte_to_size_t() -> size_t
-{
-    return ubyte_val();
-}
-
-# 34
-def ushort_to_ubyte() -> ubyte
-{
-    return ushort_val();
-}
-
-# 35
-def ushort_to_uint() -> uint
-{
-    return ushort_val();
-}
-
-# 36
-def ushort_to_ulong() -> ulong
-{
-    return ushort_val();
-}
-
-# 37
-def ushort_to_size_t() -> size_t
-{
-    return ushort_val();
-}
-
-# 38
-def uint_to_ubyte() -> ubyte
-{
-    return uint_val();
-}
-
-# 39
-def uint_to_ushort() -> ushort
-{
-    return uint_val();
-}
-
-# 40
-def uint_to_ulong() -> ulong
-{
-    return uint_val();
-}
-
-# 41
-def uint_to_size_t() -> size_t
-{
-    return uint_val();
-}
-
-# 42
-def ulong_to_ubyte() -> ubyte
-{
-    return ulong_val();
-}
-
-# 43
-def ulong_to_ushort() -> ushort
-{
-    return ulong_val();
-}
-
-# 44
-def ulong_to_uint() -> uint
-{
-    return ulong_val();
-}
-
-# 45
-def ulong_to_size_t() -> size_t
-{
-    return ulong_val();
-}
-
-# 46
-def size_t_to_ubyte() -> ubyte
-{
-    return size_t_val();
-}
-
-# 47
-def size_t_to_ushort() -> ushort
-{
-    return size_t_val();
-}
-
-# 48
-def size_t_to_int() -> int
-{
-    return size_t_val();
-}
-
-# 49
-def size_t_to_ulong() -> ulong
-{
-    return size_t_val();
-}
-
-# 50
-def main() -> int
-{
-    return int_val();
-}
-";
-    struct Expected
-    {
-        string name;
-        p_token_t token;
-    }
-    Expected[] expected = [
-        Expected("byte_val", TOKEN_byte),
-        Expected("short_val", TOKEN_short),
-        Expected("int_val", TOKEN_int),
-        Expected("long_val", TOKEN_long),
-        Expected("ssize_t_val", TOKEN_ssize_t),
-        Expected("byte_to_short", TOKEN_short),
-        Expected("byte_to_int", TOKEN_int),
-        Expected("byte_to_long", TOKEN_long),
-        Expected("byte_to_ssize_t", TOKEN_ssize_t),
-        Expected("short_to_byte", TOKEN_byte),
-        Expected("short_to_int", TOKEN_int),
-        Expected("short_to_long", TOKEN_long),
-        Expected("short_to_ssize_t", TOKEN_ssize_t),
-        Expected("int_to_byte", TOKEN_byte),
-        Expected("int_to_short", TOKEN_short),
-        Expected("int_to_long", TOKEN_long),
-        Expected("int_to_ssize_t", TOKEN_ssize_t),
-        Expected("long_to_byte", TOKEN_byte),
-        Expected("long_to_short", TOKEN_short),
-        Expected("long_to_int", TOKEN_int),
-        Expected("long_to_ssize_t", TOKEN_ssize_t),
-        Expected("ssize_t_to_byte", TOKEN_byte),
-        Expected("ssize_t_to_short", TOKEN_short),
-        Expected("ssize_t_to_int", TOKEN_int),
-        Expected("ssize_t_to_long", TOKEN_long),
-        Expected("ubyte_val", TOKEN_ubyte),
-        Expected("ushort_val", TOKEN_ushort),
-        Expected("uint_val", TOKEN_uint),
-        Expected("ulong_val", TOKEN_ulong),
-        Expected("size_t_val", TOKEN_size_t),
-        Expected("ubyte_to_ushort", TOKEN_ushort),
-        Expected("ubyte_to_uint", TOKEN_uint),
-        Expected("ubyte_to_ulong", TOKEN_ulong),
-        Expected("ubyte_to_size_t", TOKEN_size_t),
-        Expected("ushort_to_ubyte", TOKEN_ubyte),
-        Expected("ushort_to_uint", TOKEN_uint),
-        Expected("ushort_to_ulong", TOKEN_ulong),
-        Expected("ushort_to_size_t", TOKEN_size_t),
-        Expected("uint_to_ubyte", TOKEN_ubyte),
-        Expected("uint_to_ushort", TOKEN_ushort),
-        Expected("uint_to_ulong", TOKEN_ulong),
-        Expected("uint_to_size_t", TOKEN_size_t),
-        Expected("ulong_to_ubyte", TOKEN_ubyte),
-        Expected("ulong_to_ushort", TOKEN_ushort),
-        Expected("ulong_to_uint", TOKEN_uint),
-        Expected("ulong_to_size_t", TOKEN_size_t),
-        Expected("size_t_to_ubyte", TOKEN_ubyte),
-        Expected("size_t_to_ushort", TOKEN_ushort),
-        Expected("size_t_to_int", TOKEN_int),
-        Expected("size_t_to_ulong", TOKEN_ulong),
-        Expected("main", TOKEN_int),
-    ];
-    p_context_t context;
-    p_context_init(&context, input);
-    size_t result = p_parse(&context);
-    assert_eq(P_SUCCESS, result);
-    PModule * pmod = p_result(&context);
-    PModuleItems * pmis = pmod.pModuleItems;
-    PFunctionDefinition *[] pfds;
-    while (pmis !is null)
-    {
-        PModuleItem * pmi = pmis.pModuleItem;
-        if (pmi is null)
-        {
-            stderr.writeln("pmi is null!!!?");
-            assert(0);
-        }
-        PFunctionDefinition * pfd = pmi.pFunctionDefinition;
-        if (pfd !is null)
-        {
-            pfds = [pfd] ~ pfds;
-        }
-        pmis = pmis.pModuleItems;
-    }
-    assert_eq(51, pfds.length);
-    for (size_t i = 0; i < pfds.length; i++)
-    {
-        if ((expected[i].name != pfds[i].name.pvalue.s) ||
-            (expected[i].token != pfds[i].returntype.pType.pTypeBase.pToken1.token))
-        {
-            stderr.writeln("Index ", i, ": expected ", expected[i].name, "/", expected[i].token, ", got ", pfds[i].name.pvalue.s, "/", pfds[i].returntype.pType.pTypeBase.pToken1.token);
-        }
-    }
-}

spec/test_ast_ps.c

@@ -1,61 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
-    assert(start->pItems1 != NULL);
-    assert(start->pItems != NULL);
-    PItemsS * items = start->pItems;
-    assert(items->pItem != NULL);
-    assert(items->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_a, items->pItem->pToken1->token);
-    assert_eq(11, items->pItem->pToken1->pvalue);
-    assert(items->pItemsMore != NULL);
-    PItemsMoreS * itemsmore = items->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem != NULL);
-    assert(itemsmore->pItem->pItem->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pItem->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pItem->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore != NULL);
-    itemsmore = itemsmore->pItemsMore;
-    assert(itemsmore->pItem != NULL);
-    assert(itemsmore->pItem->pToken1 != NULL);
-    assert_eq(TOKEN_b, itemsmore->pItem->pToken1->token);
-    assert_eq(22, itemsmore->pItem->pToken1->pvalue);
-    assert(itemsmore->pItemsMore == NULL);
-
-    p_free_ast(start);
-
-    input = "";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start->pItems == NULL);
-
-    p_free_ast(start);
-
-    input = "2 1";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start->pItems != NULL);
-    assert(start->pItems->pItem != NULL);
-    assert(start->pItems->pItem->pDual != NULL);
-    assert(start->pItems->pItem->pDual->pTwo1 != NULL);
-    assert(start->pItems->pItem->pDual->pOne2 != NULL);
-    assert(start->pItems->pItem->pDual->pTwo2 == NULL);
-    assert(start->pItems->pItem->pDual->pOne1 == NULL);
-
-    p_free_ast(start);
-
-    return 0;
-}

spec/test_ast_ps.d

@@ -1,57 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a, ((b)), b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    PStartS * start = p_result(&context);
-    assert(start.pItems1 !is null);
-    assert(start.pItems !is null);
-    PItemsS * items = start.pItems;
-    assert(items.pItem !is null);
-    assert(items.pItem.pToken1 !is null);
-    assert_eq(TOKEN_a, items.pItem.pToken1.token);
-    assert_eq(11, items.pItem.pToken1.pvalue);
-    assert(items.pItemsMore !is null);
-    PItemsMoreS * itemsmore = items.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem !is null);
-    assert(itemsmore.pItem.pItem.pItem.pToken1 !is null);
-    assert_eq(TOKEN_b, itemsmore.pItem.pItem.pItem.pToken1.token);
-    assert_eq(22, itemsmore.pItem.pItem.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore !is null);
-    itemsmore = itemsmore.pItemsMore;
-    assert(itemsmore.pItem !is null);
-    assert(itemsmore.pItem.pToken1 !is null);
-    assert_eq(TOKEN_b, itemsmore.pItem.pToken1.token);
-    assert_eq(22, itemsmore.pItem.pToken1.pvalue);
-    assert(itemsmore.pItemsMore is null);
-
-    input = "";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start.pItems is null);
-
-    input = "2 1";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    start = p_result(&context);
-    assert(start.pItems !is null);
-    assert(start.pItems.pItem !is null);
-    assert(start.pItems.pItem.pDual !is null);
-    assert(start.pItems.pItem.pDual.pTwo1 !is null);
-    assert(start.pItems.pItem.pDual.pOne2 !is null);
-    assert(start.pItems.pItem.pDual.pTwo2 is null);
-    assert(start.pItems.pItem.pDual.pOne1 is null);
-}

spec/test_ast_token_positions.c

@@ -1,88 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(1, start->pT1->pToken->position.row);
-    assert_eq(1, start->pT1->pToken->position.col);
-    assert_eq(1, start->pT1->pToken->end_position.row);
-    assert_eq(1, start->pT1->pToken->end_position.col);
-    assert_eq(1, start->pT1->position.row);
-    assert_eq(1, start->pT1->position.col);
-    assert_eq(1, start->pT1->end_position.row);
-    assert_eq(1, start->pT1->end_position.col);
-
-    assert_eq(1, start->pT2->pToken->position.row);
-    assert_eq(2, start->pT2->pToken->position.col);
-    assert_eq(1, start->pT2->pToken->end_position.row);
-    assert_eq(3, start->pT2->pToken->end_position.col);
-    assert_eq(1, start->pT2->position.row);
-    assert_eq(2, start->pT2->position.col);
-    assert_eq(1, start->pT2->end_position.row);
-    assert_eq(3, start->pT2->end_position.col);
-
-    assert_eq(1, start->pT3->pToken->position.row);
-    assert_eq(4, start->pT3->pToken->position.col);
-    assert_eq(1, start->pT3->pToken->end_position.row);
-    assert_eq(6, start->pT3->pToken->end_position.col);
-    assert_eq(1, start->pT3->position.row);
-    assert_eq(4, start->pT3->position.col);
-    assert_eq(1, start->pT3->end_position.row);
-    assert_eq(6, start->pT3->end_position.col);
-
-    assert_eq(1, start->position.row);
-    assert_eq(1, start->position.col);
-    assert_eq(1, start->end_position.row);
-    assert_eq(6, start->end_position.col);
-
-    p_free_ast(start);
-
-    input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(3, start->pT1->pToken->position.row);
-    assert_eq(3, start->pT1->pToken->position.col);
-    assert_eq(3, start->pT1->pToken->end_position.row);
-    assert_eq(4, start->pT1->pToken->end_position.col);
-    assert_eq(3, start->pT1->position.row);
-    assert_eq(3, start->pT1->position.col);
-    assert_eq(3, start->pT1->end_position.row);
-    assert_eq(4, start->pT1->end_position.col);
-
-    assert_eq(4, start->pT2->pToken->position.row);
-    assert_eq(1, start->pT2->pToken->position.col);
-    assert_eq(5, start->pT2->pToken->end_position.row);
-    assert_eq(2, start->pT2->pToken->end_position.col);
-    assert_eq(4, start->pT2->position.row);
-    assert_eq(1, start->pT2->position.col);
-    assert_eq(5, start->pT2->end_position.row);
-    assert_eq(2, start->pT2->end_position.col);
-
-    assert_eq(7, start->pT3->pToken->position.row);
-    assert_eq(6, start->pT3->pToken->position.col);
-    assert_eq(7, start->pT3->pToken->end_position.row);
-    assert_eq(6, start->pT3->pToken->end_position.col);
-    assert_eq(7, start->pT3->position.row);
-    assert_eq(6, start->pT3->position.col);
-    assert_eq(7, start->pT3->end_position.row);
-    assert_eq(6, start->pT3->end_position.col);
-
-    assert_eq(3, start->position.row);
-    assert_eq(3, start->position.col);
-    assert_eq(7, start->end_position.row);
-    assert_eq(6, start->end_position.col);
-
-    p_free_ast(start);
-
-    return 0;
-}

spec/test_ast_token_positions.d

@@ -1,86 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "abbccc";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-
-    assert_eq(1, start.pT1.pToken.position.row);
-    assert_eq(1, start.pT1.pToken.position.col);
-    assert_eq(1, start.pT1.pToken.end_position.row);
-    assert_eq(1, start.pT1.pToken.end_position.col);
-    assert_eq(1, start.pT1.position.row);
-    assert_eq(1, start.pT1.position.col);
-    assert_eq(1, start.pT1.end_position.row);
-    assert_eq(1, start.pT1.end_position.col);
-
-    assert_eq(1, start.pT2.pToken.position.row);
-    assert_eq(2, start.pT2.pToken.position.col);
-    assert_eq(1, start.pT2.pToken.end_position.row);
-    assert_eq(3, start.pT2.pToken.end_position.col);
-    assert_eq(1, start.pT2.position.row);
-    assert_eq(2, start.pT2.position.col);
-    assert_eq(1, start.pT2.end_position.row);
-    assert_eq(3, start.pT2.end_position.col);
-
-    assert_eq(1, start.pT3.pToken.position.row);
-    assert_eq(4, start.pT3.pToken.position.col);
-    assert_eq(1, start.pT3.pToken.end_position.row);
-    assert_eq(6, start.pT3.pToken.end_position.col);
-    assert_eq(1, start.pT3.position.row);
-    assert_eq(4, start.pT3.position.col);
-    assert_eq(1, start.pT3.end_position.row);
-    assert_eq(6, start.pT3.end_position.col);
-
-    assert_eq(1, start.position.row);
-    assert_eq(1, start.position.col);
-    assert_eq(1, start.end_position.row);
-    assert_eq(6, start.end_position.col);
-
-    input = "\n\n  bb\nc\ncc\n\n     a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-
-    assert_eq(3, start.pT1.pToken.position.row);
-    assert_eq(3, start.pT1.pToken.position.col);
-    assert_eq(3, start.pT1.pToken.end_position.row);
-    assert_eq(4, start.pT1.pToken.end_position.col);
-    assert_eq(3, start.pT1.position.row);
-    assert_eq(3, start.pT1.position.col);
-    assert_eq(3, start.pT1.end_position.row);
-    assert_eq(4, start.pT1.end_position.col);
-
-    assert_eq(4, start.pT2.pToken.position.row);
-    assert_eq(1, start.pT2.pToken.position.col);
-    assert_eq(5, start.pT2.pToken.end_position.row);
-    assert_eq(2, start.pT2.pToken.end_position.col);
-    assert_eq(4, start.pT2.position.row);
-    assert_eq(1, start.pT2.position.col);
-    assert_eq(5, start.pT2.end_position.row);
-    assert_eq(2, start.pT2.end_position.col);
-
-    assert_eq(7, start.pT3.pToken.position.row);
-    assert_eq(6, start.pT3.pToken.position.col);
-    assert_eq(7, start.pT3.pToken.end_position.row);
-    assert_eq(6, start.pT3.pToken.end_position.col);
-    assert_eq(7, start.pT3.position.row);
-    assert_eq(6, start.pT3.position.col);
-    assert_eq(7, start.pT3.end_position.row);
-    assert_eq(6, start.pT3.end_position.col);
-
-    assert_eq(3, start.position.row);
-    assert_eq(3, start.position.col);
-    assert_eq(7, start.end_position.row);
-    assert_eq(6, start.end_position.col);
-}

spec/test_basic_math_grammar.c

@@ -1,29 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-
-int main()
-{
-    char const * input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
-
-    input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
-
-    input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
-
-    input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
-
-    return 0;
-}

spec/test_basic_math_grammar.d

@@ -1,32 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "1 + 2 * 3 + 4";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(&context));
-
-    input = "1 * 2 ** 4 * 3";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(&context));
-
-    input = "(1 + 2) * 3 + 4";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(&context));
-
-    input = "(2 * 2) ** 3 + 4 + 5";
-    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(&context));
-}

spec/test_d_lexer.d

@@ -0,0 +1,84 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    alias DCP = Testparser.Decoder.DecodedCodePoint;
+    string inputstring = "5+\n 66";
+    const(ubyte) * input = cast(const(ubyte) *)inputstring.ptr;
+    size_t input_length = inputstring.length;
+    DCP dcp;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP('5', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP('+', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP('\n', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(' ', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP('6', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP('6', 1u));
+    input += dcp.code_point_length;
+    input_length -= dcp.code_point_length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
+
+    inputstring = "\xf0\x1f\x27\x21";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(0x1F9E1, 4u));
+
+    inputstring = "\xf0\x1f\x27";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    inputstring = "\xf0\x1f\x27\xFF";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    inputstring = "\xfe";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+}
+
+unittest
+{
+    alias LT = Testparser.Lexer.LexedToken;
+    string input = "5 + 4 * \n677 + 567";
+    Testparser.Lexer lexer = new Testparser.Lexer(cast(const(ubyte) *)input.ptr, input.length);
+    assert(lexer.lex_token() == LT(0, 0, 1, Testparser.TOKEN_INT));
+    assert(lexer.lex_token() == LT(0, 2, 1, Testparser.TOKEN_PLUS));
+    assert(lexer.lex_token() == LT(0, 4, 1, Testparser.TOKEN_INT));
+    assert(lexer.lex_token() == LT(0, 6, 1, Testparser.TOKEN_TIMES));
+    assert(lexer.lex_token() == LT(1, 0, 3, Testparser.TOKEN_INT));
+    assert(lexer.lex_token() == LT(1, 4, 1, Testparser.TOKEN_PLUS));
+    assert(lexer.lex_token() == LT(1, 6, 3, Testparser.TOKEN_INT));
+    assert(lexer.lex_token() == LT(1, 9, 0, Testparser.TOKEN_EOF));
+
+    lexer = new Testparser.Lexer(null, 0u);
+    assert(lexer.lex_token() == LT(0, 0, 0, Testparser.TOKEN_EOF));
+}

spec/test_error_positions.c

@@ -1,42 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a 42";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "a\n123\na  a";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 4);
-    assert(p_token(&context) == TOKEN_a);
-
-    input = "12";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 1);
-    assert(p_token(&context) == TOKEN_num);
-
-    input = "a 12\n\nab";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context).row == 3);
-    assert(p_position(&context).col == 2);
-
-    input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context).row == 6);
-    assert(p_position(&context).col == 5);
-
-    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
-    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
-
-    return 0;
-}

spec/test_error_positions.d

@@ -1,40 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a 42";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "a\n123\na  a";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(3, 4));
-    assert(p_token(&context) == TOKEN_a);
-
-    input = "12";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 1));
-    assert(p_token(&context) == TOKEN_num);
-
-    input = "a 12\n\nab";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(&context) == p_position_t(3, 2));
-
-    input = "a 12\n\na\n\n77\na   \xAA";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(&context) == p_position_t(6, 5));
-
-    assert(p_token_names[TOKEN_a] == "a");
-    assert(p_token_names[TOKEN_num] == "num");
-}

spec/test_field_aliases.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    return 0;
-}

spec/test_field_aliases.d

@@ -1,15 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "foo1\nbar2";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_free_ast_token_node_memory.c

@@ -1,19 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(&context));
-    Start * start = p_result(&context);
-    assert(start->a != NULL);
-    assert(*start->a->pvalue == 1);
-    assert(start->b != NULL);
-    assert(*start->b->pvalue == 2);
-
-    p_free_ast(start);
-}

spec/test_lexer.c

@@ -1,110 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    size_t result;
-    p_code_point_t code_point;
-    uint8_t code_point_length;
-
-    result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == '5');
-    assert(code_point_length == 1u);
-
-    result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
-    assert(result == P_EOF);
-
-    result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0xA9u);
-    assert(code_point_length == 2u);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0x1F9E1u);
-    assert(code_point_length == 4u);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-
-    p_token_info_t token_info;
-    char const * input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 1u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 1u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 3u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 3u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 5u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 5u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 7u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 7u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_times);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
-    assert(token_info.position.col == 1u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 3u);
-    assert(token_info.length == 3u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
-    assert(token_info.position.col == 5u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 5u);
-    assert(token_info.length == 1u);
-    assert(token_info.token == TOKEN_plus);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
-    assert(token_info.position.col == 7u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 9u);
-    assert(token_info.length == 3u);
-    assert(token_info.token == TOKEN_int);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
-    assert(token_info.position.col == 10u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 10u);
-    assert(token_info.length == 0u);
-    assert(token_info.token == TOKEN___EOF);
-
-    p_context_init(&context, (uint8_t const *)"", 0u);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 1u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 1u);
-    assert(token_info.length == 0u);
-    assert(token_info.token == TOKEN___EOF);
-
-    return 0;
-}

spec/test_lexer.d

@@ -1,69 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    size_t result;
-    p_code_point_t code_point;
-    ubyte code_point_length;
-
-    result = p_decode_code_point("5", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == '5');
-    assert(code_point_length == 1u);
-
-    result = p_decode_code_point("", &code_point, &code_point_length);
-    assert(result == P_EOF);
-
-    result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0xA9u);
-    assert(code_point_length == 2u);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
-    assert(result == P_SUCCESS);
-    assert(code_point == 0x1F9E1u);
-    assert(code_point_length == 4u);
-
-    result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-
-    result = p_decode_code_point("\xfe", &code_point, &code_point_length);
-    assert(result == P_DECODE_ERROR);
-}
-
-unittest
-{
-    p_token_info_t token_info;
-    string input = "5 + 4 * \n677 + 567";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
-
-    p_context_init(&context, "");
-    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
-}

spec/test_lexer_match_text.c

@@ -1,15 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "identifier_123";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    return 0;
-}

spec/test_lexer_match_text.d

@@ -1,16 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `identifier_123`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    writeln("pass1");
-}

spec/test_lexer_modes.c

@@ -1,20 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "abc \"a string\" def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    input = "abc \"abc def\" def";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass2\n");
-
-    return 0;
-}

spec/test_lexer_modes.d

@@ -1,21 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `abc "a string" def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    writeln("pass1");
-
-    input = `abc "abc def" def`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    writeln("pass2");
-}

spec/test_lexer_multiple_modes.c

@@ -1,20 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "abc.def";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass1\n");
-
-    input = "abc .  abc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    printf("pass2\n");
-
-    return 0;
-}

spec/test_lexer_multiple_modes.d

@@ -1,21 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `abc.def`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    writeln("pass1");
-
-    input = `abc .  abc`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    writeln("pass2");
-}

spec/test_lexer_result_value.c

@@ -1,19 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
-
-    input = "fabulous";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
-
-    return 0;
-}

spec/test_lexer_result_value.d

@@ -1,21 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 1u);
-
-    input = `fabulous`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 8u);
-}

spec/test_lexer_unknown_character.c

@@ -1,18 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "x";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-
-    input = "123";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
-
-    return 0;
-}

spec/test_lexer_unknown_character.d

@@ -1,20 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `x`;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-
-    input = `123`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context) == 123u);
-}

spec/test_match_backslashes.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_match_backslashes.d

@@ -1,15 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "\a\b\t\n\v\f\rt";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_multiple_parsers.c

@@ -1,19 +0,0 @@
-#include "testparsermyp1.h"
-#include "testparsermyp2.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
-
-    char const * input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
-
-    return 0;
-}

spec/test_multiple_parsers.d

@@ -1,21 +0,0 @@
-import testparsermyp1;
-import testparsermyp2;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input1 = "a\n1";
-    myp1_context_t context1;
-    myp1_context_init(&context1, input1);
-    assert(myp1_parse(&context1) == MYP1_SUCCESS);
-
-    string input2 = "bcb";
-    myp2_context_t context2;
-    myp2_context_init(&context2, input2);
-    assert(myp2_parse(&context2) == MYP2_SUCCESS);
-}

spec/test_named_optional_rule_component_ast.c

@@ -1,51 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
-    assert(start->r == NULL);
-
-    p_free_ast(start);
-
-    input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start->a != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->r != NULL);
-    assert(start->pR == start->pR3);
-    assert(start->pR == start->r);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
-
-    p_free_ast(start);
-
-    input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start->a == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->r != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
-
-    p_free_ast(start);
-
-    return 0;
-}
-

spec/test_named_optional_rule_component_ast.d

@@ -1,46 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
-    assert(start.r is null);
-
-    input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start.pToken1 != null);
-    assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
-    assert(start.r != null);
-    assert(start.pR == start.pR3);
-    assert(start.pR == start.r);
-    assert_eq(TOKEN_c, start.pR.pToken1.token);
-
-    input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
-    assert_eq(TOKEN_d, start.pR.pToken1.token);
-}

spec/test_optional_rule_component.c

@@ -1,22 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}
-

spec/test_optional_rule_component.d

@@ -1,23 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_optional_rule_component_ast.c

@@ -1,48 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "b";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert_eq(TOKEN_b, start->pToken2->token);
-    assert(start->pR3 == NULL);
-    assert(start->pR == NULL);
-
-    p_free_ast(start);
-
-    input = "abcd";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start->pToken1 != NULL);
-    assert_eq(TOKEN_a, start->pToken1->token);
-    assert(start->pToken2 != NULL);
-    assert(start->pR3 != NULL);
-    assert(start->pR != NULL);
-    assert(start->pR == start->pR3);
-    assert_eq(TOKEN_c, start->pR->pToken1->token);
-
-    p_free_ast(start);
-
-    input = "bdc";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start->pToken1 == NULL);
-    assert(start->pToken2 != NULL);
-    assert(start->pR != NULL);
-    assert_eq(TOKEN_d, start->pR->pToken1->token);
-
-    p_free_ast(start);
-
-    return 0;
-}
-

spec/test_optional_rule_component_ast.d

@@ -1,43 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "b";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    Start * start = p_result(&context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert_eq(TOKEN_b, start.pToken2.token);
-    assert(start.pR3 is null);
-    assert(start.pR is null);
-
-    input = "abcd";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start.pToken1 != null);
-    assert_eq(TOKEN_a, start.pToken1.token);
-    assert(start.pToken2 != null);
-    assert(start.pR3 != null);
-    assert(start.pR != null);
-    assert(start.pR == start.pR3);
-    assert_eq(TOKEN_c, start.pR.pToken1.token);
-
-    input = "bdc";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    start = p_result(&context);
-    assert(start.pToken1 is null);
-    assert(start.pToken2 !is null);
-    assert(start.pR !is null);
-    assert_eq(TOKEN_d, start.pR.pToken1.token);
-}

spec/test_parser_identical_rules_lookahead.c

@@ -1,17 +0,0 @@
-#include "testparser.h"
-#include <string.h>
-#include <assert.h>
-
-int main()
-{
-    char const * input = "aba";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_identical_rules_lookahead.d

@@ -1,19 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "aba";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "abb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_from_multiple_states.c

@@ -1,24 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "a";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context).row == 1);
-    assert(p_position(&context).col == 2);
-    assert(context.token == TOKEN___EOF);
-
-    input = "a b";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "bb";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_rule_from_multiple_states.d

@@ -1,25 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "a";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(&context) == p_position_t(1, 2));
-    assert(context.token == TOKEN___EOF);
-
-    input = "a b";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "bb";
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parser_rule_user_code.c

@@ -1,13 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "ab";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    return 0;
-}

spec/test_parser_rule_user_code.d

@@ -1,15 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "ab";
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-}

spec/test_parsing_json.c

@@ -1,56 +0,0 @@
-#include "testparser.h"
-#include "json_types.h"
-#include <string.h>
-#include <assert.h>
-
-int main()
-{
-    char const * input = "";
-    p_context_t context;
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = "{}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_OBJECT);
-
-    input = "[]";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_ARRAY);
-
-    input = "-45.6";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == -45.6);
-
-    input = "2E-2";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(&context)->id == JSON_NUMBER);
-    assert(p_result(&context)->number == 0.02);
-
-    input = "{\"hi\":true}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    JSONValue * o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(1, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "hi") == 0);
-    assert(o->object.entries[0].value->id == JSON_TRUE);
-
-    input = "{\"ff\": false, \"nn\": null}";
-    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(&context) == P_SUCCESS);
-    o = p_result(&context);
-    assert(o->id == JSON_OBJECT);
-    assert_eq(2, o->object.size);
-    assert(strcmp(o->object.entries[0].name, "ff") == 0);
-    assert(o->object.entries[0].value->id == JSON_FALSE);
-    assert(strcmp(o->object.entries[1].name, "nn") == 0);
-    assert(o->object.entries[1].value->id == JSON_NULL);
-
-    return 0;
-}

spec/test_parsing_json.d

@@ -1,56 +0,0 @@
-import testparser;
-import std.stdio;
-import json_types;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = ``;
-    p_context_t context;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-
-    input = `{}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-
-    input = `[]`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONArray)p_result(&context));
-
-    input = `-45.6`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == -45.6);
-
-    input = `2E-2`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONNumber)p_result(&context));
-    assert((cast(JSONNumber)p_result(&context)).value == 0.02);
-
-    input = `{"hi":true}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    JSONObject o = cast(JSONObject)p_result(&context);
-    assert(o.value["hi"]);
-    assert(cast(JSONTrue)o.value["hi"]);
-
-    input = `{"ff": false, "nn": null}`;
-    p_context_init(&context, input);
-    assert(p_parse(&context) == P_SUCCESS);
-    assert(cast(JSONObject)p_result(&context));
-    o = cast(JSONObject)p_result(&context);
-    assert(o.value["ff"]);
-    assert(cast(JSONFalse)o.value["ff"]);
-    assert(o.value["nn"]);
-    assert(cast(JSONNull)o.value["nn"]);
-}