wip

.github/workflows/run-tests.yml

@@ -1,41 +0,0 @@
-name: Run Propane Tests
-
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        ruby-version: ['3.4']
-
-    steps:
-    - name: Install dependencies (Linux)
-      if: runner.os == 'Linux'
-      run: sudo apt-get update && sudo apt-get install -y gcc gdc ldc valgrind
-
-    - name: Install dependencies (macOS)
-      if: runner.os == 'macOS'
-      run: brew install gcc ldc
-
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-
-    - name: Set up Rust
-      uses: dtolnay/rust-toolchain@stable
-
-    - name: Install dependencies
-      run: bundle install
-
-    - name: Run tests
-      run: rake all

.rspec

@@ -1,2 +1,3 @@
+--format documentation
 --color
 --require spec_helper

CHANGELOG.md

@@ -1,288 +1,3 @@
-## v5.1.0
-
-### New Features
-
-- Add a `node_id()` accessor to the C++ and D tree node handle types, for node
-  identity comparison. This matches the existing `p_node_id()` macro (C) and
-  `node_id()` method (Rust).
-
-## v5.0.0
-
-### New Features
-
-- Add Rust target language output.
-- Add Rust language detection in propane.vim.
-
-### API Changes
-
-- The matched text argument passed to lexer user code blocks is now named
-  `match_text` instead of `match`, since `match` is a keyword in Rust. The
-  `match_length` argument (C and C++) is unchanged.
-- Tree generation mode now stores all tree nodes in a compact arena owned by
-  the parser context (a flat node array plus a shared child-link array).
-  This replaces the previous design of one heap allocation per node with
-  layout-punned typed structs.
-- Tree nodes are now referenced by lightweight handles rather than pointers.
-  `p_result()` and the field accessors now return handle values in tree
-  generation mode.
-- The whole tree is freed together with the context by `p_context_delete()`.
-  The `p_tree_delete()` / `p_tree_delete_XXX()` functions have been removed;
-  tree node handles are only valid while the context is alive.
-- Tree node field access changed per target language:
-  - C: per-field accessor functions (e.g. `p_Start_pItems(node)`) plus tree
-    walk macros (e.g. `p_tree_walk_Start(node, pItems, pItem, pToken1, token)`),
-    and generic accessors `p_node_valid()`, `p_node_position()`,
-    `p_node_end_position()`, `p_node_n_fields()`, `p_node_data()`, `p_node_id()`.
-  - C++: handle methods called with `()` (e.g. `node.pItems().pToken1().token()`),
-    plus the same C-style functions/macros for convenience.
-  - D: `@property` accessors preserving the previous field-access syntax
-    (e.g. `node.pItems.pToken1.token`); null checks use `.valid` instead of
-    `is null`.
-- Tree-mode parser rule user code: `$$` and `$1` etc. now yield node handles.
-  Reference child fields through the target-language accessors described above
-  rather than through struct pointer members.
-
-### Improvements
-
-- Improve D language detection in propane.vim
-- Speed up specs
-
-## v4.8.1
-
-### Fixes
-
-- Fix tree node struct type forward-declarations for C/C++
-
-## v4.8.0
-
-### New Features
-
-- Add `p_parse_inner_XXX()` APIs that accept a caller-provided set of follow
-  tokens. These behave the same as `p_parse_XXX()` by parsing starting at the
-  given start rule, but instead of expecting the rest of the input to match
-  the start rule they allow specifying a set of tokens that may follow the
-  start rule.
-- Add `p_set_position()` API to set the current text position stored in the
-  context. Useful for setting the initial text position to something other
-  than `(1, 1)` for a nested parse operation.
-- Add `p_input_index()` API to get the current input text byte offset.
-- Add `p_set_input_index()` API to set the current input text byte offset.
-  Useful together with `p_set_position()` to rewind the input part-way through
-  a parse in order to re-read an earlier section of the input.
-
-## v4.7.0
-
-### New Features
-
-- Support parser rule user code blocks in tree generation mode.
-
-### Fixes
-
-- propane.vim: do not highlight rule components as propane keywords
-
-## v4.6.0
-
-### New Features
-
-- Add lexer user code API to access matched input text positions
-- Track rule component text positions and add parser user code API to access
-
-### Fixes
-
-- Fixed a few user guide and source comments related to text input positions
-
-## v4.5.0
-
-### New Features
-
-- Add `noline` grammar statement to skip emitting `#line` directives
-- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
-
-### Fixes
-
-- Fix #line reset directives
-- Update keyword list in extra/vim/syntax/propane.vim
-- Fix propane.vim keyword detection
-
-## v4.4.0
-
-### New Features
-
-- Add p_value_get() / p_value_get_XXX() accessors
-
-## v4.3.0
-
-### New Features
-
-- Use #line for user code blocks to report input grammar position for errors.
-
-## v4.2.0
-
-### New Features
-
-- Add support for a custom lex function.
-
-## v4.1.0
-
-### New Features
-
-- Add `p_context_delete()` and `p_tree_delete()` for D targets.
-
-## v4.0.0
-
-### New Features
-
-- Add `context_user_fields` statement to allow custom context user fields.
-- Add `token_user_fields` statement to allow custom token user fields.
-- Add `on_token_node` statement to allow custom code when constructing token nodes.
-- Add `free_token_node` statement to allow custom code when freeing token nodes.
-- Add `p_context_delete()`.
-- Allow `drop` patterns to execute lexer user code blocks.
-
-### Breaking Changes
-
-- Replace `p_context_init()` with `p_context_new()` and `p_context_delete()`.
-- Renamed `p_free_tree()` to `p_tree_delete()`.
-- The `free_token_node` statement now takes a user code block instead of a
-  function name parameter.
-
-## v3.0.0
-
-### New Features
-
-- Add support for multiple starting rules (#38)
-- Add `p_free_tree()` functions to reclaim generated tree memory
-- Add `free_token_node` grammar statement to reclaim user-allocated memory stored in a Token tree node `pvalue` field
-- Add valgrind memory leak tests to unit tests
-- Fix build issues for C++ to officially support C++ target output
-
-### Improvements
-
-- Document `p_lex()` and `p_token_info_t` in user guide (#37)
-
-### Breaking Changes
-
-- Rename AST generation mode to tree generation mode (see [UPGRADING.md](UPGRADING.md))
-
-## v2.3.0
-
-### New Features
-
-- Add \D, \S, \w, \W special character classes
-
-### Improvements
-
-- Include line numbers for pattern errors
-- Improve performance in a few places
-- Parallelize parser table generation on Linux hosts
-- Add github workflow to run unit tests
-
-### Fixes
-
-- Fix a couple clang warnings for C backend
-- Fix C backend not fully initializing pvalues when multiple ptypes are used with different sizes.
-- Fix some user guide examples
-
-## v2.2.1
-
-### Fixes
-
-- Fix GC issue for D backend when AST is enabled (#36)
-
-## v2.2.0
-
-### Improvements
-
-- Allow multiple lexer modes to be specified for a lexer pattern (#35)
-- Document p_decode_code_point() API function (#34)
-
-## v2.1.1
-
-### Fixes
-
-- Field aliases for AST node fields could alias incorrect field when multiple rule alternatives present for one rule set (#33)
-
-## v2.1.0
-
-### Improvements
-
-- Report rule name and line number for conflicting AST node field positions errors (#32)
-
-## v2.0.0
-
-### Improvements
-
-- Log conflicting rules on reduce/reduce conflict (#31)
-- Use 1-based row and column values for position values (#30)
-
-### Fixes
-
-- Fix named optional rules (#29)
-
-### Upgrading
-
-- Adjust all uses of p_position_t row and col values to expect 1-based instead
-of 0-based values.
-
-## v1.5.1
-
-### Improvements
-
-- Improve performance (#28)
-
-## v1.5.0
-
-### New Features
-
-- Track start and end text positions for tokens and rules in AST node structures (#27)
-- Add warnings for shift/reduce conflicts to log file (#25)
-- Add -w command line switch to treat warnings as errors and output to stderr (#26)
-- Add rule field aliases (#24)
-
-### Improvements
-
-- Show line numbers of rules on conflict (#23)
-
-## v1.4.0
-
-### New Features
-
-- Allow user to specify AST node name prefix or suffix
-- Allow specifying the start rule name
-- Allow rule terms to be marked as optional
-
-### Improvements
-
-- Give a better error message when a referenced ptype has not been declared
-
-## v1.3.0
-
-### New Features
-
-- Add AST generation (#22)
-
-## v1.2.0
-
-### New Features
-
-- Allow one line user code blocks (#21)
-- Add backslash escape codes (#19)
-- Add API to access unexpected token found (#18)
-- Add token_names API (#17)
-- Add D example to user guide for p_context_init() (#16)
-- Allow user termination from lexer code blocks (#15)
-
-### Fixes
-
-- Fix generator hang when state transition cycle is present (#20)
-
-## v1.1.0
-
-### New Features
-
-- Add user parser terminations (#13)
-- Document generated parser API in user guide (#14)
-
 ## v1.0.0
 
 - Initial release

Gemfile

@@ -1,9 +1,7 @@
 source "https://rubygems.org"
 
-gem "base64"
 gem "rake"
 gem "rspec"
 gem "rdoc"
 gem "redcarpet"
 gem "syntax"
-gem "simplecov"

Gemfile.lock

@@ -1,54 +1,38 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    base64 (0.3.0)
+    diff-lcs (1.5.0)
-    date (3.5.1)
+    psych (5.1.0)
-    diff-lcs (1.6.2)
-    docile (1.4.1)
-    erb (6.0.4)
-    psych (5.4.0)
-      date
       stringio
-    rake (13.4.2)
+    rake (13.0.6)
-    rdoc (7.2.0)
+    rdoc (6.5.0)
-      erb
       psych (>= 4.0.0)
-      tsort
+    redcarpet (3.6.0)
-    redcarpet (3.6.1)
+    rspec (3.12.0)
-    rspec (3.13.2)
+      rspec-core (~> 3.12.0)
-      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.12.0)
-      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.12.0)
-      rspec-mocks (~> 3.13.0)
+    rspec-core (3.12.2)
-    rspec-core (3.13.6)
+      rspec-support (~> 3.12.0)
-      rspec-support (~> 3.13.0)
+    rspec-expectations (3.12.3)
-    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
+      rspec-support (~> 3.12.0)
-    rspec-mocks (3.13.8)
+    rspec-mocks (3.12.6)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.13.0)
+      rspec-support (~> 3.12.0)
-    rspec-support (3.13.7)
+    rspec-support (3.12.1)
-    simplecov (0.22.0)
+    stringio (3.0.7)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.13.2)
-    simplecov_json_formatter (0.1.4)
-    stringio (3.2.0)
     syntax (1.2.2)
-    tsort (0.2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  base64
   rake
   rdoc
   redcarpet
   rspec
-  simplecov
   syntax
 
 BUNDLED WITH
-  4.0.14
+   2.3.7

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2026 Josh Holtrop
+Copyright (c) 2010-2023 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,106 +1,21 @@
 # The Propane Parser Generator
 
-Propane is a LALR Parser Generator (LPG) which:
+Propane is an LR Parser Generator (LPG) which:
 
   * accepts LR(0), SLR, and LALR grammars
   * generates a built-in lexer to tokenize input
   * supports UTF-8 lexer inputs
-  * generates a table-driven shift/reduce parser to parse input in linear time
+  * generates a table-driven parser to parse input in linear time
-  * targets C, C++, D, or Rust language outputs
-  * optionally supports automatic full parse tree generation
-  * supports starting parsing from multiple start rules
-  * tracks input text start and end positions for all matched tokens/rules
   * is MIT-licensed
   * is distributable as a standalone Ruby script
 
 ## Installation
 
-Propane is designed to be distributed as a stand-alone single file script that
+TODO
-can be copied into and versioned in a project's source tree.
-The only requirement to run Propane is that the system has a Ruby interpreter
-installed.
-The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
-
-Simply copy the `propane` executable script into the desired location within
-the project to be built (typically the root of the repository) and mark it
-executable.
 
 ## Usage
 
-### Command Line Interface
+TODO: Write usage instructions here
-
-Propane is typically invoked from the command-line as `./propane`.
-
-    Usage: ./propane [options] <input-file> <output-file>
-    Options:
-      -h, --help  Show this usage and exit.
-      --log LOG   Write log file. This will show all parser states and their
-                  associated shifts and reduces. It can be helpful when
-                  debugging a grammar.
-      --version   Show program version and exit.
-      -w          Treat warnings as errors. This option will treat shift/reduce
-                  conflicts as fatal errors and will print them to stderr in
-                  addition to the log file.
-
-The user must specify the path to a Propane input grammar file and a path to an
-output file.
-The generated source code will be written to the output file.
-If a log file path is specified, Propane will write a log file containing
-detailed information about the parser states and transitions.
-
-### Propane Grammar File
-
-A Propane grammar file provides Propane with the patterns, tokens, grammar
-rules, and user code blocks from which to build the generated lexer and parser.
-
-Example grammar file:
-
-```
-<<
-import std.math;
->>
-
-# Parser values are unsigned integers.
-ptype ulong;
-
-# A few basic arithmetic operators.
-token plus /\+/;
-token times /\*/;
-token power /\*\*/;
-token integer /\d+/ <<
-  ulong v;
-  foreach (c; match_text)
-  {
-    v *= 10;
-    v += (c - '0');
-  }
-  $$ = v;
->>
-token lparen /\(/;
-token rparen /\)/;
-# Drop whitespace.
-drop /\s+/;
-
-Start -> E1 << $$ = $1; >>
-E1 -> E2 << $$ = $1; >>
-E1 -> E1 plus E2 << $$ = $1 + $3; >>
-E2 -> E3 << $$ = $1; >>
-E2 -> E2 times E3 << $$ = $1 * $3; >>
-E3 -> E4 << $$ = $1; >>
-E3 -> E3 power E4 <<
-  $$ = pow($1, $3);
->>
-E4 -> integer << $$ = $1; >>
-E4 -> lparen E1 rparen << $$ = $2; >>
-```
-
-Grammar files can contain comment lines beginning with `#` which are ignored.
-White space in the grammar file is also ignored.
-
-It is convention to use the extension `.propane` for the Propane grammar file,
-however any file name is accepted by Propane.
-
-See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
 
 ## Development
 

Rakefile

@@ -0,0 +1,18 @@
+require "rspec/core/rake_task"
+
+task :build_dist do
+  sh "ruby rb/build_dist.rb"
+end
+
+RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
+  if args.example_pattern
+    task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
+  end
+end
+
+task :default => :spec
+
+desc "Build user guide"
+task :user_guide do
+  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
+end

Rakefile.rb

@@ -1,59 +0,0 @@
-require "fileutils"
-require "rake/clean"
-require "rspec/core/rake_task"
-require "simplecov"
-require "stringio"
-
-CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
-
-task :build_dist do
-  sh "ruby rb/build_dist.rb"
-end
-
-RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
-  if args.example_pattern
-    ENV["partial_specs"] = "1"
-    task.rspec_opts = %W[-e "#{args.example_pattern}" -f documentation]
-  else
-    FileUtils.rm_rf("coverage")
-  end
-end
-task :spec do |task, args|
-  unless ENV["dist_specs"]
-    original_stdout = $stdout
-    sio = StringIO.new
-    $stdout = sio
-    SimpleCov.collate Dir["coverage/parts/*/.resultset.json"]
-    $stdout = original_stdout
-    sio.string.lines.each do |line|
-      $stdout.write(line) unless line =~ /Coverage report generated for/
-    end
-  end
-end
-
-task :valgrind do
-  begin
-    ENV["spec-valgrind"] = "1"
-    Rake::Task[:spec].execute
-  ensure
-    ENV.delete("spec-valgrind")
-  end
-end
-
-# dspec task is useful to test the distributable release script, but is not
-# useful for coverage information.
-desc "Dist Specs"
-task :dspec, [:example_string] => :build_dist do |task, args|
-  ENV["dist_specs"] = "1"
-  Rake::Task["spec"].execute(args)
-  ENV.delete("dist_specs")
-end
-
-task :default => :spec
-
-desc "Build user guide"
-task :user_guide do
-  system("ruby", "-Ilib", "rb/gen_user_guide.rb")
-end
-
-task :all => [:valgrind, :dspec, :user_guide]

UPGRADING.md

@@ -1,110 +0,0 @@
-## v5.0.0
-
-The generated API for tree generation mode (`tree;`) has been changed
-significantly for this version.
-Aside from the lexer user code block matched text rename described below, the
-lexer/parser value APIs for non-tree grammars are unchanged.
-
-### Lexer user code block matched text
-
-The matched text argument passed to lexer user code blocks has been renamed
-from `match` to `match_text` for all target languages.
-
-- C, C++, and D: rename references to `match` in lexer user code blocks to
-  `match_text` (for example `$$ = match[0];` becomes `$$ = match_text[0];`).
-
-The `match_length` argument (C, C++) is unchanged.
-
-### Tree memory management
-
-- Remove all calls to `p_tree_delete()` / `p_tree_delete_XXX()`. Tree nodes now
-  live in the parser context and are freed by `p_context_delete()`.
-- Tree node handles (returned by `p_result()` and the field accessors) are only
-  valid while the context is alive. Do not use them after `p_context_delete()`.
-
-### Tree node field access
-
-Tree nodes are now referenced by handle values instead of pointers, and field
-access differs per target language:
-
-- C: replace `node->field` with the accessor function `p_TYPE_field(node)`, or
-  use the tree walk macro `p_tree_walk_TYPE(node, field1, field2, ...)`. Replace
-  `x != NULL` / `x == NULL` node checks with `p_node_valid(x)` /
-  `!p_node_valid(x)`. Read positions with `p_node_position(node)` /
-  `p_node_end_position(node)`, token payload with `p_TYPE_token(node)` /
-  `p_TYPE_pvalue(node)` or `p_node_data(node)->field`, and compare node identity
-  with `p_node_id(a) == p_node_id(b)`.
-- C++: replace `node->field` with the handle method `node.field()`. Use
-  `node.valid()`, `node.position()`, `node.token()`, `node.pvalue()`, and
-  `node.data()->field` for user token fields. (The C-style functions and macros
-  above are also available.)
-- D: replace pointer declarations (`Start * s`) with value handles (`Start s`)
-  and replace `x !is null` / `x is null` with `x.valid` / `!x.valid`. Field
-  access syntax (`node.field.field`) is otherwise unchanged.
-
-### Tree-mode parser rule user code
-
-In tree generation mode `$$` and `$1`, `$2`, ... now expand to node handles.
-Reference child fields through the target-language accessors above (for example
-`$$->pA->pToken1->pvalue` becomes `p_tree_walk_Start($$, pA, pToken1, pvalue)`
-in C, `$$.pA().pToken1().pvalue()` in C++, and `$$.pA.pToken1.pvalue` in D).
-
-### Pointers into tree node storage
-
-Tree nodes previously each had their own allocation, so a pointer to a node
-stayed valid for the life of the tree. They are now held in a single array
-which is reallocated as it grows, so a pointer or reference into that array may
-be invalidated whenever a new node is created.
-
-New nodes are created while parsing, so this matters for a pointer taken in a
-tree-mode parser rule user code block, which runs before the parse has
-finished. Keep the node handle instead, which stores a node ID rather than an
-address and stays valid, and obtain the pointer from it when it is needed.
-
-For example, replace a saved pointer:
-
-```
-context_user_fields <<
-    p_node_data_t * saved;
->>
-Items -> Items a << ${context.saved} = p_node_data($$); >>
-```
-
-with a saved handle:
-
-```
-context_user_fields <<
-    Items saved_node;
->>
-Items -> Items a << ${context.saved_node} = $$; >>
-```
-
-```
-p_node_data_t * data = p_node_data(context->saved_node);
-```
-
-Once parsing has finished, no further nodes are created, so a pointer obtained
-after `p_parse()` returns stays valid until the context is deleted, as long as
-no further parsing is performed with the same context.
-
-## v4.0.0
-
-### API Changes
-
-- Replace any calls to `p_context_init()` with `p_context_new()`.
-- Replace any references to the address of a statically allocated context
-  structure with the pointer returned from `p_context_init()` (e.g. `&context`
-  -> `context`).
-- Add a call to `p_context_delete()` (for C or C++) after lexing/parsing to
-  reclaim context memory.
-- Rename `p_free_tree()` calls to `p_tree_delete()`.
-- Change `free_token_node` statement calls from taking a function name argument
-  to taking a user code block.
-
-## v3.0.0
-
-### Grammar Changes
-
-- Rename `ast;` statement to `tree;`.
-- Rename `ast_prefix;` statement to `tree_prefix;`.
-- Rename `ast_suffix;` statement to `tree_suffix;`.

assets/parser.c.erb

@@ -3,22 +3,13 @@
 #include <stdlib.h>
 #include <string.h>
 
-/**************************************************************************
- * Public data
- *************************************************************************/
-
-/** Token names. */
-const char * <%= @grammar.prefix %>token_names[] = {
-<% @grammar.tokens.each_with_index do |token, index| %>
-    "<%= token.name %>",
-<% end %>
-};
-
 /**************************************************************************
  * User code blocks
  *************************************************************************/
 
-<%= @grammar.code_blocks.fetch("", "") %>
+<% @grammar.code_blocks.each do |code| %>
+<%= code %>
+<% end %>
 
 /**************************************************************************
  * Private types
@@ -32,7 +23,6 @@ const char * <%= @grammar.prefix %>token_names[] = {
 #define P_UNEXPECTED_TOKEN 3u
 #define P_DROP 4u
 #define P_EOF 5u
-#define P_USER_TERMINATED 6u
 <% end %>
 
 /* An invalid ID value. */
@@ -43,82 +33,27 @@ const char * <%= @grammar.prefix %>token_names[] = {
  *************************************************************************/
 
 /**
- * Allocate and initialize lexer/parser context structure.
+ * Initialize lexer/parser context structure.
- *
- * Deinitialize and deallocate with <%= @grammar.prefix %>context_delete().
  *
+ * @param[out] context
+ *   Lexer/parser context structure.
  * @param input
  *   Text input.
  * @param input_length
  *   Text input length.
- *
- * @return Context structure for lexer/parser.
  */
-<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length)
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
 {
-<% if @cpp %>
+    /* New default-initialized context structure. */
-    <%= @grammar.prefix %>context_t * context = new <%= @grammar.prefix %>context_t();
+    <%= @grammar.prefix %>context_t newcontext = {0};
-<% else %>
-    <%= @grammar.prefix %>context_t * context = (<%= @grammar.prefix %>context_t *)calloc(1, sizeof(<%= @grammar.prefix %>context_t));
-<% end %>
 
     /* Lexer initialization. */
-    context->input = input;
+    newcontext.input = input;
-    context->input_length = input_length;
+    newcontext.input_length = input_length;
-    context->text_position.row = 1u;
+    newcontext.mode = <%= @lexer.mode_id("default") %>;
-    context->text_position.col = 1u;
-    context->mode = <%= @lexer.mode_id("default") %>;
-<% if @grammar.tree %>
 
-    /* Reserve node ID 0 as the null tree node. */
+    /* Copy to the user's context structure. */
-<%   if @cpp %>
+    *context = newcontext;
-    context-><%= @grammar.prefix %>tree_nodes.resize(1);
-<%   else %>
-    context-><%= @grammar.prefix %>tree_nodes_capacity = 16u;
-    context-><%= @grammar.prefix %>tree_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(16u * sizeof(<%= @grammar.prefix %>node_data_t));
-    memset(&context-><%= @grammar.prefix %>tree_nodes[0], 0, sizeof(<%= @grammar.prefix %>node_data_t));
-    context-><%= @grammar.prefix %>tree_nodes_length = 1u;
-<%   end %>
-<% end %>
-
-    return context;
-}
-
-/**
- * Deinitialize and deallocate lexer/parser context structure.
- *
- * For C++, destructors will be called for any context user fields. However, if
- * pointers are used to store allocated resources, the user should free them
- * before calling this function.
- *
- * @param context
- *   Lexer/parser context structure allocated with <%= @grammar.prefix %>context_new().
- */
-void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context)
-{
-<% if @grammar.tree && @grammar.free_token_node != "" %>
-<%   if @cpp %>
-    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes.size(); i++)
-<%   else %>
-    for (size_t i = 0u; i < context-><%= @grammar.prefix %>tree_nodes_length; i++)
-<%   end %>
-    {
-        if (context-><%= @grammar.prefix %>tree_nodes[i].is_token)
-        {
-            <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[i];
-<%= expand_code(@grammar.free_token_node, false, nil, nil) %>
-        }
-    }
-<% end %>
-<% if @cpp %>
-    delete context;
-<% else %>
-<%   if @grammar.tree %>
-    free(context-><%= @grammar.prefix %>tree_nodes);
-    free(context-><%= @grammar.prefix %>tree_children);
-<%   end %>
-    free(context);
-<% end %>
 }
 
 /**************************************************************************
@@ -281,10 +216,7 @@ typedef struct
     /** Number of bytes of input text used to match. */
     size_t length;
 
-    /** Input text position delta to end of token. */
+    /** Input text position delta. */
-    <%= @grammar.prefix %>position_t end_delta_position;
-
-    /** Input text position delta to next code point after token end. */
     <%= @grammar.prefix %>position_t delta_position;
 
     /** Accepting lexer state from the match. */
@@ -319,7 +251,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   Lexer/parser context structure.
  * @param code_id
  *   The ID of the user code block to execute.
- * @param match_text
+ * @param match
  *   Matched text for this pattern.
  * @param match_length
  *   Matched text length.
@@ -330,7 +262,7 @@ static lexer_mode_t lexer_mode_table[] = {
  *   not explicitly return a token.
  */
 static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
-    lexer_user_code_id_t code_id, uint8_t const * match_text,
+    lexer_user_code_id_t code_id, uint8_t const * match,
     size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
     switch (code_id)
@@ -378,12 +310,9 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
  *
  * @param context
  *   Lexer/parser context structure.
- * @param[out] out_match_info
+ * @param[out] out_token_info
- *   The longest match information is stored here if the return value is
+ *   The lexed token information is stored here if the return value is
- *   P_SUCCESS or P_DECODE_ERROR.
+ *   P_SUCCESS.
- * @param[out] out_unexpected_input_length
- *   The unexpected input length is stored here if the return value is
- *   P_UNEXPECTED_INPUT.
  *
  * @reval P_SUCCESS
  *   A token was successfully lexed.
@@ -397,10 +326,8 @@ static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t
 static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
     lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
 {
-    lexer_match_info_t longest_match;
+    lexer_match_info_t longest_match = {0};
-    memset(&longest_match, 0, sizeof(longest_match));
+    lexer_match_info_t attempt_match = {0};
-    lexer_match_info_t attempt_match;
-    memset(&attempt_match, 0, sizeof(attempt_match));
     *out_match_info = longest_match;
     uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
     for (;;)
@@ -414,16 +341,14 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
         switch (result)
         {
         case P_SUCCESS:
-            {
             lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
             if (transition_state != INVALID_LEXER_STATE_ID)
             {
                 attempt_match.length += code_point_length;
-                attempt_match.end_delta_position = attempt_match.delta_position;
                 if (code_point == '\n')
                 {
                     attempt_match.delta_position.row++;
-                    attempt_match.delta_position.col = 1u;
+                    attempt_match.delta_position.col = 0u;
                 }
                 else
                 {
@@ -446,7 +371,6 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 *out_unexpected_input_length = attempt_match.length + code_point_length;
                 return P_UNEXPECTED_INPUT;
             }
-            }
             break;
 
         case P_EOF:
@@ -468,6 +392,7 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
                 /* Valid EOF return. */
                 return P_EOF;
             }
+            break;
 
         case P_DECODE_ERROR:
             /* If we see a decode error, we may be partially in the middle of
@@ -499,50 +424,25 @@ static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
  *   Input text does not match any lexer pattern.
  * @retval P_DROP
  *   A drop pattern was matched so the lexer should continue.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
-    <%= @grammar.prefix %>token_info_t token_info;
+    <%= @grammar.prefix %>token_info_t token_info = {0};
-    memset(&token_info, 0, sizeof(token_info));
     token_info.position = context->text_position;
     token_info.token = INVALID_TOKEN_ID;
+    *out_token_info = token_info; // TODO: remove
     lexer_match_info_t match_info;
     size_t unexpected_input_length;
     size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
     switch (result)
     {
     case P_SUCCESS:
-        {
         <%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
-        /* Calculate the token length and start/end positions before invoking
-         * the lexer user code so that the user code can access them. The
-         * context input text position tracking is not updated until after the
-         * user code has run so that it is left unchanged if the user code
-         * requests to terminate the lexer. */
-        token_info.length = match_info.length;
-        if (match_info.end_delta_position.row != 0u)
-        {
-            token_info.end_position.row = token_info.position.row + match_info.end_delta_position.row;
-            token_info.end_position.col = match_info.end_delta_position.col;
-        }
-        else
-        {
-            token_info.end_position.row = token_info.position.row;
-            token_info.end_position.col = token_info.position.col + match_info.end_delta_position.col;
-        }
         if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
         {
-            uint8_t const * match_text = &context->input[context->input_index];
+            uint8_t const * match = &context->input[context->input_index];
             <%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
-                match_info.accepting_state->code_id, match_text, match_info.length, &token_info);
+                match_info.accepting_state->code_id, match, match_info.length, &token_info);
-            /* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
-             * that the user code is requesting to terminate the lexer. */
-            if (user_code_token == TERMINATE_TOKEN_ID)
-            {
-                return P_USER_TERMINATED;
-            }
             /* An invalid token returned from lexer_user_code() means that the
              * user code did not explicitly return a token. So only override
              * the token to return if the user code does explicitly return a
@@ -570,13 +470,12 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
             return P_DROP;
         }
         token_info.token = token_to_accept;
+        token_info.length = match_info.length;
         *out_token_info = token_info;
-        }
         return P_SUCCESS;
 
     case P_EOF:
         token_info.token = TOKEN___EOF;
-        token_info.end_position = token_info.position;
         *out_token_info = token_info;
         return P_SUCCESS;
 
@@ -614,8 +513,6 @@ static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
- * @retval P_USER_TERMINATED
- *   User code has requested to terminate the lexer.
  */
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
 {
@@ -633,9 +530,6 @@ size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%=
  * Parser
  *************************************************************************/
 
-/** Invalid position value. */
-#define INVALID_POSITION (<%= @grammar.prefix %>position_t){0u, 0u}
-
 /** Reduce ID type. */
 typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
 
@@ -695,25 +589,6 @@ typedef struct
      * reduce action.
      */
     parser_state_id_t n_states;
-<% if @grammar.tree %>
-
-    /**
-     * Map of rule components to rule set child fields.
-     */
-    uint16_t const * rule_set_node_field_index_map;
-
-    /**
-     * Number of rule set tree node fields.
-     */
-    uint16_t rule_set_node_field_array_size;
-
-    /**
-     * Whether this rule was a generated optional rule that matched the
-     * optional target. In this case, propagate the matched target node up
-     * instead of making a new node for this rule.
-     */
-    bool propagate_optional_target;
-<% end %>
 } reduce_t;
 
 /** Parser state entry. */
@@ -742,51 +617,21 @@ typedef struct
     /** Parser state ID. */
     size_t state_id;
 
-<% if @grammar.tree %>
-    /** Tree node ID. */
-    <%= @grammar.prefix %>node_id_t node_id;
-<% else %>
-    <%= @grammar.prefix %>position_t position;
-    <%= @grammar.prefix %>position_t end_position;
     /** Parser value from this state. */
     <%= @grammar.prefix %>value_t pvalue;
-<% end %>
 } state_value_t;
 
 /** Parser shift table. */
 static const shift_t parser_shift_table[] = {
 <%   @parser.shift_table.each do |shift| %>
-    {<%= shift[:symbol].id %>u, <%= shift[:state_id] %>u},
+    {<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
 <%   end %>
 };
 
-<% if @grammar.tree %>
-<%   @grammar.rules.each do |rule| %>
-<%     unless rule.flat_rule_set_node_field_index_map? %>
-const uint16_t r_<%= rule.name.gsub("$", "_") %><%= rule.id %>_node_field_index_map[<%= rule.rule_set_node_field_index_map.size %>] = {<%= rule.rule_set_node_field_index_map.map {|v| v.to_s}.join(", ") %>};
-<%     end %>
-<%   end %>
-<% end %>
-
 /** Parser reduce table. */
 static const reduce_t parser_reduce_table[] = {
 <%   @parser.reduce_table.each do |reduce| %>
-    {
+    {<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
-        <%= reduce[:token_id] %>u, /* Token: <%= reduce[:token] ? reduce[:token].name : "(any)" %> */
-        <%= reduce[:rule_id] %>u, /* Rule ID */
-        <%= reduce[:rule_set_id] %>u, /* Rule set ID (<%= reduce[:rule].rule_set.name %>) */
-<%     if @grammar.tree %>
-        <%= reduce[:n_states] %>u, /* Number of states */
-<%       if reduce[:rule].flat_rule_set_node_field_index_map? %>
-        NULL, /* No rule set node field index map (flat map) */
-<%       else %>
-        &r_<%= reduce[:rule].name.gsub("$", "_") %><%= reduce[:rule].id %>_node_field_index_map[0], /* Rule set node field index map */
-<%       end %>
-        <%= reduce[:rule].rule_set.tree_fields.size %>, /* Number of tree fields */
-        <%= reduce[:propagate_optional_target] %>}, /* Propagate optional target? */
-<%     else %>
-        <%= reduce[:n_states] %>u},
-<%     end %>
 <%   end %>
 };
 
@@ -856,7 +701,7 @@ static void state_values_stack_push(state_values_stack_t * stack)
     if (current_length >= current_capacity)
     {
         size_t const new_capacity = current_capacity * 2u;
-        state_value_t * new_entries = (state_value_t *)malloc(new_capacity * sizeof(state_value_t));
+        state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
         memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
         free(stack->entries);
         stack->capacity = new_capacity;
@@ -890,143 +735,17 @@ static void state_values_stack_free(state_values_stack_t * stack)
      free(stack->entries);
 }
 
-<% if @grammar.tree %>
-/* Tree arena helpers. */
-
-/**
- * Allocate a new (zeroed) tree node in the context arena.
- *
- * @return The new node ID.
- */
-static <%= @grammar.prefix %>node_id_t tree_new_node(<%= @grammar.prefix %>context_t * context)
-{
-<% if @cpp %>
-    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes.size();
-    context-><%= @grammar.prefix %>tree_nodes.emplace_back();
-    return id;
-<% else %>
-    if (context-><%= @grammar.prefix %>tree_nodes_length >= context-><%= @grammar.prefix %>tree_nodes_capacity)
-    {
-        size_t new_capacity = context-><%= @grammar.prefix %>tree_nodes_capacity * 2u;
-        <%= @grammar.prefix %>node_data_t * new_nodes = (<%= @grammar.prefix %>node_data_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_data_t));
-        memcpy(new_nodes, context-><%= @grammar.prefix %>tree_nodes, context-><%= @grammar.prefix %>tree_nodes_length * sizeof(<%= @grammar.prefix %>node_data_t));
-        free(context-><%= @grammar.prefix %>tree_nodes);
-        context-><%= @grammar.prefix %>tree_nodes = new_nodes;
-        context-><%= @grammar.prefix %>tree_nodes_capacity = new_capacity;
-    }
-    <%= @grammar.prefix %>node_id_t id = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_nodes_length;
-    memset(&context-><%= @grammar.prefix %>tree_nodes[id], 0, sizeof(<%= @grammar.prefix %>node_data_t));
-    context-><%= @grammar.prefix %>tree_nodes_length += 1u;
-    return id;
-<% end %>
-}
-
-/**
- * Reserve n contiguous (zeroed) child slots in the shared children array.
- *
- * @return The offset of the first reserved slot.
- */
-static <%= @grammar.prefix %>node_id_t tree_reserve_children(<%= @grammar.prefix %>context_t * context, size_t n)
-{
-<% if @cpp %>
-    <%= @grammar.prefix %>node_id_t offset = (<%= @grammar.prefix %>node_id_t)context-><%= @grammar.prefix %>tree_children.size();
-    context-><%= @grammar.prefix %>tree_children.resize(context-><%= @grammar.prefix %>tree_children.size() + n);
-    return offset;
-<% else %>
-    size_t offset = context-><%= @grammar.prefix %>tree_children_length;
-    size_t needed = offset + n;
-    if (needed > context-><%= @grammar.prefix %>tree_children_capacity)
-    {
-        size_t new_capacity = context-><%= @grammar.prefix %>tree_children_capacity ? context-><%= @grammar.prefix %>tree_children_capacity : 1u;
-        while (new_capacity < needed)
-        {
-            new_capacity *= 2u;
-        }
-        <%= @grammar.prefix %>node_id_t * new_children = (<%= @grammar.prefix %>node_id_t *)malloc(new_capacity * sizeof(<%= @grammar.prefix %>node_id_t));
-        if (context-><%= @grammar.prefix %>tree_children != NULL)
-        {
-            memcpy(new_children, context-><%= @grammar.prefix %>tree_children, context-><%= @grammar.prefix %>tree_children_length * sizeof(<%= @grammar.prefix %>node_id_t));
-            free(context-><%= @grammar.prefix %>tree_children);
-        }
-        context-><%= @grammar.prefix %>tree_children = new_children;
-        context-><%= @grammar.prefix %>tree_children_capacity = new_capacity;
-    }
-    memset(&context-><%= @grammar.prefix %>tree_children[offset], 0, n * sizeof(<%= @grammar.prefix %>node_id_t));
-    context-><%= @grammar.prefix %>tree_children_length = needed;
-    return (<%= @grammar.prefix %>node_id_t)offset;
-<% end %>
-}
-
-/* Tree node field accessor functions. */
-<%= c_tree_accessor_defs %>
-<% end %>
-
-<% unless @grammar.tree %>
-/**
- * Get the rule position (start or end) for the currently matched rule.
- */
-static <%= @grammar.prefix %>position_t get_rule_position(state_values_stack_t * statevalues, size_t i, size_t n_states, bool get_end)
-{
-    if (n_states > 0u)
-    {
-        if (i == 0u)
-        {
-            if (get_end)
-            {
-                int stack_index = -1;
-                for (size_t j = 0u; j < n_states; j++)
-                {
-                    state_value_t * sv = state_values_stack_index(statevalues, stack_index - (int)j);
-                    if (<%= @grammar.prefix %>position_valid(sv->end_position))
-                    {
-                        return sv->end_position;
-                    }
-                }
-            }
-            else
-            {
-                int stack_index = -(int)n_states;
-                for (size_t j = 0u; j < n_states; j++)
-                {
-                    state_value_t * sv = state_values_stack_index(statevalues, stack_index + (int)j);
-                    if (<%= @grammar.prefix %>position_valid(sv->position))
-                    {
-                        return sv->position;
-                    }
-                }
-            }
-        }
-        else
-        {
-            if (get_end)
-            {
-                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->end_position;
-            }
-            else
-            {
-                return state_values_stack_index(statevalues, -1 - (int)n_states + (int)i)->position;
-            }
-        }
-    }
-    <%= @grammar.prefix %>position_t empty_pos;
-    memset(&empty_pos, 0, sizeof(empty_pos));
-    return empty_pos;
-}
-<% end %>
-
-<% if !@grammar.tree || @grammar.parser_user_code_used? %>
 /**
  * Execute user code associated with a parser rule.
  *
  * @param rule The ID of the rule.
  *
- * @retval P_SUCCESS
+ * @return Parse value.
- *   Continue parsing.
- * @retval P_USER_TERMINATED
- *   User requested to terminate parsing.
  */
-static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t _node_id" : "#{@grammar.prefix}value_t * _pvalue" %>, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
+static <%= @grammar.prefix %>value_t parser_user_code(uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states)
 {
+    <%= @grammar.prefix %>value_t _pvalue = {0};
+
     switch (rule)
     {
 <% @grammar.rules.each do |rule| %>
@@ -1039,9 +758,8 @@ static size_t parser_user_code(<%= @grammar.tree ? "#{@grammar.prefix}node_id_t
     default: break;
     }
 
-    return P_SUCCESS;
+    return _pvalue;
 }
-<% end %>
 
 /**
  * Check if the parser should shift to a new state.
@@ -1075,7 +793,7 @@ static size_t check_shift(size_t state_id, size_t symbol_id)
  * @param token
  *   Incoming token.
  *
- * @return Reduce table index to reduce with, or INVALID_ID if none.
+ * @return State to reduce to, or INVALID_ID if none.
  */
 static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
 {
@@ -1097,54 +815,33 @@ static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
  *
  * @param context
  *   Lexer/parser context structure.
- * @param start_state_id
- *   ID of the state in which to start.
- * @param start_rule_set_id
- *   Rule set ID for the requested start rule. Only used when
- *   @p follow_tokens is non-NULL, to gate follow-token shift success.
- * @param follow_tokens
- *   Optional array of caller-provided follow tokens (tokens expected to
- *   appear immediately after the start rule in some outer context). Used to
- *   drive the "parse inner" retry logic. May be NULL for a standard parse.
- * @param n_follow_tokens
- *   Number of entries in @p follow_tokens.
  *
  * @retval P_SUCCESS
  *   The parser successfully matched the input text. The parse result value
  *   can be accessed with <%= @grammar.prefix %>result().
  * @retval P_UNEXPECTED_TOKEN
  *   An unexpected token was encountered that does not match any grammar rule.
- *   The function p_token(&context) can be used to get the unexpected token.
+ *   The value context->token holds the unexpected token.
  * @reval P_DECODE_ERROR
  *   The decoder encountered invalid text encoding.
  * @reval P_UNEXPECTED_INPUT
  *   Input text does not match any lexer pattern.
  */
-static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start_state_id,
+size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
-    size_t start_rule_set_id,
-    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
 {
     <%= @grammar.prefix %>token_info_t token_info;
     <%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
     state_values_stack_t statevalues;
     size_t reduced_rule_set = INVALID_ID;
-    size_t last_shifted_rule_set_id = INVALID_ID;
-<% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t reduced_parser_node;
-<% else %>
-    <%= @grammar.prefix %>position_t reduced_position;
-    <%= @grammar.prefix %>position_t reduced_end_position;
     <%= @grammar.prefix %>value_t reduced_parser_value;
-<% end %>
     state_values_stack_init(&statevalues);
     state_values_stack_push(&statevalues);
-    state_values_stack_index(&statevalues, -1)->state_id = start_state_id;
     size_t result;
     for (;;)
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= lex_fn %>(context, &token_info);
+            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;
@@ -1152,18 +849,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             }
             token = token_info.token;
         }
-        /* For a "parse inner" operation, determine once per iteration whether
-         * the current token is a member of the caller-provided follow token
-         * set. Used by both the shift-side and reduce-side retries below. */
-        bool token_is_follow = false;
-        for (size_t i = 0u; i < n_follow_tokens; i++)
-        {
-            if (token == follow_tokens[i])
-            {
-                token_is_follow = true;
-                break;
-            }
-        }
         size_t shift_state = INVALID_ID;
         if (reduced_rule_set != INVALID_ID)
         {
@@ -1175,184 +860,38 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
             if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
             {
                 /* Successful parse. */
-<% if @grammar.tree %>
-                context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
-<% else %>
                 context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
-<% end %>
                 result = P_SUCCESS;
                 break;
             }
-            if ((shift_state == INVALID_ID) && token_is_follow)
-            {
-                /* For a "parse inner" operation, if the incoming token is one
-                 * of the caller's follow tokens, retry the shift as
-                 * TOKEN___EOF. Only consider the parse complete if the reduced
-                 * start rule is the only thing on the parse stack (i.e. the
-                 * initial state plus a single shifted start rule set entry). */
-                size_t retry_shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
-                if ((retry_shift_state != INVALID_ID) &&
-                    (statevalues.length == 2u) &&
-                    (last_shifted_rule_set_id == start_rule_set_id))
-                {
-                    /* Successful parse via follow token. Rewind the input
-                     * position so that the follow token is not consumed from
-                     * the input stream and remains available for a subsequent
-                     * call to <%= @grammar.prefix %>lex() or a
-                     * <%= @grammar.prefix %>parse*() function. */
-                    context->input_index -= token_info.length;
-                    context->text_position = token_info.position;
-<% if @grammar.tree %>
-                    context->parse_result = state_values_stack_index(&statevalues, -1)->node_id;
-<% else %>
-                    context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
-<% end %>
-                    result = P_SUCCESS;
-                    break;
-                }
-            }
         }
         if (shift_state != INVALID_ID)
         {
-            /* We have something to shift. Track the last shifted rule set ID
+            /* We have something to shift. */
-             * (INVALID_ID if we just shifted a token) so the follow-token
-             * shift retry can gate success on the reduced start rule being the
-             * only thing on top of the initial state. */
-            last_shifted_rule_set_id = reduced_rule_set;
             state_values_stack_push(&statevalues);
-            state_value_t * new_state_info = state_values_stack_index(&statevalues, -1);
+            state_values_stack_index(&statevalues, -1)->state_id = shift_state;
-            new_state_info->state_id = shift_state;
             if (reduced_rule_set == INVALID_ID)
             {
                 /* We shifted a token, mark it consumed. */
-<% if @grammar.tree %>
-                <%= @grammar.prefix %>node_id_t token_node_id = tree_new_node(context);
-                <%= @grammar.prefix %>node_data_t * token_tree_node = &context-><%= @grammar.prefix %>tree_nodes[token_node_id];
-                token_tree_node->position = token_info.position;
-                token_tree_node->end_position = token_info.end_position;
-                token_tree_node->n_fields = 0u;
-                token_tree_node->is_token = 1u;
-                token_tree_node->token = token;
-                token_tree_node->pvalue = token_info.pvalue;
-<%= expand_code(@grammar.on_token_node, false, nil, nil) %>
-                new_state_info->node_id = token_node_id;
-<% else %>
-                new_state_info->position = token_info.position;
-                new_state_info->end_position = token_info.end_position;
-                new_state_info->pvalue = token_info.pvalue;
-<% end %>
                 token = INVALID_TOKEN_ID;
+                state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
             }
             else
             {
                 /* We shifted a RuleSet. */
-<% if @grammar.tree %>
+                state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
-                new_state_info->node_id = reduced_parser_node;
+                <%= @grammar.prefix %>value_t new_parse_result = {0};
-<% else %>
-                new_state_info->pvalue = reduced_parser_value;
-                new_state_info->position = reduced_position;
-                new_state_info->end_position = reduced_end_position;
-                <%= @grammar.prefix %>value_t new_parse_result;
-                memset(&new_parse_result, 0, sizeof(new_parse_result));
                 reduced_parser_value = new_parse_result;
-<% end %>
                 reduced_rule_set = INVALID_ID;
             }
             continue;
         }
 
         size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
-        if ((reduce_index == INVALID_ID) && token_is_follow)
-        {
-            /* For a "parse inner" operation, if the incoming token is one of
-             * the caller's follow tokens, retry the reduce lookup as
-             * TOKEN___EOF. Whatever reduce_index results (if any) is used
-             * regardless of which rule set it reduces to; this allows chains
-             * of reductions leading up to the start rule. */
-            reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, TOKEN___EOF);
-        }
         if (reduce_index != INVALID_ID)
         {
             /* We have something to reduce. */
-<% if @grammar.tree %>
+            reduced_parser_value = parser_user_code(parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states);
-            if (parser_reduce_table[reduce_index].propagate_optional_target)
-            {
-                reduced_parser_node = state_values_stack_index(&statevalues, -1)->node_id;
-            }
-            else if (parser_reduce_table[reduce_index].n_states > 0)
-            {
-                uint16_t n_fields = parser_reduce_table[reduce_index].rule_set_node_field_array_size;
-                /* Reserve child slots. New slots are zero-initialized
-                 * (null node ID) so absent optional children remain null. */
-                <%= @grammar.prefix %>node_id_t child_offset = tree_reserve_children(context, n_fields);
-                if (parser_reduce_table[reduce_index].rule_set_node_field_index_map == NULL)
-                {
-                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
-                    {
-                        context-><%= @grammar.prefix %>tree_children[child_offset + i] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
-                    }
-                }
-                else
-                {
-                    for (size_t i = 0; i < parser_reduce_table[reduce_index].n_states; i++)
-                    {
-                        context-><%= @grammar.prefix %>tree_children[child_offset + parser_reduce_table[reduce_index].rule_set_node_field_index_map[i]] = state_values_stack_index(&statevalues, -(int)parser_reduce_table[reduce_index].n_states + (int)i)->node_id;
-                    }
-                }
-                <%= @grammar.prefix %>node_id_t node_id = tree_new_node(context);
-                <%= @grammar.prefix %>node_data_t * node = &context-><%= @grammar.prefix %>tree_nodes[node_id];
-                node->position = INVALID_POSITION;
-                node->end_position = INVALID_POSITION;
-                node->child_offset = child_offset;
-                node->n_fields = n_fields;
-                node->is_token = 0u;
-                bool position_found = false;
-                for (uint16_t i = 0; i < n_fields; i++)
-                {
-                    <%= @grammar.prefix %>node_id_t child_id = context-><%= @grammar.prefix %>tree_children[child_offset + i];
-                    if ((child_id != 0u) && <%= @grammar.prefix %>position_valid(context-><%= @grammar.prefix %>tree_nodes[child_id].position))
-                    {
-                        if (!position_found)
-                        {
-                            node->position = context-><%= @grammar.prefix %>tree_nodes[child_id].position;
-                            position_found = true;
-                        }
-                        node->end_position = context-><%= @grammar.prefix %>tree_nodes[child_id].end_position;
-                    }
-                }
-                reduced_parser_node = node_id;
-            }
-            else
-            {
-                reduced_parser_node = 0u;
-            }
-<%   if @grammar.parser_user_code_used? %>
-            if (parser_user_code(reduced_parser_node, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
-            {
-                state_values_stack_free(&statevalues);
-                return P_USER_TERMINATED;
-            }
-<%   end %>
-<% else %>
-            <%= @grammar.prefix %>value_t reduced_parser_value2;
-            memset(&reduced_parser_value2, 0, sizeof(reduced_parser_value2));
-            if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
-            {
-                state_values_stack_free(&statevalues);
-                return P_USER_TERMINATED;
-            }
-            reduced_parser_value = reduced_parser_value2;
-            if (parser_reduce_table[reduce_index].n_states > 0u)
-            {
-                reduced_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, false);
-                reduced_end_position = get_rule_position(&statevalues, 0u, parser_reduce_table[reduce_index].n_states, true);
-            }
-            else
-            {
-                memset(&reduced_position, 0, sizeof(reduced_position));
-                memset(&reduced_end_position, 0, sizeof(reduced_end_position));
-            }
-<% end %>
             reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
             state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
             continue;
@@ -1372,25 +911,6 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     return result;
 }
 
-size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
-{
-    return parse_from(context, 0u, <%= @parser.rule_sets[@grammar.start_rules[0]].id %>u, NULL, 0u);
-}
-
-<% @grammar.start_rules.each_with_index do |start_rule, i| %>
-
-size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
-{
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, NULL, 0u);
-}
-
-size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
-    <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens)
-{
-    return parse_from(context, <%= i %>u, <%= @parser.rule_sets[start_rule].id %>u, follow_tokens, n_follow_tokens);
-}
-<% end %>
-
 /**
  * Get the parse result value.
  *
@@ -1399,29 +919,10 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
  *
  * @return Parse result value.
  */
-<% if @grammar.tree %>
-<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
-{
-    return <%= tree_handle(h_type(@grammar.start_rules[0]), "context->parse_result") %>;
-}
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
-{
-    return <%= tree_handle(h_type(start_rule), "context->parse_result") %>;
-}
-<%   end %>
-<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
 {
     return context->parse_result.v_<%= start_rule_type[0] %>;
 }
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context)
-{
-    return context->parse_result.v_<%= start_rule_type(i)[0] %>;
-}
-<%   end %>
-<% end %>
 
 /**
  * Get the current text input position.
@@ -1435,78 +936,3 @@ size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %
 {
     return context->text_position;
 }
-
-/**
- * Set the current text input position.
- *
- * This can be used to set the initial text position to something other than
- * (1, 1) for a nested parse operation so that error positions reported by
- * subsequent lexer/parser calls are relative to a larger enclosing document.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param position
- *   Text position to set.
- */
-void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position)
-{
-    context->text_position = position;
-}
-
-/**
- * Get the current input text byte offset.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return Current input text byte offset (measured from the start of the
- *   input text passed to <%= @grammar.prefix %>context_new()).
- */
-size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context)
-{
-    return context->input_index;
-}
-
-/**
- * Set the current input text byte offset.
- *
- * This moves the lexer's read cursor to the given byte offset (measured from
- * the start of the input text passed to <%= @grammar.prefix %>context_new()).
- * It can be used together with <%= @grammar.prefix %>set_position() to rewind
- * the input part-way through a parse in order to re-read an earlier section of
- * the input. The byte offset is not validated; the caller is responsible for
- * providing an offset within the bounds of the input text. A value previously
- * returned by <%= @grammar.prefix %>input_index() is a suitable argument.
- *
- * @param context
- *   Lexer/parser context structure.
- * @param input_index
- *   Input text byte offset to set.
- */
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index)
-{
-    context->input_index = input_index;
-}
-
-/**
- * Get the user terminate code.
- *
- * @param context
- *   Lexer/parser context structure.
- *
- * @return User terminate code.
- */
-size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
-{
-    return context->user_terminate_code;
-}
-
-/**
- * Get the parse token.
- *
- * @return Parse token.
- */
-<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
-{
-    return context->token;
-}

assets/parser.h.erb

@@ -8,9 +8,6 @@
 
 #include <stdint.h>
 #include <stddef.h>
-<% if @cpp %>
-#include <vector>
-<% end %>
 
 /**************************************************************************
  * Public types
@@ -23,10 +20,9 @@
 #define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
 #define <%= @grammar.prefix.upcase %>DROP 4u
 #define <%= @grammar.prefix.upcase %>EOF 5u
-#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
 
 /** Token type. */
-typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
+typedef <%= get_type_for(@grammar.invalid_token_id) %> <%= @grammar.prefix %>token_t;
 
 /** Token IDs. */
 <% @grammar.tokens.each_with_index do |token, index| %>
@@ -36,35 +32,10 @@ typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>t
 <%   end %>
 <% end %>
 #define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
-#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
 
 /** Code point type. */
 typedef uint32_t <%= @grammar.prefix %>code_point_t;
 
-/**
- * A structure to keep track of input position.
- *
- * This is useful for reporting errors, etc...
- */
-typedef struct
-{
-    /** Input text row (1-based). */
-    uint32_t row;
-
-    /** Input text column (1-based). */
-    uint32_t col;
-} <%= @grammar.prefix %>position_t;
-
-/** Return whether the position is valid. */
-#define <%= @grammar.prefix %>position_valid(p) ((p).row != 0u)
-
-/** User header code blocks. */
-<%= @grammar.code_blocks.fetch("header", "") %>
-
-<% if @grammar.tree %>
-/** Parser values type. */
-typedef <%= @grammar.ptype %> <%= @grammar.prefix %>value_t;
-<% else %>
 /** Parser values type(s). */
 typedef union
 {
@@ -73,58 +44,26 @@ typedef union
 <% end %>
 } <%= @grammar.prefix %>value_t;
 
-/** Parser value constructor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
-{
-    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
-}
-<%   end %>
-
-/** Parser value accessor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
-{
-    return pvalue->v_<%= name %>;
-}
-<%   end %>
-<% end %>
-
-<% if @grammar.tree %>
-/** Tree node ID type (index into the context node arena). ID 0 is null. */
-typedef uint32_t <%= @grammar.prefix %>node_id_t;
-
 /**
- * Tree node record.
+ * A structure to keep track of parser position.
  *
- * All tree nodes are stored contiguously in the context node arena. Child
+ * This is useful for reporting errors, etc...
- * links are stored in a shared children array: a node's children
- * occupy children[child_offset .. child_offset + n_fields]. Token payload
- * fields (token, pvalue, and any user fields) are only meaningful when
- * is_token is nonzero.
  */
 typedef struct
 {
-    <%= @grammar.prefix %>position_t position;
+    /** Input text row (0-based). */
-    <%= @grammar.prefix %>position_t end_position;
+    uint32_t row;
-    <%= @grammar.prefix %>node_id_t child_offset;
+
-    uint16_t n_fields;
+    /** Input text column (0-based). */
-    uint8_t is_token;
+    uint32_t col;
-    <%= @grammar.prefix %>token_t token;
+} <%= @grammar.prefix %>position_t;
-    <%= @grammar.prefix %>value_t pvalue;
-<%= @grammar.token_user_fields %>
-} <%= @grammar.prefix %>node_data_t;
-<% end %>
 
 /** Lexed token information. */
 typedef struct
 {
-    /** Text position of first code point in token. */
+    /** Text position where the token was found. */
     <%= @grammar.prefix %>position_t position;
 
-    /** Text position of last code point in token. */
-    <%= @grammar.prefix %>position_t end_position;
-
     /** Number of input bytes used by the token. */
     size_t length;
 
@@ -135,19 +74,13 @@ typedef struct
     <%= @grammar.prefix %>value_t pvalue;
 } <%= @grammar.prefix %>token_info_t;
 
-typedef struct <%= @grammar.prefix %>context_s <%= @grammar.prefix %>context_t;
-
-<% if @grammar.tree %>
-<%= c_tree_handle_types_header %>
-
-<% end %>
 /**
  * Lexer and parser context.
  *
  * The user must allocate an instance of this structure and pass it to any
  * public API function.
  */
-struct <%= @grammar.prefix %>context_s
+typedef struct
 {
     /* Lexer context data. */
 
@@ -169,53 +102,13 @@ struct <%= @grammar.prefix %>context_s
     /* Parser context data. */
 
     /** Parse result value. */
-<% if @grammar.tree %>
-    <%= @grammar.prefix %>node_id_t parse_result;
-
-<%   if @cpp %>
-    /** Tree node arena. Node ID 0 is reserved as the null node. */
-    std::vector<<%= @grammar.prefix %>node_data_t> <%= @grammar.prefix %>tree_nodes;
-
-    /** Shared tree child links. */
-    std::vector<<%= @grammar.prefix %>node_id_t> <%= @grammar.prefix %>tree_children;
-<%   else %>
-    /** Tree node arena. Node ID 0 is reserved as the null node. */
-    <%= @grammar.prefix %>node_data_t * <%= @grammar.prefix %>tree_nodes;
-    size_t <%= @grammar.prefix %>tree_nodes_length;
-    size_t <%= @grammar.prefix %>tree_nodes_capacity;
-
-    /** Shared tree child links. */
-    <%= @grammar.prefix %>node_id_t * <%= @grammar.prefix %>tree_children;
-    size_t <%= @grammar.prefix %>tree_children_length;
-    size_t <%= @grammar.prefix %>tree_children_capacity;
-<%   end %>
-<% else %>
     <%= @grammar.prefix %>value_t parse_result;
-<% end %>
 
     /** Unexpected token received. */
     <%= @grammar.prefix %>token_t token;
+} <%= @grammar.prefix %>context_t;
 
-    /** User terminate code. */
+void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
-    size_t user_terminate_code;
-
-<%= @grammar.context_user_fields %>
-};
-
-<% if @grammar.tree %>
-<%= c_tree_types_header %>
-<% end %>
-
-/**************************************************************************
- * Public data
- *************************************************************************/
-
-/** Token names. */
-extern const char * <%= @grammar.prefix %>token_names[];
-
-<%= @grammar.prefix %>context_t * <%= @grammar.prefix %>context_new(uint8_t const * input, size_t input_length);
-
-void <%= @grammar.prefix %>context_delete(<%= @grammar.prefix %>context_t * context);
 
 size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
         <%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
@@ -223,32 +116,7 @@ size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t inp
 size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
 
 size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-size_t <%= @grammar.prefix %>parse_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-size_t <%= @grammar.prefix %>parse_inner_<%= start_rule %>(<%= @grammar.prefix %>context_t * context,
-        <%= @grammar.prefix %>token_t const * follow_tokens, size_t n_follow_tokens);
-<%   end %>
 
-<% if @grammar.tree %>
-<%= h_type(@grammar.start_rules[0]) %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= h_type(start_rule) %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-<%   end %>
-<% else %>
 <%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
-<%   @grammar.start_rules.each_with_index do |start_rule, i| %>
-<%= start_rule_type(i)[1] %> <%= @grammar.prefix %>result_<%= start_rule %>(<%= @grammar.prefix %>context_t * context);
-<%   end %>
-<% end %>
 
 <%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
-
-void <%= @grammar.prefix %>set_position(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>position_t position);
-
-size_t <%= @grammar.prefix %>input_index(<%= @grammar.prefix %>context_t * context);
-
-void <%= @grammar.prefix %>set_input_index(<%= @grammar.prefix %>context_t * context, size_t input_index);
-
-size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
-
-<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);

extra/vim/ftdetect/propane.vim

@@ -1 +0,0 @@
-au BufNewFile,BufRead *.propane set filetype=propane

extra/vim/syntax/propane.vim

@@ -1,85 +0,0 @@
-" Vim syntax file for Propane
-" Language: propane
-" Maintainer: Josh Holtrop
-" URL: https://github.com/holtrop/propane
-
-if exists("b:current_syntax")
-  finish
-endif
-
-" Guess the language of the user code blocks from their contents so that the
-" matching syntax file can be included below. b:propane_subtype may also be set
-" before this file is sourced to select the language explicitly.
-if !exists("b:propane_subtype")
-  " Rust markers. Each keyword requires the syntax that follows it in Rust so
-  " that a plain identifier of the same name in another language does not match
-  " (`int fn = 3;' in C, for example). Type names are only accepted within a
-  " `ptype' statement for the same reason.
-  let s:rust = '\<let\s\+\%(mut\s\+\)\?\w'
-  let s:rust .= '\|\<fn\s\+\w\+\s*('
-  let s:rust .= '\|&mut\>\|\<pub\s\+\w\|\<impl\s\+\w'
-  let s:rust .= '\|#\[\|\<use\s\+\%(std\|core\)::'
-  let s:rust .= '\|\<ptype\>[^;]*\<\%(isize\|usize\|i8\|i16\|i32\|i64\|i128'
-  let s:rust .= '\|u8\|u16\|u32\|u64\|u128\|f32\|f64\|String\)\>'
-  " D markers. These are spellings that have no valid C, C++, or Rust
-  " equivalent, so `import' is deliberately not among them: it is a D keyword
-  " but is also a C++20 module declaration.
-  let s:d = '\<foreach\%(_reverse\)\?\s*([^)]*;'
-  let s:d .= '\|\~=\|\<static\s\+if\s*(\|\<version\s*(\s*\w\+\s*)'
-  let s:d .= '\|\<scope\s*(\s*\%(exit\|failure\|success\)\s*)'
-  let s:d .= '\|\<\%(unittest\|mixin\|immutable\|__gshared\|invariant\)\>'
-  let s:d .= '\|\<alias\s\+\w\+\s*=\|\<enum\s\+\w\+\s*='
-  let s:d .= '\|@\%(property\|safe\|trusted\|system\|nogc\|disable\)\>'
-  let s:d .= '\|\<is\s\+null\>\|\<cast\s*(\s*\w\+\s*)'
-  let s:d .= '\|\<write\%(ln\|fln\|f\)\s*('
-  let s:d .= '\|\<\%(dchar\|dstring\|wstring\|cent\|ucent\)\>'
-  " A module import on its own is ambiguous between D and C++20, so only take
-  " it as D when nothing else in the file looks like C++.
-  let s:import = '\<import\s\+[A-Za-z_][A-Za-z0-9_.]*\s*;'
-  let s:cpp = '::\|\<template\s*<\|\<namespace\>\|\<nullptr\>\|#include\s*[<"]'
-  if search(s:rust, 'nw') > 0
-    let b:propane_subtype = "rust"
-  elseif search(s:d, 'nw') > 0
-    let b:propane_subtype = "d"
-  elseif search(s:import, 'nw') > 0 && search(s:cpp, 'nw') == 0
-    let b:propane_subtype = "d"
-  else
-    let b:propane_subtype = "cpp"
-  endif
-  unlet s:rust s:d s:import s:cpp
-endif
-
-exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
-
-syn region propaneTarget matchgroup=propaneDelimiter start="<<" end=">>$" contains=@propaneTarget keepend
-
-syn match propaneComment "#.*"
-syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
-syn match propaneFieldOperator ":" contained
-syn match propaneOperator "?"
-" The right-hand side of a rule (after '->' up to '<<' or ';') lists symbol
-" names that may coincide with propane keywords (e.g. 'start', 'token',
-" 'tree'). Wrap it in a region that excludes keyword matches so those names
-" are not highlighted as keywords. The '<<' is left unconsumed so the
-" propaneTarget region can still match it.
-syn region propaneRuleRhs matchgroup=propaneOperator start="->" end="\ze<<" end=";" contains=propaneFieldAlias,propaneRuleOperator,propaneComment keepend
-syn match propaneRuleOperator "?" contained
-" Keywords that introduce a user-defined name. The name is consumed by
-" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
-" is not highlighted as a keyword. These must be a match (not syn keyword)
-" because a syn keyword always wins over a contained nextgroup match.
-syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
-syn match propaneName "\<\h\w*\>" contained
-syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
-
-syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
-
-hi def link propaneComment Comment
-hi def link propaneKeyword Keyword
-hi def link propaneNameDecl Keyword
-hi def link propaneRegex String
-hi def link propaneOperator Operator
-hi def link propaneRuleOperator Operator
-hi def link propaneFieldOperator Operator
-hi def link propaneDelimiter Delimiter
-hi def link propaneFieldAlias Identifier

lib/propane.rb

@@ -1,7 +1,6 @@
 require "erb"
 require "set"
 require "stringio"
-require_relative "propane/assets"
 require_relative "propane/cli"
 require_relative "propane/code_point_range"
 require_relative "propane/fa"
@@ -31,10 +30,10 @@ class Propane
 
   class << self
 
-    def run(input_file, output_file, log_file, options)
+    def run(input_file, output_file, log_file)
       begin
-        grammar = Grammar.new(File.read(input_file), input_file)
+        grammar = Grammar.new(File.read(input_file))
-        generator = Generator.new(grammar, output_file, log_file, options)
+        generator = Generator.new(grammar, output_file, log_file)
         generator.generate
       rescue Error => e
         $stderr.puts e.message

lib/propane/assets.rb

@@ -1,10 +0,0 @@
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
-        File.binread(path)
-      end
-    end
-  end
-end

lib/propane/cli.rb

@@ -4,21 +4,15 @@ class Propane
     USAGE = <<EOF
 Usage: #{$0} [options] <input-file> <output-file>
 Options:
-  -h, --help  Show this usage and exit.
+  --log LOG   Write log file
-  --log LOG   Write log file. This will show all parser states and their
+  --version   Show program version and exit
-              associated shifts and reduces. It can be helpful when
+  -h, --help  Show this usage and exit
-              debugging a grammar.
-  --version   Show program version and exit.
-  -w          Treat warnings as errors. This option will treat shift/reduce
-              conflicts as fatal errors and will print them to stderr in
-              addition to the log file.
 EOF
 
     class << self
 
       def run(args)
         params = []
-        options = {}
         log_file = nil
         i = 0
         while i < args.size
@@ -30,13 +24,11 @@ EOF
               log_file = args[i]
             end
           when "--version"
-            puts "propane version #{VERSION}"
+            puts "propane v#{VERSION}"
             return 0
           when "-h", "--help"
             puts USAGE
             return 0
-          when "-w"
-            options[:warnings_as_errors] = true
           when /^-/
             $stderr.puts "Error: unknown option #{arg}"
             return 1
@@ -53,7 +45,7 @@ EOF
           $stderr.puts "Error: cannot read #{params[0]}"
           return 2
         end
-        Propane.run(*params, log_file, options)
+        Propane.run(*params, log_file)
       end
 
     end

lib/propane/generator.rb

@@ -2,7 +2,7 @@ class Propane
 
   class Generator
 
-    def initialize(grammar, output_file, log_file, options)
+    def initialize(grammar, output_file, log_file)
       @grammar = grammar
       @output_file = output_file
       if log_file
@@ -10,60 +10,25 @@ class Propane
       else
         @log = StringIO.new
       end
+      @classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
       @language =
-        if output_file.end_with?(".d")
+        if output_file =~ /\.([a-z]+)$/
-          "d"
+          $1
-        elsif output_file.end_with?(".c")
-          "c"
-        elsif output_file =~ %r{\.(cc|cpp|cxx)$}
-          @cpp = true
-          "c"
-        elsif output_file.end_with?(".rs")
-          "rust"
         else
-          raise Error.new("Could not determine target language from output file name (#{output_file})")
+          "d"
         end
-      @options = options
       process_grammar!
     end
 
     def generate
-      extensions = [nil]
+      extensions = [@language]
       if @language == "c"
         extensions += %w[h]
       end
       extensions.each do |extension|
-        template_language = @language == "rust" ? "rs" : @language
+        erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.#{extension}.erb")), trim_mode: "<>")
-        template = Assets.get("parser.#{extension || template_language}.erb")
-        if extension
         output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
-        else
+        result = erb.result(binding.clone)
-          output_file = @output_file
-        end
-        erb = ERB.new(template, trim_mode: "<>")
-        # Rust has no #line directive support. For a Rust target the directives
-        # that the grammar embeds around user code blocks are replaced with
-        # comments naming the grammar file and line number the code came from,
-        # so that the origin of a section of user code can still be found by
-        # reading up from a compiler diagnostic pointing into the generated
-        # module.
-        user_code_origin = nil
-        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
-          if @language == "rust"
-            if md = line.match(/^#line (\d+) "([^"]*)"/)
-              user_code_origin = "#{md[2]} line #{md[1]}"
-              line.sub(/^#line \d+ "[^"]*"/, %[/* Begin user code from #{user_code_origin}. */])
-            elsif line == "#linereset\n"
-              %[/* End user code from #{user_code_origin}. */\n]
-            else
-              line
-            end
-          elsif line == "#linereset\n"
-            %[#line #{i + 2} "#{output_file}"\n]
-          else
-            line
-          end
-        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -77,8 +42,8 @@ class Propane
       # Assign default pattern mode to patterns without a mode assigned.
       found_default = false
       @grammar.patterns.each do |pattern|
-        if pattern.modes.empty?
+        if pattern.mode.nil?
-          pattern.modes << "default"
+          pattern.mode = "default"
           found_default = true
         end
         pattern.ptypename ||= "default"
@@ -86,7 +51,6 @@ class Propane
       unless found_default
         raise Error.new("No patterns found for default mode")
       end
-      check_ptypes!
       # Add EOF token.
       @grammar.tokens << Token.new("$EOF", nil, nil)
       tokens_by_name = {}
@@ -101,18 +65,12 @@ class Propane
         end
         tokens_by_name[token.name] = token
       end
-      # Create real start rule(s).
+      # Check for user start rule.
-      real_start_rules = @grammar.start_rules.map do |start_rule|
+      unless @grammar.rules.find {|rule| rule.name == "Start"}
-        unless @grammar.rules.find {|rule| rule.name == start_rule}
+        raise Error.new("Start rule not found")
-          raise Error.new("Start rule `#{start_rule}` not found")
       end
-        Rule.new("$#{start_rule}", [start_rule, "$EOF"], nil, nil, nil)
+      # Add "real" start rule.
-      end
+      @grammar.rules.unshift(Rule.new("$Start", ["Start", "$EOF"], nil, nil, nil))
-      # Add real start rules before user-given rules.
-      @grammar.rules = real_start_rules + @grammar.rules
-      # Generate and add rules for optional components.
-      generate_optional_component_rules!(tokens_by_name)
-      # Build rule sets.
       rule_sets = {}
       rule_set_id = @grammar.tokens.size
       @grammar.rules.each_with_index do |rule, rule_id|
@@ -161,56 +119,10 @@ class Propane
         end
       end
       determine_possibly_empty_rulesets!(rule_sets)
-      rule_sets.each do |name, rule_set|
-        rule_set.finalize(@grammar)
-      end
       # Generate the lexer.
       @lexer = Lexer.new(@grammar)
       # Generate the parser.
-      @parser = Parser.new(@grammar, rule_sets, @log, @options)
+      @parser = Parser.new(@grammar, rule_sets, @log)
-    end
-
-    # Check that any referenced ptypes have been defined.
-    def check_ptypes!
-      (@grammar.patterns + @grammar.tokens + @grammar.rules).each do |potor|
-        if potor.ptypename
-          unless @grammar.ptypes.include?(potor.ptypename)
-            raise Error.new("Error: Line #{potor.line_number}: ptype #{potor.ptypename} not declared. Declare with `ptype` statement.")
-          end
-        end
-      end
-    end
-
-    # Generate and add rules for any optional components.
-    def generate_optional_component_rules!(tokens_by_name)
-      optional_rules_added = Set.new
-      @grammar.rules.each do |rule|
-        rule.components.each do |component|
-          if component =~ /^(.*)\?$/
-            c = $1
-            unless optional_rules_added.include?(component)
-              # Create two rules for the optional component: one empty and
-              # one just matching the component.
-              # We need to find the ptypename for the optional component in
-              # order to copy it to the generated rules.
-              if tokens_by_name[c]
-                # The optional component is a token.
-                ptypename = tokens_by_name[c].ptypename
-              else
-                # The optional component must be a rule, so find any instance
-                # of that rule that specifies a ptypename.
-                ptypename = @grammar.rules.reduce(nil) do |result, rule|
-                  rule.name == c && rule.ptypename ? rule.ptypename : result
-                end
-              end
-              @grammar.rules << Rule.new(component, [], nil, ptypename, rule.line_number)
-              optcode = @grammar.tree ? nil : "$$ = $1;\n"
-              @grammar.rules << Rule.new(component, [c], optcode, ptypename, rule.line_number)
-              optional_rules_added << component
-            end
-          end
-        end
-      end
     end
 
     # Determine which grammar rules could expand to empty sequences.
@@ -286,126 +198,26 @@ class Propane
       code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
         "TOKEN_#{Token.code_name($1)}"
       end
-      code = code.gsub(/\$terminate\((.*)\);/) do |match|
-        user_terminate_code = $1
-        retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
-        case @language
-        when "c"
-          "context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        when "d"
-          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        when "rust"
-          "context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
-        end
-      end
-      code = code.gsub(/\$\{context\.(\w+)\}/) do |match|
-        fieldname = $1
-        case @language
-        when "c"
-          "context->#{fieldname}"
-        when "d"
-          "context.#{fieldname}"
-        when "rust"
-          "context.#{fieldname}"
-        end
-      end
-      code = code.gsub(/\$\{token\.(\w+)\}/) do |match|
-        fieldname = $1
-        case @language
-        when "c"
-          "token_tree_node->#{fieldname}"
-        when "d"
-          "token_tree_node.#{fieldname}"
-        when "rust"
-          "token_tree_node.#{fieldname}"
-        end
-      end
       if parser
         code = code.gsub(/\$\$/) do |match|
-          if @grammar.tree
-            typename = "#{@grammar.tree_prefix}#{rule.name}#{@grammar.tree_suffix}"
-            case @language
-            when "c"
-              tree_handle(typename, "_node_id")
-            when "d"
-              tree_handle(typename, "_node_id")
-            when "rust"
-              tree_handle(typename, "_node_id")
-            end
-          else
-            case @language
-            when "c"
-              "_pvalue->v_#{rule.ptypename}"
-            when "d"
           "_pvalue.v_#{rule.ptypename}"
-            when "rust"
-              "(*_pvalue.v_#{rule.ptypename}_mut())"
-            end
-          end
         end
         code = code.gsub(/\$(\d+)/) do |match|
-          parser_component_reference(rule, $1.to_i)
-        end
-        code = code.gsub(/\$\{(\$|\d+)\.position\}/) do |match|
           index = $1.to_i
-          "get_rule_position(statevalues, #{index}, n_states, false)"
+          case @language
-        end
+          when "c"
-        code = code.gsub(/\$\{(\$|\d+)\.end_position\}/) do |match|
+            "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
-          index = $1.to_i
+          when "d"
-          "get_rule_position(statevalues, #{index}, n_states, true)"
+            "statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
-        end
-        code = code.gsub(/\$\{(\w+)\}/) do |match|
-          aliasname = $1
-          if index = rule.aliases[aliasname]
-            # Field aliases are just a named reference to a positional rule
-            # component, so reuse the same expansion as `$1', `$2', etc. Note
-            # that rule.aliases stores a 0-based component index, so add 1 to
-            # convert it to the 1-based index used for positional references.
-            parser_component_reference(rule, index + 1)
-          else
-            raise Error.new("Field alias '#{aliasname}' not found")
           end
         end
       else
         code = code.gsub(/\$\$/) do |match|
-          if @grammar.tree
-            case @language
-            when "c"
-              "out_token_info->pvalue"
-            when "d"
-              "out_token_info.pvalue"
-            when "rust"
-              "out_token_info.pvalue"
-            end
-          else
           case @language
           when "c"
             "out_token_info->pvalue.v_#{pattern.ptypename}"
           when "d"
             "out_token_info.pvalue.v_#{pattern.ptypename}"
-            when "rust"
-              "(*out_token_info.pvalue.v_#{pattern.ptypename}_mut())"
-            end
-          end
-        end
-        code = code.gsub(/\$\{position\}/) do |match|
-          case @language
-          when "c"
-            "out_token_info->position"
-          when "d"
-            "out_token_info.position"
-          when "rust"
-            "out_token_info.position"
-          end
-        end
-        code = code.gsub(/\$\{end_position\}/) do |match|
-          case @language
-          when "c"
-            "out_token_info->end_position"
-          when "d"
-            "out_token_info.end_position"
-          when "rust"
-            "out_token_info.end_position"
           end
         end
         code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
@@ -419,418 +231,19 @@ class Propane
             "context->mode = #{mode_id}u"
           when "d"
             "context.mode = #{mode_id}u"
-          when "rust"
-            "context.mode = #{mode_id}"
           end
         end
       end
       code
     end
 
-    # Expand a positional reference to a parser rule component.
-    #
-    # This is used to expand `$1', `$2', etc. as well as field aliases (which
-    # are just named references to a positional rule component).
-    #
-    # @param rule [Rule]
-    #   The Rule containing the user code.
-    # @param index [Integer]
-    #   1-based index of the rule component to reference.
-    #
-    # @return [String]
-    #   Expanded rule component reference.
-    def parser_component_reference(rule, index)
-      component = rule.components[index - 1]
-      if @grammar.tree
-        # In tree mode a component reference yields a handle to that
-        # component's tree node. An optional component propagates its target
-        # node (or null), so use the optional target's node type.
-        if component.is_a?(RuleSet) && component.optional?
-          component = component.option_target
-        end
-        node_name = component.is_a?(Token) ? "Token" : component.name
-        typename = "#{@grammar.tree_prefix}#{node_name}#{@grammar.tree_suffix}"
-        case @language
-        when "c"
-          tree_handle(typename, "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->node_id")
-        when "d"
-          tree_handle(typename, "statevalues[$-1-n_states+#{index}].node_id")
-        when "rust"
-          tree_handle(typename, "statevalues[statevalues.len() - 1 - n_states + #{index}].node_id")
-        end
-      else
-        case @language
-        when "c"
-          "state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{component.ptypename}"
-        when "d"
-          "statevalues[$-1-n_states+#{index}].pvalue.v_#{component.ptypename}"
-        when "rust"
-          "statevalues[statevalues.len() - 1 - n_states + #{index}].pvalue.get_v_#{component.ptypename}()"
-        end
-      end
-    end
-
-    # Construct a tree node handle expression for the target language.
-    #
-    # A handle is a small value pairing the parser context with a node ID
-    # (an index into the context's node arena). All handle types share this
-    # layout; the distinct types exist for documentation and, in C, to drive
-    # the tree walk macro's type threading.
-    #
-    # @param typename [String]
-    #   Handle type name.
-    # @param id_expr [String]
-    #   Expression yielding the node ID.
-    # @param parenthesize [Boolean]
-    #   Whether to parenthesize the expression. Parentheses are required where
-    #   the expression is substituted into a user code block, since the
-    #   expression could be followed there by a field access or appear in a
-    #   position where a bare Rust struct literal is not accepted. They are
-    #   unnecessary where the expression stands alone, and Rust warns about
-    #   them there, so this can be disabled for those uses.
-    #
-    # @return [String]
-    #   Handle constructor expression.
-    def tree_handle(typename, id_expr, parenthesize = true)
-      if @cpp
-        "(#{typename}{context, #{id_expr}})"
-      elsif @language == "c"
-        "((#{typename}){context, #{id_expr}})"
-      elsif @language == "rust"
-        expr = "#{typename} { context, id: #{id_expr} }"
-        parenthesize ? "(#{expr})" : expr
-      else
-        "#{typename}(context, #{id_expr})"
-      end
-    end
-
-    # Get the list of non-optional, non-internal rule sets that get a tree node
-    # handle type generated for them.
-    #
-    # @return [Array<Propane::RuleSet>]
-    #   Rule sets with generated tree node handle types.
-    def tree_node_rule_sets
-      @parser.rule_sets.reject do |name, rule_set|
-        name.start_with?("$") || rule_set.optional?
-      end.map {|name, rule_set| rule_set}
-    end
-
-    # Maximum number of chained fields supported by a single C tree walk macro
-    # invocation. Deeper navigation can be expressed by nesting walk calls.
-    C_TREE_WALK_MAX = 16
-
-    # Get the tree node handle type name for a node name.
-    #
-    # @param name [String]
-    #   Rule set name, or "Token".
-    #
-    # @return [String]
-    #   Handle type name.
-    def h_type(name)
-      "#{@grammar.tree_prefix}#{name}#{@grammar.tree_suffix}"
-    end
-
-    # Get the list of all tree node handle type names (Token plus rule sets).
-    #
-    # @return [Array<String>]
-    #   Handle type names.
-    def tree_handle_types
-      [h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}
-    end
-
-    # Enumerate the navigation fields of a rule set's tree node.
-    #
-    # @yield [rtype, field_name, child_type, slot]
-    #   Handle type name, field accessor name, child handle type, and child
-    #   slot index.
-    def each_tree_field(rule_set)
-      rtype = h_type(rule_set.name)
-      rule_set.tree_fields.each_with_index do |fields, slot|
-        fields.each do |field_name, child_type|
-          yield rtype, field_name, child_type, slot
-        end
-      end
-    end
-
-    # Generate the tree node handle type declarations for the header.
-    #
-    # These are emitted before the context structure definition so that a
-    # context_user_fields block can declare a field of a handle type.
-    #
-    # @return [String]
-    #   Handle type declarations.
-    def c_tree_handle_types_header
-      @cpp ? cpp_tree_handle_types_header : c_only_tree_handle_types_header
-    end
-
-    # Generate the remainder of the tree node section for the header.
-    #
-    # This is emitted after the context structure definition since it
-    # dereferences the context and so requires the complete type.
-    #
-    # @return [String]
-    #   Accessors, macros, and out-of-line handle method definitions.
-    def c_tree_types_header
-      @cpp ? cpp_tree_types_header : c_only_tree_types_header
-    end
-
-    # Generate the C (non-C++) tree node handle type section for the header.
-    def c_only_tree_handle_types_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Tree node handle types. @{ */"
-      tree_handle_types.each do |t|
-        out << "typedef struct { #{p}context_t * __context; #{p}node_id_t __id; } #{t};"
-      end
-      out << ""
-      out.join("\n")
-    end
-
-    def c_only_tree_types_header
-      out = []
-      out << c_common_accessors_header
-      out << "/** @} */"
-      out.join("\n")
-    end
-
-    # Generate the C-style (function + macro) tree node accessors shared by the
-    # C and C++ headers. In C++ these are provided in addition to the handle
-    # methods so that C-style code (and the tree walk macros) also works.
-    def c_common_accessors_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Generic tree node accessors (usable on any handle type). */"
-      out << "#define #{p}node_valid(h) ((h).__id != 0u)"
-      out << "#define #{p}node_id(h) ((h).__id)"
-      out << "#define #{p}node_data(h) (&(h).__context->#{p}tree_nodes[(h).__id])"
-      out << "#define #{p}node_position(h) ((h).__context->#{p}tree_nodes[(h).__id].position)"
-      out << "#define #{p}node_end_position(h) ((h).__context->#{p}tree_nodes[(h).__id].end_position)"
-      out << "#define #{p}node_n_fields(h) ((h).__id ? (h).__context->#{p}tree_nodes[(h).__id].n_fields : (uint16_t)0u)"
-      out << ""
-      out << "/** Tree node field accessor functions. */"
-      out << "#{p}token_t #{p}#{h_type("Token")}_token(#{h_type("Token")} node);"
-      out << "#{p}value_t #{p}#{h_type("Token")}_pvalue(#{h_type("Token")} node);"
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node);"
-        end
-      end
-      out << ""
-      out << c_tree_walk_macros
-      out.join("\n")
-    end
-
-    # Generate the C tree walk macro machinery.
-    def c_tree_walk_macros
-      p = @grammar.prefix
-      max = C_TREE_WALK_MAX
-      out = []
-      out << "/* Tree walk macros: p_tree_walk_<Type>(handle, field, ...). */"
-      out << "#define #{p}CAT_(a, b) a##b"
-      out << "#define #{p}CAT(a, b) #{p}CAT_(a, b)"
-      out << "#define #{p}TA(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}TYPEAFTER_, t), _), f)"
-      out << "#define #{p}ACC(t, f) #{p}CAT(#{p}CAT(#{p}CAT(#{p}, t), _), f)"
-      argn = (1..max).map {|i| "_#{i}"}.join(", ")
-      rseq = (0..max).to_a.reverse.join(", ")
-      out << "#define #{p}ARG_N(#{argn}, N, ...) N"
-      out << "#define #{p}NARG(...) #{p}ARG_N(__VA_ARGS__, #{rseq})"
-      (1..max).each do |n|
-        fparams = (1..n).map {|k| "f#{k}"}.join(", ")
-        call = "h"
-        (1..n).each do |k|
-          texpr = "R"
-          (1...k).each {|j| texpr = "#{p}TA(#{texpr}, f#{j})"}
-          call = "#{p}ACC(#{texpr}, f#{k})(#{call})"
-        end
-        out << "#define #{p}tree_walk_#{n}(R, h, #{fparams}) #{call}"
-      end
-      out << "#define #{p}tree_walk_dispatch(R, h, ...) #{p}CAT(#{p}tree_walk_, #{p}NARG(__VA_ARGS__))(R, h, __VA_ARGS__)"
-      # Type transition map (navigation fields only).
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << "#define #{p}TYPEAFTER_#{rtype}_#{field_name} #{child_type}"
-        end
-      end
-      # Per-handle-type walk entry points.
-      tree_handle_types.each do |t|
-        out << "#define #{p}tree_walk_#{t}(...) #{p}tree_walk_dispatch(#{t}, __VA_ARGS__)"
-      end
-      out.join("\n")
-    end
-
-    # Generate the C tree node accessor function definitions for the source.
-    #
-    # @return [String]
-    #   Accessor function definitions.
-    def c_tree_accessor_defs
-      p = @grammar.prefix
-      tt = h_type("Token")
-      out = []
-      out << "#{p}token_t #{p}#{tt}_token(#{tt} node)"
-      out << "{"
-      out << "    return node.__context->#{p}tree_nodes[node.__id].token;"
-      out << "}"
-      out << ""
-      out << "#{p}value_t #{p}#{tt}_pvalue(#{tt} node)"
-      out << "{"
-      out << "    return node.__context->#{p}tree_nodes[node.__id].pvalue;"
-      out << "}"
-      tree_node_rule_sets.each do |rule_set|
-        each_tree_field(rule_set) do |rtype, field_name, child_type, slot|
-          out << ""
-          out << "#{child_type} #{p}#{rtype}_#{field_name}(#{rtype} node)"
-          out << "{"
-          out << "    #{child_type} result;"
-          out << "    result.__context = node.__context;"
-          out << "    if (node.__id == 0u)"
-          out << "    {"
-          out << "        result.__id = 0u;"
-          out << "        return result;"
-          out << "    }"
-          out << "    result.__id = node.__context->#{p}tree_children[node.__context->#{p}tree_nodes[node.__id].child_offset + #{slot}u];"
-          out << "    return result;"
-          out << "}"
-        end
-      end
-      out.join("\n")
-    end
-
-    # Generate the C++ tree node handle class declarations for the header.
-    # Only valid() and node_id() are defined inline; every other method
-    # dereferences the context, which is still an incomplete type here, so
-    # those are declared and defined out of line once the context is
-    # complete.
-    def cpp_tree_handle_types_header
-      p = @grammar.prefix
-      out = []
-      out << "/** Tree node handle types. @{ */"
-      tree_handle_types.each {|t| out << "struct #{t};"}
-      out << ""
-      tt = h_type("Token")
-      out << "struct #{tt}"
-      out << "{"
-      out << "    #{p}context_t * __context;"
-      out << "    #{p}node_id_t __id;"
-      out << "    bool valid() const { return __id != 0u; }"
-      out << "    #{p}node_id_t node_id() const { return __id; }"
-      out << "    #{p}node_data_t * data() const;"
-      out << "    #{p}position_t position() const;"
-      out << "    #{p}position_t end_position() const;"
-      out << "    uint16_t n_fields() const;"
-      out << "    #{p}token_t token() const;"
-      out << "    #{p}value_t pvalue() const;"
-      out << "};"
-      out << ""
-      tree_node_rule_sets.each do |rule_set|
-        rtype = h_type(rule_set.name)
-        out << "struct #{rtype}"
-        out << "{"
-        out << "    #{p}context_t * __context;"
-        out << "    #{p}node_id_t __id;"
-        out << "    bool valid() const { return __id != 0u; }"
-        out << "    #{p}node_id_t node_id() const { return __id; }"
-        out << "    #{p}node_data_t * data() const;"
-        out << "    #{p}position_t position() const;"
-        out << "    #{p}position_t end_position() const;"
-        out << "    uint16_t n_fields() const;"
-        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
-          out << "    #{child_type} #{field_name}() const;"
-        end
-        out << "};"
-        out << ""
-      end
-      out.join("\n")
-    end
-
-    # Generate the out-of-line C++ handle method definitions plus the C-style
-    # accessors. Emitted after the context structure definition.
-    def cpp_tree_types_header
-      p = @grammar.prefix
-      out = []
-      # Common node methods, now that the context type is complete.
-      ([h_type("Token")] + tree_node_rule_sets.map {|rs| h_type(rs.name)}).each do |ht|
-        out << "inline #{p}node_data_t * #{ht}::data() const { return &__context->#{p}tree_nodes[__id]; }"
-        out << "inline #{p}position_t #{ht}::position() const { return __context->#{p}tree_nodes[__id].position; }"
-        out << "inline #{p}position_t #{ht}::end_position() const { return __context->#{p}tree_nodes[__id].end_position; }"
-        out << "inline uint16_t #{ht}::n_fields() const { return __id ? __context->#{p}tree_nodes[__id].n_fields : (uint16_t)0u; }"
-      end
-      tt = h_type("Token")
-      out << "inline #{p}token_t #{tt}::token() const { return __context->#{p}tree_nodes[__id].token; }"
-      out << "inline #{p}value_t #{tt}::pvalue() const { return __context->#{p}tree_nodes[__id].pvalue; }"
-      out << ""
-      # Out-of-line navigation method bodies (all handle types now complete).
-      tree_node_rule_sets.each do |rule_set|
-        rtype = h_type(rule_set.name)
-        each_tree_field(rule_set) do |rt, field_name, child_type, slot|
-          out << "inline #{child_type} #{rtype}::#{field_name}() const"
-          out << "{"
-          out << "    if (__id == 0u)"
-          out << "    {"
-          out << "        return #{child_type}{__context, 0u};"
-          out << "    }"
-          out << "    return #{child_type}{__context, __context->#{p}tree_children[__context->#{p}tree_nodes[__id].child_offset + #{slot}u]};"
-          out << "}"
-        end
-      end
-      out << ""
-      out << "/*"
-      out << " * C-style function and macro accessors, provided in addition to the handle"
-      out << " * methods above so that C-style code and the tree walk macros also work."
-      out << " */"
-      out << c_common_accessors_header
-      out << "/** @} */"
-      out.join("\n")
-    end
-
-    # Rust keywords that must be escaped as raw identifiers when used as a
-    # generated identifier (e.g. a field alias named `type`).
-    RUST_KEYWORDS = %w[
-      as break const continue dyn else enum extern false fn for if impl in let
-      loop match mod move mut pub ref return static struct trait true type
-      unsafe use where while async await abstract become box do final macro
-      override priv typeof unsized virtual yield try gen
-    ]
-
-    # Escape a name as a Rust raw identifier if it is a reserved keyword.
-    #
-    # @param name [String]
-    #   Identifier name.
-    #
-    # @return [String]
-    #   Name, escaped as a raw identifier if necessary.
-    def rust_ident(name)
-      RUST_KEYWORDS.include?(name) ? "r##{name}" : name
-    end
-
-    # Map a ptype type string to a valid Rust type.
-    #
-    # The default ptype is a C "void *"; for Rust with no declared ptype we use
-    # the unit type instead.
-    #
-    # @param typestring [String]
-    #   ptype type string.
-    #
-    # @return [String]
-    #   Rust type string.
-    def rust_ptype(typestring)
-      typestring == "void *" ? "()" : typestring
-    end
-
-    # Get the lex function to use.
-    #
-    # @return [String]
-    #   Lex function to use.
-    def lex_fn
-      @grammar.lex_fn || "#{@grammar.prefix}lex"
-    end
-
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]
     #   Start rule parser value type name and type string.
-    def start_rule_type(start_rule_index = 0)
+    def start_rule_type
       start_rule = @grammar.rules.find do |rule|
-        rule.name == @grammar.start_rules[start_rule_index]
+        rule.name == "Start"
       end
       [start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
     end
@@ -849,8 +262,6 @@ class Propane
           "uint8_t"
         when "d"
           "ubyte"
-        when "rust"
-          "u8"
         end
       elsif max <= 0xFFFF
         case @language
@@ -858,15 +269,11 @@ class Propane
           "uint16_t"
         when "d"
           "ushort"
-        when "rust"
-          "u16"
         end
       else
         case @language
         when "c"
           "uint32_t"
-        when "rust"
-          "u32"
         else
           "uint"
         end

lib/propane/grammar.rb

@@ -5,45 +5,27 @@ class Propane
     # Reserve identifiers beginning with a double-underscore for internal use.
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
-    attr_reader :context_user_fields
+    attr_reader :classname
-    attr_reader :lex_fn
-    attr_reader :tree
-    attr_reader :tree_prefix
-    attr_reader :tree_suffix
-    attr_reader :free_token_node
     attr_reader :modulename
     attr_reader :patterns
-    attr_accessor :rules
+    attr_reader :rules
-    attr_reader :start_rules
     attr_reader :tokens
     attr_reader :code_blocks
     attr_reader :ptypes
     attr_reader :prefix
-    attr_reader :on_token_node
-    attr_reader :token_user_fields
 
-    def initialize(input, filename)
+    def initialize(input)
-      @filename = filename
       @patterns = []
-      @start_rules = []
       @tokens = []
       @rules = []
-      @code_blocks = {}
+      @code_blocks = []
       @line_number = 1
       @next_line_number = @line_number
-      @modeline = nil
+      @mode = nil
       @input = input.gsub("\r\n", "\n")
       @ptypes = {"default" => "void *"}
       @prefix = "p_"
-      @tree = false
-      @tree_prefix = ""
-      @tree_suffix = ""
-      @free_token_node = ""
-      @context_user_fields = nil
-      @on_token_node = ""
-      @token_user_fields = nil
       parse_grammar!
-      @start_rules << "Start" if @start_rules.empty?
     end
 
     def ptype
@@ -54,14 +36,6 @@ class Propane
       @tokens.size
     end
 
-    def terminate_token_id
-      @tokens.size + 1
-    end
-
-    def parser_user_code_used?
-      @rules.any? {|r| r.code}
-    end
-
     private
 
     def parse_grammar!
@@ -73,26 +47,17 @@ class Propane
     def parse_statement!
       if parse_white_space!
       elsif parse_comment_line!
-      elsif @modeline.nil? && parse_mode_label!
+      elsif @mode.nil? && parse_mode_label!
-      elsif parse_context_user_fields_statement!
-      elsif parse_lex_fn!
-      elsif parse_tree_statement!
-      elsif parse_tree_prefix_statement!
-      elsif parse_tree_suffix_statement!
-      elsif parse_free_token_node_statement!
       elsif parse_module_statement!
-      elsif parse_on_token_node_statement!
+      elsif parse_class_statement!
-      elsif parse_token_user_fields_statement!
       elsif parse_ptype_statement!
       elsif parse_pattern_statement!
-      elsif parse_start_statement!
       elsif parse_token_statement!
       elsif parse_tokenid_statement!
       elsif parse_drop_statement!
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
-      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -102,8 +67,8 @@ class Propane
     end
 
     def parse_mode_label!
-      if md = consume!(/(#{IDENTIFIER_REGEX}(?:\s*,\s*#{IDENTIFIER_REGEX})*)\s*:/)
+      if md = consume!(/(#{IDENTIFIER_REGEX})\s*:/)
-        @modeline = md[1]
+        @mode = md[1]
       end
     end
 
@@ -115,75 +80,23 @@ class Propane
       consume!(/#.*\n/)
     end
 
-    def parse_context_user_fields_statement!
-      if md = consume!(/context_user_fields\b\s*/)
-        unless code = parse_code_block!
-          raise Error.new("Line #{@line_number}: expected code block")
-        end
-        @context_user_fields ||= ""
-        @context_user_fields += code
-      end
-    end
-
-    def parse_lex_fn!
-      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
-        @lex_fn = md[1]
-      end
-    end
-
-    def parse_tree_statement!
-      if consume!(/tree\s*;/)
-        @tree = true
-      end
-    end
-
-    def parse_tree_prefix_statement!
-      if md = consume!(/tree_prefix\s+(\w+)\s*;/)
-        @tree_prefix = md[1]
-      end
-    end
-
-    def parse_tree_suffix_statement!
-      if md = consume!(/tree_suffix\s+(\w+)\s*;/)
-        @tree_suffix = md[1]
-      end
-    end
-
     def parse_module_statement!
       if consume!(/module\s+/)
         md = consume!(/([\w.]+)\s*/, "expected module name")
         @modulename = md[1]
         consume!(/;/, "expected `;'")
-        @modeline = nil
+        @mode = nil
         true
       end
     end
 
-    def parse_on_token_node_statement!
+    def parse_class_statement!
-      if md = consume!(/on_token_node\b\s*/)
+      if consume!(/class\s+/)
-        unless code = parse_code_block!
+        md = consume!(/([\w.]+)\s*/, "expected class name")
-          raise Error.new("Line #{@line_number}: expected code block")
+        @classname = md[1]
-        end
+        consume!(/;/, "expected `;'")
-        @on_token_node += code
+        @mode = nil
-      end
+        true
-    end
-
-    def parse_token_user_fields_statement!
-      if md = consume!(/token_user_fields\b\s*/)
-        unless code = parse_code_block!
-          raise Error.new("Line #{@line_number}: expected code block")
-        end
-        @token_user_fields ||= ""
-        @token_user_fields += code
-      end
-    end
-
-    def parse_free_token_node_statement!
-      if md = consume!(/free_token_node\b\s*/)
-        unless code = parse_code_block!
-          raise Error.new("Line #{@line_number}: expected code block")
-        end
-        @free_token_node += code
       end
     end
 
@@ -191,9 +104,6 @@ class Propane
       if consume!(/ptype\s+/)
         name = "default"
         if md = consume!(/(#{IDENTIFIER_REGEX})\s*=\s*/)
-          if @tree
-            raise Error.new("Multiple ptypes are unsupported in tree mode")
-          end
           name = md[1]
         end
         md = consume!(/([^;]+);/, "expected parser result type expression")
@@ -206,21 +116,18 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @tree
-            raise Error.new("Multiple ptypes are unsupported in tree mode")
-          end
           ptypename = md[1]
         end
         pattern = parse_pattern! || name
         consume!(/\s+/)
         unless code = parse_code_block!
-          consume!(/;/, "expected `;' or code block")
+          consume!(/;/, "expected pattern or `;' or code block")
         end
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
+        pattern = Pattern.new(pattern: pattern, token: token, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
         @patterns << pattern
-        @modeline = nil
+        @mode = nil
         true
       end
     end
@@ -230,15 +137,12 @@ class Propane
         md = consume!(/(#{IDENTIFIER_REGEX})\s*/, "expected token name")
         name = md[1]
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @tree
-            raise Error.new("Multiple ptypes are unsupported in tree mode")
-          end
           ptypename = md[1]
         end
         consume!(/;/, "expected `;'");
         token = Token.new(name, ptypename, @line_number)
         @tokens << token
-        @modeline = nil
+        @mode = nil
         true
       end
     end
@@ -250,11 +154,9 @@ class Propane
           raise Error.new("Line #{@line_number}: expected pattern to follow `drop'")
         end
         consume!(/\s+/)
-        unless code = parse_code_block!
+        consume!(/;/, "expected `;'")
-          consume!(/;/, "expected `;' or code block")
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
-        end
+        @mode = nil
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline)
-        @modeline = nil
         true
       end
     end
@@ -262,16 +164,13 @@ class Propane
     def parse_rule_statement!
       if md = consume!(/(#{IDENTIFIER_REGEX})\s*(?:\((#{IDENTIFIER_REGEX})\))?\s*->\s*/)
         rule_name, ptypename = *md[1, 2]
-        if @tree && ptypename
+        md = consume!(/((?:#{IDENTIFIER_REGEX}\s*)*)\s*/, "expected rule component list")
-          raise Error.new("Multiple ptypes are unsupported in tree mode")
-        end
-        md = consume!(/((?:#{IDENTIFIER_REGEX}\??(?::#{IDENTIFIER_REGEX})?\s*)*)\s*/, "expected rule component list")
         components = md[1].strip.split(/\s+/)
         unless code = parse_code_block!
-          consume!(/;/, "expected `;' or code block")
+          consume!(/;/, "expected pattern or `;' or code block")
         end
         @rules << Rule.new(rule_name, components, code, ptypename, @line_number)
-        @modeline = nil
+        @mode = nil
         true
       end
     end
@@ -280,46 +179,21 @@ class Propane
       if pattern = parse_pattern!
         consume!(/\s+/)
         if md = consume!(/\((#{IDENTIFIER_REGEX})\)\s*/)
-          if @tree
-            raise Error.new("Multiple ptypes are unsupported in tree mode")
-          end
           ptypename = md[1]
         end
         unless code = parse_code_block!
           raise Error.new("Line #{@line_number}: expected code block to follow pattern")
         end
-        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, modes: get_modes_from_modeline, ptypename: ptypename)
+        @patterns << Pattern.new(pattern: pattern, line_number: @line_number, code: code, mode: @mode, ptypename: ptypename)
-        @modeline = nil
+        @mode = nil
         true
       end
     end
 
-    def parse_start_statement!
-      if md = consume!(/start\s+([\w\s]*);/)
-        start_rules = md[1].split(/\s+/).map(&:strip)
-        start_rules.each do |start_rule|
-          @start_rules << start_rule unless @start_rules.include?(start_rule)
-        end
-      end
-    end
-
     def parse_code_block_statement!
-      if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
+      if code = parse_code_block!
-        name, code = md[1..2]
+        @code_blocks << code
-        code = code.chomp
+        @mode = nil
-        unless @noline
-          if code.start_with?("\n")
-            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          else
-            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
-          end
-        end
-        if @code_blocks[name]
-          @code_blocks[name] += code
-        else
-          @code_blocks[name] = code
-        end
-        @modeline = nil
         true
       end
     end
@@ -331,13 +205,6 @@ class Propane
       end
     end
 
-    def parse_noline_statement!
-      if md = consume!(/noline\s*;/)
-        @noline = true
-        true
-      end
-    end
-
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -351,8 +218,6 @@ class Propane
             end
           elsif md = consume!(%r{(.)})
             pattern += md[1]
-          elsif @input == "" || @input.start_with?("\n")
-            raise Error.new("Line #{@line_number}: Unterminated pattern; expected `/`")
           end
         end
         pattern
@@ -360,16 +225,8 @@ class Propane
     end
 
     def parse_code_block!
-      if md = consume!(/<<(.*?)>>\n/m)
+      if md = consume!(/<<\n(.*?)^>>\n/m)
-        code = md[1].chomp
+        md[1]
-        unless @noline
-          if code.start_with?("\n")
-            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          else
-            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
-          end
-        end
-        code
       end
     end
 
@@ -401,14 +258,6 @@ class Propane
       end
     end
 
-    def get_modes_from_modeline
-      if @modeline
-        Set[*@modeline.split(",").map(&:strip)]
-      else
-        Set.new
-      end
-    end
-
   end
 
 end

lib/propane/lexer.rb

@@ -26,14 +26,8 @@ class Propane
     private
 
     def build_tables!
-      modenames = @grammar.patterns.reduce(Set.new) do |result, pattern|
+      @modes = @grammar.patterns.group_by do |pattern|
-        result + pattern.modes
+        pattern.mode
-      end
-      @modes = modenames.reduce({}) do |result, modename|
-        result[modename] = @grammar.patterns.select do |pattern|
-          pattern.modes.include?(modename)
-        end
-        result
       end.transform_values do |patterns|
         {dfa: DFA.new(patterns)}
       end

lib/propane/parser.rb

@@ -7,38 +7,26 @@ class Propane
     attr_reader :reduce_table
     attr_reader :rule_sets
 
-    def initialize(grammar, rule_sets, log, options)
+    def initialize(grammar, rule_sets, log)
       @grammar = grammar
       @rule_sets = rule_sets
       @log = log
       @item_sets = []
       @item_sets_set = {}
-      @warnings = Set.new
+      start_item = Item.new(grammar.rules.first, 0)
-      @errors = Set.new
+      eval_item_sets = Set[ItemSet.new([start_item])]
-      @options = options
-      start_items = grammar.rules[0...grammar.start_rules.length].map do |start_rule|
-        Item.new(start_rule, 0)
-      end
-      start_item_sets = start_items.map {|item| ItemSet.new([item])}
-      eval_item_sets = Set[*start_item_sets]
 
       while eval_item_sets.size > 0
-        item_set =
+        item_set = eval_item_sets.first
-          if start_item_sets.size > 0
-            # Ensure we evaluate start_item_sets first in order
-            start_item_sets.slice!(0)
-          else
-            eval_item_sets.first
-          end
         eval_item_sets.delete(item_set)
         unless @item_sets_set.include?(item_set)
           item_set.id = @item_sets.size
           @item_sets << item_set
           @item_sets_set[item_set] = item_set
-          item_set.next_symbols.each do |next_symbol|
+          item_set.following_symbols.each do |following_symbol|
-            unless next_symbol.name == "$EOF"
+            unless following_symbol.name == "$EOF"
-              next_item_set = item_set.build_next_item_set(next_symbol)
+              following_set = item_set.build_following_item_set(following_symbol)
-              eval_item_sets << next_item_set
+              eval_item_sets << following_set
             end
           end
         end
@@ -49,21 +37,8 @@ class Propane
       end
 
       build_reduce_actions!
-      build_tables!
       write_log!
-      errormessage = ""
+      build_tables!
-      if @errors.size > 0
-        errormessage += @errors.join("\n")
-      end
-      if @warnings.size > 0 && @options[:warnings_as_errors]
-        if errormessage != ""
-          errormessage += "\n"
-        end
-        errormessage += "Fatal errors (-w):\n" + @warnings.join("\n")
-      end
-      if errormessage != ""
-        raise Error.new(errormessage)
-      end
     end
 
     private
@@ -73,47 +48,48 @@ class Propane
       @shift_table = []
       @reduce_table = []
       @item_sets.each do |item_set|
-        unless item_set.reduce_rules.empty?
+        shift_entries = item_set.following_symbols.map do |following_symbol|
-          item_set.shift_entries.each do |shift_entry|
+          state_id =
-            token = shift_entry[:symbol]
+            if following_symbol.name == "$EOF"
-            if item_set.reduce_actions
+              0
-              if rule = item_set.reduce_actions[token]
+            else
-                @warnings << "Shift/Reduce conflict (state #{item_set.id}) between token #{token.name} and rule #{rule.name} (defined on line #{rule.line_number})"
+              item_set.following_item_set[following_symbol].id
-              end
-            end
             end
+          {
+            symbol_id: following_symbol.id,
+            state_id: state_id,
+          }
         end
         reduce_entries =
-          if rule = item_set.reduce_rule
+          case ra = item_set.reduce_actions
-            [{token_id: @grammar.invalid_token_id, rule_id: rule.id, rule: rule,
+          when Rule
-              rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+            [{token_id: @grammar.invalid_token_id, rule_id: ra.id,
-              propagate_optional_target: rule.optional? && rule.components.size == 1}]
+              rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
-          elsif reduce_actions = item_set.reduce_actions
+          when Hash
-            reduce_actions.map do |token, rule|
+            ra.map do |token, rule|
-              {token: token, token_id: token.id, rule_id: rule.id, rule: rule,
+              {token_id: token.id, rule_id: rule.id,
-               rule_set_id: rule.rule_set.id, n_states: rule.components.size,
+               rule_set_id: rule.rule_set.id, n_states: rule.components.size}
-               propagate_optional_target: rule.optional? && rule.components.size == 1}
             end
           else
             []
           end
         @state_table << {
           shift_index: @shift_table.size,
-          n_shifts: item_set.shift_entries.size,
+          n_shifts: shift_entries.size,
           reduce_index: @reduce_table.size,
           n_reduces: reduce_entries.size,
         }
-        @shift_table += item_set.shift_entries
+        @shift_table += shift_entries
         @reduce_table += reduce_entries
       end
     end
 
     def process_item_set(item_set)
-      item_set.next_symbols.each do |next_symbol|
+      item_set.following_symbols.each do |following_symbol|
-        unless next_symbol.name == "$EOF"
+        unless following_symbol.name == "$EOF"
-          next_item_set = @item_sets_set[item_set.build_next_item_set(next_symbol)]
+          following_set = @item_sets_set[item_set.build_following_item_set(following_symbol)]
-          item_set.next_item_set[next_symbol] = next_item_set
+          item_set.following_item_set[following_symbol] = following_set
-          next_item_set.in_sets << item_set
+          following_set.in_sets << item_set
         end
       end
     end
@@ -123,109 +99,7 @@ class Propane
     # @return [void]
     def build_reduce_actions!
       @item_sets.each do |item_set|
-        build_shift_entries(item_set)
+        item_set.reduce_actions = build_reduce_actions_for_item_set(item_set)
-        build_reduce_actions_for_item_set(item_set)
-      end
-      item_sets_to_process = @item_sets.select do |item_set|
-        # We need lookahead reduce actions if:
-        # 1) There is more than one possible rule to reduce. In this case the
-        #    lookahead token can help choose which rule to reduce.
-        # 2) There is at least one shift action and one reduce action for
-        #    this item set. In this case the lookahead reduce actions are
-        #    needed to test for a Shift/Reduce conflict.
-        item_set.reduce_rules.size > 1 ||
-          (item_set.reduce_rules.size > 0 && item_set.shift_entries.size > 0)
-      end
-      if RbConfig::CONFIG["host_os"] =~ /linux/
-        item_sets_by_id = {}
-        item_sets_to_process.each do |item_set|
-          item_sets_by_id[item_set.object_id] = item_set
-        end
-        tokens_by_id = {}
-        @grammar.tokens.each do |token|
-          tokens_by_id[token.object_id] = token
-        end
-        rules_by_id = {}
-        @grammar.rules.each do |rule|
-          rules_by_id[rule.object_id] = rule
-        end
-        n_threads = Util.determine_n_threads
-        semaphore = Mutex.new
-        queue = Queue.new
-        threads = {}
-        n_threads.times do
-          piper, pipew = IO.pipe
-          thread = Thread.new do
-            loop do
-              item_set = nil
-              semaphore.synchronize do
-                item_set = item_sets_to_process.slice!(0)
-              end
-              break if item_set.nil?
-              fork do
-                piper.close
-                build_lookahead_reduce_actions_for_item_set(item_set, pipew)
-              end
-            end
-            queue.push(Thread.current)
-          end
-          threads[thread] = [piper, pipew]
-        end
-        until threads.empty?
-          thread = queue.pop
-          piper, pipew = threads[thread]
-          pipew.close
-          thread_txt = piper.read
-          thread_txt.each_line do |line|
-            if line.start_with?("RA,")
-              parts = line.split(",")
-              item_set_id, token_id, rule_id = parts[1..3].map(&:to_i)
-              item_set = item_sets_by_id[item_set_id]
-              unless item_set
-                raise "Internal error: could not find item set from thread"
-              end
-              token = tokens_by_id[token_id]
-              unless item_set
-                raise "Internal error: could not find token from thread"
-              end
-              rule = rules_by_id[rule_id]
-              unless item_set
-                raise "Internal error: could not find rule from thread"
-              end
-              item_set.reduce_actions ||= {}
-              item_set.reduce_actions[token] = rule
-            elsif line.start_with?("Error: ")
-              @errors << line.chomp
-            else
-              raise "Internal error: unhandled thread line #{line}"
-            end
-          end
-          thread.join
-          threads.delete(thread)
-        end
-      else
-        # Fall back to single threaded algorithm.
-        item_sets_to_process.each do |item_set|
-          item_set.reduce_actions = build_lookahead_reduce_actions_for_item_set(item_set)
-        end
-      end
-    end
-
-    # Build the shift entries for a single item set.
-    #
-    # @return [void]
-    def build_shift_entries(item_set)
-      item_set.shift_entries = item_set.next_symbols.map do |next_symbol|
-        state_id =
-          if next_symbol.name == "$EOF"
-            0
-          else
-            item_set.next_item_set[next_symbol].id
-          end
-        {
-          symbol: next_symbol,
-          state_id: state_id,
-        }
       end
     end
 
@@ -234,45 +108,40 @@ class Propane
     # @param item_set [ItemSet]
     #   ItemSet (parser state)
     #
-    # @return [void]
+    # @return [nil, Rule, Hash]
+    #   If no reduce actions are possible for the given item set, nil.
+    #   If only one reduce action is possible for the given item set, the Rule
+    #   to reduce.
+    #   Otherwise, a mapping of lookahead Tokens to the Rules to reduce.
     def build_reduce_actions_for_item_set(item_set)
       # To build the reduce actions, we start by looking at any
       # "complete" items, i.e., items where the parse position is at the
       # end of a rule. These are the only rules that are candidates for
       # reduction in the current ItemSet.
-      item_set.reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
+      reduce_rules = Set.new(item_set.items.select(&:complete?).map(&:rule))
 
-      if item_set.reduce_rules.size == 1
+      # If there are no rules to reduce for this ItemSet, we're done here.
-        item_set.reduce_rule = item_set.reduce_rules.first
+      return nil if reduce_rules.size == 0
-      end
+
-    end
+      # If there is exactly one rule to reduce for this ItemSet, then do not
+      # figure out the lookaheads; just reduce it.
+      return reduce_rules.first if reduce_rules.size == 1
+
+      # Otherwise, we have more than one possible rule to reduce.
 
-    # Build the reduce actions for a single item set (parser state).
-    #
-    # @param item_set [ItemSet]
-    #   ItemSet (parser state)
-    # @param fh [File]
-    #   Output file handle for multiprocessing mode.
-    #
-    # @return [Hash]
-    #   Mapping of lookahead Tokens to the Rules to reduce.
-    def build_lookahead_reduce_actions_for_item_set(item_set, fh = nil)
       # We will be looking for all possible tokens that can follow instances of
       # these rules. Rather than looking through the entire grammar for the
       # possible following tokens, we will only look in the item sets leading
       # up to this one. This restriction gives us a more precise lookahead set,
       # and allows us to parse LALR grammars.
-      item_sets = Set[item_set] + item_set.leading_item_sets
+      item_sets = item_set.leading_item_sets
-      item_set.reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
+      reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
         lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
         lookahead_tokens_for_rule.each do |lookahead_token|
           if existing_reduce_rule = reduce_actions[lookahead_token]
-            error = "Error: reduce/reduce conflict (state #{item_set.id}) between rule #{existing_reduce_rule.name}##{existing_reduce_rule.id} (defined on line #{existing_reduce_rule.line_number}) and rule #{reduce_rule.name}##{reduce_rule.id} (defined on line #{reduce_rule.line_number}) for lookahead token #{lookahead_token}"
+            raise Error.new("Error: reduce/reduce conflict between rule #{existing_reduce_rule.id} (#{existing_reduce_rule.name}) and rule #{reduce_rule.id} (#{reduce_rule.name})")
-            @errors << error
-            fh.puts(error) if fh
           end
           reduce_actions[lookahead_token] = reduce_rule
-          fh.puts "RA,#{item_set.object_id},#{lookahead_token.object_id},#{reduce_rule.object_id}" if fh
         end
         reduce_actions
       end
@@ -312,14 +181,13 @@ class Propane
         # tokens to form the lookahead token set.
         item_sets.each do |item_set|
           item_set.items.each do |item|
-            if item.next_symbol == rule_set
+            if item.following_symbol == rule_set
               (1..).each do |offset|
-                case symbol = item.next_symbol(offset)
+                case symbol = item.following_symbol(offset)
                 when nil
                   rule_set = item.rule.rule_set
                   unless checked_rule_sets.include?(rule_set)
                     rule_sets_to_check_after << rule_set
-                    checked_rule_sets << rule_set
                   end
                   break
                 when Token
@@ -372,26 +240,20 @@ class Propane
         @log.puts
         @log.puts "  Incoming states: #{incoming_ids.join(", ")}"
         @log.puts "  Outgoing states:"
-        item_set.next_item_set.each do |next_symbol, next_item_set|
+        item_set.following_item_set.each do |following_symbol, following_item_set|
-          @log.puts "    #{next_symbol.name} => #{next_item_set.id}"
+          @log.puts "    #{following_symbol.name} => #{following_item_set.id}"
         end
         @log.puts
         @log.puts "  Reduce actions:"
-        if item_set.reduce_rule
+        case item_set.reduce_actions
-          @log.puts "    * => rule #{item_set.reduce_rule.id}, rule set #{@rule_sets[item_set.reduce_rule.name].id} (#{item_set.reduce_rule.name})"
+        when Rule
-        elsif item_set.reduce_actions
+          @log.puts "    * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
+        when Hash
           item_set.reduce_actions.each do |token, rule|
             @log.puts "    lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
           end
         end
       end
-      if @warnings.size > 0
-        @log.puts
-        @log.puts "Warnings:"
-        @warnings.each do |warning|
-          @log.puts "  #{warning}"
-        end
-      end
     end
 
   end

lib/propane/parser/item.rb

@@ -22,7 +22,6 @@ class Propane
       def initialize(rule, position)
         @rule = rule
         @position = position
-        @_hash = [@rule, @position].hash
       end
 
       # Hash function.
@@ -30,7 +29,7 @@ class Propane
       # @return [Integer]
       #   Hash code.
       def hash
-        @_hash
+        [@rule, @position].hash
       end
 
       # Compare Item objects.
@@ -57,7 +56,7 @@ class Propane
 
       # Return the set of Items obtained by "closing" the current item.
       #
-      # If the next symbol for the current item is another Rule name, then
+      # If the following symbol for the current item is another Rule name, then
       # this method will return all Items for that Rule with a position of 0.
       # Otherwise, an empty Array is returned.
       #
@@ -82,17 +81,17 @@ class Propane
         @position == @rule.components.size
       end
 
-      # Get the next symbol for the Item.
+      # Get the following symbol for the Item.
       #
-      # That is, the symbol which is after the parse position marker in the
+      # That is, the symbol which follows the parse position marker in the
       # current Item.
       #
       # @param offset [Integer]
       #   Offset from current parse position to examine.
       #
       # @return [Token, RuleSet, nil]
-      #   Next symbol for the Item.
+      #   Following symbol for the Item.
-      def next_symbol(offset = 0)
+      def following_symbol(offset = 0)
         @rule.components[@position + offset]
       end
 
@@ -109,25 +108,25 @@ class Propane
         end
       end
 
-      # Get whether this Item's next symbol is the given symbol.
+      # Get whether this Item is followed by the provided symbol.
       #
       # @param symbol [Token, RuleSet]
       #   Symbol to query.
       #
       # @return [Boolean]
-      #   Whether this Item's next symbol is the given symbol.
+      #   Whether this Item is followed by the provided symbol.
-      def next_symbol?(symbol)
+      def followed_by?(symbol)
-        next_symbol == symbol
+        following_symbol == symbol
       end
 
-      # Get the next item for this Item.
+      # Get the following item for this Item.
       #
       # That is, the Item formed by moving the parse position marker one place
       # forward from its position in this Item.
       #
       # @return [Item]
-      #   The next item for this Item.
+      #   The following item for this Item.
-      def next_item
+      def following_item
         Item.new(@rule, @position + 1)
       end
 

lib/propane/parser/item_set.rb

@@ -2,7 +2,7 @@ class Propane
   class Parser
 
     # Represent a parser "item set", which is a set of possible items that the
-    # parser could currently be parsing. This is equivalent to a parser state.
+    # parser could currently be parsing.
     class ItemSet
 
       # @return [Set<Item>]
@@ -14,58 +14,45 @@ class Propane
       attr_accessor :id
 
       # @return [Hash]
-      #   Maps a next symbol to its ItemSet.
+      #   Maps a following symbol to its ItemSet.
-      attr_reader :next_item_set
+      attr_reader :following_item_set
 
       # @return [Set<ItemSet>]
       #   ItemSets leading to this item set.
       attr_reader :in_sets
 
-      # @return [nil, Rule]
+      # @return [nil, Rule, Hash]
-      #   Rule to reduce if there is only one possibility.
+      #   Reduce actions, mapping lookahead tokens to rules.
-      attr_accessor :reduce_rule
-
-      # @return [Set<Rule>]
-      #   Set of rules that could be reduced in this parser state.
-      attr_accessor :reduce_rules
-
-      # @return [nil, Hash]
-      #   Reduce actions, mapping lookahead tokens to rules, if there is
-      #   more than one rule that could be reduced.
       attr_accessor :reduce_actions
 
-      # @return [Array<Hash>]
-      #   Shift table entries.
-      attr_accessor :shift_entries
-
       # Build an ItemSet.
       #
       # @param items [Array<Item>]
       #   Items in this ItemSet.
       def initialize(items)
         @items = Set.new(items)
-        @next_item_set = {}
+        @following_item_set = {}
         @in_sets = Set.new
         close!
       end
 
-      # Get the set of next symbols for all Items in this ItemSet.
+      # Get the set of following symbols for all Items in this ItemSet.
       #
       # @return [Set<Token, RuleSet>]
-      #   Set of next symbols for all Items in this ItemSet.
+      #   Set of following symbols for all Items in this ItemSet.
-      def next_symbols
+      def following_symbols
-        @_next_symbols ||= Set.new(@items.map(&:next_symbol).compact)
+        Set.new(@items.map(&:following_symbol).compact)
       end
 
-      # Build a next ItemSet for the given next symbol.
+      # Build a following ItemSet for the given following symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Next symbol to build the next ItemSet for.
+      #   Following symbol to build the following ItemSet for.
       #
       # @return [ItemSet]
-      #   Next ItemSet for the given next symbol.
+      #   Following ItemSet for the given following symbol.
-      def build_next_item_set(symbol)
+      def build_following_item_set(symbol)
-        ItemSet.new(items_with_next(symbol).map(&:next_item))
+        ItemSet.new(items_followed_by(symbol).map(&:following_item))
       end
 
       # Hash function.
@@ -100,26 +87,13 @@ class Propane
 
       # Set of ItemSets that lead to this ItemSet.
       #
+      # This set includes this ItemSet.
+      #
       # @return [Set<ItemSet>]
       #   Set of all ItemSets that lead up to this ItemSet.
       def leading_item_sets
-        @_leading_item_sets ||=
+        @in_sets.reduce(Set[self]) do |result, item_set|
-          begin
+          result + item_set.leading_item_sets
-            result = Set.new
-            eval_sets = Set[self]
-            evaled = Set.new
-            while eval_sets.size > 0
-              eval_set = eval_sets.first
-              eval_sets.delete(eval_set)
-              evaled << eval_set
-              eval_set.in_sets.each do |in_set|
-                result << in_set
-                unless evaled.include?(in_set)
-                  eval_sets << in_set
-                end
-              end
-            end
-            result
         end
       end
 
@@ -153,16 +127,16 @@ class Propane
         end
       end
 
-      # Get the Items with the given next symbol.
+      # Get the Items followed by the given following symbol.
       #
       # @param symbol [Token, RuleSet]
-      #   Next symbol.
+      #   Following symbol.
       #
       # @return [Array<Item>]
-      #   Items with the given next symbol.
+      #   Items followed by the given following symbol.
-      def items_with_next(symbol)
+      def items_followed_by(symbol)
         @items.select do |item|
-          item.next_symbol?(symbol)
+          item.followed_by?(symbol)
         end
       end
 

lib/propane/pattern.rb

@@ -26,9 +26,9 @@ class Propane
     #   Regex NFA for matching the pattern.
     attr_reader :nfa
 
-    # @return [Set]
+    # @return [String, nil]
-    #   Lexer modes for this pattern.
+    #   Lexer mode for this pattern.
-    attr_accessor :modes
+    attr_accessor :mode
 
     # @return [String, nil]
     #   Parser value type name.
@@ -46,16 +46,16 @@ class Propane
     #   Token to be returned by this pattern.
     # @option options [Integer, nil] :line_number
     #   Line number where the token was defined in the input grammar.
-    # @option options [String, nil] :modes
+    # @option options [String, nil] :mode
-    #   Lexer modes for this pattern.
+    #   Lexer mode for this pattern.
     def initialize(options)
       @code = options[:code]
       @pattern = options[:pattern]
       @token = options[:token]
       @line_number = options[:line_number]
-      @modes = options[:modes]
+      @mode = options[:mode]
       @ptypename = options[:ptypename]
-      regex = Regex.new(@pattern, @line_number)
+      regex = Regex.new(@pattern)
       regex.nfa.end_state.accepts = self
       @nfa = regex.nfa
     end

lib/propane/regex.rb

@@ -4,13 +4,12 @@ class Propane
     attr_reader :unit
     attr_reader :nfa
 
-    def initialize(pattern, line_number)
+    def initialize(pattern)
       @pattern = pattern.dup
-      @line_number = line_number
       @unit = parse_alternates
       @nfa = @unit.to_nfa
       if @pattern != ""
-        raise Error.new(%[Line #{@line_number}: unexpected "#{@pattern}" in pattern])
+        raise Error.new(%[Unexpected "#{@pattern}" in pattern])
       end
     end
 
@@ -42,7 +41,7 @@ class Propane
             mu = MultiplicityUnit.new(last_unit, min_count, max_count)
             au.replace_last!(mu)
           else
-            raise Error.new("Line #{@line_number}: #{c} follows nothing")
+            raise Error.new("#{c} follows nothing")
           end
         when "|"
           au.new_alternate!
@@ -60,7 +59,7 @@ class Propane
     def parse_group
       au = parse_alternates
       if @pattern[0] != ")"
-        raise Error.new("Line #{@line_number}: unterminated group in pattern")
+        raise Error.new("Unterminated group in pattern")
       end
       @pattern.slice!(0)
       au
@@ -71,7 +70,7 @@ class Propane
       index = 0
       loop do
         if @pattern == ""
-          raise Error.new("Line #{@line_number}: unterminated character class")
+          raise Error.new("Unterminated character class")
         end
         c = @pattern.slice!(0)
         if c == "]"
@@ -85,13 +84,13 @@ class Propane
         elsif c == "-" && @pattern[0] != "]"
           begin_cu = ccu.last_unit
           unless begin_cu.is_a?(CharacterRangeUnit) && begin_cu.code_point_range.size == 1
-            raise Error.new("Line #{@line_number}: character range must be between single characters")
+            raise Error.new("Character range must be between single characters")
           end
           if @pattern[0] == "\\"
             @pattern.slice!(0)
             end_cu = parse_backslash
             unless end_cu.is_a?(CharacterRangeUnit) && end_cu.code_point_range.size == 1
-              raise Error.new("Line #{@line_number}: character range must be between single characters")
+              raise Error.new("Character range must be between single characters")
             end
             max_code_point = end_cu.code_point
           else
@@ -117,7 +116,7 @@ class Propane
         elsif max_count.to_s != ""
           max_count = max_count.to_i
           if max_count < min_count
-            raise Error.new("Line #{@line_number}: maximum repetition count cannot be less than minimum repetition count")
+            raise Error.new("Maximum repetition count cannot be less than minimum repetition count")
           end
         else
           max_count = nil
@@ -125,33 +124,18 @@ class Propane
         @pattern = pattern
         [min_count, max_count]
       else
-        raise Error.new("Line #{@line_number}: unexpected match count following {")
+        raise Error.new("Unexpected match count at #{@pattern}")
       end
     end
 
     def parse_backslash
       if @pattern == ""
-        raise Error.new("Line #{@line_number}: error: unfollowed \\")
+        raise Error.new("Error: unfollowed \\")
       else
         c = @pattern.slice!(0)
         case c
-        when "a"
-          CharacterRangeUnit.new("\a")
-        when "b"
-          CharacterRangeUnit.new("\b")
         when "d"
           CharacterRangeUnit.new("0", "9")
-        when "D"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu.negate = true
-          ccu
-        when "f"
-          CharacterRangeUnit.new("\f")
-        when "n"
-          CharacterRangeUnit.new("\n")
-        when "r"
-          CharacterRangeUnit.new("\r")
         when "s"
           ccu = CharacterClassUnit.new
           ccu << CharacterRangeUnit.new(" ")
@@ -161,35 +145,6 @@ class Propane
           ccu << CharacterRangeUnit.new("\f")
           ccu << CharacterRangeUnit.new("\v")
           ccu
-        when "S"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new(" ")
-          ccu << CharacterRangeUnit.new("\t")
-          ccu << CharacterRangeUnit.new("\r")
-          ccu << CharacterRangeUnit.new("\n")
-          ccu << CharacterRangeUnit.new("\f")
-          ccu << CharacterRangeUnit.new("\v")
-          ccu.negate = true
-          ccu
-        when "t"
-          CharacterRangeUnit.new("\t")
-        when "v"
-          CharacterRangeUnit.new("\v")
-        when "w"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("_")
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu << CharacterRangeUnit.new("a", "z")
-          ccu << CharacterRangeUnit.new("A", "Z")
-          ccu
-        when "W"
-          ccu = CharacterClassUnit.new
-          ccu << CharacterRangeUnit.new("_")
-          ccu << CharacterRangeUnit.new("0", "9")
-          ccu << CharacterRangeUnit.new("a", "z")
-          ccu << CharacterRangeUnit.new("A", "Z")
-          ccu.negate = true
-          ccu
         else
           CharacterRangeUnit.new(c)
         end

lib/propane/regex/unit.rb

@@ -92,20 +92,17 @@ class Propane
         @units = []
         @negate = false
       end
-      def method_missing(*args, &block)
+      def initialize
-        @units.__send__(*args, &block)
+        @units = []
+      end
+      def method_missing(*args)
+        @units.__send__(*args)
       end
       def <<(thing)
         if thing.is_a?(CharacterClassUnit)
-          if thing.negate
-            CodePointRange.invert_ranges(thing.map(&:code_point_range)).each do |cpr|
-              CharacterRangeUnit.new(cpr.first, cpr.last)
-            end
-          else
           thing.each do |ccu_unit|
             @units << ccu_unit
           end
-          end
         else
           @units << thing
         end

lib/propane/rule.rb

@@ -6,10 +6,6 @@ class Propane
     #   Rule components.
     attr_reader :components
 
-    # @return [Hash]
-    #   Field aliases.
-    attr_reader :aliases
-
     # @return [String]
     #   User code associated with the rule.
     attr_reader :code
@@ -34,11 +30,6 @@ class Propane
     #   The RuleSet that this Rule is a part of.
     attr_accessor :rule_set
 
-    # @return [Array<Integer>]
-    #   Map this rule's components to their positions in the parent RuleSet's
-    #   node field pointer array. This is used for tree construction.
-    attr_accessor :rule_set_node_field_index_map
-
     # Construct a Rule.
     #
     # @param name [String]
@@ -53,20 +44,7 @@ class Propane
     #   Line number where the rule was defined in the input grammar.
     def initialize(name, components, code, ptypename, line_number)
       @name = name
-      @aliases = {}
+      @components = components
-      @components = components.each_with_index.map do |component, i|
-        if component =~ /(\S+):(\S+)/
-          c, aliasname = $1, $2
-          if @aliases[aliasname]
-            raise Error.new("Error: duplicate field alias `#{aliasname}` for rule #{name} defined on line #{line_number}")
-          end
-          @aliases[aliasname] = i
-          c
-        else
-          component
-        end
-      end
-      @rule_set_node_field_index_map = components.map {0}
       @code = code
       @ptypename = ptypename
       @line_number = line_number
@@ -82,14 +60,6 @@ class Propane
       @components.empty?
     end
 
-    # Return whether this is an optional Rule.
-    #
-    # @return [Boolean]
-    #   Whether this is an optional Rule.
-    def optional?
-      @name.end_with?("?")
-    end
-
     # Represent the Rule as a String.
     #
     # @return [String]
@@ -98,17 +68,6 @@ class Propane
       "#{@name} -> #{@components.map(&:name).join(" ")}"
     end
 
-    # Check whether the rule set node field index map is just a 1:1 mapping.
-    #
-    # @return [Boolean]
-    #   Boolean indicating whether the rule set node field index map is just a
-    #   1:1 mapping.
-    def flat_rule_set_node_field_index_map?
-      @rule_set_node_field_index_map.each_with_index.all? do |v, i|
-        v == i
-      end
-    end
-
   end
 
 end

lib/propane/rule_set.rb

@@ -1,12 +1,7 @@
 class Propane
 
-  # A RuleSet collects all grammar rules of the same name.
   class RuleSet
 
-    # @return [Array<Hash>]
-    #   tree fields.
-    attr_reader :tree_fields
-
     # @return [Integer]
     #   ID of the RuleSet.
     attr_reader :id
@@ -56,24 +51,6 @@ class Propane
       @could_be_empty
     end
 
-    # Return whether this is an optional RuleSet.
-    #
-    # @return [Boolean]
-    #   Whether this is an optional RuleSet.
-    def optional?
-      @name.end_with?("?")
-    end
-
-    # For optional rule sets, return the underlying component that is optional.
-    def option_target
-      @rules.each do |rule|
-        if rule.components.size > 0
-          return rule.components[0]
-        end
-      end
-      raise "Optional rule target not found"
-    end
-
     # Build the start token set for the RuleSet.
     #
     # @return [Set<Token>]
@@ -98,72 +75,6 @@ class Propane
       @_start_token_set
     end
 
-    # Finalize a RuleSet after adding all Rules to it.
-    def finalize(grammar)
-      if grammar.tree
-        build_tree_fields(grammar)
-      end
-    end
-
-    private
-
-    # Build the set of tree fields for this RuleSet.
-    #
-    # This is an Array of Hashes. Each entry in the Array corresponds to a
-    # field location in the tree node. The entry is a Hash. It could have one or
-    # two keys. It will always have the field name with a positional suffix as
-    # a key. It may also have the field name without the positional suffix if
-    # that field only exists in one position across all Rules in the RuleSet.
-    #
-    # @return [void]
-    def build_tree_fields(grammar)
-      field_tree_node_indexes = {}
-      field_indexes_across_all_rules = {}
-      # Stores the index into @tree_fields by field alias name.
-      field_aliases = {}
-      @tree_fields = []
-      @rules.each do |rule|
-        rule.components.each_with_index do |component, i|
-          if component.is_a?(RuleSet) && component.optional?
-            component = component.option_target
-          end
-          if component.is_a?(Token)
-            node_name = "Token"
-          else
-            node_name = component.name
-          end
-          struct_name = "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
-          field_name = "p#{node_name}#{i + 1}"
-          unless field_tree_node_indexes[field_name]
-            field_tree_node_indexes[field_name] = @tree_fields.size
-            @tree_fields << {field_name => struct_name}
-          end
-          rule.aliases.each do |alias_name, index|
-            if index == i
-              alias_tree_fields_index = field_tree_node_indexes[field_name]
-              if field_aliases[alias_name] && field_aliases[alias_name] != alias_tree_fields_index
-                raise Error.new("Error: conflicting tree node field positions for alias `#{alias_name}` in rule #{rule.name} defined on line #{rule.line_number}")
-              end
-              field_aliases[alias_name] = alias_tree_fields_index
-              @tree_fields[alias_tree_fields_index][alias_name] = @tree_fields[alias_tree_fields_index].first[1]
-            end
-          end
-          field_indexes_across_all_rules[node_name] ||= Set.new
-          field_indexes_across_all_rules[node_name] << field_tree_node_indexes[field_name]
-          rule.rule_set_node_field_index_map[i] = field_tree_node_indexes[field_name]
-        end
-      end
-      field_indexes_across_all_rules.each do |node_name, indexes_across_all_rules|
-        if indexes_across_all_rules.size == 1
-          # If this field was only seen in one position across all rules,
-          # then add an alias to the positional field name that does not
-          # include the position.
-          @tree_fields[indexes_across_all_rules.first]["p#{node_name}"] =
-            "#{grammar.tree_prefix}#{node_name}#{grammar.tree_suffix}"
-        end
-      end
-    end
-
   end
 
 end

lib/propane/util.rb

@@ -10,32 +10,6 @@ class Propane
         "#{s}\n* #{message} *\n#{s}\n"
       end
 
-      # Determine the number of threads to use.
-      #
-      # @return [Integer]
-      #   The number of threads to use.
-      def determine_n_threads
-        # Try to figure out how many threads are available on the host hardware.
-        begin
-          case RbConfig::CONFIG["host_os"]
-          when /linux/
-            return File.read("/proc/cpuinfo").scan(/^processor\s*:/).size
-          when /mswin|mingw|msys/
-            if `wmic cpu get NumberOfLogicalProcessors -value` =~ /NumberOfLogicalProcessors=(\d+)/
-              return $1.to_i
-            end
-          when /darwin/
-            if `sysctl -n hw.ncpu` =~ /(\d+)/
-              return $1.to_i
-            end
-          end
-        rescue
-        end
-
-        # If we can't figure it out, default to 4.
-        4
-      end
-
     end
 
   end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "5.1.0"
+  VERSION = "0.1.0"
 end

propane.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec bundle exec ruby -Ilib bin/propane "$@"

rb/build_dist.rb

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require "erb"
 require "fileutils"
 require "digest/md5"
 
@@ -14,24 +13,6 @@ START_FILE = "bin/#{PROG_NAME}"
 LIB_DIR = "lib"
 DIST = "dist"
 
-ASSETS_TEMPLATE = <<EOF
-class Propane
-  module Assets
-    class << self
-      def get(name)
-        case name
-<% Dir.glob("assets/*").each  do |asset_file| %>
-        when <%= File.basename(asset_file).inspect %>
-          <%= File.binread(asset_file).inspect %>
-<% end  %>
-        end
-      end
-    end
-  end
-end
-EOF
-
-assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
 files_processed = {}
 combined_file = []
 
@@ -44,12 +25,8 @@ combine_files = lambda do |file|
         if File.exist?(path)
           unless files_processed[path]
             files_processed[path] = true
-            if require_name == "propane/assets"
-              combined_file << assets_module
-            else
             combine_files[path]
           end
-          end
         else
           raise "require path #{path.inspect} not found"
         end

spec/json_parser.c.propane

@@ -1,183 +0,0 @@
-<<header
-#include "json_types.h"
-#include "testutils.h"
->>
-<<
-#include "math.h"
-#include <stdbool.h>
-static str_t string_value;
->>
-
-ptype JSONValue *;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    double n = 0.0;
-    bool negative = false;
-    size_t i = 0u;
-    if (match_text[i] == '-')
-    {
-        negative = true;
-        i++;
-    }
-    while ('0' <= match_text[i] && match_text[i] <= '9')
-    {
-        n *= 10.0;
-        n += (match_text[i] - '0');
-        i++;
-    }
-    if (match_text[i] == '.')
-    {
-        i++;
-        double mult = 0.1;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
-        {
-            n += mult * (match_text[i] - '0');
-            mult /= 10.0;
-            i++;
-        }
-    }
-    if (match_text[i] == 'e' || match_text[i] == 'E')
-    {
-        bool exp_negative = false;
-        i++;
-        if (match_text[i] == '-')
-        {
-            exp_negative = true;
-            i++;
-        }
-        else if (match_text[i] == '+')
-        {
-            i++;
-        }
-        long exp = 0.0;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
-        {
-            exp *= 10;
-            exp += (match_text[i] - '0');
-            i++;
-        }
-        if (exp_negative)
-        {
-            exp = -exp;
-        }
-        n = pow(n, exp);
-    }
-    if (negative)
-    {
-        n = -n;
-    }
-    $$ = JSONValue_new(JSON_NUMBER);
-    $$->number = n;
->>
-token true <<
-  $$ = JSONValue_new(JSON_TRUE);
->>
-token false <<
-  $$ = JSONValue_new(JSON_FALSE);
->>
-token null <<
-  $$ = JSONValue_new(JSON_NULL);
->>
-/"/ <<
-  $mode(string);
-  str_init(&string_value, "");
->>
-string: token string /"/ <<
-  $$ = JSONValue_new(JSON_STRING);
-  $$->string = string_value;
-  $mode(default);
->>
-string: /\\"/ <<
-  str_append(&string_value, "\"");
->>
-string: /\\\\/ <<
-  str_append(&string_value, "\\");
->>
-string: /\\\// <<
-  str_append(&string_value, "/");
->>
-string: /\\b/ <<
-  str_append(&string_value, "\b");
->>
-string: /\\f/ <<
-  str_append(&string_value, "\f");
->>
-string: /\\n/ <<
-  str_append(&string_value, "\n");
->>
-string: /\\r/ <<
-  str_append(&string_value, "\r");
->>
-string: /\\t/ <<
-  str_append(&string_value, "\t");
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  char s[] = {'{', (char)match_text[2], (char)match_text[3], (char)match_text[4], (char)match_text[5], '}', 0};
-  str_append(&string_value, s);
->>
-string: /[^\\]/ <<
-  char s[] = {(char)match_text[0], 0};
-  str_append(&string_value, s);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONObject_new();
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
-  $$ = $1;
->>
-KeyValue -> string colon Value <<
-  $$ = JSONObject_new();
-  JSONObject_append($$, str_cstr(&$1->string), $3);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONArray_new();
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  JSONArray_append($1, $3);
-  $$ = $1;
->>

spec/json_parser.propane

@@ -20,46 +20,46 @@ token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
     double n;
     bool negative;
     size_t i = 0u;
-    if (match_text[i] == '-')
+    if (match[i] == '-')
     {
         negative = true;
         i++;
     }
-    while ('0' <= match_text[i] && match_text[i] <= '9')
+    while ('0' <= match[i] && match[i] <= '9')
     {
         n *= 10.0;
-        n += (match_text[i] - '0');
+        n += (match[i] - '0');
         i++;
     }
-    if (match_text[i] == '.')
+    if (match[i] == '.')
     {
         i++;
         double mult = 0.1;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
-            n += mult * (match_text[i] - '0');
+            n += mult * (match[i] - '0');
             mult /= 10.0;
             i++;
         }
     }
-    if (match_text[i] == 'e' || match_text[i] == 'E')
+    if (match[i] == 'e' || match[i] == 'E')
     {
         bool exp_negative;
         i++;
-        if (match_text[i] == '-')
+        if (match[i] == '-')
         {
             exp_negative = true;
             i++;
         }
-        else if (match_text[i] == '+')
+        else if (match[i] == '+')
         {
             i++;
         }
         long exp;
-        while ('0' <= match_text[i] && match_text[i] <= '9')
+        while ('0' <= match[i] && match[i] <= '9')
         {
             exp *= 10;
-            exp += (match_text[i] - '0');
+            exp += (match[i] - '0');
             i++;
         }
         if (exp_negative)
@@ -117,10 +117,10 @@ string: /\\t/ <<
 >>
 string: /\\u[0-9a-fA-F]{4}/ <<
   /* Not actually going to encode the code point for this example... */
-  string_value ~= "{" ~ match_text[2..6] ~ "}";
+  string_value ~= "{" ~ match[2..6] ~ "}";
 >>
 string: /[^\\]/ <<
-  string_value ~= match_text;
+  string_value ~= match;
 >>
 Start -> Value <<
   $$ = $1;

spec/json_parser.rust.propane

@@ -1,176 +0,0 @@
-<<
-pub const JSON_OBJECT: usize = 0;
-pub const JSON_ARRAY: usize = 1;
-pub const JSON_NUMBER: usize = 2;
-pub const JSON_STRING: usize = 3;
-pub const JSON_TRUE: usize = 4;
-pub const JSON_FALSE: usize = 5;
-pub const JSON_NULL: usize = 6;
-
-#[derive(Clone, Default)]
-pub enum JSONValue {
-    #[default]
-    Null,
-    Object(Vec<(String, JSONValue)>),
-    Array(Vec<JSONValue>),
-    Number(f64),
-    StringVal(String),
-    True,
-    False,
-}
-
-impl JSONValue {
-    pub fn id(&self) -> usize {
-        match self {
-            JSONValue::Object(_) => JSON_OBJECT,
-            JSONValue::Array(_) => JSON_ARRAY,
-            JSONValue::Number(_) => JSON_NUMBER,
-            JSONValue::StringVal(_) => JSON_STRING,
-            JSONValue::True => JSON_TRUE,
-            JSONValue::False => JSON_FALSE,
-            JSONValue::Null => JSON_NULL,
-        }
-    }
-    pub fn number(&self) -> f64 {
-        if let JSONValue::Number(n) = self { *n } else { 0.0 }
-    }
-    pub fn string(&self) -> &str {
-        if let JSONValue::StringVal(s) = self { s.as_str() } else { "" }
-    }
-    pub fn object_len(&self) -> usize {
-        if let JSONValue::Object(e) = self { e.len() } else { 0 }
-    }
-    pub fn array_len(&self) -> usize {
-        if let JSONValue::Array(e) = self { e.len() } else { 0 }
-    }
-}
->>
-
-context_user_fields <<
-    pub string_value: String,
->>
-
-ptype JSONValue;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token lbracket /\[/;
-token rbracket /\]/;
-token comma /,/;
-token colon /:/;
-token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
-    let n: f64 = std::str::from_utf8(match_text).unwrap().parse().unwrap();
-    $$ = JSONValue::Number(n);
->>
-token true <<
-  $$ = JSONValue::True;
->>
-token false <<
-  $$ = JSONValue::False;
->>
-token null <<
-  $$ = JSONValue::Null;
->>
-/"/ <<
-  $mode(string);
-  ${context.string_value} = String::new();
->>
-string: token string /"/ <<
-  $$ = JSONValue::StringVal(std::mem::take(&mut ${context.string_value}));
-  $mode(default);
->>
-string: /\\"/ <<
-  ${context.string_value}.push('"');
->>
-string: /\\\\/ <<
-  ${context.string_value}.push('\\');
->>
-string: /\\\// <<
-  ${context.string_value}.push('/');
->>
-string: /\\b/ <<
-  ${context.string_value}.push('\u{0008}');
->>
-string: /\\f/ <<
-  ${context.string_value}.push('\u{000C}');
->>
-string: /\\n/ <<
-  ${context.string_value}.push('\n');
->>
-string: /\\r/ <<
-  ${context.string_value}.push('\r');
->>
-string: /\\t/ <<
-  ${context.string_value}.push('\t');
->>
-string: /\\u[0-9a-fA-F]{4}/ <<
-  /* Not actually going to encode the code point for this example... */
-  let s: String = ['{', match_text[2] as char, match_text[3] as char, match_text[4] as char, match_text[5] as char, '}'].iter().collect();
-  ${context.string_value}.push_str(&s);
->>
-string: /[^\\]/ <<
-  ${context.string_value}.push(match_text[0] as char);
->>
-Start -> Value <<
-  $$ = $1;
->>
-Value -> string <<
-  $$ = $1;
->>
-Value -> number <<
-  $$ = $1;
->>
-Value -> Object <<
-  $$ = $1;
->>
-Value -> Array <<
-  $$ = $1;
->>
-Value -> true <<
-  $$ = $1;
->>
-Value -> false <<
-  $$ = $1;
->>
-Value -> null <<
-  $$ = $1;
->>
-Object -> lbrace rbrace <<
-  $$ = JSONValue::Object(Vec::new());
->>
-Object -> lbrace KeyValues rbrace <<
-  $$ = $2;
->>
-KeyValues -> KeyValue <<
-  $$ = $1;
->>
-KeyValues -> KeyValues comma KeyValue <<
-  let mut obj = $1;
-  if let JSONValue::Object(kve) = $3 {
-      if let JSONValue::Object(entries) = &mut obj {
-          entries.extend(kve);
-      }
-  }
-  $$ = obj;
->>
-KeyValue -> string colon Value <<
-  let name = if let JSONValue::StringVal(s) = $1 { s } else { String::new() };
-  $$ = JSONValue::Object(vec![(name, $3)]);
->>
-Array -> lbracket rbracket <<
-  $$ = JSONValue::Array(Vec::new());
->>
-Array -> lbracket Values rbracket <<
-  $$ = $2;
->>
-Values -> Value <<
-  $$ = $1;
->>
-Values -> Values comma Value <<
-  let mut arr = $1;
-  if let JSONValue::Array(elems) = &mut arr {
-      elems.push($3);
-  }
-  $$ = arr;
->>

spec/json_types.c

@@ -1,64 +0,0 @@
-#include "json_types.h"
-#include <string.h>
-#include <stdlib.h>
-#include "testutils.h"
-
-JSONValue * JSONValue_new(size_t id)
-{
-    JSONValue * jv = (JSONValue *)calloc(1, sizeof(JSONValue));
-    jv->id = id;
-    return jv;
-}
-
-JSONValue * JSONObject_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_OBJECT);
-    jv->object.size = 0u;
-    return jv;
-}
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
-{
-    size_t const size = object->object.size;
-    for (size_t i = 0u; i < size; i++)
-    {
-        if (strcmp(name, object->object.entries[i].name) == 0)
-        {
-            object->object.entries[i].value = value;
-            return;
-        }
-    }
-    size_t const new_size = size + 1;
-    JSONObjectEntry * new_entries = (JSONObjectEntry *)malloc(sizeof(object->object.entries[0]) * new_size);
-    if (size > 0)
-    {
-        memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
-        free(object->object.entries);
-    }
-    object->object.entries = new_entries;
-    object->object.entries[size].name = name;
-    object->object.entries[size].value = value;
-    object->object.size = new_size;
-}
-
-JSONValue * JSONArray_new(void)
-{
-    JSONValue * jv = JSONValue_new(JSON_ARRAY);
-    jv->array.size = 0u;
-    return jv;
-}
-
-void JSONArray_append(JSONValue * array, JSONValue * value)
-{
-    size_t const size = array->array.size;
-    size_t const new_size = size + 1;
-    JSONValue ** new_entries = (JSONValue **)malloc(sizeof(JSONValue *) * new_size);
-    if (array->array.size > 0)
-    {
-        memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
-        free(array->array.entries);
-    }
-    array->array.entries = new_entries;
-    array->array.entries[size] = value;
-    array->array.size = new_size;
-}

spec/json_types.h

@@ -1,48 +0,0 @@
-#pragma once
-
-#include <stddef.h>
-#include "testutils.h"
-
-#define JSON_OBJECT 0u
-#define JSON_ARRAY 1u
-#define JSON_NUMBER 2u
-#define JSON_STRING 3u
-#define JSON_TRUE 4u
-#define JSON_FALSE 5u
-#define JSON_NULL 6u
-
-typedef struct JSONObjectEntry_s
-{
-    char const * name;
-    struct JSONValue_s * value;
-} JSONObjectEntry;
-
-typedef struct JSONValue_s
-{
-    size_t id;
-    union
-    {
-        struct
-        {
-            size_t size;
-            JSONObjectEntry * entries;
-        } object;
-        struct
-        {
-            size_t size;
-            struct JSONValue_s ** entries;
-        } array;
-        double number;
-        str_t string;
-    };
-} JSONValue;
-
-JSONValue * JSONValue_new(size_t id);
-
-JSONValue * JSONObject_new(void);
-
-void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
-
-JSONValue * JSONArray_new(void);
-
-void JSONArray_append(JSONValue * array, JSONValue * value);

spec/macros.c.propane

@@ -1,25 +0,0 @@
-<<
-#include <stdlib.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
-void record(int v);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ << char b[100]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; record($$); >>
-MacroStart -> macro macroname lbrace;

spec/macros.d.propane

@@ -1,31 +0,0 @@
-<<
-import test_macros;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ <<
-    int n = 0;
-    foreach (c; match_text)
-    {
-        n *= 10;
-        n += (c - '0');
-    }
-    $$ = n;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; record($$); >>
-MacroStart -> macro macroname lbrace;

spec/macros.rust.propane

@@ -1,80 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    loop {
-        if context.expanding {
-            let ei = context.expand_i;
-            context.expand_i += 1;
-            if context.expand_i >= context.token_infos.len() {
-                context.expanding = false;
-            }
-            *out_token_info = context.token_infos[ei].clone();
-            return P_SUCCESS;
-        }
-
-        let lex_result = p_lex(context, out_token_info);
-        if lex_result != P_SUCCESS {
-            return lex_result;
-        }
-
-        if out_token_info.token == TOKEN_macro {
-            context.defining = true;
-        } else if out_token_info.token == TOKEN_macroname {
-            if !context.defining {
-                context.expanding = true;
-                context.expand_i = 0;
-                continue;
-            }
-        } else if out_token_info.token == TOKEN_lbrace {
-            if context.defining {
-                /* Capture the macro body tokens (up to the closing '}'). */
-                let mut infos: Vec<p_token_info_t> = Vec::new();
-                loop {
-                    let mut ti = p_token_info_t::default();
-                    assert_eq!(P_SUCCESS, p_lex(context, &mut ti));
-                    if ti.token == TOKEN_rbrace {
-                        break;
-                    }
-                    infos.push(ti);
-                }
-                context.token_infos = infos;
-                context.defining = false;
-            }
-        } else {
-            context.defining = false;
-        }
-        return lex_result;
-    }
-}
->>
-
-context_user_fields <<
-    pub defining: bool,
-    pub expanding: bool,
-    pub expand_i: usize,
-    pub token_infos: Vec<p_token_info_t>,
-    pub nums: Vec<i64>,
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token macro;
-token macroname /@[a-zA-Z_]\w*/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Statement -> MacroStart;
-Add -> num plus num << $$ = $1 + $3; ${context.nums}.push($$); >>
-MacroStart -> macro macroname lbrace;

spec/parse_inner_nested.c.propane

@@ -1,19 +0,0 @@
-<<
-#include <stdlib.h>
-#include <string.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.d.propane

@@ -1,17 +0,0 @@
-<<
-import test_parse_inner_nested;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested.rust.propane

@@ -1,41 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    let result = p_lex(context, out_token_info);
-    if result != P_SUCCESS {
-        return result;
-    }
-    if out_token_info.token == TOKEN_lparen {
-        /* Reentrant nested parse of the parenthesized sub-expression. */
-        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
-        if inner_result != P_SUCCESS {
-            return inner_result;
-        }
-        let value = p_result_Start(context);
-        /* p_parse_inner rewound the input so ')' was not consumed; consume it. */
-        let mut rparen_info = p_token_info_t::default();
-        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
-        assert_eq!(TOKEN_rparen, rparen_info.token);
-        out_token_info.token = TOKEN_num;
-        out_token_info.pvalue = p_value(value);
-    }
-    P_SUCCESS
-}
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Expr << $$ = $1; >>
-Expr -> num << $$ = $1; >>
-Expr -> Expr plus num << $$ = $1 + $3; >>

spec/parse_inner_nested_tree.c.propane

@@ -1,17 +0,0 @@
-<<
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/parse_inner_nested_tree.d.propane

@@ -1,17 +0,0 @@
-<<
-import test_parse_inner_nested_tree;
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/parse_inner_nested_tree.rust.propane

@@ -1,44 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    let result = p_lex(context, out_token_info);
-    if result != P_SUCCESS {
-        return result;
-    }
-    if out_token_info.token == TOKEN_lparen {
-        let start_position = out_token_info.position;
-        let inner_result = p_parse_inner_Start(context, &[TOKEN_rparen]);
-        if inner_result != P_SUCCESS {
-            return inner_result;
-        }
-        /* Read the inner subtree's span before re-borrowing context to lex. */
-        let inner = p_result_Start(context);
-        assert!(inner.valid());
-        let inner_start_col = inner.position().col;
-        let inner_end_col = inner.end_position().col;
-        let mut rparen_info = p_token_info_t::default();
-        assert_eq!(P_SUCCESS, p_lex(context, &mut rparen_info));
-        assert_eq!(TOKEN_rparen, rparen_info.token);
-        assert_eq!(start_position.col + 1, inner_start_col);
-        assert_eq!(rparen_info.position.col - 1, inner_end_col);
-        /* Synthesize a num token spanning the whole "( ... )" group. */
-        out_token_info.token = TOKEN_num;
-        out_token_info.position = start_position;
-        out_token_info.end_position = rparen_info.end_position;
-    }
-    P_SUCCESS
-}
->>
-
-tree;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token lparen /\(/;
-token rparen /\)/;
-token plus /\+/;
-token num /\d+/;
-
-Start -> Expr;
-Expr -> num;
-Expr -> Expr plus num;

spec/propane/grammar_spec.rb

@@ -5,6 +5,7 @@ class Propane
 # Comment line
 
 module a.b;
+class Foobar;
 ptype   XYZ  *  ;
 
 token while;
@@ -28,7 +29,8 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
+      expect(grammar.classname).to eq "Foobar"
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -36,44 +38,44 @@ EOF
 
       o = grammar.tokens.find {|token| token.name == "while"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 6
+      expect(o.line_number).to eq 7
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "while"
-      expect(o.line_number).to eq 6
+      expect(o.line_number).to eq 7
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "id"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 9
+      expect(o.line_number).to eq 10
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
-      expect(o.line_number).to eq 9
+      expect(o.line_number).to eq 10
       expect(o.code).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "token_with_code"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 11
+      expect(o.line_number).to eq 12
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
-      expect(o.line_number).to eq 11
+      expect(o.line_number).to eq 12
-      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
+      expect(o.code).to eq "Code for the token\n"
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 15
+      expect(o.line_number).to eq 16
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
       expect(o).to_not be_nil
-      expect(o.line_number).to eq 17
+      expect(o.line_number).to eq 18
       expect(o.token).to be_nil
       expect(o.code).to be_nil
 
@@ -82,20 +84,20 @@ EOF
       o = grammar.rules[0]
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
-      expect(o.line_number).to eq 19
+      expect(o.line_number).to eq 20
-      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
+      expect(o.code).to eq "  a = 42;\n"
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
       expect(o.components).to eq %w[C while id]
-      expect(o.line_number).to eq 22
+      expect(o.line_number).to eq 23
       expect(o.code).to be_nil
 
       o = grammar.rules[2]
       expect(o.name).to eq "B"
       expect(o.components).to eq []
-      expect(o.line_number).to eq 23
+      expect(o.line_number).to eq 24
-      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
+      expect(o.code).to eq "  b = 0;\n"
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +115,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +124,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
+      expect(o.code).to eq "  a = b;\n  return c;\n"
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,42 +132,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+      expect(o.code).to eq %[  writeln("Hello there");\n]
-    end
-
-    it "does not emit #line directives with noline statement" do
-      input = <<EOF
-noline;
-
-token code1 <<
-  a = b;
-  return c;
->>
-
-token code2 <<
-  writeln("Hello there");
->>
-
-tokenid token_with_no_pattern;
-
-prefix myparser_;
-EOF
-      grammar = Grammar.new(input, "test.propane")
-      expect(grammar.prefix).to eq "myparser_"
-
-      o = grammar.tokens.find {|token| token.name == "code1"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq %[\n  a = b;\n  return c;]
-
-      o = grammar.tokens.find {|token| token.name == "code2"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -179,37 +146,37 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.modes).to be_empty
+      expect(o.mode).to be_nil
 
       o = grammar.tokens.find {|token| token.name == "b"}
       expect(o).to_not be_nil
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m1"]
+      expect(o.mode).to eq "m1"
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "foo"}
       expect(o).to_not be_nil
-      expect(o.modes).to be_empty
+      expect(o.mode).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "bar"}
       expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m2"]
+      expect(o.mode).to eq "m2"
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "q"}
       expect(o).to_not be_nil
-      expect(o.modes).to be_empty
+      expect(o.mode).to be_nil
 
       o = grammar.patterns.find {|pattern| pattern.pattern == "r"}
       expect(o).to_not be_nil
-      expect(o.modes).to eq Set["m3"]
+      expect(o.mode).to eq "m3"
     end
 
     it "allows assigning ptypes to tokens and rules" do
@@ -232,7 +199,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar, "test.propane")
+  grammar = Propane::Grammar.new(grammar)
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)
@@ -126,74 +126,6 @@ EOF
     ]
     expect(run(<<EOF, ";")).to eq expected
 token semicolon /;/;
-EOF
-  end
-
-  it "matches a negated character class" do
-    expected = [
-      ["pattern", "/abc/"],
-    ]
-    expect(run(<<EOF, "/abc/")).to eq expected
-token pattern /\\/[^\\s]*\\//;
-EOF
-  end
-
-  it "matches special character classes " do
-    expected = [
-      ["a", "abc123_FOO"],
-    ]
-    expect(run(<<EOF, "abc123_FOO")).to eq expected
-token a /\\w+/;
-EOF
-    expected = [
-      ["b", "FROG*%$#"],
-    ]
-    expect(run(<<EOF, "FROG*%$#")).to eq expected
-token b /FROG\\D{1,4}/;
-EOF
-    expected = [
-      ["c", "$883366"],
-    ]
-    expect(run(<<EOF, "$883366")).to eq expected
-token c /$\\d+/;
-EOF
-    expected = [
-      ["d", "^&$@"],
-    ]
-    expect(run(<<EOF, "^&$@")).to eq expected
-token d /^\\W+/;
-EOF
-    expected = [
-      ["a", "abc123_FOO"],
-      [nil, " "],
-      ["b", "FROG*%$#"],
-      [nil, " "],
-      ["c", "$883366"],
-      [nil, " "],
-      ["d", "^&$@"],
-    ]
-    expect(run(<<EOF, "abc123_FOO FROG*%$# $883366 ^&$@")).to eq expected
-token a /\\w+/;
-token b /FROG\\D{1,4}/;
-token c /$\\d+/;
-token d /^\\W+/;
-drop /\\s+/;
-EOF
-  end
-
-  it "matches a negated character class with a nested inner negated character class" do
-    expected = [
-      ["t", "$&*"],
-    ]
-    expect(run(<<EOF, "$&*")).to eq expected
-token t /[^%\\W]+/;
-EOF
-  end
-
-  it "\\s matches a newline" do
-    expected = [["s", "\n"]]
-    expect(run(<<EOF, "\n")).to eq expected
-token s /\\s/;
 EOF
   end
 end

spec/propane/regex_spec.rb

@@ -2,14 +2,14 @@ class Propane
   RSpec.describe Regex do
 
     it "parses an empty expression" do
-      regex = Regex.new("", 1)
+      regex = Regex.new("")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0].size).to eq 0
     end
 
     it "parses a single character unit expression" do
-      regex = Regex.new("a", 1)
+      regex = Regex.new("a")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -19,7 +19,7 @@ class Propane
     end
 
     it "parses a group with a single character unit expression" do
-      regex = Regex.new("(a)", 1)
+      regex = Regex.new("(a)")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -33,7 +33,7 @@ class Propane
     end
 
     it "parses a *" do
-      regex = Regex.new("a*", 1)
+      regex = Regex.new("a*")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -47,7 +47,7 @@ class Propane
     end
 
     it "parses a +" do
-      regex = Regex.new("a+", 1)
+      regex = Regex.new("a+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -61,7 +61,7 @@ class Propane
     end
 
     it "parses a ?" do
-      regex = Regex.new("a?", 1)
+      regex = Regex.new("a?")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -75,7 +75,7 @@ class Propane
     end
 
     it "parses a multiplicity count" do
-      regex = Regex.new("a{5}", 1)
+      regex = Regex.new("a{5}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -89,7 +89,7 @@ class Propane
     end
 
     it "parses a minimum-only multiplicity count" do
-      regex = Regex.new("a{5,}", 1)
+      regex = Regex.new("a{5,}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -103,7 +103,7 @@ class Propane
     end
 
     it "parses a minimum and maximum multiplicity count" do
-      regex = Regex.new("a{5,8}", 1)
+      regex = Regex.new("a{5,8}")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -118,7 +118,7 @@ class Propane
     end
 
     it "parses an escaped *" do
-      regex = Regex.new("a\\*", 1)
+      regex = Regex.new("a\\*")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -131,7 +131,7 @@ class Propane
     end
 
     it "parses an escaped +" do
-      regex = Regex.new("a\\+", 1)
+      regex = Regex.new("a\\+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -144,7 +144,7 @@ class Propane
     end
 
     it "parses an escaped \\" do
-      regex = Regex.new("\\\\d", 1)
+      regex = Regex.new("\\\\d")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -157,7 +157,7 @@ class Propane
     end
 
     it "parses a character class" do
-      regex = Regex.new("[a-z_]", 1)
+      regex = Regex.new("[a-z_]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -175,7 +175,7 @@ class Propane
     end
 
     it "parses a negated character class" do
-      regex = Regex.new("[^xyz]", 1)
+      regex = Regex.new("[^xyz]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -189,25 +189,8 @@ class Propane
       expect(ccu[0].first).to eq "x".ord
     end
 
-    it "parses a negated character class with inner character classes" do
-      regex = Regex.new("[^x\\sz]", 1)
-      expect(regex.unit).to be_a Regex::AlternatesUnit
-      expect(regex.unit.alternates.size).to eq 1
-      expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
-      seq_unit = regex.unit.alternates[0]
-      expect(seq_unit.size).to eq 1
-      expect(seq_unit[0]).to be_a Regex::CharacterClassUnit
-      ccu = seq_unit[0]
-      expect(ccu.negate).to be_truthy
-      expect(ccu.size).to eq 8
-      expect(ccu[0]).to be_a Regex::CharacterRangeUnit
-      expect(ccu[0].first).to eq "x".ord
-      expect(ccu[1].first).to eq " ".ord
-      expect(ccu[7].first).to eq "z".ord
-    end
-
     it "parses - as a plain character at beginning of a character class" do
-      regex = Regex.new("[-9]", 1)
+      regex = Regex.new("[-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -221,7 +204,7 @@ class Propane
     end
 
     it "parses - as a plain character at end of a character class" do
-      regex = Regex.new("[0-]", 1)
+      regex = Regex.new("[0-]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -237,7 +220,7 @@ class Propane
     end
 
     it "parses - as a plain character at beginning of a negated character class" do
-      regex = Regex.new("[^-9]", 1)
+      regex = Regex.new("[^-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -252,7 +235,7 @@ class Propane
     end
 
     it "parses . as a plain character in a character class" do
-      regex = Regex.new("[.]", 1)
+      regex = Regex.new("[.]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -267,7 +250,7 @@ class Propane
     end
 
     it "parses - as a plain character when escaped in middle of character class" do
-      regex = Regex.new("[0\\-9]", 1)
+      regex = Regex.new("[0\\-9]")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -286,7 +269,7 @@ class Propane
     end
 
     it "parses alternates" do
-      regex = Regex.new("ab|c", 1)
+      regex = Regex.new("ab|c")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 2
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -296,7 +279,7 @@ class Propane
     end
 
     it "parses a ." do
-      regex = Regex.new("a.b", 1)
+      regex = Regex.new("a.b")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 1
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit
@@ -307,7 +290,7 @@ class Propane
     end
 
     it "parses something complex" do
-      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+", 1)
+      regex = Regex.new("(a|)*|[^^]|\\|v|[x-y]+")
       expect(regex.unit).to be_a Regex::AlternatesUnit
       expect(regex.unit.alternates.size).to eq 4
       expect(regex.unit.alternates[0]).to be_a Regex::SequenceUnit

spec/rewind.c.propane

@@ -1,23 +0,0 @@
-<<
-#include <stdlib.h>
-#include <string.h>
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info);
-void record(int value);
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << char b[32]; memcpy(b, match_text, match_length); b[match_length] = '\0'; $$ = atoi(b); >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/rewind.d.propane

@@ -1,20 +0,0 @@
-<<
-import test_rewind;
->>
-
-ptype int;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ << int n = 0; foreach (ch; match_text) { n *= 10; n += (ch - '0'); } $$ = n; >>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << record($1 + $3); >>

spec/rewind.rust.propane

@@ -1,67 +0,0 @@
-<<
-fn mylexfn(context: &mut p_context_t, out_token_info: &mut p_token_info_t) -> usize {
-    loop {
-        let result = p_lex(context, out_token_info);
-        if result != P_SUCCESS {
-            return result;
-        }
-        if out_token_info.token == TOKEN_repeat {
-            let mut count_info = p_token_info_t::default();
-            assert_eq!(P_SUCCESS, p_lex(context, &mut count_info));
-            assert_eq!(TOKEN_num, count_info.token);
-            let mut brace_info = p_token_info_t::default();
-            assert_eq!(P_SUCCESS, p_lex(context, &mut brace_info));
-            assert_eq!(TOKEN_lbrace, brace_info.token);
-            context.remaining = p_value_get(&count_info.pvalue);
-            context.body_index = p_input_index(context);
-            context.body_position = p_position(context);
-            continue;
-        }
-        if out_token_info.token == TOKEN_rbrace {
-            if context.remaining > 1 {
-                context.remaining -= 1;
-                let bi = context.body_index;
-                let bp = context.body_position;
-                p_set_input_index(context, bi);
-                p_set_position(context, bp);
-                continue;
-            }
-            context.remaining = 0;
-            continue;
-        }
-        if out_token_info.token == TOKEN_num {
-            context.num_cols.push(out_token_info.position.col);
-        }
-        return result;
-    }
-}
->>
-
-context_user_fields <<
-    pub nums: Vec<i64>,
-    pub num_cols: Vec<u32>,
-    pub remaining: i64,
-    pub body_index: usize,
-    pub body_position: p_position_t,
->>
-
-ptype i64;
-
-lex_fn mylexfn;
-
-drop /\s+/;
-token repeat /repeat/;
-token lbrace /\{/;
-token rbrace /\}/;
-token plus /\+/;
-token num /\d+/ <<
-  let mut v: i64 = 0;
-  for c in match_text { v = v * 10 + (*c - b'0') as i64; }
-  $$ = v;
->>
-
-Start -> Statements;
-Statements -> ;
-Statements -> Statement Statements;
-Statement -> Add;
-Add -> num plus num << ${context.nums}.push($1 + $3); >>

spec/spec_helper.rb

@@ -1,36 +1,11 @@
-unless ENV["dist_specs"]
+require "bundler/setup"
-  require "bundler/setup"
+require "propane"
-  require "simplecov"
 
-  class MyFormatter
+RSpec.configure do |config|
-    def format(*args)
-    end
-  end
-  SimpleCov.start do
-    add_filter "/spec/"
-    add_filter "/.bundle/"
-    if ENV["partial_specs"]
-      command_name "RSpec-partial"
-    else
-      command_name "RSpec"
-    end
-    project_name "Propane"
-    # Keep this process's results separate from the propane subprocess results
-    # so that nothing has to merge on the fly; the spec Rake task collates all
-    # of the parts once the suite is done.
-    coverage_dir "coverage/parts/rspec"
-    merge_timeout 3600
-    formatter(MyFormatter)
-  end
-
-  RSpec.configure do |config|
   # Enable flags like --only-failures and --next-failure
   config.example_status_persistence_file_path = ".rspec_status"
 
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
-  end
 end
-
-require "propane"

spec/test_basic_math_grammar.c

@@ -5,29 +5,25 @@
 int main()
 {
     char const * input = "1 + 2 * 3 + 4";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(context));
+    assert_eq(11, p_result(&context));
-    p_context_delete(context);
 
     input = "1 * 2 ** 4 * 3";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(context));
+    assert_eq(48, p_result(&context));
-    p_context_delete(context);
 
     input = "(1 + 2) * 3 + 4";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(context));
+    assert_eq(13, p_result(&context));
-    p_context_delete(context);
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(context));
+    assert_eq(73, p_result(&context));
-    p_context_delete(context);
 
     return 0;
 }

spec/test_basic_math_grammar.d

@@ -10,23 +10,23 @@ int main()
 unittest
 {
     string input = "1 + 2 * 3 + 4";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(11, p_result(context));
+    assert_eq(11, p_result(&context));
 
     input = "1 * 2 ** 4 * 3";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(48, p_result(context));
+    assert_eq(48, p_result(&context));
 
     input = "(1 + 2) * 3 + 4";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(13, p_result(context));
+    assert_eq(13, p_result(&context));
 
     input = "(2 * 2) ** 3 + 4 + 5";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert_eq(P_SUCCESS, p_parse(context));
+    assert_eq(P_SUCCESS, p_parse(&context));
-    assert_eq(73, p_result(context));
+    assert_eq(73, p_result(&context));
 }

spec/test_basic_math_grammar.rs

@@ -1,16 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let cases: [(&[u8], u64); 4] = [
-        (b"1 + 2 * 3 + 4", 11),
-        (b"1 * 2 ** 4 * 3", 48),
-        (b"(1 + 2) * 3 + 4", 13),
-        (b"(2 * 2) ** 3 + 4 + 5", 73),
-    ];
-    for (input, expected) in cases {
-        let mut context = p_context_new(input);
-        assert_eq!(P_SUCCESS, p_parse(&mut context));
-        assert_eq!(expected, p_result(&context));
-        p_context_delete(context);
-    }
-}

spec/test_context_field_handle.c

@@ -1,39 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char input[128];
-    size_t i;
-    p_context_t * context;
-    Token token;
-
-    /* Enough tokens that the tree node arena is reallocated during the parse. */
-    memset(input, 0, sizeof(input));
-    for (i = 0u; i < 40u; i++)
-    {
-        input[i] = 'a';
-    }
-
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1u, context->have_first);
-    assert(p_node_valid(context->first_item));
-    token = p_Item_pToken1(context->first_item);
-    assert(p_node_valid(token));
-    assert_eq(TOKEN_a, p_Token_token(token));
-    assert_eq(7u, p_Token_pvalue(token));
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, p_node_position(context->first_item).row);
-    assert_eq(1u, p_node_position(context->first_item).col);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_context_field_handle.cpp

@@ -1,36 +0,0 @@
-#include "testparser.h"
-#include <cassert>
-#include <cstring>
-#include "testutils.h"
-
-int main()
-{
-    char input[128];
-
-    /* Enough tokens that the tree node arena is reallocated during the parse. */
-    memset(input, 0, sizeof(input));
-    for (size_t i = 0u; i < 40u; i++)
-    {
-        input[i] = 'a';
-    }
-
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1u, context->have_first);
-    assert(context->first_item.valid());
-    Token token = context->first_item.pToken1();
-    assert(token.valid());
-    assert_eq(TOKEN_a, token.token());
-    assert_eq(7u, token.pvalue());
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, context->first_item.position().row);
-    assert_eq(1u, context->first_item.position().col);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_context_field_handle.d

@@ -1,35 +0,0 @@
-import testparser;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* Enough tokens that the tree node array is reallocated during the parse. */
-    string input;
-    foreach (i; 0 .. 40)
-    {
-        input ~= "a";
-    }
-
-    p_context_t * context = p_context_new(input);
-    assert_eq(P_SUCCESS, p_parse(context));
-
-    /* The handle was stored in a context user field during the parse, before
-     * the remaining nodes were created. It still refers to the same node. */
-    assert_eq(1, context.have_first);
-    assert(context.first_item.valid);
-    Token token = context.first_item.pToken1;
-    assert(token.valid);
-    assert_eq(TOKEN_a, token.token);
-    assert_eq(7, token.pvalue);
-
-    /* The stored handle refers to the first Item, which starts at column 1. */
-    assert_eq(1u, context.first_item.position.row);
-    assert_eq(1u, context.first_item.position.col);
-
-    p_context_delete(context);
-}

spec/test_custom_lex_fn.c

@@ -1,15 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-
-int main()
-{
-    char const * input = "cbacba";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-    size_t result = p_result(context);
-    assert_eq(0x932187932187, result);
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_custom_lex_fn.d

@@ -1,18 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "cbacba";
-    p_context_t * context = p_context_new(input);
-    assert_eq(P_SUCCESS, p_parse(context));
-    size_t result = p_result(context);
-    assert_eq(0x932187932187, result);
-    p_context_delete(context);
-}

spec/test_custom_lex_fn.rs

@@ -1,8 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"cbacba");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    assert_eq!(0x932187932187, p_result(&c));
-    p_context_delete(c);
-}

spec/test_drop_code_block.c

@@ -1,15 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "    # comment 1\n#    comment 2\na\n";
-    p_context_t * context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_drop_code_block.d

@@ -1,16 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "    # comment 1\n#    comment 2\na\n";
-    p_context_t * context;
-    context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-}

spec/test_drop_code_block.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"    # comment 1\n#    comment 2\na\n");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_error_positions.c

@@ -5,43 +5,35 @@
 int main()
 {
     char const * input = "a 42";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    p_context_delete(context);
 
     input = "a\n123\na  a";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(context).row == 3);
+    assert(p_position(&context).row == 2);
-    assert(p_position(context).col == 4);
+    assert(p_position(&context).col == 3);
-    assert(p_token(context) == TOKEN_a);
+    assert(context.token == TOKEN_a);
-    p_context_delete(context);
 
     input = "12";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(context).row == 1);
+    assert(p_position(&context).row == 0);
-    assert(p_position(context).col == 1);
+    assert(p_position(&context).col == 0);
-    assert(p_token(context) == TOKEN_num);
+    assert(context.token == TOKEN_num);
-    p_context_delete(context);
 
     input = "a 12\n\nab";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(context).row == 3);
+    assert(p_position(&context).row == 2);
-    assert(p_position(context).col == 2);
+    assert(p_position(&context).col == 1);
-    p_context_delete(context);
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(context).row == 6);
+    assert(p_position(&context).row == 5);
-    assert(p_position(context).col == 5);
+    assert(p_position(&context).col == 4);
-
-    assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
-    assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
-    p_context_delete(context);
 
     return 0;
 }

spec/test_error_positions.d

@@ -9,32 +9,29 @@ int main()
 unittest
 {
     string input = "a 42";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
 
     input = "a\n123\na  a";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(context) == p_position_t(3, 4));
+    assert(p_position(&context) == p_position_t(2, 3));
-    assert(p_token(context) == TOKEN_a);
+    assert(context.token == TOKEN_a);
 
     input = "12";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_UNEXPECTED_TOKEN);
+    assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
-    assert(p_position(context) == p_position_t(1, 1));
+    assert(p_position(&context) == p_position_t(0, 0));
-    assert(p_token(context) == TOKEN_num);
+    assert(context.token == TOKEN_num);
 
     input = "a 12\n\nab";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    assert(p_position(context) == p_position_t(3, 2));
+    assert(p_position(&context) == p_position_t(2, 1));
 
     input = "a 12\n\na\n\n77\na   \xAA";
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_DECODE_ERROR);
+    assert(p_parse(&context) == P_DECODE_ERROR);
-    assert(p_position(context) == p_position_t(6, 5));
+    assert(p_position(&context) == p_position_t(5, 4));
-
-    assert(p_token_names[TOKEN_a] == "a");
-    assert(p_token_names[TOKEN_num] == "num");
 }

spec/test_error_positions.rs

@@ -1,35 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"a 42");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a\n123\na  a");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(4, p_position(&c).col);
-    assert_eq!(TOKEN_a, p_token(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"12");
-    assert_eq!(P_UNEXPECTED_TOKEN, p_parse(&mut c));
-    assert_eq!(1, p_position(&c).row);
-    assert_eq!(1, p_position(&c).col);
-    assert_eq!(TOKEN_num, p_token(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a 12\n\nab");
-    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(2, p_position(&c).col);
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a 12\n\na\n\n77\na   \xAA");
-    assert_eq!(P_DECODE_ERROR, p_parse(&mut c));
-    assert_eq!(6, p_position(&c).row);
-    assert_eq!(5, p_position(&c).col);
-    assert_eq!("a", p_token_names[TOKEN_a as usize]);
-    assert_eq!("num", p_token_names[TOKEN_num as usize]);
-    p_context_delete(c);
-}

spec/test_field_aliases.c

@@ -1,14 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    char const * input = "foo1\nbar2";
-    p_context_t * context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-    return 0;
-}

spec/test_field_aliases.d

@@ -1,15 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "foo1\nbar2";
-    p_context_t * context;
-    context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-}

spec/test_field_aliases.rs

@@ -1,7 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"foo1\nbar2");
-    assert_eq!(P_SUCCESS, p_parse(&mut c));
-    p_context_delete(c);
-}

spec/test_input_index.c

@@ -1,60 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include "testutils.h"
-
-int main()
-{
-    /* Grammar (simple):
-     *   drop /\\s+/;
-     *   token a; token b;
-     *   Start -> a b;
-     *
-     * Verifies that p_input_index() reports the parser/lexer's current byte
-     * offset into the input text. */
-
-    /* Fresh context: input_index starts at 0. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        assert_eq(0u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* After each successful lex the byte offset advances past the token. */
-    {
-        char const * input = "a b";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_a, (size_t)token_info.token);
-        assert_eq(1u, p_input_index(context));
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert_eq((size_t)TOKEN_b, (size_t)token_info.token);
-        /* The dropped space between `a` and `b` advances input_index too. */
-        assert_eq(3u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* After a full successful parse, input_index has reached the end. */
-    {
-        char const * input = "ab";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        assert(p_parse_Start(context) == P_SUCCESS);
-        assert_eq(2u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    /* When parse_inner completes via a follow token, the follow token is not
-     * consumed, so input_index points at the start of the follow token. */
-    {
-        char const * input = "abb";
-        p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-        p_token_t follow_tokens[] = { TOKEN_b };
-        assert(p_parse_inner_Start(context, follow_tokens, 1u) == P_SUCCESS);
-        assert_eq(2u, p_input_index(context));
-        p_context_delete(context);
-    }
-
-    return 0;
-}

spec/test_input_index.d

@@ -1,51 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    /* See test_input_index.c for details on the grammar and cases. */
-
-    /* Fresh context: input_index starts at 0. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        assert(p_input_index(context) == 0);
-    }
-
-    /* After each successful lex the byte offset advances past the token. */
-    {
-        string input = "a b";
-        p_context_t * context = p_context_new(input);
-        p_token_info_t token_info;
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_a);
-        assert(p_input_index(context) == 1);
-        assert(p_lex(context, &token_info) == P_SUCCESS);
-        assert(token_info.token == TOKEN_b);
-        assert(p_input_index(context) == 3);
-    }
-
-    /* After a full successful parse, input_index has reached the end. */
-    {
-        string input = "ab";
-        p_context_t * context = p_context_new(input);
-        assert(p_parse_Start(context) == P_SUCCESS);
-        assert(p_input_index(context) == 2);
-    }
-
-    /* When parse_inner completes via a follow token, the follow token is not
-     * consumed, so input_index points at the start of the follow token. */
-    {
-        string input = "abb";
-        p_context_t * context = p_context_new(input);
-        p_token_t[] follow_tokens = [TOKEN_b];
-        assert(p_parse_inner_Start(context, follow_tokens) == P_SUCCESS);
-        assert(p_input_index(context) == 2);
-    }
-}

spec/test_input_index.rs

@@ -1,28 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let c = p_context_new(b"ab");
-    assert_eq!(0, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"a b");
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_a, ti.token);
-    assert_eq!(1, p_input_index(&c));
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_b, ti.token);
-    assert_eq!(3, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"ab");
-    assert_eq!(P_SUCCESS, p_parse_Start(&mut c));
-    assert_eq!(2, p_input_index(&c));
-    p_context_delete(c);
-
-    let mut c = p_context_new(b"abb");
-    let follow = [TOKEN_b];
-    assert_eq!(P_SUCCESS, p_parse_inner_Start(&mut c, &follow));
-    assert_eq!(2, p_input_index(&c));
-    p_context_delete(c);
-}

spec/test_lexer.c

@@ -38,75 +38,55 @@ int main()
 
     p_token_info_t token_info;
     char const * input = "5 + 4 * \n677 + 567";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
+    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 1u);
+    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 1u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
+    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 3u);
+    assert(token_info.position.col == 2u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
+    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 5u);
+    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 1u);
+    assert(token_info.position.row == 0u);
-    assert(token_info.position.col == 7u);
+    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 1u);
-    assert(token_info.end_position.col == 7u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_times);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
+    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 1u);
+    assert(token_info.position.col == 0u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 3u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
+    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 5u);
+    assert(token_info.position.col == 4u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 5u);
     assert(token_info.length == 1u);
     assert(token_info.token == TOKEN_plus);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
+    assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 7u);
+    assert(token_info.position.col == 6u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 9u);
     assert(token_info.length == 3u);
     assert(token_info.token == TOKEN_int);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info.position.row == 2u);
-    assert(token_info.position.col == 10u);
-    assert(token_info.end_position.row == 2u);
-    assert(token_info.end_position.col == 10u);
-    assert(token_info.length == 0u);
-    assert(token_info.token == TOKEN___EOF);
-    p_context_delete(context);
-
-    context = p_context_new((uint8_t const *)"", 0u);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
     assert(token_info.position.row == 1u);
-    assert(token_info.position.col == 1u);
+    assert(token_info.position.col == 9u);
-    assert(token_info.end_position.row == 1u);
+    assert(token_info.length == 0u);
-    assert(token_info.end_position.col == 1u);
+    assert(token_info.token == TOKEN___EOF);
+
+    p_context_init(&context, (uint8_t const *)"", 0u);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
+    assert(token_info.position.row == 0u);
+    assert(token_info.position.col == 0u);
     assert(token_info.length == 0u);
     assert(token_info.token == TOKEN___EOF);
-    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer.d

@@ -44,26 +44,26 @@ unittest
 {
     p_token_info_t token_info;
     string input = "5 + 4 * \n677 + 567";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 3), p_position_t(1, 3), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 5), p_position_t(1, 5), 1, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 7), p_position_t(1, 7), 1, TOKEN_times));
+    assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 1), p_position_t(2, 3), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 5), p_position_t(2, 5), 1, TOKEN_plus));
+    assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 7), p_position_t(2, 9), 3, TOKEN_int));
+    assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(2, 10), p_position_t(2, 10), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
 
-    context = p_context_new("");
+    p_context_init(&context, "");
-    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(p_lex(&context, &token_info) == P_SUCCESS);
-    assert(token_info == p_token_info_t(p_position_t(1, 1), p_position_t(1, 1), 0, TOKEN___EOF));
+    assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
 }

spec/test_lexer.rs

@@ -1,49 +0,0 @@
-use testparser::*;
-
-fn chk(ti: &p_token_info_t, row: u32, col: u32, erow: u32, ecol: u32, len: usize, token: p_token_t) {
-    assert_eq!(row, ti.position.row);
-    assert_eq!(col, ti.position.col);
-    assert_eq!(erow, ti.end_position.row);
-    assert_eq!(ecol, ti.end_position.col);
-    assert_eq!(len, ti.length);
-    assert_eq!(token, ti.token);
-}
-
-fn main() {
-    let mut cp: p_code_point_t = 0;
-    let mut cpl: u8 = 0;
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"5", &mut cp, &mut cpl));
-    assert_eq!('5' as u32, cp);
-    assert_eq!(1, cpl);
-
-    assert_eq!(P_EOF, p_decode_code_point(b"", &mut cp, &mut cpl));
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xC2\xA9", &mut cp, &mut cpl));
-    assert_eq!(0xA9, cp);
-    assert_eq!(2, cpl);
-
-    assert_eq!(P_SUCCESS, p_decode_code_point(b"\xf0\x9f\xa7\xa1", &mut cp, &mut cpl));
-    assert_eq!(0x1F9E1, cp);
-    assert_eq!(4, cpl);
-
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\x27", &mut cp, &mut cpl));
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xf0\x9f\xa7\xFF", &mut cp, &mut cpl));
-    assert_eq!(P_DECODE_ERROR, p_decode_code_point(b"\xfe", &mut cp, &mut cpl));
-
-    let mut context = p_context_new(b"5 + 4 * \n677 + 567");
-    let mut ti = p_token_info_t::default();
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 1, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 3, 1, 3, 1, TOKEN_plus);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 5, 1, 5, 1, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 7, 1, 7, 1, TOKEN_times);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 1, 2, 3, 3, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 5, 2, 5, 1, TOKEN_plus);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 7, 2, 9, 3, TOKEN_int);
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 2, 10, 2, 10, 0, TOKEN___EOF);
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"");
-    assert_eq!(P_SUCCESS, p_lex(&mut context, &mut ti)); chk(&ti, 1, 1, 1, 1, 0, TOKEN___EOF);
-    p_context_delete(context);
-}

spec/test_lexer_match_text.c

@@ -6,11 +6,10 @@
 int main()
 {
     char const * input = "identifier_123";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     printf("pass1\n");
-    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_match_text.d

@@ -9,8 +9,8 @@ int main()
 unittest
 {
     string input = `identifier_123`;
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 }

spec/test_lexer_match_text.rs

@@ -1,8 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"identifier_123");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-}

spec/test_lexer_modes.c

@@ -6,17 +6,15 @@
 int main()
 {
     char const * input = "abc \"a string\" def";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     printf("pass1\n");
-    p_context_delete(context);
 
     input = "abc \"abc def\" def";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     printf("pass2\n");
-    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_modes.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `abc "a string" def`;
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass1");
 
     input = `abc "abc def" def`;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
     writeln("pass2");
 }

spec/test_lexer_modes.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"abc \"a string\" def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abc \"abc def\" def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass2");
-    p_context_delete(context);
-}

spec/test_lexer_multiple_modes.c

@@ -1,22 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-#include <stdio.h>
-
-int main()
-{
-    char const * input = "abc.def";
-    p_context_t * context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    printf("pass1\n");
-    p_context_delete(context);
-
-    input = "abc .  abc";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    printf("pass2\n");
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_lexer_multiple_modes.d

@@ -1,21 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = `abc.def`;
-    p_context_t * context;
-    context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    writeln("pass1");
-
-    input = `abc .  abc`;
-    context = p_context_new(input);
-    assert(p_parse(context) == P_SUCCESS);
-    writeln("pass2");
-}

spec/test_lexer_multiple_modes.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"abc.def");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass1");
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"abc .  abc");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    println!("pass2");
-    p_context_delete(context);
-}

spec/test_lexer_positions.c

@@ -1,50 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    char const * input = "abc\n  defg hi\n!";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    p_token_info_t token_info;
-
-    /* First token "abc" on row 1, cols 1-3. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context->last_start.row == 1u);
-    assert(context->last_start.col == 1u);
-    assert(context->last_end.row == 1u);
-    assert(context->last_end.col == 3u);
-    /* The lexer code block observed the same positions reported to the caller. */
-    assert(context->last_start.row == token_info.position.row);
-    assert(context->last_start.col == token_info.position.col);
-    assert(context->last_end.row == token_info.end_position.row);
-    assert(context->last_end.col == token_info.end_position.col);
-
-    /* Second token "defg" on row 2, cols 3-6. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context->last_start.row == 2u);
-    assert(context->last_start.col == 3u);
-    assert(context->last_end.row == 2u);
-    assert(context->last_end.col == 6u);
-
-    /* Third token "hi" on row 2, cols 8-9. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context->last_start.row == 2u);
-    assert(context->last_start.col == 8u);
-    assert(context->last_end.row == 2u);
-    assert(context->last_end.col == 9u);
-
-    /* The "!" stop token terminates the lexer. The context input text position
-     * must not be updated when the lexer user code requests termination, so it
-     * still points at the "!" token on row 3, col 1. */
-    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(context) == 42u);
-    assert(context->text_position.row == 3u);
-    assert(context->text_position.col == 1u);
-
-    p_context_delete(context);
-    return 0;
-}

spec/test_lexer_positions.d

@@ -1,42 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "abc\n  defg hi\n!";
-    p_context_t * context = p_context_new(input);
-    p_token_info_t token_info;
-
-    /* First token "abc" on row 1, cols 1-3. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context.last_start == p_position_t(1, 1));
-    assert(context.last_end == p_position_t(1, 3));
-    /* The lexer code block observed the same positions reported to the caller. */
-    assert(context.last_start == token_info.position);
-    assert(context.last_end == token_info.end_position);
-
-    /* Second token "defg" on row 2, cols 3-6. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context.last_start == p_position_t(2, 3));
-    assert(context.last_end == p_position_t(2, 6));
-
-    /* Third token "hi" on row 2, cols 8-9. */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_word);
-    assert(context.last_start == p_position_t(2, 8));
-    assert(context.last_end == p_position_t(2, 9));
-
-    /* The "!" stop token terminates the lexer. The context input text position
-     * must not be updated when the lexer user code requests termination, so it
-     * still points at the "!" token on row 3, col 1. */
-    assert(p_lex(context, &token_info) == P_USER_TERMINATED);
-    assert(p_user_terminate_code(context) == 42u);
-    assert(context.text_position == p_position_t(3, 1));
-}

spec/test_lexer_positions.rs

@@ -1,38 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut c = p_context_new(b"abc\n  defg hi\n!");
-    let mut ti = p_token_info_t::default();
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(1, c.last_start.row);
-    assert_eq!(1, c.last_start.col);
-    assert_eq!(1, c.last_end.row);
-    assert_eq!(3, c.last_end.col);
-    assert_eq!(c.last_start.row, ti.position.row);
-    assert_eq!(c.last_start.col, ti.position.col);
-    assert_eq!(c.last_end.row, ti.end_position.row);
-    assert_eq!(c.last_end.col, ti.end_position.col);
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(2, c.last_start.row);
-    assert_eq!(3, c.last_start.col);
-    assert_eq!(2, c.last_end.row);
-    assert_eq!(6, c.last_end.col);
-
-    assert_eq!(P_SUCCESS, p_lex(&mut c, &mut ti));
-    assert_eq!(TOKEN_word, ti.token);
-    assert_eq!(2, c.last_start.row);
-    assert_eq!(8, c.last_start.col);
-    assert_eq!(2, c.last_end.row);
-    assert_eq!(9, c.last_end.col);
-
-    assert_eq!(P_USER_TERMINATED, p_lex(&mut c, &mut ti));
-    assert_eq!(42, p_user_terminate_code(&c));
-    assert_eq!(3, p_position(&c).row);
-    assert_eq!(1, p_position(&c).col);
-
-    p_context_delete(c);
-}

spec/test_lexer_result_value.c

@@ -5,17 +5,15 @@
 int main()
 {
     char const * input = "x";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 1u);
+    assert(p_result(&context) == 1u);
-    p_context_delete(context);
 
     input = "fabulous";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 8u);
+    assert(p_result(&context) == 8u);
-    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_result_value.d

@@ -9,13 +9,13 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 1u);
+    assert(p_result(&context) == 1u);
 
     input = `fabulous`;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 8u);
+    assert(p_result(&context) == 8u);
 }

spec/test_lexer_result_value.rs

@@ -1,13 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"x");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(1, p_result(&context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"fabulous");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(8, p_result(&context));
-    p_context_delete(context);
-}

spec/test_lexer_unknown_character.c

@@ -5,16 +5,14 @@
 int main()
 {
     char const * input = "x";
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
-    p_context_delete(context);
 
     input = "123";
-    context = p_context_new((uint8_t const *)input, strlen(input));
+    p_context_init(&context, (uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 123u);
+    assert(p_result(&context) == 123u);
-    p_context_delete(context);
 
     return 0;
 }

spec/test_lexer_unknown_character.d

@@ -9,12 +9,12 @@ int main()
 unittest
 {
     string input = `x`;
-    p_context_t * context;
+    p_context_t context;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_UNEXPECTED_INPUT);
+    assert(p_parse(&context) == P_UNEXPECTED_INPUT);
 
     input = `123`;
-    context = p_context_new(input);
+    p_context_init(&context, input);
-    assert(p_parse(context) == P_SUCCESS);
+    assert(p_parse(&context) == P_SUCCESS);
-    assert(p_result(context) == 123u);
+    assert(p_result(&context) == 123u);
 }

spec/test_lexer_unknown_character.rs

@@ -1,12 +0,0 @@
-use testparser::*;
-
-fn main() {
-    let mut context = p_context_new(b"x");
-    assert_eq!(P_UNEXPECTED_INPUT, p_parse(&mut context));
-    p_context_delete(context);
-
-    let mut context = p_context_new(b"123");
-    assert_eq!(P_SUCCESS, p_parse(&mut context));
-    assert_eq!(123, p_result(&context));
-    p_context_delete(context);
-}

spec/test_macros.c

@@ -1,118 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-#include <assert.h>
-#include <stddef.h>
-#include <stdbool.h>
-
-static p_context_t * context;
-size_t n_tokens;
-p_token_info_t token_infos[10];
-
-/* Capture the macro body tokens (everything up to the closing '}') into
- * token_infos[]. Called from mylexfn() right after the definition's '{' has
- * been lexed, so the input cursor is positioned at the first body token. */
-static void capture_macro_body(void)
-{
-    n_tokens = 0u;
-    for (;;)
-    {
-        size_t result = p_lex(context, &token_infos[n_tokens]);
-        assert_eq(result, P_SUCCESS);
-        if (token_infos[n_tokens].token == TOKEN_rbrace)
-        {
-            break;
-        }
-        n_tokens++;
-        assert(n_tokens < sizeof(token_infos) / sizeof(token_infos[0]));
-    }
-}
-
-size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static bool defining;
-    static bool expanding;
-    static size_t expand_i;
-
-    for (;;)
-    {
-        if (expanding)
-        {
-            size_t ei = expand_i++;
-            if (expand_i >= n_tokens)
-            {
-                expanding = false;
-            }
-            *out_token_info = token_infos[ei];
-            return P_SUCCESS;
-        }
-
-        size_t lex_result = p_lex(context, out_token_info);
-        if (lex_result != P_SUCCESS)
-        {
-            return lex_result;
-        }
-
-        switch (out_token_info->token)
-        {
-        case TOKEN_macro:
-            /* Start of a macro definition: "macro macroname { ... }". */
-            defining = true;
-            break;
-        case TOKEN_macroname:
-            if (!defining)
-            {
-                /* Use of a macro: replay its captured body tokens instead of
-                 * returning the macroname to the parser. */
-                expanding = true;
-                expand_i = 0u;
-                continue;
-            }
-            /* Definition name: pass through and keep waiting for '{'. */
-            break;
-        case TOKEN_lbrace:
-            if (defining)
-            {
-                /* Consume and store the macro body now, before the parser gets
-                 * a chance to read its lookahead token (which would otherwise
-                 * swallow the first body token). */
-                capture_macro_body();
-                defining = false;
-            }
-            break;
-        default:
-            defining = false;
-            break;
-        }
-        return lex_result;
-    }
-}
-
-size_t n_nums;
-int nums[10];
-
-void record(int v)
-{
-    nums[n_nums++] = v;
-}
-
-int main()
-{
-    char const * input =
-        "macro @m { 23 + 200 }\n"
-        "66 + 100\n"
-        "@m\n"
-        "33 + 55\n"
-        "@m\n";
-    context = p_context_new((uint8_t const *)input, strlen(input));
-    assert(p_parse(context) == P_SUCCESS);
-    p_context_delete(context);
-
-    assert_eq(n_nums, 4);
-    assert_eq(nums[0], 166);
-    assert_eq(nums[1], 223);
-    assert_eq(nums[2], 88);
-    assert_eq(nums[3], 223);
-
-    return 0;
-}