Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 josh
|
d466189982 |
No files matched your search
@@ -1,25 +1,3 @@
|
||||
## v1.2.0
|
||||
|
||||
### New Features
|
||||
|
||||
- Allow one line user code blocks (#21)
|
||||
- Add backslash escape codes (#19)
|
||||
- Add API to access unexpected token found (#18)
|
||||
- Add token_names API (#17)
|
||||
- Add D example to user guide for p_context_init() (#16)
|
||||
- Allow user termination from lexer code blocks (#15)
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix generator hang when state transition cycle is present (#20)
|
||||
|
||||
## v1.1.0
|
||||
|
||||
### New Features
|
||||
|
||||
- Add user parser terminations (#13)
|
||||
- Document generated parser API in user guide (#14)
|
||||
|
||||
## v1.0.0
|
||||
|
||||
- Initial release
|
||||
@@ -5,4 +5,3 @@ gem "rspec"
|
||||
gem "rdoc"
|
||||
gem "redcarpet"
|
||||
gem "syntax"
|
||||
gem "simplecov"
|
||||
+14
-22
@@ -2,33 +2,26 @@ GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
diff-lcs (1.5.0)
|
||||
docile (1.4.0)
|
||||
psych (5.1.0)
|
||||
psych (5.0.1)
|
||||
stringio
|
||||
rake (13.0.6)
|
||||
rdoc (6.5.0)
|
||||
psych (>= 4.0.0)
|
||||
redcarpet (3.6.0)
|
||||
rspec (3.12.0)
|
||||
rspec-core (~> 3.12.0)
|
||||
rspec-expectations (~> 3.12.0)
|
||||
rspec-mocks (~> 3.12.0)
|
||||
rspec-core (3.12.2)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-expectations (3.12.3)
|
||||
redcarpet (3.5.1)
|
||||
rspec (3.11.0)
|
||||
rspec-core (~> 3.11.0)
|
||||
rspec-expectations (~> 3.11.0)
|
||||
rspec-mocks (~> 3.11.0)
|
||||
rspec-core (3.11.0)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-expectations (3.11.0)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-mocks (3.12.6)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-mocks (3.11.1)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-support (3.12.1)
|
||||
simplecov (0.22.0)
|
||||
docile (~> 1.1)
|
||||
simplecov-html (~> 0.11)
|
||||
simplecov_json_formatter (~> 0.1)
|
||||
simplecov-html (0.12.3)
|
||||
simplecov_json_formatter (0.1.4)
|
||||
stringio (3.0.7)
|
||||
rspec-support (~> 3.11.0)
|
||||
rspec-support (3.11.0)
|
||||
stringio (3.0.4)
|
||||
syntax (1.2.2)
|
||||
|
||||
PLATFORMS
|
||||
@@ -39,7 +32,6 @@ DEPENDENCIES
|
||||
rdoc
|
||||
redcarpet
|
||||
rspec
|
||||
simplecov
|
||||
syntax
|
||||
|
||||
BUNDLED WITH
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2010-2024 Josh Holtrop
|
||||
Copyright (c) 2010-2023 Josh Holtrop
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -1,113 +1,21 @@
|
||||
# The Propane Parser Generator
|
||||
|
||||
Propane is a LALR Parser Generator (LPG) which:
|
||||
Propane is an LR Parser Generator (LPG) which:
|
||||
|
||||
* accepts LR(0), SLR, and LALR grammars
|
||||
* generates a built-in lexer to tokenize input
|
||||
* supports UTF-8 lexer inputs
|
||||
* generates a table-driven shift/reduce parser to parse input in linear time
|
||||
* target C or D language outputs
|
||||
* generates a table-driven parser to parse input in linear time
|
||||
* is MIT-licensed
|
||||
* is distributable as a standalone Ruby script
|
||||
|
||||
## Installation
|
||||
|
||||
Propane is designed to be distributed as a stand-alone single file script that
|
||||
can be copied into and versioned in a project's source tree.
|
||||
The only requirement to run Propane is that the system has a Ruby interpreter
|
||||
installed.
|
||||
The latest release can be downloaded from [https://github.com/holtrop/propane/releases](https://github.com/holtrop/propane/releases).
|
||||
Simply copy the `propane` executable script into the desired location within
|
||||
the project to be built (typically the root of the repository) and mark it
|
||||
executable.
|
||||
TODO
|
||||
|
||||
## Usage
|
||||
|
||||
### Command Line Interface
|
||||
|
||||
Propane is typically invoked from the command-line as `./propane`.
|
||||
|
||||
Usage: ./propane [options] <input-file> <output-file>
|
||||
Options:
|
||||
--log LOG Write log file
|
||||
--version Show program version and exit
|
||||
-h, --help Show this usage and exit
|
||||
|
||||
The user must specify the path to a Propane input grammar file and a path to an
|
||||
output file.
|
||||
The generated source code will be written to the output file.
|
||||
If a log file path is specified, Propane will write a log file containing
|
||||
detailed information about the parser states and transitions.
|
||||
|
||||
### Propane Grammar File
|
||||
|
||||
A Propane grammar file provides Propane with the patterns, tokens, grammar
|
||||
rules, and user code blocks from which to build the generated lexer and parser.
|
||||
|
||||
Example grammar file:
|
||||
|
||||
```
|
||||
<<
|
||||
import std.math;
|
||||
>>
|
||||
|
||||
# Parser values are unsigned integers.
|
||||
ptype ulong;
|
||||
|
||||
# A few basic arithmetic operators.
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
token power /\\*\\*/;
|
||||
token integer /\\d+/ <<
|
||||
ulong v;
|
||||
foreach (c; match)
|
||||
{
|
||||
v *= 10;
|
||||
v += (c - '0');
|
||||
}
|
||||
$$ = v;
|
||||
>>
|
||||
token lparen /\\(/;
|
||||
token rparen /\\)/;
|
||||
# Drop whitespace.
|
||||
drop /\\s+/;
|
||||
|
||||
Start -> E1 <<
|
||||
$$ = $1;
|
||||
>>
|
||||
E1 -> E2 <<
|
||||
$$ = $1;
|
||||
>>
|
||||
E1 -> E1 plus E2 <<
|
||||
$$ = $1 + $3;
|
||||
>>
|
||||
E2 -> E3 <<
|
||||
$$ = $1;
|
||||
>>
|
||||
E2 -> E2 times E3 <<
|
||||
$$ = $1 * $3;
|
||||
>>
|
||||
E3 -> E4 <<
|
||||
$$ = $1;
|
||||
>>
|
||||
E3 -> E3 power E4 <<
|
||||
$$ = pow($1, $3);
|
||||
>>
|
||||
E4 -> integer <<
|
||||
$$ = $1;
|
||||
>>
|
||||
E4 -> lparen E1 rparen <<
|
||||
$$ = $2;
|
||||
>>
|
||||
```
|
||||
|
||||
Grammar files can contain comment lines beginning with `#` which are ignored.
|
||||
White space in the grammar file is also ignored.
|
||||
|
||||
It is convention to use the extension `.propane` for the Propane grammar file,
|
||||
however any file name is accepted by Propane.
|
||||
|
||||
See [https://holtrop.github.io/propane/index.html](https://holtrop.github.io/propane/index.html) for the full User Guide.
|
||||
TODO: Write usage instructions here
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
require "rake/clean"
|
||||
require "rspec/core/rake_task"
|
||||
|
||||
CLEAN.include %w[spec/run gen .yardoc yard coverage dist]
|
||||
|
||||
task :build_dist do
|
||||
sh "ruby rb/build_dist.rb"
|
||||
end
|
||||
@@ -13,20 +10,9 @@ RSpec::Core::RakeTask.new(:spec, :example_pattern) do |task, args|
|
||||
end
|
||||
end
|
||||
|
||||
# dspec task is useful to test the distributable release script, but is not
|
||||
# useful for coverage information.
|
||||
desc "Dist Specs"
|
||||
task :dspec, [:example_string] => :build_dist do |task, args|
|
||||
ENV["dist_specs"] = "1"
|
||||
Rake::Task["spec"].execute(args)
|
||||
ENV.delete("dist_specs")
|
||||
end
|
||||
|
||||
task :default => :spec
|
||||
|
||||
desc "Build user guide"
|
||||
task :user_guide do
|
||||
system("ruby", "-Ilib", "rb/gen_user_guide.rb")
|
||||
end
|
||||
|
||||
task :all => [:spec, :dspec, :user_guide]
|
||||
@@ -1,986 +0,0 @@
|
||||
#include "<%= File.basename(output_file).sub(%r{\.[a-z]+$}, "") %>.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/**************************************************************************
|
||||
* Public data
|
||||
*************************************************************************/
|
||||
|
||||
/** Token names. */
|
||||
const char * <%= @grammar.prefix %>token_names[] = {
|
||||
<% @grammar.tokens.each_with_index do |token, index| %>
|
||||
"<%= token.name %>",
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/**************************************************************************
|
||||
* User code blocks
|
||||
*************************************************************************/
|
||||
|
||||
<%= @grammar.code_blocks.fetch("", "") %>
|
||||
|
||||
/**************************************************************************
|
||||
* Private types
|
||||
*************************************************************************/
|
||||
|
||||
<% if @grammar.prefix.upcase != "P_" %>
|
||||
/* Result codes. */
|
||||
#define P_SUCCESS 0u
|
||||
#define P_DECODE_ERROR 1u
|
||||
#define P_UNEXPECTED_INPUT 2u
|
||||
#define P_UNEXPECTED_TOKEN 3u
|
||||
#define P_DROP 4u
|
||||
#define P_EOF 5u
|
||||
#define P_USER_TERMINATED 6u
|
||||
<% end %>
|
||||
|
||||
/* An invalid ID value. */
|
||||
#define INVALID_ID ((size_t)-1)
|
||||
|
||||
/**************************************************************************
|
||||
* State initialization
|
||||
*************************************************************************/
|
||||
|
||||
/**
|
||||
* Initialize lexer/parser context structure.
|
||||
*
|
||||
* @param[out] context
|
||||
* Lexer/parser context structure.
|
||||
* @param input
|
||||
* Text input.
|
||||
* @param input_length
|
||||
* Text input length.
|
||||
*/
|
||||
void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length)
|
||||
{
|
||||
/* New default-initialized context structure. */
|
||||
<%= @grammar.prefix %>context_t newcontext = {0};
|
||||
|
||||
/* Lexer initialization. */
|
||||
newcontext.input = input;
|
||||
newcontext.input_length = input_length;
|
||||
newcontext.mode = <%= @lexer.mode_id("default") %>;
|
||||
|
||||
/* Copy to the user's context structure. */
|
||||
*context = newcontext;
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
* Decoder
|
||||
*************************************************************************/
|
||||
|
||||
/**
|
||||
* Decode a UTF-8 code point.
|
||||
*
|
||||
* @param input
|
||||
* Text input to decode.
|
||||
* @param input_length
|
||||
* Input text length.
|
||||
* @param[out] out_code_point
|
||||
* The decoded code point is stored here if the return value is P_SUCCESS.
|
||||
* @param[out] out_code_point_length
|
||||
* The number of bytes the code point used is stored here if the return value
|
||||
* is P_SUCCESS.
|
||||
*
|
||||
* @retval P_SUCCESS on a successful code point decode
|
||||
* @retval P_DECODE_ERROR when an encoding error is observed
|
||||
* @retval P_EOF when the end of the text input is reached
|
||||
*/
|
||||
size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
|
||||
<%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length)
|
||||
{
|
||||
if (input_length == 0u)
|
||||
{
|
||||
return P_EOF;
|
||||
}
|
||||
char c = input[0];
|
||||
<%= @grammar.prefix %>code_point_t code_point;
|
||||
uint8_t code_point_length;
|
||||
if ((c & 0x80u) == 0u)
|
||||
{
|
||||
code_point = c;
|
||||
code_point_length = 1u;
|
||||
}
|
||||
else
|
||||
{
|
||||
uint8_t following_bytes;
|
||||
if ((c & 0xE0u) == 0xC0u)
|
||||
{
|
||||
code_point = c & 0x1Fu;
|
||||
following_bytes = 1u;
|
||||
}
|
||||
else if ((c & 0xF0u) == 0xE0u)
|
||||
{
|
||||
code_point = c & 0x0Fu;
|
||||
following_bytes = 2u;
|
||||
}
|
||||
else if ((c & 0xF8u) == 0xF0u)
|
||||
{
|
||||
code_point = c & 0x07u;
|
||||
following_bytes = 3u;
|
||||
}
|
||||
else if ((c & 0xFCu) == 0xF8u)
|
||||
{
|
||||
code_point = c & 0x03u;
|
||||
following_bytes = 4u;
|
||||
}
|
||||
else if ((c & 0xFEu) == 0xFCu)
|
||||
{
|
||||
code_point = c & 0x01u;
|
||||
following_bytes = 5u;
|
||||
}
|
||||
else
|
||||
{
|
||||
return P_DECODE_ERROR;
|
||||
}
|
||||
if (input_length <= following_bytes)
|
||||
{
|
||||
return P_DECODE_ERROR;
|
||||
}
|
||||
code_point_length = (uint8_t)(following_bytes + 1u);
|
||||
for (size_t i = 0u; i < following_bytes; i++)
|
||||
{
|
||||
char b = input[i + 1u];
|
||||
if ((b & 0xC0u) != 0x80u)
|
||||
{
|
||||
return P_DECODE_ERROR;
|
||||
}
|
||||
code_point = (code_point << 6u) | (b & 0x3Fu);
|
||||
}
|
||||
}
|
||||
*out_code_point = code_point;
|
||||
*out_code_point_length = code_point_length;
|
||||
return P_SUCCESS;
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
* Lexer
|
||||
*************************************************************************/
|
||||
|
||||
/** Lexer state ID type. */
|
||||
typedef <%= get_type_for(@lexer.state_table.size) %> lexer_state_id_t;
|
||||
|
||||
/** Invalid lexer state ID. */
|
||||
#define INVALID_LEXER_STATE_ID <%= @lexer.state_table.size %>u
|
||||
|
||||
/** Lexer user code ID type. */
|
||||
<% user_code_id_count = (@grammar.patterns.map(&:code_id).compact.max || 0) + 1 %>
|
||||
typedef <%= get_type_for(user_code_id_count) %> lexer_user_code_id_t;
|
||||
|
||||
/** Invalid lexer user code ID. */
|
||||
#define INVALID_USER_CODE_ID <%= user_code_id_count %>u
|
||||
|
||||
/**
|
||||
* Lexer transition table entry.
|
||||
*
|
||||
* An incoming code point matching the range for a transition entry will cause
|
||||
* the lexer to progress to the destination state.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
/** First code point in the range for this transition. */
|
||||
<%= @grammar.prefix %>code_point_t first;
|
||||
|
||||
/** Last code point in the range for this transition. */
|
||||
<%= @grammar.prefix %>code_point_t last;
|
||||
|
||||
/** Destination lexer state ID for this transition. */
|
||||
lexer_state_id_t destination_state;
|
||||
} lexer_transition_t;
|
||||
|
||||
/** Lexer state table entry. */
|
||||
typedef struct
|
||||
{
|
||||
/** Index to the transition table for this state. */
|
||||
<%= get_type_for(@lexer.transition_table.size - 1) %> transition_table_index;
|
||||
|
||||
/** Number of transition table entries for this state. */
|
||||
<%= get_type_for(@lexer.state_table.map {|ste| ste[:n_transitions]}.max) %> n_transitions;
|
||||
|
||||
/** Lexer token formed at this state. */
|
||||
<%= @grammar.prefix %>token_t token;
|
||||
|
||||
/** Lexer user code ID to execute at this state. */
|
||||
lexer_user_code_id_t code_id;
|
||||
|
||||
/** Whether this state matches a lexer pattern. */
|
||||
bool accepts;
|
||||
} lexer_state_t;
|
||||
|
||||
/** Lexer mode table entry. */
|
||||
typedef struct
|
||||
{
|
||||
/** Offset in the state table to be used for this mode. */
|
||||
uint32_t state_table_offset;
|
||||
} lexer_mode_t;
|
||||
|
||||
/**
|
||||
* Lexer match info structure.
|
||||
*
|
||||
* This structure holds output values from the lexer upon a successful pattern
|
||||
* match.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
/** Number of bytes of input text used to match. */
|
||||
size_t length;
|
||||
|
||||
/** Input text position delta. */
|
||||
<%= @grammar.prefix %>position_t delta_position;
|
||||
|
||||
/** Accepting lexer state from the match. */
|
||||
lexer_state_t const * accepting_state;
|
||||
} lexer_match_info_t;
|
||||
|
||||
/** Lexer transition table. */
|
||||
static lexer_transition_t lexer_transition_table[] = {
|
||||
<% @lexer.transition_table.each do |transition_table_entry| %>
|
||||
{<%= transition_table_entry[:first] %>u, <%= transition_table_entry[:last] %>u, <%= transition_table_entry[:destination] %>u},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/** Lexer state table. */
|
||||
static lexer_state_t lexer_state_table[] = {
|
||||
<% @lexer.state_table.each do |state_table_entry| %>
|
||||
{<%= state_table_entry[:transition_table_index] %>u, <%= state_table_entry[:n_transitions] %>u, <%= state_table_entry[:token] || "INVALID_TOKEN_ID" %>, <%= state_table_entry[:code_id] || "INVALID_USER_CODE_ID" %>, <%= state_table_entry[:accepts] %>},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/** Lexer mode table. */
|
||||
static lexer_mode_t lexer_mode_table[] = {
|
||||
<% @lexer.mode_table.each do |mode_table_entry| %>
|
||||
{<%= mode_table_entry[:state_table_offset] %>},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/**
|
||||
* Execute user code associated with a lexer pattern.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
* @param code_id
|
||||
* The ID of the user code block to execute.
|
||||
* @param match
|
||||
* Matched text for this pattern.
|
||||
* @param match_length
|
||||
* Matched text length.
|
||||
* @param out_token_info
|
||||
* Lexer token info in progress.
|
||||
*
|
||||
* @return Token to accept, or invalid token if the user code does
|
||||
* not explicitly return a token.
|
||||
*/
|
||||
static <%= @grammar.prefix %>token_t lexer_user_code(<%= @grammar.prefix %>context_t * context,
|
||||
lexer_user_code_id_t code_id, uint8_t const * match,
|
||||
size_t match_length, <%= @grammar.prefix %>token_info_t * out_token_info)
|
||||
{
|
||||
switch (code_id)
|
||||
{
|
||||
<% @grammar.patterns.each do |pattern| %>
|
||||
<% if pattern.code_id %>
|
||||
case <%= pattern.code_id %>u: {
|
||||
<%= expand_code(pattern.code, false, nil, pattern) %>
|
||||
} break;
|
||||
<% end %>
|
||||
<% end %>
|
||||
default: break;
|
||||
}
|
||||
|
||||
return INVALID_TOKEN_ID;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there is a transition from the current lexer state to another
|
||||
* based on the given input code point.
|
||||
*
|
||||
* @param current_state
|
||||
* Current lexer state.
|
||||
* @param code_point
|
||||
* Input code point.
|
||||
*
|
||||
* @return Lexer state to transition to, or INVALID_LEXER_STATE_ID if none.
|
||||
*/
|
||||
static lexer_state_id_t check_lexer_transition(uint32_t current_state, uint32_t code_point)
|
||||
{
|
||||
uint32_t transition_table_index = lexer_state_table[current_state].transition_table_index;
|
||||
for (uint32_t i = 0u; i < lexer_state_table[current_state].n_transitions; i++)
|
||||
{
|
||||
if ((lexer_transition_table[transition_table_index + i].first <= code_point) &&
|
||||
(code_point <= lexer_transition_table[transition_table_index + i].last))
|
||||
{
|
||||
return lexer_transition_table[transition_table_index + i].destination_state;
|
||||
}
|
||||
}
|
||||
return INVALID_LEXER_STATE_ID;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the longest lexer pattern match at the current position.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
* @param[out] out_token_info
|
||||
* The lexed token information is stored here if the return value is
|
||||
* P_SUCCESS.
|
||||
*
|
||||
* @reval P_SUCCESS
|
||||
* A token was successfully lexed.
|
||||
* @reval P_DECODE_ERROR
|
||||
* The decoder encountered invalid text encoding.
|
||||
* @reval P_UNEXPECTED_INPUT
|
||||
* Input text does not match any lexer pattern.
|
||||
* @retval P_EOF
|
||||
* The end of the text input was reached.
|
||||
*/
|
||||
static size_t find_longest_match(<%= @grammar.prefix %>context_t * context,
|
||||
lexer_match_info_t * out_match_info, size_t * out_unexpected_input_length)
|
||||
{
|
||||
lexer_match_info_t longest_match = {0};
|
||||
lexer_match_info_t attempt_match = {0};
|
||||
*out_match_info = longest_match;
|
||||
uint32_t current_state = lexer_mode_table[context->mode].state_table_offset;
|
||||
for (;;)
|
||||
{
|
||||
size_t const input_index = context->input_index + attempt_match.length;
|
||||
uint8_t const * input = &context->input[input_index];
|
||||
size_t input_length = context->input_length - input_index;
|
||||
<%= @grammar.prefix %>code_point_t code_point;
|
||||
uint8_t code_point_length;
|
||||
size_t result = <%= @grammar.prefix %>decode_code_point(input, input_length, &code_point, &code_point_length);
|
||||
switch (result)
|
||||
{
|
||||
case P_SUCCESS:
|
||||
lexer_state_id_t transition_state = check_lexer_transition(current_state, code_point);
|
||||
if (transition_state != INVALID_LEXER_STATE_ID)
|
||||
{
|
||||
attempt_match.length += code_point_length;
|
||||
if (code_point == '\n')
|
||||
{
|
||||
attempt_match.delta_position.row++;
|
||||
attempt_match.delta_position.col = 0u;
|
||||
}
|
||||
else
|
||||
{
|
||||
attempt_match.delta_position.col++;
|
||||
}
|
||||
current_state = transition_state;
|
||||
if (lexer_state_table[current_state].accepts)
|
||||
{
|
||||
attempt_match.accepting_state = &lexer_state_table[current_state];
|
||||
longest_match = attempt_match;
|
||||
}
|
||||
}
|
||||
else if (longest_match.length > 0)
|
||||
{
|
||||
*out_match_info = longest_match;
|
||||
return P_SUCCESS;
|
||||
}
|
||||
else
|
||||
{
|
||||
*out_unexpected_input_length = attempt_match.length + code_point_length;
|
||||
return P_UNEXPECTED_INPUT;
|
||||
}
|
||||
break;
|
||||
|
||||
case P_EOF:
|
||||
/* We hit EOF. */
|
||||
if (longest_match.length > 0)
|
||||
{
|
||||
/* We have a match, so use it. */
|
||||
*out_match_info = longest_match;
|
||||
return P_SUCCESS;
|
||||
}
|
||||
else if (attempt_match.length != 0)
|
||||
{
|
||||
/* There is a partial match - error! */
|
||||
*out_unexpected_input_length = attempt_match.length;
|
||||
return P_UNEXPECTED_INPUT;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Valid EOF return. */
|
||||
return P_EOF;
|
||||
}
|
||||
|
||||
case P_DECODE_ERROR:
|
||||
/* If we see a decode error, we may be partially in the middle of
|
||||
* matching a pattern, so return the attempted match info so that
|
||||
* the input text position can be updated. */
|
||||
*out_match_info = attempt_match;
|
||||
return result;
|
||||
|
||||
default:
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to lex the next token in the input stream.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
* @param[out] out_token_info
|
||||
* The lexed token information is stored here if the return value is
|
||||
* P_SUCCESS.
|
||||
*
|
||||
* @reval P_SUCCESS
|
||||
* A token was successfully lexed.
|
||||
* @reval P_DECODE_ERROR
|
||||
* The decoder encountered invalid text encoding.
|
||||
* @reval P_UNEXPECTED_INPUT
|
||||
* Input text does not match any lexer pattern.
|
||||
* @retval P_DROP
|
||||
* A drop pattern was matched so the lexer should continue.
|
||||
* @retval P_USER_TERMINATED
|
||||
* User code has requested to terminate the lexer.
|
||||
*/
|
||||
static size_t attempt_lex_token(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
||||
{
|
||||
<%= @grammar.prefix %>token_info_t token_info = {0};
|
||||
token_info.position = context->text_position;
|
||||
token_info.token = INVALID_TOKEN_ID;
|
||||
*out_token_info = token_info; // TODO: remove
|
||||
lexer_match_info_t match_info;
|
||||
size_t unexpected_input_length;
|
||||
size_t result = find_longest_match(context, &match_info, &unexpected_input_length);
|
||||
switch (result)
|
||||
{
|
||||
case P_SUCCESS:
|
||||
<%= @grammar.prefix %>token_t token_to_accept = match_info.accepting_state->token;
|
||||
if (match_info.accepting_state->code_id != INVALID_USER_CODE_ID)
|
||||
{
|
||||
uint8_t const * match = &context->input[context->input_index];
|
||||
<%= @grammar.prefix %>token_t user_code_token = lexer_user_code(context,
|
||||
match_info.accepting_state->code_id, match, match_info.length, &token_info);
|
||||
/* A TERMINATE_TOKEN_ID return code from lexer_user_code() means
|
||||
* that the user code is requesting to terminate the lexer. */
|
||||
if (user_code_token == TERMINATE_TOKEN_ID)
|
||||
{
|
||||
return P_USER_TERMINATED;
|
||||
}
|
||||
/* An invalid token returned from lexer_user_code() means that the
|
||||
* user code did not explicitly return a token. So only override
|
||||
* the token to return if the user code does explicitly return a
|
||||
* token. */
|
||||
if (user_code_token != INVALID_TOKEN_ID)
|
||||
{
|
||||
token_to_accept = user_code_token;
|
||||
}
|
||||
}
|
||||
|
||||
/* Update the input position tracking. */
|
||||
context->input_index += match_info.length;
|
||||
context->text_position.row += match_info.delta_position.row;
|
||||
if (match_info.delta_position.row != 0u)
|
||||
{
|
||||
context->text_position.col = match_info.delta_position.col;
|
||||
}
|
||||
else
|
||||
{
|
||||
context->text_position.col += match_info.delta_position.col;
|
||||
}
|
||||
|
||||
if (token_to_accept == INVALID_TOKEN_ID)
|
||||
{
|
||||
return P_DROP;
|
||||
}
|
||||
token_info.token = token_to_accept;
|
||||
token_info.length = match_info.length;
|
||||
*out_token_info = token_info;
|
||||
return P_SUCCESS;
|
||||
|
||||
case P_EOF:
|
||||
token_info.token = TOKEN___EOF;
|
||||
*out_token_info = token_info;
|
||||
return P_SUCCESS;
|
||||
|
||||
case P_DECODE_ERROR:
|
||||
/* Update the input position tracking. */
|
||||
context->input_index += match_info.length;
|
||||
context->text_position.row += match_info.delta_position.row;
|
||||
if (match_info.delta_position.row != 0u)
|
||||
{
|
||||
context->text_position.col = match_info.delta_position.col;
|
||||
}
|
||||
else
|
||||
{
|
||||
context->text_position.col += match_info.delta_position.col;
|
||||
}
|
||||
return result;
|
||||
|
||||
default:
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lex the next token in the input stream.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
* @param[out] out_token_info
|
||||
* The lexed token information is stored here if the return value is
|
||||
* P_SUCCESS.
|
||||
*
|
||||
* @reval P_SUCCESS
|
||||
* A token was successfully lexed.
|
||||
* @reval P_DECODE_ERROR
|
||||
* The decoder encountered invalid text encoding.
|
||||
* @reval P_UNEXPECTED_INPUT
|
||||
* Input text does not match any lexer pattern.
|
||||
* @retval P_USER_TERMINATED
|
||||
* User code has requested to terminate the lexer.
|
||||
*/
|
||||
size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info)
|
||||
{
|
||||
for (;;)
|
||||
{
|
||||
size_t result = attempt_lex_token(context, out_token_info);
|
||||
if (result != P_DROP)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
* Parser
|
||||
*************************************************************************/
|
||||
|
||||
/** Reduce ID type. */
|
||||
typedef <%= get_type_for(@parser.reduce_table.size) %> reduce_id_t;
|
||||
|
||||
/**
|
||||
* A symbol ID can hold either a token ID or a rule set ID.
|
||||
*
|
||||
* Token IDs and rule set IDs share the same namespace, with rule set IDs
|
||||
* beginning after token IDs end.
|
||||
*/
|
||||
typedef <%= get_type_for(@parser.rule_sets.map(&:last).map(&:id).max) %> symbol_id_t;
|
||||
|
||||
/** Parser state ID type. */
|
||||
typedef <%= get_type_for(@parser.state_table.size) %> parser_state_id_t;
|
||||
|
||||
/** Parser rule ID type. */
|
||||
typedef <%= get_type_for(@grammar.rules.size) %> rule_id_t;
|
||||
|
||||
/** Parser shift ID type. */
|
||||
typedef <%= get_type_for(@parser.shift_table.size) %> shift_id_t;
|
||||
|
||||
/** Shift table entry. */
|
||||
typedef struct
|
||||
{
|
||||
/** Token or rule set ID. */
|
||||
symbol_id_t symbol_id;
|
||||
|
||||
/** Parser state to shift to. */
|
||||
parser_state_id_t state_id;
|
||||
} shift_t;
|
||||
|
||||
/** Reduce table entry. */
|
||||
typedef struct
|
||||
{
|
||||
/** Lookahead token. */
|
||||
<%= @grammar.prefix %>token_t token;
|
||||
|
||||
/**
|
||||
* Rule ID.
|
||||
*
|
||||
* This is used to execute the parser user code block associated with a
|
||||
* grammar rule.
|
||||
*/
|
||||
rule_id_t rule;
|
||||
|
||||
/**
|
||||
* Rule set ID.
|
||||
*
|
||||
* This is used as the new top symbol ID of the parse stack after this
|
||||
* reduce action.
|
||||
*/
|
||||
symbol_id_t rule_set;
|
||||
|
||||
/**
|
||||
* Number of states leading to this reduce action.
|
||||
*
|
||||
* This is the number of entries popped from the parse stack after this
|
||||
* reduce action.
|
||||
*/
|
||||
parser_state_id_t n_states;
|
||||
} reduce_t;
|
||||
|
||||
/** Parser state entry. */
|
||||
typedef struct
|
||||
{
|
||||
/** First shift table entry for this parser state. */
|
||||
shift_id_t shift_table_index;
|
||||
|
||||
/** Number of shift table entries for this parser state. */
|
||||
shift_id_t n_shift_entries;
|
||||
|
||||
/** First reduce table entry for this parser state. */
|
||||
reduce_id_t reduce_table_index;
|
||||
|
||||
/** Number of reduce table entries for this parser state. */
|
||||
reduce_id_t n_reduce_entries;
|
||||
} parser_state_t;
|
||||
|
||||
/**
|
||||
* Structure to hold a state ID and value pair.
|
||||
*
|
||||
* A stack of these structures makes up the parse stack.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
/** Parser state ID. */
|
||||
size_t state_id;
|
||||
|
||||
/** Parser value from this state. */
|
||||
<%= @grammar.prefix %>value_t pvalue;
|
||||
} state_value_t;
|
||||
|
||||
/** Parser shift table. */
|
||||
static const shift_t parser_shift_table[] = {
|
||||
<% @parser.shift_table.each do |shift| %>
|
||||
{<%= shift[:symbol_id] %>u, <%= shift[:state_id] %>u},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/** Parser reduce table. */
|
||||
static const reduce_t parser_reduce_table[] = {
|
||||
<% @parser.reduce_table.each do |reduce| %>
|
||||
{<%= reduce[:token_id] %>u, <%= reduce[:rule_id] %>u, <%= reduce[:rule_set_id] %>u, <%= reduce[:n_states] %>u},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/** Parser state table. */
|
||||
static const parser_state_t parser_state_table[] = {
|
||||
<% @parser.state_table.each do |state| %>
|
||||
{<%= state[:shift_index] %>u, <%= state[:n_shifts] %>u, <%= state[:reduce_index] %>u, <%= state[:n_reduces] %>u},
|
||||
<% end %>
|
||||
};
|
||||
|
||||
/* state_values stack functionality */
|
||||
|
||||
/** state_values stack type. */
|
||||
typedef struct
|
||||
{
|
||||
size_t length;
|
||||
size_t capacity;
|
||||
state_value_t * entries;
|
||||
} state_values_stack_t;
|
||||
|
||||
/**
|
||||
* Initialize state_values stack structure.
|
||||
*
|
||||
* @param stack
|
||||
* state_values stack structure.
|
||||
*/
|
||||
static void state_values_stack_init(state_values_stack_t * stack)
|
||||
{
|
||||
const size_t initial_capacity = 10u;
|
||||
stack->length = 0u;
|
||||
stack->capacity = initial_capacity;
|
||||
stack->entries = (state_value_t *)malloc(initial_capacity * sizeof(state_value_t));
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a state_values stack.
|
||||
*
|
||||
* @param stack
|
||||
* state_values stack structure.
|
||||
* @param index
|
||||
* Index to the stack.
|
||||
*
|
||||
* @return Pointer to the state value structure at the given index.
|
||||
*/
|
||||
static state_value_t * state_values_stack_index(state_values_stack_t * stack, int index)
|
||||
{
|
||||
if (index >= 0)
|
||||
{
|
||||
return &stack->entries[index];
|
||||
}
|
||||
else
|
||||
{
|
||||
return &stack->entries[stack->length - (size_t)(unsigned int)(-index)];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Push a new state_value to the state_values stack.
|
||||
*
|
||||
* @param stack
|
||||
* state_values stack structure.
|
||||
*/
|
||||
static void state_values_stack_push(state_values_stack_t * stack)
|
||||
{
|
||||
size_t const current_capacity = stack->capacity;
|
||||
size_t const current_length = stack->length;
|
||||
if (current_length >= current_capacity)
|
||||
{
|
||||
size_t const new_capacity = current_capacity * 2u;
|
||||
state_value_t * new_entries = malloc(new_capacity * sizeof(state_value_t));
|
||||
memcpy(new_entries, stack->entries, current_length * sizeof(state_value_t));
|
||||
free(stack->entries);
|
||||
stack->capacity = new_capacity;
|
||||
stack->entries = new_entries;
|
||||
}
|
||||
memset(&stack->entries[current_length], 0, sizeof(state_value_t));
|
||||
stack->length = current_length + 1u;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pop entries from a state_values stack.
|
||||
*
|
||||
* @param stack
|
||||
* state_values stack structure.
|
||||
* @param n
|
||||
* Number of states to pop.
|
||||
*/
|
||||
static void state_values_stack_pop(state_values_stack_t * stack, size_t n)
|
||||
{
|
||||
stack->length -= n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Free memory for a state_values stack structure.
|
||||
*
|
||||
* @param stack
|
||||
* state_values stack structure.
|
||||
*/
|
||||
static void state_values_stack_free(state_values_stack_t * stack)
|
||||
{
|
||||
free(stack->entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute user code associated with a parser rule.
|
||||
*
|
||||
* @param rule The ID of the rule.
|
||||
*
|
||||
* @retval P_SUCCESS
|
||||
* Continue parsing.
|
||||
* @retval P_USER_TERMINATED
|
||||
* User requested to terminate parsing.
|
||||
*/
|
||||
static size_t parser_user_code(<%= @grammar.prefix %>value_t * _pvalue, uint32_t rule, state_values_stack_t * statevalues, uint32_t n_states, <%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
switch (rule)
|
||||
{
|
||||
<% @grammar.rules.each do |rule| %>
|
||||
<% if rule.code %>
|
||||
case <%= rule.id %>u: {
|
||||
<%= expand_code(rule.code, true, rule, nil) %>
|
||||
} break;
|
||||
<% end %>
|
||||
<% end %>
|
||||
default: break;
|
||||
}
|
||||
|
||||
return P_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the parser should shift to a new state.
|
||||
*
|
||||
* @param state_id
|
||||
* Parser state ID.
|
||||
* @param symbol_id
|
||||
* Incoming token/rule set ID.
|
||||
*
|
||||
* @return State to shift to, or INVALID_ID if none.
|
||||
*/
|
||||
static size_t check_shift(size_t state_id, size_t symbol_id)
|
||||
{
|
||||
uint32_t start = parser_state_table[state_id].shift_table_index;
|
||||
uint32_t end = start + parser_state_table[state_id].n_shift_entries;
|
||||
for (uint32_t i = start; i < end; i++)
|
||||
{
|
||||
if (parser_shift_table[i].symbol_id == symbol_id)
|
||||
{
|
||||
return parser_shift_table[i].state_id;
|
||||
}
|
||||
}
|
||||
return INVALID_ID;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the parser should reduce to a new state.
|
||||
*
|
||||
* @param state_id
|
||||
* Parser state ID.
|
||||
* @param token
|
||||
* Incoming token.
|
||||
*
|
||||
* @return State to reduce to, or INVALID_ID if none.
|
||||
*/
|
||||
static size_t check_reduce(size_t state_id, <%= @grammar.prefix %>token_t token)
|
||||
{
|
||||
size_t start = parser_state_table[state_id].reduce_table_index;
|
||||
size_t end = start + parser_state_table[state_id].n_reduce_entries;
|
||||
for (size_t i = start; i < end; i++)
|
||||
{
|
||||
if ((parser_reduce_table[i].token == token) ||
|
||||
(parser_reduce_table[i].token == INVALID_TOKEN_ID))
|
||||
{
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return INVALID_ID;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the parser.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
*
|
||||
* @retval P_SUCCESS
|
||||
* The parser successfully matched the input text. The parse result value
|
||||
* can be accessed with <%= @grammar.prefix %>result().
|
||||
* @retval P_UNEXPECTED_TOKEN
|
||||
* An unexpected token was encountered that does not match any grammar rule.
|
||||
* The function p_token(&context) can be used to get the unexpected token.
|
||||
* @reval P_DECODE_ERROR
|
||||
* The decoder encountered invalid text encoding.
|
||||
* @reval P_UNEXPECTED_INPUT
|
||||
* Input text does not match any lexer pattern.
|
||||
*/
|
||||
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
<%= @grammar.prefix %>token_info_t token_info;
|
||||
<%= @grammar.prefix %>token_t token = INVALID_TOKEN_ID;
|
||||
state_values_stack_t statevalues;
|
||||
size_t reduced_rule_set = INVALID_ID;
|
||||
<%= @grammar.prefix %>value_t reduced_parser_value;
|
||||
state_values_stack_init(&statevalues);
|
||||
state_values_stack_push(&statevalues);
|
||||
size_t result;
|
||||
for (;;)
|
||||
{
|
||||
if (token == INVALID_TOKEN_ID)
|
||||
{
|
||||
size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
|
||||
if (lexer_result != P_SUCCESS)
|
||||
{
|
||||
result = lexer_result;
|
||||
break;
|
||||
}
|
||||
token = token_info.token;
|
||||
}
|
||||
size_t shift_state = INVALID_ID;
|
||||
if (reduced_rule_set != INVALID_ID)
|
||||
{
|
||||
shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, reduced_rule_set);
|
||||
}
|
||||
if (shift_state == INVALID_ID)
|
||||
{
|
||||
shift_state = check_shift(state_values_stack_index(&statevalues, -1)->state_id, token);
|
||||
if ((shift_state != INVALID_ID) && (token == TOKEN___EOF))
|
||||
{
|
||||
/* Successful parse. */
|
||||
context->parse_result = state_values_stack_index(&statevalues, -1)->pvalue;
|
||||
result = P_SUCCESS;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (shift_state != INVALID_ID)
|
||||
{
|
||||
/* We have something to shift. */
|
||||
state_values_stack_push(&statevalues);
|
||||
state_values_stack_index(&statevalues, -1)->state_id = shift_state;
|
||||
if (reduced_rule_set == INVALID_ID)
|
||||
{
|
||||
/* We shifted a token, mark it consumed. */
|
||||
token = INVALID_TOKEN_ID;
|
||||
state_values_stack_index(&statevalues, -1)->pvalue = token_info.pvalue;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* We shifted a RuleSet. */
|
||||
state_values_stack_index(&statevalues, -1)->pvalue = reduced_parser_value;
|
||||
<%= @grammar.prefix %>value_t new_parse_result = {0};
|
||||
reduced_parser_value = new_parse_result;
|
||||
reduced_rule_set = INVALID_ID;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t reduce_index = check_reduce(state_values_stack_index(&statevalues, -1)->state_id, token);
|
||||
if (reduce_index != INVALID_ID)
|
||||
{
|
||||
/* We have something to reduce. */
|
||||
<%= @grammar.prefix %>value_t reduced_parser_value2 = {0};
|
||||
if (parser_user_code(&reduced_parser_value2, parser_reduce_table[reduce_index].rule, &statevalues, parser_reduce_table[reduce_index].n_states, context) == P_USER_TERMINATED)
|
||||
{
|
||||
return P_USER_TERMINATED;
|
||||
}
|
||||
reduced_parser_value = reduced_parser_value2;
|
||||
reduced_rule_set = parser_reduce_table[reduce_index].rule_set;
|
||||
state_values_stack_pop(&statevalues, parser_reduce_table[reduce_index].n_states);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* A token was successfully lexed, so the input text position was
|
||||
* advanced. However, this is an unexpected token, so we want to reset
|
||||
* the context text position to point to the token rather than the text
|
||||
* after it, so that if the caller wants to report the error position,
|
||||
* it will point to the correct position of the unexpected token. */
|
||||
context->text_position = token_info.position;
|
||||
context->token = token;
|
||||
result = P_UNEXPECTED_TOKEN;
|
||||
break;
|
||||
}
|
||||
state_values_stack_free(&statevalues);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the parse result value.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
*
|
||||
* @return Parse result value.
|
||||
*/
|
||||
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
return context->parse_result.v_<%= start_rule_type[0] %>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current text input position.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
*
|
||||
* @return Current text position.
|
||||
*/
|
||||
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
return context->text_position;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the user terminate code.
|
||||
*
|
||||
* @param context
|
||||
* Lexer/parser context structure.
|
||||
*
|
||||
* @return User terminate code.
|
||||
*/
|
||||
size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
return context->user_terminate_code;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the parse token.
|
||||
*
|
||||
* @return Parse token.
|
||||
*/
|
||||
<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context)
|
||||
{
|
||||
return context->token;
|
||||
}
|
||||
+708
-916
File diff suppressed because it is too large.
Load diff
@@ -1,141 +0,0 @@
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* This file is generated by Propane.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/**************************************************************************
|
||||
* Public types
|
||||
*************************************************************************/
|
||||
|
||||
/* Result codes. */
|
||||
#define <%= @grammar.prefix.upcase %>SUCCESS 0u
|
||||
#define <%= @grammar.prefix.upcase %>DECODE_ERROR 1u
|
||||
#define <%= @grammar.prefix.upcase %>UNEXPECTED_INPUT 2u
|
||||
#define <%= @grammar.prefix.upcase %>UNEXPECTED_TOKEN 3u
|
||||
#define <%= @grammar.prefix.upcase %>DROP 4u
|
||||
#define <%= @grammar.prefix.upcase %>EOF 5u
|
||||
#define <%= @grammar.prefix.upcase %>USER_TERMINATED 6u
|
||||
|
||||
/** Token type. */
|
||||
typedef <%= get_type_for(@grammar.terminate_token_id) %> <%= @grammar.prefix %>token_t;
|
||||
|
||||
/** Token IDs. */
|
||||
<% @grammar.tokens.each_with_index do |token, index| %>
|
||||
#define TOKEN_<%= token.code_name %> <%= index %>u
|
||||
<% unless token.id == index %>
|
||||
<% raise "Token ID (#{token.id}) does not match index (#{index}) for token #{token.name}!" %>
|
||||
<% end %>
|
||||
<% end %>
|
||||
#define INVALID_TOKEN_ID <%= @grammar.invalid_token_id %>u
|
||||
#define TERMINATE_TOKEN_ID <%= @grammar.terminate_token_id %>u
|
||||
|
||||
/** Code point type. */
|
||||
typedef uint32_t <%= @grammar.prefix %>code_point_t;
|
||||
|
||||
/** User header code blocks. */
|
||||
<%= @grammar.code_blocks.fetch("header", "") %>
|
||||
|
||||
/** Parser values type(s). */
|
||||
typedef union
|
||||
{
|
||||
<% @grammar.ptypes.each do |name, typestring| %>
|
||||
<%= typestring %> v_<%= name %>;
|
||||
<% end %>
|
||||
} <%= @grammar.prefix %>value_t;
|
||||
|
||||
/**
|
||||
* A structure to keep track of parser position.
|
||||
*
|
||||
* This is useful for reporting errors, etc...
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
/** Input text row (0-based). */
|
||||
uint32_t row;
|
||||
|
||||
/** Input text column (0-based). */
|
||||
uint32_t col;
|
||||
} <%= @grammar.prefix %>position_t;
|
||||
|
||||
/** Lexed token information. */
|
||||
typedef struct
|
||||
{
|
||||
/** Text position where the token was found. */
|
||||
<%= @grammar.prefix %>position_t position;
|
||||
|
||||
/** Number of input bytes used by the token. */
|
||||
size_t length;
|
||||
|
||||
/** Token that was lexed. */
|
||||
<%= @grammar.prefix %>token_t token;
|
||||
|
||||
/** Parser value associated with the token. */
|
||||
<%= @grammar.prefix %>value_t pvalue;
|
||||
} <%= @grammar.prefix %>token_info_t;
|
||||
|
||||
/**
|
||||
* Lexer and parser context.
|
||||
*
|
||||
* The user must allocate an instance of this structure and pass it to any
|
||||
* public API function.
|
||||
*/
|
||||
typedef struct
|
||||
{
|
||||
/* Lexer context data. */
|
||||
|
||||
/** Input text. */
|
||||
uint8_t const * input;
|
||||
|
||||
/** Input text length. */
|
||||
size_t input_length;
|
||||
|
||||
/** Input text index (byte offset). */
|
||||
size_t input_index;
|
||||
|
||||
/** Input text position (row/column). */
|
||||
<%= @grammar.prefix %>position_t text_position;
|
||||
|
||||
/** Current lexer mode. */
|
||||
size_t mode;
|
||||
|
||||
/* Parser context data. */
|
||||
|
||||
/** Parse result value. */
|
||||
<%= @grammar.prefix %>value_t parse_result;
|
||||
|
||||
/** Unexpected token received. */
|
||||
<%= @grammar.prefix %>token_t token;
|
||||
|
||||
/** User terminate code. */
|
||||
size_t user_terminate_code;
|
||||
} <%= @grammar.prefix %>context_t;
|
||||
|
||||
/**************************************************************************
|
||||
* Public data
|
||||
*************************************************************************/
|
||||
|
||||
/** Token names. */
|
||||
extern const char * <%= @grammar.prefix %>token_names[];
|
||||
|
||||
void <%= @grammar.prefix %>context_init(<%= @grammar.prefix %>context_t * context, uint8_t const * input, size_t input_length);
|
||||
|
||||
size_t <%= @grammar.prefix %>decode_code_point(uint8_t const * input, size_t input_length,
|
||||
<%= @grammar.prefix %>code_point_t * out_code_point, uint8_t * out_code_point_length);
|
||||
|
||||
size_t <%= @grammar.prefix %>lex(<%= @grammar.prefix %>context_t * context, <%= @grammar.prefix %>token_info_t * out_token_info);
|
||||
|
||||
size_t <%= @grammar.prefix %>parse(<%= @grammar.prefix %>context_t * context);
|
||||
|
||||
<%= start_rule_type[1] %> <%= @grammar.prefix %>result(<%= @grammar.prefix %>context_t * context);
|
||||
|
||||
<%= @grammar.prefix %>position_t <%= @grammar.prefix %>position(<%= @grammar.prefix %>context_t * context);
|
||||
|
||||
size_t <%= @grammar.prefix %>user_terminate_code(<%= @grammar.prefix %>context_t * context);
|
||||
|
||||
<%= @grammar.prefix %>token_t <%= @grammar.prefix %>token(<%= @grammar.prefix %>context_t * context);
|
||||
+3
-667
@@ -7,15 +7,15 @@ ${/remove}
|
||||
|
||||
#> Overview
|
||||
|
||||
Propane is a LALR Parser Generator (LPG) which:
|
||||
Propane is an LR Parser Generator (LPG) which:
|
||||
|
||||
* accepts LR(0), SLR, and LALR grammars
|
||||
* generates a built-in lexer to tokenize input
|
||||
* supports UTF-8 lexer inputs
|
||||
* generates a table-driven shift/reduce parser to parse input in linear time
|
||||
* targets C or D language outputs
|
||||
* generates a table-driven parser to parse input in linear time
|
||||
* is MIT-licensed
|
||||
* is distributable as a standalone Ruby script
|
||||
* supports D language
|
||||
|
||||
#> Installation
|
||||
|
||||
@@ -49,670 +49,6 @@ detailed information about the parser states and transitions.
|
||||
A Propane grammar file provides Propane with the patterns, tokens, grammar
|
||||
rules, and user code blocks from which to build the generated lexer and parser.
|
||||
|
||||
Example grammar file:
|
||||
|
||||
```
|
||||
<<
|
||||
import std.math;
|
||||
>>
|
||||
|
||||
# Parser values are unsigned integers.
|
||||
ptype ulong;
|
||||
|
||||
# A few basic arithmetic operators.
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
token power /\\*\\*/;
|
||||
token integer /\\d+/ <<
|
||||
ulong v;
|
||||
foreach (c; match)
|
||||
{
|
||||
v *= 10;
|
||||
v += (c - '0');
|
||||
}
|
||||
$$ = v;
|
||||
>>
|
||||
token lparen /\\(/;
|
||||
token rparen /\\)/;
|
||||
# Drop whitespace.
|
||||
drop /\\s+/;
|
||||
|
||||
Start -> E1 << $$ = $1; >>
|
||||
E1 -> E2 << $$ = $1; >>
|
||||
E1 -> E1 plus E2 << $$ = $1 + $3; >>
|
||||
E2 -> E3 << $$ = $1; >>
|
||||
E2 -> E2 times E3 << $$ = $1 * $3; >>
|
||||
E3 -> E4 << $$ = $1; >>
|
||||
E3 -> E3 power E4 << $$ = pow($1, $3); >>
|
||||
E4 -> integer << $$ = $1; >>
|
||||
E4 -> lparen E1 rparen << $$ = $2; >>
|
||||
```
|
||||
|
||||
Grammar files can contain comment lines beginning with `#` which are ignored.
|
||||
White space in the grammar file is also ignored.
|
||||
|
||||
It is convention to use the extension `.propane` for the Propane grammar file,
|
||||
however any file name is accepted by Propane.
|
||||
|
||||
This user guide follows the convention of beginning a token name with a
|
||||
lowercase character and beginning a rule name with an uppercase character.
|
||||
|
||||
##> User Code Blocks
|
||||
|
||||
User code blocks begin following a "<<" token and end with a ">>" token found
|
||||
at the end of a line.
|
||||
All text lines in the code block are copied verbatim into the output file.
|
||||
|
||||
### Standalone Code Blocks
|
||||
|
||||
C example:
|
||||
|
||||
```
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
```
|
||||
|
||||
D example:
|
||||
|
||||
```
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
```
|
||||
|
||||
Standalone code blocks are emitted early in the output file as top-level code
|
||||
outside the context of any function.
|
||||
Standalone code blocks are a good place to include/import any other necessary
|
||||
supporting code modules.
|
||||
They can also define helper functions that can be reused by lexer or parser
|
||||
user code blocks.
|
||||
They are emitted in the order they are defined in the grammar file.
|
||||
|
||||
For a C target, the word "header" may immediately follow the "<<" token to
|
||||
cause Propane to emit the code block in the generated header file rather than
|
||||
the generated implementation file.
|
||||
This allows including another header that may be necessary to define any types
|
||||
needed by a `ptype` directive, for example:
|
||||
|
||||
```
|
||||
<<header
|
||||
#include "mytypes.h"
|
||||
>>
|
||||
```
|
||||
|
||||
### Lexer pattern code blocks
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
ptype ulong;
|
||||
|
||||
token integer /\\d+/ <<
|
||||
ulong v;
|
||||
foreach (c; match)
|
||||
{
|
||||
v *= 10;
|
||||
v += (c - '0');
|
||||
}
|
||||
$$ = v;
|
||||
>>
|
||||
```
|
||||
|
||||
Lexer code blocks appear following a `token` or pattern expression.
|
||||
User code in a lexer code block will be executed when the lexer matches the
|
||||
given pattern.
|
||||
Assignment to the `$$` symbol will associate a parser value with the lexed
|
||||
token.
|
||||
This parser value can then be used later in a parser rule.
|
||||
|
||||
### Parser rule code blocks
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
E1 -> E1 plus E2 << $$ = $1 + $3; >>
|
||||
```
|
||||
|
||||
Parser rule code blocks appear following a rule expression.
|
||||
User code in a parser rule code block will be executed when the parser reduces
|
||||
the given rule.
|
||||
Assignment to the `$$` symbol will associate a parser value with the reduced
|
||||
rule.
|
||||
Parser values for the rules or tokens in the rule pattern can be accessed
|
||||
positionally with tokens `$1`, `$2`, `$3`, etc...
|
||||
|
||||
##> Specifying tokens - the `token` statement
|
||||
|
||||
The `token` statement allows defining a lexer token and a pattern to match that
|
||||
token.
|
||||
The name of the token must be specified immediately following the `token`
|
||||
keyword.
|
||||
A regular expression pattern may optionally follow the token name.
|
||||
If a regular expression pattern is not specified, the name of the token is
|
||||
taken to be the pattern.
|
||||
See also: ${#Regular expression syntax}.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
token for;
|
||||
```
|
||||
|
||||
In this example, the token name is `for` and the pattern to match it is
|
||||
`/for/`.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
token lbrace /\{/;
|
||||
```
|
||||
|
||||
In this example, the token name is `lbrace` and a single left curly brace will
|
||||
match it.
|
||||
|
||||
The `token` statement can also include a user code block.
|
||||
The user code block will be executed whenever the token is matched by the
|
||||
lexer.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
token if << writeln("'if' keyword lexed"); >>
|
||||
```
|
||||
|
||||
The `token` statement is actually a shortcut statement for a combination of a
|
||||
`tokenid` statement and a pattern statement.
|
||||
To define a lexer token without an associated pattern to match it, use a
|
||||
`tokenid` statement.
|
||||
To define a lexer pattern that may or may not result in a matched token, use
|
||||
a pattern statement.
|
||||
|
||||
##> Defining tokens without a matching pattern - the `tokenid` statement
|
||||
|
||||
The `tokenid` statement can be used to define a token without associating it
|
||||
with a lexer pattern that matches it.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
tokenid string;
|
||||
```
|
||||
|
||||
The `tokenid` statement can be useful when defining a token that may optionally
|
||||
be returned by user code associated with a pattern.
|
||||
|
||||
It is also useful when lexer modes and multiple lexer patterns are required to
|
||||
build up a full token.
|
||||
A common example is parsing a string.
|
||||
See the ${#Lexer modes} chapter for more information.
|
||||
|
||||
##> Specifying a lexer pattern - the pattern statement
|
||||
|
||||
A pattern statement is used to define a lexer pattern that can execute user
|
||||
code but may not result in a matched token.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
/foo+/ << writeln("saw a foo pattern"); >>
|
||||
```
|
||||
|
||||
This can be especially useful with ${#Lexer modes}.
|
||||
|
||||
See also ${#Regular expression syntax}.
|
||||
|
||||
##> Ignoring input sections - the `drop` statement
|
||||
|
||||
A `drop` statement can be used to specify a lexer pattern that when matched
|
||||
should result in the matched input being dropped and lexing continuing after
|
||||
the matched input.
|
||||
|
||||
A common use for a `drop` statement would be to ignore whitespace sequences in
|
||||
the user input.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
drop /\s+/;
|
||||
```
|
||||
|
||||
See also ${#Regular expression syntax}.
|
||||
|
||||
##> Regular expression syntax
|
||||
|
||||
A regular expression ("regex") is used to define lexer patterns in `token`,
|
||||
pattern, and `drop` statements.
|
||||
A regular expression begins and ends with a `/` character.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
/#.*$/
|
||||
```
|
||||
|
||||
Regular expressions can include many special characters:
|
||||
|
||||
* The `.` character matches any input character other than a newline.
|
||||
* The `*` character matches any number of the previous regex element.
|
||||
* The `+` character matches one or more of the previous regex element.
|
||||
* The `?` character matches 0 or 1 of the previous regex element.
|
||||
* The `[` character begins a character class.
|
||||
* The `(` character begins a matching group.
|
||||
* The `{` character begins a count qualifier.
|
||||
* The `\` character escapes the following character and changes its meaning:
|
||||
* The `\a` sequence matches an ASCII bell character (0x07).
|
||||
* The `\b` sequence matches an ASCII backspace character (0x08).
|
||||
* The `\d` sequence matches any character `0` through `9`.
|
||||
* The `\f` sequence matches an ASCII form feed character (0x0C).
|
||||
* The `\n` sequence matches an ASCII new line character (0x0A).
|
||||
* The `\r` sequence matches an ASCII carriage return character (0x0D).
|
||||
* The `\s` sequence matches a space, horizontal tab `\t`, carriage return
|
||||
`\r`, a form feed `\f`, or a vertical tab `\v` character.
|
||||
* The `\t` sequence matches an ASCII tab character (0x09).
|
||||
* The `\v` sequence matches an ASCII vertical tab character (0x0B).
|
||||
* Any other character matches itself.
|
||||
* The `|` character creates an alternate match.
|
||||
|
||||
Any other character just matches itself in the input stream.
|
||||
|
||||
A character class consists of a list of character alternates or character
|
||||
ranges that can be matched by the character class.
|
||||
For example `[a-zA-Z_]` matches any lowercase character between `a` and `z` or
|
||||
any uppercase character between `A` and `Z` or the underscore `_` character.
|
||||
Character classes can also be negative character classes if the first character
|
||||
after the `[` is a `^` character.
|
||||
In this case, the set of characters matched by the character class is the
|
||||
inverse of what it otherwise would have been.
|
||||
For example, `[^0-9]` matches any character other than 0 through 9.
|
||||
|
||||
A matching group can be used to override the pattern sequence that multiplicity
|
||||
specifiers apply to.
|
||||
For example, the pattern `/foo+/` matches "foo" or "foooo", while the pattern
|
||||
`/(foo)+/` matches "foo" or "foofoofoo", but not "foooo".
|
||||
|
||||
A count qualifier in curly braces can be used to restrict the number of matches
|
||||
of the preceding atom to an explicit minimum and maximum range.
|
||||
For example, the pattern `\d{3}` matches exactly 3 digits 0-9.
|
||||
Both a minimum and maximum multiplicity count can be specified and separated by
|
||||
a comma.
|
||||
For example, `/a{1,5}/` matches between 1 and 5 `a` characters.
|
||||
Either the minimum or maximum count can be omitted to omit the corresponding
|
||||
restriction in the number of matches allowed.
|
||||
|
||||
An alternate match is created with the `|` character.
|
||||
For example, the pattern `/foo|bar/` matches either the sequence "foo" or the
|
||||
sequence "bar".
|
||||
|
||||
##> Lexer modes
|
||||
|
||||
Lexer modes can be used to change the set of patterns that are matched by the
|
||||
lexer.
|
||||
A common use for lexer modes is to match strings.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
<<
|
||||
string mystringvalue;
|
||||
>>
|
||||
|
||||
tokenid str;
|
||||
|
||||
# String processing
|
||||
/"/ <<
|
||||
mystringvalue = "";
|
||||
$mode(string);
|
||||
>>
|
||||
string: /[^"]+/ << mystringvalue += match; >>
|
||||
string: /"/ <<
|
||||
$mode(default);
|
||||
return $token(str);
|
||||
>>
|
||||
```
|
||||
|
||||
A lexer mode is defined by placing the name before a colon (`:`) character that
|
||||
precedes a token or pattern statement.
|
||||
The token or pattern statement is restricted to only applying if the named mode
|
||||
is active.
|
||||
|
||||
By default, the active lexer mode is named `default`.
|
||||
A `$mode()` call within a lexer code block can be used to change lexer modes.
|
||||
|
||||
In the above example, when the lexer in the default mode sees a doublequote
|
||||
(`"`) character, the lexer code block will clear the `mystringvalue` variable
|
||||
and will set the lexer mode to `string`.
|
||||
When the lexer begins looking for patterns to match against the input, it will
|
||||
now look only for patterns tagged for the `string` lexer mode.
|
||||
Any non-`"` character will be appended to the `mystringvalue` string.
|
||||
A `"` character will end the `string` lexer mode and return to the `default`
|
||||
lexer mode.
|
||||
It also returns the `str` token now that the token is complete.
|
||||
|
||||
Note that the token name `str` above could have been `string` instead - the
|
||||
namespace for token names is distinct from the namespace for lexer modes.
|
||||
|
||||
##> Specifying parser value types - the `ptype` statement
|
||||
|
||||
The `ptype` statement is used to define parser value type(s).
|
||||
Example:
|
||||
|
||||
```
|
||||
ptype void *;
|
||||
```
|
||||
|
||||
This defines the default parser value type to be `void *` (this is, in fact,
|
||||
the default parser value type if the grammar file does not specify otherwise).
|
||||
|
||||
Each defined lexer token type and parser rule has an associated parser value
|
||||
type.
|
||||
When the lexer runs, each lexed token has a parser value associated with it.
|
||||
When the parser runs, each instance of a reduced rule has a parser value
|
||||
associated with it.
|
||||
Propane supports using different parser value types for different rules and
|
||||
token types.
|
||||
The example `ptype` statement above defines the default parser value type.
|
||||
A parser value type name can optionally be specified following the `ptype`
|
||||
keyword.
|
||||
For example:
|
||||
|
||||
```
|
||||
ptype Value;
|
||||
ptype array = Value[];
|
||||
ptype dict = Value[string];
|
||||
|
||||
Object -> lbrace rbrace << $$ = new Value(); >>
|
||||
|
||||
Values (array) -> Value << $$ = [$1]; >>
|
||||
Values -> Values comma Value << $$ = $1 ~ [$3]; >>
|
||||
|
||||
KeyValue (dict) -> string colon Value << $$ = [$1: $3]; >>
|
||||
```
|
||||
|
||||
In this example, the default parser value type is `Value`.
|
||||
A parser value type named `array` is defined to mean `Value[]`.
|
||||
A parser value type named `dict` is defined to mean `Value[string]`.
|
||||
Any defined tokens or rules that do not specify a parser value type will have
|
||||
the default parser value type associated with them.
|
||||
To associate a different parser value type with a token or rule, write the
|
||||
parser value type name in parentheses following the name of the token or rule.
|
||||
In this example:
|
||||
|
||||
* a reduced `Object`'s parser value has a type of `Value`.
|
||||
* a reduced `Values`'s parser value has a type of `Value[]`.
|
||||
* a reduced `KeyValue`'s parser value has a type of `Value[string]`.
|
||||
|
||||
##> Specifying a parser rule - the rule statement
|
||||
|
||||
Rule statements create parser rules which define the grammar that will be
|
||||
parsed by the generated parser.
|
||||
|
||||
Multiple rules with the same name can be specified.
|
||||
Rules with the same name define a rule set for that name and act as
|
||||
alternatives that the parser can accept when attempting to match a reference to
|
||||
that rule.
|
||||
|
||||
The grammar file must define a rule with the name `Start` which will be used as
|
||||
the top-level starting rule that the parser attempts to reduce.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
ptype ulong;
|
||||
token word /[a-z]+/ << $$ = match.length; >>
|
||||
Start -> word << $$ = $1; >>
|
||||
```
|
||||
|
||||
In the above example the `Start` rule is defined to match a single `word`
|
||||
token.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
Start -> E1 << $$ = $1; >>
|
||||
E1 -> E2 << $$ = $1; >>
|
||||
E1 -> E1 plus E2 << $$ = $1 + $3; >>
|
||||
E2 -> E3 << $$ = $1; >>
|
||||
E2 -> E2 times E3 << $$ = $1 * $3; >>
|
||||
E3 -> E4 << $$ = $1; >>
|
||||
E3 -> E3 power E4 << $$ = pow($1, $3); >>
|
||||
E4 -> integer << $$ = $1; >>
|
||||
E4 -> lparen E1 rparen << $$ = $2; >>
|
||||
```
|
||||
|
||||
A parser rule has zero or more terms on the right side of its definition.
|
||||
Each of these terms is either a token name or a rule name.
|
||||
|
||||
In a parser rule code block, parser values for the right side terms are
|
||||
accessible as `$1` for the first term's parser value, `$2` for the second
|
||||
term's parser value, etc...
|
||||
The `$$` symbol accesses the output parser value for this rule.
|
||||
The above examples demonstrate how the parser values for the rule components
|
||||
can be used to produce the parser value for the accepted rule.
|
||||
|
||||
##> Specifying the parser module name - the `module` statement
|
||||
|
||||
The `module` statement can be used to specify the module name for a generated
|
||||
D module.
|
||||
|
||||
```
|
||||
module proj.parser;
|
||||
```
|
||||
|
||||
If a module statement is not present, then the generated D module will not
|
||||
contain a module statement and the default module name will be used.
|
||||
|
||||
##> Specifying the generated API prefix - the `prefix` statement
|
||||
|
||||
By default the public API (types, constants, and functions) of the generated
|
||||
lexer and parser uses a prefix of `p_`.
|
||||
|
||||
This prefix can be changed with the `prefix` statement.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
prefix myparser_;
|
||||
```
|
||||
|
||||
With a parser generated with this `prefix` statement, instead of calling
|
||||
`p_context_init()` you would call `myparser_context_init()`.
|
||||
|
||||
The `prefix` statement can be optionally used if you would like to change the
|
||||
prefix used by your generated lexer and parser to something other than the
|
||||
default.
|
||||
|
||||
It can also be used when generating multiple lexers/parsers to be used in the
|
||||
same program to avoid symbol collisions.
|
||||
|
||||
##> User termination of the lexer or parser
|
||||
|
||||
Propane supports allowing lexer or parser user code blocks to terminate
|
||||
execution of the parser.
|
||||
Some example uses of this functionality could be to:
|
||||
|
||||
* Detect integer overflow when lexing an integer literal constant.
|
||||
* Detect and report an error as soon as possible during parsing before continuing to parse any more of the input.
|
||||
* Determine whether parsing should stop and instead be performed using a different parser version.
|
||||
|
||||
To terminate parsing from a lexer or parser user code block, use the
|
||||
`$terminate(code)` function, passing an integer expression argument.
|
||||
For example:
|
||||
|
||||
```
|
||||
NewExpression -> new Expression << $terminate(42); >>
|
||||
```
|
||||
|
||||
The value passed to the `$terminate()` function is known as the "user terminate
|
||||
code".
|
||||
If the parser returns a `P_USER_TERMINATED` result code, then the user
|
||||
terminate code can be accessed using the `p_user_terminate_code()` API
|
||||
function.
|
||||
|
||||
#> Propane generated API
|
||||
|
||||
By default, Propane uses a prefix of `p_` when generating a lexer/parser.
|
||||
This prefix is used for all publicly declared types and functions.
|
||||
The uppercase version of the prefix is used for all constant values.
|
||||
|
||||
This section documents the generated API using the default `p_` or `P_` names.
|
||||
|
||||
##> Constants
|
||||
|
||||
Propane generates the following result code constants:
|
||||
|
||||
* `P_SUCCESS`: A successful decode/lex/parse operation has taken place.
|
||||
* `P_DECODE_ERROR`: An error occurred when decoding UTF-8 input.
|
||||
* `P_UNEXPECTED_INPUT`: Input was received by the lexer that does not match any lexer pattern.
|
||||
* `P_UNEXPECTED_TOKEN`: A token was seen in a location that does not match any parser rule.
|
||||
* `P_DROP`: The lexer matched a drop pattern.
|
||||
* `P_EOF`: The lexer reached the end of the input string.
|
||||
* `P_USER_TERMINATED`: A parser user code block has requested to terminate the parser.
|
||||
|
||||
Result codes are returned by the functions `p_decode_input()`, `p_lex()`, and `p_parse()`.
|
||||
|
||||
##> Types
|
||||
|
||||
### `p_context_t`
|
||||
|
||||
Propane defines a `p_context_t` structure type.
|
||||
The structure is intended to be used opaquely and stores information related to
|
||||
the state of the lexer and parser.
|
||||
Integrating code must define an instance of the `p_context_t` structure.
|
||||
A pointer to this instance is passed to the generated functions.
|
||||
|
||||
### `p_position_t`
|
||||
|
||||
The `p_position_t` structure contains two fields `row` and `col`.
|
||||
These fields contain the 0-based row and column describing a parser position.
|
||||
|
||||
##> Functions
|
||||
|
||||
### `p_context_init`
|
||||
|
||||
The `p_context_init()` function must be called to initialize the context
|
||||
structure.
|
||||
The input to be used for lexing/parsing is passed in when initializing the
|
||||
context structure.
|
||||
|
||||
C example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input, input_length);
|
||||
```
|
||||
|
||||
D example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
```
|
||||
|
||||
### `p_parse`
|
||||
|
||||
The `p_parse()` function is the main entry point to the parser.
|
||||
It must be passed a pointer to an initialized context structure.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input, input_length);
|
||||
size_t result = p_parse(&context);
|
||||
```
|
||||
|
||||
### `p_result`
|
||||
|
||||
The `p_result()` function can be used to retrieve the final parse value after
|
||||
`p_parse()` returns a `P_SUCCESS` value.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input, input_length);
|
||||
size_t result = p_parse(&context);
|
||||
if (p_parse(&context) == P_SUCCESS)
|
||||
{
|
||||
result = p_result(&context);
|
||||
}
|
||||
```
|
||||
|
||||
### `p_position`
|
||||
|
||||
The `p_position()` function can be used to retrieve the parser position where
|
||||
an error occurred.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input, input_length);
|
||||
size_t result = p_parse(&context);
|
||||
if (p_parse(&context) == P_UNEXPECTED_TOKEN)
|
||||
{
|
||||
p_position_t error_position = p_position(&context);
|
||||
fprintf(stderr, "Error: unexpected token at row %u column %u\n",
|
||||
error_position.row + 1, error_position.col + 1);
|
||||
}
|
||||
```
|
||||
|
||||
### `p_user_terminate_code`
|
||||
|
||||
The `p_user_terminate_code()` function can be used to retrieve the user
|
||||
terminate code after `p_parse()` returns a `P_USER_TERMINATED` value.
|
||||
User terminate codes are arbitrary values that can be defined by the user to
|
||||
be returned when the user requests to terminate parsing.
|
||||
They have no particular meaning to Propane.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
if (p_parse(&context) == P_USER_TERMINATED)
|
||||
{
|
||||
size_t user_terminate_code = p_user_terminate_code(&context);
|
||||
}
|
||||
```
|
||||
|
||||
### `p_token`
|
||||
|
||||
The `p_token()` function can be used to retrieve the current parse token.
|
||||
This is useful after `p_parse()` returns a `P_UNEXPECTED_TOKEN` value.
|
||||
terminate code after `p_parse()` returns a `P_USER_TERMINATED` value to
|
||||
indicate what token the parser was not expecting.
|
||||
|
||||
Example:
|
||||
|
||||
```
|
||||
if (p_parse(&context) == P_UNEXPECTED_TOKEN)
|
||||
{
|
||||
p_token_t unexpected_token = p_token(&context);
|
||||
}
|
||||
```
|
||||
|
||||
##> Data
|
||||
|
||||
### `p_token_names`
|
||||
|
||||
The `p_token_names` array contains the grammar-specified token names.
|
||||
It is indexed by the token ID.
|
||||
|
||||
C example:
|
||||
|
||||
```
|
||||
p_context_t context;
|
||||
p_context_init(&context, input, input_length);
|
||||
size_t result = p_parse(&context);
|
||||
if (p_parse(&context) == P_UNEXPECTED_TOKEN)
|
||||
{
|
||||
p_position_t error_position = p_position(&context);
|
||||
fprintf(stderr, "Error: unexpected token `%s' at row %u column %u\n",
|
||||
p_token_names[context.token],
|
||||
error_position.row + 1, error_position.col + 1);
|
||||
}
|
||||
```
|
||||
|
||||
#> License
|
||||
|
||||
Propane is licensed under the terms of the MIT License:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
require "erb"
|
||||
require "set"
|
||||
require "stringio"
|
||||
require_relative "propane/assets"
|
||||
require_relative "propane/cli"
|
||||
require_relative "propane/code_point_range"
|
||||
require_relative "propane/fa"
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
class Propane
|
||||
module Assets
|
||||
class << self
|
||||
def get(name)
|
||||
path = File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/#{name}")
|
||||
File.binread(path)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
+10
-85
@@ -10,28 +10,15 @@ class Propane
|
||||
else
|
||||
@log = StringIO.new
|
||||
end
|
||||
@language =
|
||||
if output_file =~ /\.([a-z]+)$/
|
||||
$1
|
||||
else
|
||||
"d"
|
||||
end
|
||||
@classname = @grammar.classname || File.basename(output_file).sub(%r{[^a-zA-Z0-9].*}, "").capitalize
|
||||
process_grammar!
|
||||
end
|
||||
|
||||
def generate
|
||||
extensions = [@language]
|
||||
if @language == "c"
|
||||
extensions += %w[h]
|
||||
end
|
||||
extensions.each do |extension|
|
||||
template = Assets.get("parser.#{extension}.erb")
|
||||
erb = ERB.new(template, trim_mode: "<>")
|
||||
output_file = @output_file.sub(%r{\.[a-z]+$}, ".#{extension}")
|
||||
result = erb.result(binding.clone)
|
||||
File.open(output_file, "wb") do |fh|
|
||||
fh.write(result)
|
||||
end
|
||||
erb = ERB.new(File.read(File.join(File.dirname(File.expand_path(__FILE__)), "../../assets/parser.d.erb")), trim_mode: "<>")
|
||||
result = erb.result(binding.clone)
|
||||
File.open(@output_file, "wb") do |fh|
|
||||
fh.write(result)
|
||||
end
|
||||
@log.close
|
||||
end
|
||||
@@ -196,44 +183,19 @@ class Propane
|
||||
# Expanded user code block.
|
||||
def expand_code(code, parser, rule, pattern)
|
||||
code = code.gsub(/\$token\(([$\w]+)\)/) do |match|
|
||||
"TOKEN_#{Token.code_name($1)}"
|
||||
end
|
||||
code = code.gsub(/\$terminate\((.*)\);/) do |match|
|
||||
user_terminate_code = $1
|
||||
retval = rule ? "P_USER_TERMINATED" : "TERMINATE_TOKEN_ID"
|
||||
case @language
|
||||
when "c"
|
||||
"context->user_terminate_code = (#{user_terminate_code}); return #{retval};"
|
||||
when "d"
|
||||
"context.user_terminate_code = (#{user_terminate_code}); return #{retval};"
|
||||
end
|
||||
"Token(TOKEN_#{Token.code_name($1)})"
|
||||
end
|
||||
if parser
|
||||
code = code.gsub(/\$\$/) do |match|
|
||||
case @language
|
||||
when "c"
|
||||
"_pvalue->v_#{rule.ptypename}"
|
||||
when "d"
|
||||
"_pvalue.v_#{rule.ptypename}"
|
||||
end
|
||||
"_pvalue.v_#{rule.ptypename}"
|
||||
end
|
||||
code = code.gsub(/\$(\d+)/) do |match|
|
||||
index = $1.to_i
|
||||
case @language
|
||||
when "c"
|
||||
"state_values_stack_index(statevalues, -1 - (int)n_states + #{index})->pvalue.v_#{rule.components[index - 1].ptypename}"
|
||||
when "d"
|
||||
"statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
|
||||
end
|
||||
"statevalues[$-1-n_states+#{index}].pvalue.v_#{rule.components[index - 1].ptypename}"
|
||||
end
|
||||
else
|
||||
code = code.gsub(/\$\$/) do |match|
|
||||
case @language
|
||||
when "c"
|
||||
"out_token_info->pvalue.v_#{pattern.ptypename}"
|
||||
when "d"
|
||||
"out_token_info.pvalue.v_#{pattern.ptypename}"
|
||||
end
|
||||
"result.pvalue.v_#{pattern.ptypename}"
|
||||
end
|
||||
code = code.gsub(/\$mode\(([a-zA-Z_][a-zA-Z_0-9]*)\)/) do |match|
|
||||
mode_name = $1
|
||||
@@ -241,12 +203,7 @@ class Propane
|
||||
unless mode_id
|
||||
raise Error.new("Lexer mode '#{mode_name}' not found")
|
||||
end
|
||||
case @language
|
||||
when "c"
|
||||
"context->mode = #{mode_id}u"
|
||||
when "d"
|
||||
"context.mode = #{mode_id}u"
|
||||
end
|
||||
"m_mode = #{mode_id}u"
|
||||
end
|
||||
end
|
||||
code
|
||||
@@ -263,38 +220,6 @@ class Propane
|
||||
[start_rule.ptypename, @grammar.ptypes[start_rule.ptypename]]
|
||||
end
|
||||
|
||||
# Get an unsigned integer type that can hold the given maximum value.
|
||||
#
|
||||
# @param max [Integer]
|
||||
# Maximum value to store.
|
||||
#
|
||||
# @return [String]
|
||||
# Type.
|
||||
def get_type_for(max)
|
||||
if max <= 0xFF
|
||||
case @language
|
||||
when "c"
|
||||
"uint8_t"
|
||||
when "d"
|
||||
"ubyte"
|
||||
end
|
||||
elsif max <= 0xFFFF
|
||||
case @language
|
||||
when "c"
|
||||
"uint16_t"
|
||||
when "d"
|
||||
"ushort"
|
||||
end
|
||||
else
|
||||
case @language
|
||||
when "c"
|
||||
"uint32_t"
|
||||
else
|
||||
"uint"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
+18
-34
@@ -5,25 +5,24 @@ class Propane
|
||||
# Reserve identifiers beginning with a double-underscore for internal use.
|
||||
IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
|
||||
|
||||
attr_reader :classname
|
||||
attr_reader :modulename
|
||||
attr_reader :patterns
|
||||
attr_reader :rules
|
||||
attr_reader :tokens
|
||||
attr_reader :code_blocks
|
||||
attr_reader :ptypes
|
||||
attr_reader :prefix
|
||||
|
||||
def initialize(input)
|
||||
@patterns = []
|
||||
@tokens = []
|
||||
@rules = []
|
||||
@code_blocks = {}
|
||||
@code_blocks = []
|
||||
@line_number = 1
|
||||
@next_line_number = @line_number
|
||||
@mode = nil
|
||||
@input = input.gsub("\r\n", "\n")
|
||||
@ptypes = {"default" => "void *"}
|
||||
@prefix = "p_"
|
||||
parse_grammar!
|
||||
end
|
||||
|
||||
@@ -31,14 +30,6 @@ class Propane
|
||||
@ptypes["default"]
|
||||
end
|
||||
|
||||
def invalid_token_id
|
||||
@tokens.size
|
||||
end
|
||||
|
||||
def terminate_token_id
|
||||
@tokens.size + 1
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_grammar!
|
||||
@@ -52,6 +43,7 @@ class Propane
|
||||
elsif parse_comment_line!
|
||||
elsif @mode.nil? && parse_mode_label!
|
||||
elsif parse_module_statement!
|
||||
elsif parse_class_statement!
|
||||
elsif parse_ptype_statement!
|
||||
elsif parse_pattern_statement!
|
||||
elsif parse_token_statement!
|
||||
@@ -59,7 +51,6 @@ class Propane
|
||||
elsif parse_drop_statement!
|
||||
elsif parse_rule_statement!
|
||||
elsif parse_code_block_statement!
|
||||
elsif parse_prefix_statement!
|
||||
else
|
||||
if @input.size > 25
|
||||
@input = @input.slice(0..20) + "..."
|
||||
@@ -92,6 +83,16 @@ class Propane
|
||||
end
|
||||
end
|
||||
|
||||
def parse_class_statement!
|
||||
if consume!(/class\s+/)
|
||||
md = consume!(/([\w.]+)\s*/, "expected class name")
|
||||
@classname = md[1]
|
||||
consume!(/;/, "expected `;'")
|
||||
@mode = nil
|
||||
true
|
||||
end
|
||||
end
|
||||
|
||||
def parse_ptype_statement!
|
||||
if consume!(/ptype\s+/)
|
||||
name = "default"
|
||||
@@ -147,7 +148,7 @@ class Propane
|
||||
end
|
||||
consume!(/\s+/)
|
||||
consume!(/;/, "expected `;'")
|
||||
@patterns << Pattern.new(pattern: pattern, line_number: @line_number, mode: @mode)
|
||||
@patterns << Pattern.new(pattern: pattern, line_number: @line_number, drop: true, mode: @mode)
|
||||
@mode = nil
|
||||
true
|
||||
end
|
||||
@@ -183,27 +184,13 @@ class Propane
|
||||
end
|
||||
|
||||
def parse_code_block_statement!
|
||||
if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
|
||||
name, code = md[1..2]
|
||||
code.sub!(/\A\n/, "")
|
||||
code += "\n" unless code.end_with?("\n")
|
||||
if @code_blocks[name]
|
||||
@code_blocks[name] += code
|
||||
else
|
||||
@code_blocks[name] = code
|
||||
end
|
||||
if code = parse_code_block!
|
||||
@code_blocks << code
|
||||
@mode = nil
|
||||
true
|
||||
end
|
||||
end
|
||||
|
||||
def parse_prefix_statement!
|
||||
if md = consume!(/prefix\s+(#{IDENTIFIER_REGEX})\s*;/)
|
||||
@prefix = md[1]
|
||||
true
|
||||
end
|
||||
end
|
||||
|
||||
def parse_pattern!
|
||||
if md = consume!(%r{/})
|
||||
pattern = ""
|
||||
@@ -224,11 +211,8 @@ class Propane
|
||||
end
|
||||
|
||||
def parse_code_block!
|
||||
if md = consume!(/<<(.*?)>>\n/m)
|
||||
code = md[1]
|
||||
code.sub!(/\A\n/, "")
|
||||
code += "\n" unless code.end_with?("\n")
|
||||
code
|
||||
if md = consume!(/<<\n(.*?)^>>\n/m)
|
||||
md[1]
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
+47
-46
@@ -1,13 +1,55 @@
|
||||
class Propane
|
||||
class Lexer
|
||||
|
||||
attr_reader :state_table
|
||||
attr_reader :transition_table
|
||||
attr_reader :mode_table
|
||||
|
||||
def initialize(grammar)
|
||||
@grammar = grammar
|
||||
build_tables!
|
||||
end
|
||||
|
||||
def build_tables
|
||||
@modes = @grammar.patterns.group_by do |pattern|
|
||||
pattern.mode
|
||||
end.transform_values do |patterns|
|
||||
{dfa: DFA.new(patterns)}
|
||||
end
|
||||
@modes.each_with_index do |(mode_name, mode_info), index|
|
||||
mode_info[:id] = index
|
||||
end
|
||||
transition_table = []
|
||||
state_table = []
|
||||
mode_table = []
|
||||
@modes.each do |mode_name, mode_info|
|
||||
state_table_offset = state_table.size
|
||||
mode_table << {
|
||||
state_table_offset: state_table_offset,
|
||||
}
|
||||
states = mode_info[:dfa].enumerate
|
||||
states.each do |state, id|
|
||||
drop = state.accepts && state.accepts.drop?
|
||||
token =
|
||||
if state.accepts && state.accepts.token
|
||||
state.accepts.token.id
|
||||
end
|
||||
code_id =
|
||||
if state.accepts && state.accepts.code_id
|
||||
state.accepts.code_id
|
||||
end
|
||||
state_table << {
|
||||
transition_table_index: transition_table.size,
|
||||
n_transitions: state.transitions.size,
|
||||
drop: drop,
|
||||
token: token,
|
||||
code_id: code_id,
|
||||
}
|
||||
state.transitions.each do |transition|
|
||||
transition_table << {
|
||||
first: transition.code_point_range.first,
|
||||
last: transition.code_point_range.last,
|
||||
destination: states[transition.destination] + state_table_offset,
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
[transition_table, state_table, mode_table]
|
||||
end
|
||||
|
||||
# Get ID for a mode.
|
||||
@@ -23,46 +65,5 @@ class Propane
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_tables!
|
||||
@modes = @grammar.patterns.group_by do |pattern|
|
||||
pattern.mode
|
||||
end.transform_values do |patterns|
|
||||
{dfa: DFA.new(patterns)}
|
||||
end
|
||||
@modes.each_with_index do |(mode_name, mode_info), index|
|
||||
mode_info[:id] = index
|
||||
end
|
||||
@state_table = []
|
||||
@transition_table = []
|
||||
@mode_table = []
|
||||
@modes.each do |mode_name, mode_info|
|
||||
state_table_offset = @state_table.size
|
||||
@mode_table << {
|
||||
state_table_offset: state_table_offset,
|
||||
}
|
||||
states = mode_info[:dfa].enumerate
|
||||
states.each do |state, id|
|
||||
token = state.accepts && state.accepts.token && state.accepts.token.id
|
||||
code_id = state.accepts && state.accepts.code_id && state.accepts.code_id
|
||||
@state_table << {
|
||||
transition_table_index: @transition_table.size,
|
||||
n_transitions: state.transitions.size,
|
||||
accepts: !!state.accepts,
|
||||
token: token,
|
||||
code_id: code_id,
|
||||
}
|
||||
state.transitions.each do |transition|
|
||||
@transition_table << {
|
||||
first: transition.code_point_range.first,
|
||||
last: transition.code_point_range.last,
|
||||
destination: states[transition.destination] + state_table_offset,
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
+16
-21
@@ -2,11 +2,6 @@ class Propane
|
||||
|
||||
class Parser
|
||||
|
||||
attr_reader :state_table
|
||||
attr_reader :shift_table
|
||||
attr_reader :reduce_table
|
||||
attr_reader :rule_sets
|
||||
|
||||
def initialize(grammar, rule_sets, log)
|
||||
@grammar = grammar
|
||||
@rule_sets = rule_sets
|
||||
@@ -38,15 +33,12 @@ class Propane
|
||||
|
||||
build_reduce_actions!
|
||||
write_log!
|
||||
build_tables!
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_tables!
|
||||
@state_table = []
|
||||
@shift_table = []
|
||||
@reduce_table = []
|
||||
def build_tables
|
||||
shift_table = []
|
||||
state_table = []
|
||||
reduce_table = []
|
||||
@item_sets.each do |item_set|
|
||||
shift_entries = item_set.following_symbols.map do |following_symbol|
|
||||
state_id =
|
||||
@@ -56,14 +48,14 @@ class Propane
|
||||
item_set.following_item_set[following_symbol].id
|
||||
end
|
||||
{
|
||||
symbol_id: following_symbol.id,
|
||||
token_id: following_symbol.id,
|
||||
state_id: state_id,
|
||||
}
|
||||
end
|
||||
reduce_entries =
|
||||
case ra = item_set.reduce_actions
|
||||
when Rule
|
||||
[{token_id: @grammar.invalid_token_id, rule_id: ra.id,
|
||||
[{token_id: @grammar.tokens.size, rule_id: ra.id,
|
||||
rule_set_id: ra.rule_set.id, n_states: ra.components.size}]
|
||||
when Hash
|
||||
ra.map do |token, rule|
|
||||
@@ -73,17 +65,20 @@ class Propane
|
||||
else
|
||||
[]
|
||||
end
|
||||
@state_table << {
|
||||
shift_index: @shift_table.size,
|
||||
state_table << {
|
||||
shift_index: shift_table.size,
|
||||
n_shifts: shift_entries.size,
|
||||
reduce_index: @reduce_table.size,
|
||||
reduce_index: reduce_table.size,
|
||||
n_reduces: reduce_entries.size,
|
||||
}
|
||||
@shift_table += shift_entries
|
||||
@reduce_table += reduce_entries
|
||||
shift_table += shift_entries
|
||||
reduce_table += reduce_entries
|
||||
end
|
||||
[state_table, shift_table, reduce_table]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def process_item_set(item_set)
|
||||
item_set.following_symbols.each do |following_symbol|
|
||||
unless following_symbol.name == "$EOF"
|
||||
@@ -134,7 +129,7 @@ class Propane
|
||||
# possible following tokens, we will only look in the item sets leading
|
||||
# up to this one. This restriction gives us a more precise lookahead set,
|
||||
# and allows us to parse LALR grammars.
|
||||
item_sets = Set[item_set] + item_set.leading_item_sets
|
||||
item_sets = item_set.leading_item_sets
|
||||
reduce_rules.reduce({}) do |reduce_actions, reduce_rule|
|
||||
lookahead_tokens_for_rule = build_lookahead_tokens_to_reduce(reduce_rule, item_sets)
|
||||
lookahead_tokens_for_rule.each do |lookahead_token|
|
||||
@@ -247,7 +242,7 @@ class Propane
|
||||
@log.puts " Reduce actions:"
|
||||
case item_set.reduce_actions
|
||||
when Rule
|
||||
@log.puts " * => rule #{item_set.reduce_actions.id}, rule set #{@rule_sets[item_set.reduce_actions.name].id} (#{item_set.reduce_actions.name})"
|
||||
@log.puts " * => #{item_set.reduce_actions.id} (#{item_set.reduce_actions.name})"
|
||||
when Hash
|
||||
item_set.reduce_actions.each do |token, rule|
|
||||
@log.puts " lookahead #{token.name} => #{rule.name} (#{rule.id}), rule set ##{rule.rule_set.id}"
|
||||
|
||||
@@ -87,24 +87,14 @@ class Propane
|
||||
|
||||
# Set of ItemSets that lead to this ItemSet.
|
||||
#
|
||||
# This set includes this ItemSet.
|
||||
#
|
||||
# @return [Set<ItemSet>]
|
||||
# Set of all ItemSets that lead up to this ItemSet.
|
||||
def leading_item_sets
|
||||
result = Set.new
|
||||
eval_sets = Set[self]
|
||||
evaled = Set.new
|
||||
while eval_sets.size > 0
|
||||
eval_set = eval_sets.first
|
||||
eval_sets.delete(eval_set)
|
||||
evaled << eval_set
|
||||
eval_set.in_sets.each do |in_set|
|
||||
result << in_set
|
||||
unless evaled.include?(in_set)
|
||||
eval_sets << in_set
|
||||
end
|
||||
end
|
||||
@in_sets.reduce(Set[self]) do |result, item_set|
|
||||
result + item_set.leading_item_sets
|
||||
end
|
||||
result
|
||||
end
|
||||
|
||||
# Represent the ItemSet as a String.
|
||||
|
||||
@@ -40,6 +40,8 @@ class Propane
|
||||
# Optional parameters.
|
||||
# @option options [String, nil] :code
|
||||
# Code block to execute when the pattern is matched.
|
||||
# @option options [Boolean] :drop
|
||||
# Whether this is a drop pattern.
|
||||
# @option options [String, nil] :pattern
|
||||
# Pattern.
|
||||
# @option options [Token, nil] :token
|
||||
@@ -50,6 +52,7 @@ class Propane
|
||||
# Lexer mode for this pattern.
|
||||
def initialize(options)
|
||||
@code = options[:code]
|
||||
@drop = options[:drop]
|
||||
@pattern = options[:pattern]
|
||||
@token = options[:token]
|
||||
@line_number = options[:line_number]
|
||||
@@ -60,6 +63,14 @@ class Propane
|
||||
@nfa = regex.nfa
|
||||
end
|
||||
|
||||
# Whether the pattern is a drop pattern.
|
||||
#
|
||||
# @return [Boolean]
|
||||
# Whether the pattern is a drop pattern.
|
||||
def drop?
|
||||
@drop
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
@@ -134,18 +134,8 @@ class Propane
|
||||
else
|
||||
c = @pattern.slice!(0)
|
||||
case c
|
||||
when "a"
|
||||
CharacterRangeUnit.new("\a", "\a")
|
||||
when "b"
|
||||
CharacterRangeUnit.new("\b", "\b")
|
||||
when "d"
|
||||
CharacterRangeUnit.new("0", "9")
|
||||
when "f"
|
||||
CharacterRangeUnit.new("\f", "\f")
|
||||
when "n"
|
||||
CharacterRangeUnit.new("\n", "\n")
|
||||
when "r"
|
||||
CharacterRangeUnit.new("\r", "\r")
|
||||
when "s"
|
||||
ccu = CharacterClassUnit.new
|
||||
ccu << CharacterRangeUnit.new(" ")
|
||||
@@ -155,10 +145,6 @@ class Propane
|
||||
ccu << CharacterRangeUnit.new("\f")
|
||||
ccu << CharacterRangeUnit.new("\v")
|
||||
ccu
|
||||
when "t"
|
||||
CharacterRangeUnit.new("\t", "\t")
|
||||
when "v"
|
||||
CharacterRangeUnit.new("\v", "\v")
|
||||
else
|
||||
CharacterRangeUnit.new(c)
|
||||
end
|
||||
|
||||
@@ -4,7 +4,7 @@ class Propane
|
||||
|
||||
# @return [Integer]
|
||||
# ID of the RuleSet.
|
||||
attr_reader :id
|
||||
attr_accessor :id
|
||||
|
||||
# @return [String]
|
||||
# Name of the RuleSet.
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
class Propane
|
||||
VERSION = "1.2.0"
|
||||
VERSION = "0.1.0"
|
||||
end
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exec bundle exec ruby -Ilib bin/propane "$@"
|
||||
+1
-24
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
require "erb"
|
||||
require "fileutils"
|
||||
require "digest/md5"
|
||||
|
||||
@@ -14,24 +13,6 @@ START_FILE = "bin/#{PROG_NAME}"
|
||||
LIB_DIR = "lib"
|
||||
DIST = "dist"
|
||||
|
||||
ASSETS_TEMPLATE = <<EOF
|
||||
class Propane
|
||||
module Assets
|
||||
class << self
|
||||
def get(name)
|
||||
case name
|
||||
<% Dir.glob("assets/*").each do |asset_file| %>
|
||||
when <%= File.basename(asset_file).inspect %>
|
||||
<%= File.binread(asset_file).inspect %>
|
||||
<% end %>
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
EOF
|
||||
|
||||
assets_module = ERB.new(ASSETS_TEMPLATE, trim_mode: "<>").result
|
||||
files_processed = {}
|
||||
combined_file = []
|
||||
|
||||
@@ -44,11 +25,7 @@ combine_files = lambda do |file|
|
||||
if File.exist?(path)
|
||||
unless files_processed[path]
|
||||
files_processed[path] = true
|
||||
if require_name == "propane/assets"
|
||||
combined_file << assets_module
|
||||
else
|
||||
combine_files[path]
|
||||
end
|
||||
combine_files[path]
|
||||
end
|
||||
else
|
||||
raise "require path #{path.inspect} not found"
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
<<header
|
||||
#include "json_types.h"
|
||||
#include "testutils.h"
|
||||
>>
|
||||
<<
|
||||
#include "math.h"
|
||||
#include <stdbool.h>
|
||||
static str_t string_value;
|
||||
>>
|
||||
|
||||
ptype JSONValue *;
|
||||
|
||||
drop /\s+/;
|
||||
token lbrace /\{/;
|
||||
token rbrace /\}/;
|
||||
token lbracket /\[/;
|
||||
token rbracket /\]/;
|
||||
token comma /,/;
|
||||
token colon /:/;
|
||||
token number /-?(0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/ <<
|
||||
double n = 0.0;
|
||||
bool negative = false;
|
||||
size_t i = 0u;
|
||||
if (match[i] == '-')
|
||||
{
|
||||
negative = true;
|
||||
i++;
|
||||
}
|
||||
while ('0' <= match[i] && match[i] <= '9')
|
||||
{
|
||||
n *= 10.0;
|
||||
n += (match[i] - '0');
|
||||
i++;
|
||||
}
|
||||
if (match[i] == '.')
|
||||
{
|
||||
i++;
|
||||
double mult = 0.1;
|
||||
while ('0' <= match[i] && match[i] <= '9')
|
||||
{
|
||||
n += mult * (match[i] - '0');
|
||||
mult /= 10.0;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if (match[i] == 'e' || match[i] == 'E')
|
||||
{
|
||||
bool exp_negative = false;
|
||||
i++;
|
||||
if (match[i] == '-')
|
||||
{
|
||||
exp_negative = true;
|
||||
i++;
|
||||
}
|
||||
else if (match[i] == '+')
|
||||
{
|
||||
i++;
|
||||
}
|
||||
long exp = 0.0;
|
||||
while ('0' <= match[i] && match[i] <= '9')
|
||||
{
|
||||
exp *= 10;
|
||||
exp += (match[i] - '0');
|
||||
i++;
|
||||
}
|
||||
if (exp_negative)
|
||||
{
|
||||
exp = -exp;
|
||||
}
|
||||
n = pow(n, exp);
|
||||
}
|
||||
if (negative)
|
||||
{
|
||||
n = -n;
|
||||
}
|
||||
$$ = JSONValue_new(JSON_NUMBER);
|
||||
$$->number = n;
|
||||
>>
|
||||
token true <<
|
||||
$$ = JSONValue_new(JSON_TRUE);
|
||||
>>
|
||||
token false <<
|
||||
$$ = JSONValue_new(JSON_FALSE);
|
||||
>>
|
||||
token null <<
|
||||
$$ = JSONValue_new(JSON_NULL);
|
||||
>>
|
||||
/"/ <<
|
||||
$mode(string);
|
||||
str_init(&string_value, "");
|
||||
>>
|
||||
string: token string /"/ <<
|
||||
$$ = JSONValue_new(JSON_STRING);
|
||||
$$->string = string_value;
|
||||
$mode(default);
|
||||
>>
|
||||
string: /\\"/ <<
|
||||
str_append(&string_value, "\"");
|
||||
>>
|
||||
string: /\\\\/ <<
|
||||
str_append(&string_value, "\\");
|
||||
>>
|
||||
string: /\\\// <<
|
||||
str_append(&string_value, "/");
|
||||
>>
|
||||
string: /\\b/ <<
|
||||
str_append(&string_value, "\b");
|
||||
>>
|
||||
string: /\\f/ <<
|
||||
str_append(&string_value, "\f");
|
||||
>>
|
||||
string: /\\n/ <<
|
||||
str_append(&string_value, "\n");
|
||||
>>
|
||||
string: /\\r/ <<
|
||||
str_append(&string_value, "\r");
|
||||
>>
|
||||
string: /\\t/ <<
|
||||
str_append(&string_value, "\t");
|
||||
>>
|
||||
string: /\\u[0-9a-fA-F]{4}/ <<
|
||||
/* Not actually going to encode the code point for this example... */
|
||||
char s[] = {'{', match[2], match[3], match[4], match[5], '}', 0};
|
||||
str_append(&string_value, s);
|
||||
>>
|
||||
string: /[^\\]/ <<
|
||||
char s[] = {match[0], 0};
|
||||
str_append(&string_value, s);
|
||||
>>
|
||||
Start -> Value <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> string <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> number <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> Object <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> Array <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> true <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> false <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Value -> null <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Object -> lbrace rbrace <<
|
||||
$$ = JSONObject_new();
|
||||
>>
|
||||
Object -> lbrace KeyValues rbrace <<
|
||||
$$ = $2;
|
||||
>>
|
||||
KeyValues -> KeyValue <<
|
||||
$$ = $1;
|
||||
>>
|
||||
KeyValues -> KeyValues comma KeyValue <<
|
||||
JSONObject_append($1, $3->object.entries[0].name, $3->object.entries[0].value);
|
||||
$$ = $1;
|
||||
>>
|
||||
KeyValue -> string colon Value <<
|
||||
$$ = JSONObject_new();
|
||||
JSONObject_append($$, str_cstr(&$1->string), $3);
|
||||
>>
|
||||
Array -> lbracket rbracket <<
|
||||
$$ = JSONArray_new();
|
||||
>>
|
||||
Array -> lbracket Values rbracket <<
|
||||
$$ = $2;
|
||||
>>
|
||||
Values -> Value <<
|
||||
$$ = $1;
|
||||
>>
|
||||
Values -> Values comma Value <<
|
||||
JSONArray_append($1, $3);
|
||||
$$ = $1;
|
||||
>>
|
||||
File renamed without changes.
@@ -1,64 +0,0 @@
|
||||
#include "json_types.h"
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include "testutils.h"
|
||||
|
||||
JSONValue * JSONValue_new(size_t id)
|
||||
{
|
||||
JSONValue * jv = calloc(1, sizeof(JSONValue));
|
||||
jv->id = id;
|
||||
return jv;
|
||||
}
|
||||
|
||||
JSONValue * JSONObject_new(void)
|
||||
{
|
||||
JSONValue * jv = JSONValue_new(JSON_OBJECT);
|
||||
jv->object.size = 0u;
|
||||
return jv;
|
||||
}
|
||||
|
||||
void JSONObject_append(JSONValue * object, char const * name, JSONValue * value)
|
||||
{
|
||||
size_t const size = object->object.size;
|
||||
for (size_t i = 0u; i < size; i++)
|
||||
{
|
||||
if (strcmp(name, object->object.entries[i].name) == 0)
|
||||
{
|
||||
object->object.entries[i].value = value;
|
||||
return;
|
||||
}
|
||||
}
|
||||
size_t const new_size = size + 1;
|
||||
void * new_entries = malloc(sizeof(object->object.entries[0]) * new_size);
|
||||
if (size > 0)
|
||||
{
|
||||
memcpy(new_entries, object->object.entries, size * sizeof(object->object.entries[0]));
|
||||
free(object->object.entries);
|
||||
}
|
||||
object->object.entries = new_entries;
|
||||
object->object.entries[size].name = name;
|
||||
object->object.entries[size].value = value;
|
||||
object->object.size = new_size;
|
||||
}
|
||||
|
||||
JSONValue * JSONArray_new(void)
|
||||
{
|
||||
JSONValue * jv = JSONValue_new(JSON_ARRAY);
|
||||
jv->array.size = 0u;
|
||||
return jv;
|
||||
}
|
||||
|
||||
void JSONArray_append(JSONValue * array, JSONValue * value)
|
||||
{
|
||||
size_t const size = array->array.size;
|
||||
size_t const new_size = size + 1;
|
||||
JSONValue ** new_entries = malloc(sizeof(JSONValue *) * new_size);
|
||||
if (array->array.size > 0)
|
||||
{
|
||||
memcpy(new_entries, array->array.entries, sizeof(JSONValue *) * size);
|
||||
free(array->array.entries);
|
||||
}
|
||||
array->array.entries = new_entries;
|
||||
array->array.entries[size] = value;
|
||||
array->array.size = new_size;
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <stddef.h>
|
||||
#include "testutils.h"
|
||||
|
||||
#define JSON_OBJECT 0u
|
||||
#define JSON_ARRAY 1u
|
||||
#define JSON_NUMBER 2u
|
||||
#define JSON_STRING 3u
|
||||
#define JSON_TRUE 4u
|
||||
#define JSON_FALSE 5u
|
||||
#define JSON_NULL 6u
|
||||
|
||||
typedef struct JSONValue_s
|
||||
{
|
||||
size_t id;
|
||||
union
|
||||
{
|
||||
struct
|
||||
{
|
||||
size_t size;
|
||||
struct
|
||||
{
|
||||
char const * name;
|
||||
struct JSONValue_s * value;
|
||||
} * entries;
|
||||
} object;
|
||||
struct
|
||||
{
|
||||
size_t size;
|
||||
struct JSONValue_s ** entries;
|
||||
} array;
|
||||
double number;
|
||||
str_t string;
|
||||
};
|
||||
} JSONValue;
|
||||
|
||||
JSONValue * JSONValue_new(size_t id);
|
||||
|
||||
JSONValue * JSONObject_new(void);
|
||||
|
||||
void JSONObject_append(JSONValue * object, char const * name, JSONValue * value);
|
||||
|
||||
JSONValue * JSONArray_new(void);
|
||||
|
||||
void JSONArray_append(JSONValue * array, JSONValue * value);
|
||||
@@ -5,6 +5,7 @@ class Propane
|
||||
# Comment line
|
||||
|
||||
module a.b;
|
||||
class Foobar;
|
||||
ptype XYZ * ;
|
||||
|
||||
token while;
|
||||
@@ -29,51 +30,51 @@ B -> <<
|
||||
>>
|
||||
EOF
|
||||
grammar = Grammar.new(input)
|
||||
expect(grammar.classname).to eq "Foobar"
|
||||
expect(grammar.modulename).to eq "a.b"
|
||||
expect(grammar.ptype).to eq "XYZ *"
|
||||
expect(grammar.ptypes).to eq("default" => "XYZ *")
|
||||
expect(grammar.prefix).to eq "p_"
|
||||
|
||||
o = grammar.tokens.find {|token| token.name == "while"}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.line_number).to eq 6
|
||||
expect(o.line_number).to eq 7
|
||||
|
||||
o = grammar.patterns.find {|pattern| pattern.token == o}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.pattern).to eq "while"
|
||||
expect(o.line_number).to eq 6
|
||||
expect(o.line_number).to eq 7
|
||||
expect(o.code).to be_nil
|
||||
|
||||
o = grammar.tokens.find {|token| token.name == "id"}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.line_number).to eq 9
|
||||
expect(o.line_number).to eq 10
|
||||
|
||||
o = grammar.patterns.find {|pattern| pattern.token == o}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.pattern).to eq "[a-zA-Z_][a-zA-Z_0-9]*"
|
||||
expect(o.line_number).to eq 9
|
||||
expect(o.line_number).to eq 10
|
||||
expect(o.code).to be_nil
|
||||
|
||||
o = grammar.tokens.find {|token| token.name == "token_with_code"}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.line_number).to eq 11
|
||||
expect(o.line_number).to eq 12
|
||||
|
||||
o = grammar.patterns.find {|pattern| pattern.token == o}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.pattern).to eq "token_with_code"
|
||||
expect(o.line_number).to eq 11
|
||||
expect(o.line_number).to eq 12
|
||||
expect(o.code).to eq "Code for the token\n"
|
||||
|
||||
o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.line_number).to eq 15
|
||||
expect(o.line_number).to eq 16
|
||||
|
||||
o = grammar.patterns.find {|pattern| pattern.token == o}
|
||||
expect(o).to be_nil
|
||||
|
||||
o = grammar.patterns.find {|pattern| pattern.pattern == "\\s+"}
|
||||
expect(o).to_not be_nil
|
||||
expect(o.line_number).to eq 17
|
||||
expect(o.line_number).to eq 18
|
||||
expect(o.token).to be_nil
|
||||
expect(o.code).to be_nil
|
||||
|
||||
@@ -82,19 +83,19 @@ EOF
|
||||
o = grammar.rules[0]
|
||||
expect(o.name).to eq "A"
|
||||
expect(o.components).to eq %w[B]
|
||||
expect(o.line_number).to eq 19
|
||||
expect(o.line_number).to eq 20
|
||||
expect(o.code).to eq " a = 42;\n"
|
||||
|
||||
o = grammar.rules[1]
|
||||
expect(o.name).to eq "B"
|
||||
expect(o.components).to eq %w[C while id]
|
||||
expect(o.line_number).to eq 22
|
||||
expect(o.line_number).to eq 23
|
||||
expect(o.code).to be_nil
|
||||
|
||||
o = grammar.rules[2]
|
||||
expect(o.name).to eq "B"
|
||||
expect(o.components).to eq []
|
||||
expect(o.line_number).to eq 23
|
||||
expect(o.line_number).to eq 24
|
||||
expect(o.code).to eq " b = 0;\n"
|
||||
end
|
||||
|
||||
@@ -110,11 +111,8 @@ token code2 <<
|
||||
>>
|
||||
|
||||
tokenid token_with_no_pattern;
|
||||
|
||||
prefix myparser_;
|
||||
EOF
|
||||
grammar = Grammar.new(input)
|
||||
expect(grammar.prefix).to eq "myparser_"
|
||||
|
||||
o = grammar.tokens.find {|token| token.name == "code1"}
|
||||
expect(o).to_not be_nil
|
||||
|
||||
+162
-590
@@ -4,52 +4,12 @@ require "open3"
|
||||
Results = Struct.new(:stdout, :stderr, :status)
|
||||
|
||||
describe Propane do
|
||||
before(:all) do
|
||||
@statics = {}
|
||||
end
|
||||
|
||||
def write_grammar(grammar, options = {})
|
||||
options[:name] ||= ""
|
||||
File.write("spec/run/testparser#{options[:name]}.propane", grammar)
|
||||
def write_grammar(grammar)
|
||||
File.write("spec/run/testparser.propane", grammar)
|
||||
end
|
||||
|
||||
def build_parser(options = {})
|
||||
@statics[:build_test_id] ||= 0
|
||||
@statics[:build_test_id] += 1
|
||||
if ENV["dist_specs"]
|
||||
command = %W[dist/propane]
|
||||
else
|
||||
command = %W[ruby -I spec/run -r _simplecov_setup -I lib bin/propane]
|
||||
command_prefix =
|
||||
if ENV["partial_specs"]
|
||||
"p"
|
||||
else
|
||||
"b"
|
||||
end
|
||||
command_name = "#{command_prefix}#{@statics[:build_test_id]}"
|
||||
File.open("spec/run/_simplecov_setup.rb", "w") do |fh|
|
||||
fh.puts <<EOF
|
||||
require "bundler"
|
||||
Bundler.setup
|
||||
require "simplecov"
|
||||
class MyFormatter
|
||||
def format(*args)
|
||||
end
|
||||
end
|
||||
SimpleCov.start do
|
||||
command_name(#{command_name.inspect})
|
||||
filters.clear
|
||||
add_filter do |src|
|
||||
!(src.filename[SimpleCov.root])
|
||||
end
|
||||
formatter(MyFormatter)
|
||||
end
|
||||
# force color off
|
||||
ENV["TERM"] = nil
|
||||
EOF
|
||||
end
|
||||
end
|
||||
command += %W[spec/run/testparser#{options[:name]}.propane spec/run/testparser#{options[:name]}.#{options[:language]} --log spec/run/testparser#{options[:name]}.log]
|
||||
command = %w[./propane.sh spec/run/testparser.propane spec/run/testparser.d --log spec/run/testparser.log]
|
||||
if (options[:capture])
|
||||
stdout, stderr, status = Open3.capture3(*command)
|
||||
Results.new(stdout, stderr, status)
|
||||
@@ -59,18 +19,8 @@ EOF
|
||||
end
|
||||
end
|
||||
|
||||
def compile(test_files, options = {})
|
||||
test_files = Array(test_files)
|
||||
options[:parsers] ||= [""]
|
||||
parsers = options[:parsers].map do |name|
|
||||
"spec/run/testparser#{name}.#{options[:language]}"
|
||||
end
|
||||
case options[:language]
|
||||
when "c"
|
||||
result = system(*%w[gcc -Wall -o spec/run/testparser -Ispec -Ispec/run], *parsers, *test_files, "spec/testutils.c", "-lm")
|
||||
when "d"
|
||||
result = system(*%w[ldc2 --unittest -of spec/run/testparser -Ispec], *parsers, *test_files, "spec/testutils.d")
|
||||
end
|
||||
def compile(*test_files)
|
||||
result = system(*%w[ldc2 --unittest -of spec/run/testparser spec/run/testparser.d -Ispec], *test_files)
|
||||
expect(result).to be_truthy
|
||||
end
|
||||
|
||||
@@ -112,67 +62,27 @@ EOF
|
||||
FileUtils.mkdir_p("spec/run")
|
||||
end
|
||||
|
||||
%w[d c].each do |language|
|
||||
|
||||
context "#{language.upcase} language" do
|
||||
|
||||
it "generates a lexer" do
|
||||
write_grammar <<EOF
|
||||
it "generates a D lexer" do
|
||||
write_grammar <<EOF
|
||||
token int /\\d+/;
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
drop /\\s+/;
|
||||
Start -> Foo;
|
||||
Foo -> int <<>>
|
||||
Foo -> plus <<>>
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_lexer.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "detects a lexer error when an unknown character is seen" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
ptype int;
|
||||
token int /\\d+/ <<
|
||||
int v = 0;
|
||||
for (size_t i = 0u; i < match_length; i++)
|
||||
{
|
||||
v *= 10;
|
||||
v += (match[i] - '0');
|
||||
}
|
||||
$$ = v;
|
||||
Foo -> int <<
|
||||
>>
|
||||
Start -> int << $$ = $1; >>
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
ptype int;
|
||||
token int /\\d+/ <<
|
||||
int v;
|
||||
foreach (c; match)
|
||||
{
|
||||
v *= 10;
|
||||
v += (c - '0');
|
||||
}
|
||||
$$ = v;
|
||||
Foo -> plus <<
|
||||
>>
|
||||
Start -> int << $$ = $1; >>
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_lexer_unknown_character.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_d_lexer.d")
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "generates a parser" do
|
||||
write_grammar <<EOF
|
||||
it "generates a parser" do
|
||||
write_grammar <<EOF
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
token zero /0/;
|
||||
@@ -184,99 +94,21 @@ E -> B;
|
||||
B -> zero;
|
||||
B -> one;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
end
|
||||
build_parser
|
||||
end
|
||||
|
||||
it "generates a parser that does basic math - user guide example" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <math.h>
|
||||
>>
|
||||
|
||||
ptype size_t;
|
||||
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
token power /\\*\\*/;
|
||||
token integer /\\d+/ <<
|
||||
size_t v = 0u;
|
||||
for (size_t i = 0u; i < match_length; i++)
|
||||
{
|
||||
v *= 10;
|
||||
v += (match[i] - '0');
|
||||
}
|
||||
$$ = v;
|
||||
>>
|
||||
token lparen /\\(/;
|
||||
token rparen /\\)/;
|
||||
drop /\\s+/;
|
||||
|
||||
Start -> E1 << $$ = $1; >>
|
||||
E1 -> E2 << $$ = $1; >>
|
||||
E1 -> E1 plus E2 << $$ = $1 + $3; >>
|
||||
E2 -> E3 << $$ = $1; >>
|
||||
E2 -> E2 times E3 << $$ = $1 * $3; >>
|
||||
E3 -> E4 << $$ = $1; >>
|
||||
E3 -> E3 power E4 << $$ = (size_t)pow($1, $3); >>
|
||||
E4 -> integer << $$ = $1; >>
|
||||
E4 -> lparen E1 rparen << $$ = $2; >>
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.math;
|
||||
>>
|
||||
|
||||
ptype ulong;
|
||||
|
||||
token plus /\\+/;
|
||||
token times /\\*/;
|
||||
token power /\\*\\*/;
|
||||
token integer /\\d+/ <<
|
||||
ulong v;
|
||||
foreach (c; match)
|
||||
{
|
||||
v *= 10;
|
||||
v += (c - '0');
|
||||
}
|
||||
$$ = v;
|
||||
>>
|
||||
token lparen /\\(/;
|
||||
token rparen /\\)/;
|
||||
drop /\\s+/;
|
||||
|
||||
Start -> E1 << $$ = $1; >>
|
||||
E1 -> E2 << $$ = $1; >>
|
||||
E1 -> E1 plus E2 << $$ = $1 + $3; >>
|
||||
E2 -> E3 << $$ = $1; >>
|
||||
E2 -> E2 times E3 << $$ = $1 * $3; >>
|
||||
E3 -> E4 << $$ = $1; >>
|
||||
E3 -> E3 power E4 << $$ = pow($1, $3); >>
|
||||
E4 -> integer << $$ = $1; >>
|
||||
E4 -> lparen E1 rparen << $$ = $2; >>
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_basic_math_grammar.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "generates an SLR parser" do
|
||||
write_grammar <<EOF
|
||||
it "generates an SLR parser" do
|
||||
write_grammar <<EOF
|
||||
token one /1/;
|
||||
Start -> E;
|
||||
E -> one E;
|
||||
E -> one;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
end
|
||||
build_parser
|
||||
end
|
||||
|
||||
it "distinguishes between multiple identical rules with lookahead symbol" do
|
||||
write_grammar <<EOF
|
||||
it "distinguishes between multiple identical rules with lookahead symbol" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b;
|
||||
Start -> R1 a;
|
||||
@@ -284,14 +116,14 @@ Start -> R2 b;
|
||||
R1 -> a b;
|
||||
R2 -> a b;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_parser_identical_rules_lookahead.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_d_parser_identical_rules_lookahead.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "handles reducing a rule that could be arrived at from multiple states" do
|
||||
write_grammar <<EOF
|
||||
it "handles reducing a rule that could be arrived at from multiple states" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b;
|
||||
drop /\\s+/;
|
||||
@@ -299,32 +131,14 @@ Start -> a R1;
|
||||
Start -> b R1;
|
||||
R1 -> b;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_parser_rule_from_multiple_states.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_d_parser_rule_from_multiple_states.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "executes user code when matching lexer token" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
token abc <<
|
||||
printf("abc!\\n");
|
||||
>>
|
||||
token def;
|
||||
Start -> Abcs def;
|
||||
Abcs -> ;
|
||||
Abcs -> abc Abcs;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
it "executes user code when matching lexer token" do
|
||||
write_grammar <<EOF
|
||||
token abc <<
|
||||
writeln("abc!");
|
||||
>>
|
||||
@@ -333,125 +147,65 @@ Start -> Abcs def;
|
||||
Abcs -> ;
|
||||
Abcs -> abc Abcs;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_user_code.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"abc!",
|
||||
"pass1",
|
||||
"abc!",
|
||||
"abc!",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_user_code.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"abc!",
|
||||
"pass1",
|
||||
"abc!",
|
||||
"abc!",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
|
||||
it "supports a pattern statement" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
it "supports a pattern statement" do
|
||||
write_grammar <<EOF
|
||||
token abc;
|
||||
/def/ <<
|
||||
printf("def!\\n");
|
||||
writeln("def!");
|
||||
>>
|
||||
Start -> abc;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
token abc;
|
||||
/def/ << writeln("def!"); >>
|
||||
Start -> abc;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_pattern.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"def!",
|
||||
"pass1",
|
||||
"def!",
|
||||
"def!",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_pattern.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"def!",
|
||||
"pass1",
|
||||
"def!",
|
||||
"def!",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
|
||||
it "supports returning tokens from pattern code blocks" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
it "supports returning tokens from pattern code blocks" do
|
||||
write_grammar <<EOF
|
||||
token abc;
|
||||
/def/ << printf("def!\\n"); >>
|
||||
/ghi/ <<
|
||||
printf("ghi!\\n");
|
||||
return $token(abc);
|
||||
/def/ <<
|
||||
writeln("def!");
|
||||
>>
|
||||
Start -> abc;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
token abc;
|
||||
/def/ << writeln("def!"); >>
|
||||
/ghi/ <<
|
||||
writeln("ghi!");
|
||||
return $token(abc);
|
||||
>>
|
||||
Start -> abc;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_return_token_from_pattern.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"def!",
|
||||
"ghi!",
|
||||
"def!",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_return_token_from_pattern.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"def!",
|
||||
"ghi!",
|
||||
"def!",
|
||||
])
|
||||
end
|
||||
|
||||
it "supports lexer modes" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
token abc;
|
||||
token def;
|
||||
tokenid string;
|
||||
drop /\\s+/;
|
||||
/"/ <<
|
||||
printf("begin string mode\\n");
|
||||
$mode(string);
|
||||
>>
|
||||
string: /[^"]+/ <<
|
||||
printf("captured string\\n");
|
||||
>>
|
||||
string: /"/ <<
|
||||
$mode(default);
|
||||
return $token(string);
|
||||
>>
|
||||
Start -> abc string def;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
it "supports lexer modes" do
|
||||
write_grammar <<EOF
|
||||
token abc;
|
||||
token def;
|
||||
tokenid string;
|
||||
@@ -469,74 +223,68 @@ string: /"/ <<
|
||||
>>
|
||||
Start -> abc string def;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_lexer_modes.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"begin string mode",
|
||||
"captured string",
|
||||
"pass1",
|
||||
"begin string mode",
|
||||
"captured string",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_lexer_modes.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"begin string mode",
|
||||
"captured string",
|
||||
"pass1",
|
||||
"begin string mode",
|
||||
"captured string",
|
||||
"pass2",
|
||||
])
|
||||
end
|
||||
|
||||
it "executes user code associated with a parser rule" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
it "executes user code associated with a parser rule" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b;
|
||||
Start -> A B << printf("Start!\\n"); >>
|
||||
A -> a << printf("A!\\n"); >>
|
||||
B -> b << printf("B!\\n"); >>
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
Start -> A B <<
|
||||
writeln("Start!");
|
||||
>>
|
||||
A -> a <<
|
||||
writeln("A!");
|
||||
>>
|
||||
B -> b <<
|
||||
writeln("B!");
|
||||
>>
|
||||
token a;
|
||||
token b;
|
||||
Start -> A B << writeln("Start!"); >>
|
||||
A -> a << writeln("A!"); >>
|
||||
B -> b << writeln("B!"); >>
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_parser_rule_user_code.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"A!",
|
||||
"B!",
|
||||
"Start!",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_parser_rule_user_code.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"A!",
|
||||
"B!",
|
||||
"Start!",
|
||||
])
|
||||
end
|
||||
|
||||
it "parses lists" do
|
||||
write_grammar <<EOF
|
||||
ptype #{language == "c" ? "uint32_t" : "uint"};
|
||||
it "parses lists" do
|
||||
write_grammar <<EOF
|
||||
ptype uint;
|
||||
token a;
|
||||
Start -> As << $$ = $1; >>
|
||||
As -> << $$ = 0u; >>
|
||||
As -> As a << $$ = $1 + 1u; >>
|
||||
Start -> As <<
|
||||
$$ = $1;
|
||||
>>
|
||||
As -> <<
|
||||
$$ = 0u;
|
||||
>>
|
||||
As -> As a <<
|
||||
$$ = $1 + 1u;
|
||||
>>
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_parsing_lists.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
expect(results.stderr).to eq ""
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_parsing_lists.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
expect(results.stderr).to eq ""
|
||||
end
|
||||
|
||||
it "fails to generate a parser for a LR(1) grammar that is not LALR" do
|
||||
write_grammar <<EOF
|
||||
it "fails to generate a parser for a LR(1) grammar that is not LALR" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b;
|
||||
token c;
|
||||
@@ -549,62 +297,30 @@ Start -> b E d;
|
||||
E -> e;
|
||||
F -> e;
|
||||
EOF
|
||||
results = build_parser(capture: true, language: language)
|
||||
expect(results.status).to_not eq 0
|
||||
expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
|
||||
end
|
||||
results = build_parser(capture: true)
|
||||
expect(results.status).to_not eq 0
|
||||
expect(results.stderr).to match %r{reduce/reduce conflict.*\(E\).*\(F\)}
|
||||
end
|
||||
|
||||
it "provides matched text to user code blocks" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
>>
|
||||
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
|
||||
char * t = malloc(match_length + 1);
|
||||
strncpy(t, (char *)match, match_length);
|
||||
printf("Matched token is %s\\n", t);
|
||||
free(t);
|
||||
>>
|
||||
Start -> id;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
it "provides matched text to user code blocks" do
|
||||
write_grammar <<EOF
|
||||
token id /[a-zA-Z_][a-zA-Z0-9_]*/ <<
|
||||
writeln("Matched token is ", match);
|
||||
>>
|
||||
Start -> id;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_lexer_match_text.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"Matched token is identifier_123",
|
||||
"pass1",
|
||||
])
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_lexer_match_text.d")
|
||||
results = run
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"Matched token is identifier_123",
|
||||
"pass1",
|
||||
])
|
||||
end
|
||||
|
||||
it "allows storing a result value for the lexer" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
ptype uint64_t;
|
||||
token word /[a-z]+/ <<
|
||||
$$ = match_length;
|
||||
>>
|
||||
Start -> word <<
|
||||
$$ = $1;
|
||||
>>
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
it "allows storing a result value for the lexer" do
|
||||
write_grammar <<EOF
|
||||
ptype ulong;
|
||||
token word /[a-z]+/ <<
|
||||
$$ = match.length;
|
||||
@@ -613,160 +329,16 @@ Start -> word <<
|
||||
$$ = $1;
|
||||
>>
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_lexer_result_value.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
build_parser
|
||||
compile("spec/test_lexer_result_value.d")
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "tracks position of parser errors" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token num /\\d+/;
|
||||
drop /\\s+/;
|
||||
Start -> a num Start;
|
||||
Start -> a num;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_error_positions.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "allows creating a JSON parser" do
|
||||
write_grammar(File.read("spec/json_parser.#{language}.propane"))
|
||||
build_parser(language: language)
|
||||
compile(["spec/test_parsing_json.#{language}", "spec/json_types.#{language}"], language: language)
|
||||
end
|
||||
|
||||
it "allows generating multiple parsers in the same program" do
|
||||
write_grammar(<<EOF, name: "myp1")
|
||||
prefix myp1_;
|
||||
token a;
|
||||
token num /\\d+/;
|
||||
drop /\\s+/;
|
||||
Start -> a num;
|
||||
EOF
|
||||
build_parser(name: "myp1", language: language)
|
||||
write_grammar(<<EOF, name: "myp2")
|
||||
prefix myp2_;
|
||||
token b;
|
||||
token c;
|
||||
Start -> b c b;
|
||||
EOF
|
||||
build_parser(name: "myp2", language: language)
|
||||
compile("spec/test_multiple_parsers.#{language}", parsers: %w[myp1 myp2], language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "allows the user to terminate the lexer" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b <<
|
||||
$terminate(8675309);
|
||||
>>
|
||||
token c;
|
||||
Start -> Any;
|
||||
Any -> a;
|
||||
Any -> b;
|
||||
Any -> c;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_user_terminate_lexer.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "allows the user to terminate the parser" do
|
||||
write_grammar <<EOF
|
||||
token a;
|
||||
token b;
|
||||
token c;
|
||||
Start -> Any;
|
||||
Any -> a Any;
|
||||
Any -> b Any << $terminate(4200); >>
|
||||
Any -> c Any;
|
||||
Any -> ;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
compile("spec/test_user_terminate.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
end
|
||||
|
||||
it "matches backslash escape sequences" do
|
||||
case language
|
||||
when "c"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
#include <stdio.h>
|
||||
>>
|
||||
tokenid t;
|
||||
/\\a/ << printf("A\\n"); >>
|
||||
/\\b/ << printf("B\\n"); >>
|
||||
/\\t/ << printf("T\\n"); >>
|
||||
/\\n/ << printf("N\\n"); >>
|
||||
/\\v/ << printf("V\\n"); >>
|
||||
/\\f/ << printf("F\\n"); >>
|
||||
/\\r/ << printf("R\\n"); >>
|
||||
/t/ << return $token(t); >>
|
||||
Start -> t;
|
||||
EOF
|
||||
when "d"
|
||||
write_grammar <<EOF
|
||||
<<
|
||||
import std.stdio;
|
||||
>>
|
||||
tokenid t;
|
||||
/\\a/ << writeln("A"); >>
|
||||
/\\b/ << writeln("B"); >>
|
||||
/\\t/ << writeln("T"); >>
|
||||
/\\n/ << writeln("N"); >>
|
||||
/\\v/ << writeln("V"); >>
|
||||
/\\f/ << writeln("F"); >>
|
||||
/\\r/ << writeln("R"); >>
|
||||
/t/ <<
|
||||
return $token(t);
|
||||
>>
|
||||
Start -> t;
|
||||
EOF
|
||||
end
|
||||
build_parser(language: language)
|
||||
compile("spec/test_match_backslashes.#{language}", language: language)
|
||||
results = run
|
||||
expect(results.stderr).to eq ""
|
||||
expect(results.status).to eq 0
|
||||
verify_lines(results.stdout, [
|
||||
"A",
|
||||
"B",
|
||||
"T",
|
||||
"N",
|
||||
"V",
|
||||
"F",
|
||||
"R",
|
||||
])
|
||||
end
|
||||
|
||||
it "handles when an item set leads to itself" do
|
||||
write_grammar <<EOF
|
||||
token one;
|
||||
token two;
|
||||
|
||||
Start -> Opt one Start;
|
||||
Start -> ;
|
||||
|
||||
Opt -> two;
|
||||
Opt -> ;
|
||||
EOF
|
||||
build_parser(language: language)
|
||||
end
|
||||
end
|
||||
it "allows creating a JSON parser" do
|
||||
write_grammar(File.read("spec/json_parser.propane"))
|
||||
build_parser
|
||||
compile("spec/test_parsing_json.d", "spec/json_types.d")
|
||||
end
|
||||
end
|
||||
+7
-23
@@ -1,27 +1,11 @@
|
||||
unless ENV["dist_specs"]
|
||||
require "bundler/setup"
|
||||
require "simplecov"
|
||||
require "bundler/setup"
|
||||
require "propane"
|
||||
|
||||
SimpleCov.start do
|
||||
add_filter "/spec/"
|
||||
add_filter "/.bundle/"
|
||||
if ENV["partial_specs"]
|
||||
command_name "RSpec-partial"
|
||||
else
|
||||
command_name "RSpec"
|
||||
end
|
||||
project_name "Propane"
|
||||
merge_timeout 3600
|
||||
end
|
||||
RSpec.configure do |config|
|
||||
# Enable flags like --only-failures and --next-failure
|
||||
config.example_status_persistence_file_path = ".rspec_status"
|
||||
|
||||
RSpec.configure do |config|
|
||||
# Enable flags like --only-failures and --next-failure
|
||||
config.example_status_persistence_file_path = ".rspec_status"
|
||||
|
||||
config.expect_with :rspec do |c|
|
||||
c.syntax = :expect
|
||||
end
|
||||
config.expect_with :rspec do |c|
|
||||
c.syntax = :expect
|
||||
end
|
||||
end
|
||||
|
||||
require "propane"
|
||||
@@ -1,29 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include "testutils.h"
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "1 + 2 * 3 + 4";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(11, p_result(&context));
|
||||
|
||||
input = "1 * 2 ** 4 * 3";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(48, p_result(&context));
|
||||
|
||||
input = "(1 + 2) * 3 + 4";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(13, p_result(&context));
|
||||
|
||||
input = "(2 * 2) ** 3 + 4 + 5";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(73, p_result(&context));
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
import testutils;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "1 + 2 * 3 + 4";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(11, p_result(&context));
|
||||
|
||||
input = "1 * 2 ** 4 * 3";
|
||||
p_context_init(&context, input);
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(48, p_result(&context));
|
||||
|
||||
input = "(1 + 2) * 3 + 4";
|
||||
p_context_init(&context, input);
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(13, p_result(&context));
|
||||
|
||||
input = "(2 * 2) ** 3 + 4 + 5";
|
||||
p_context_init(&context, input);
|
||||
assert_eq(P_SUCCESS, p_parse(&context));
|
||||
assert_eq(73, p_result(&context));
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
alias Result = Testparser.Decoder.Result;
|
||||
Result result;
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("5");
|
||||
assert(result == Result.success('5', 1u));
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("");
|
||||
assert(result == Result.eof());
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("\xC2\xA9");
|
||||
assert(result == Result.success(0xA9u, 2u));
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xa1");
|
||||
assert(result == Result.success(0x1F9E1, 4u));
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("\xf0\x9f\x27");
|
||||
assert(result == Result.decode_error());
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("\xf0\x9f\xa7\xFF");
|
||||
assert(result == Result.decode_error());
|
||||
|
||||
result = Testparser.Decoder.decode_code_point("\xfe");
|
||||
assert(result == Result.decode_error());
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
alias Result = Testparser.Lexer.Result;
|
||||
string input = "5 + 4 * \n677 + 567";
|
||||
Testparser.Lexer lexer = new Testparser.Lexer(input);
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 1, Testparser.TOKEN_int));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 2, 1, Testparser.TOKEN_plus));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 4, 1, Testparser.TOKEN_int));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 6, 1, Testparser.TOKEN_times));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 0, 3, Testparser.TOKEN_int));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 4, 1, Testparser.TOKEN_plus));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 6, 3, Testparser.TOKEN_int));
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 1, 9, 0, Testparser.TOKEN___EOF));
|
||||
|
||||
lexer = new Testparser.Lexer("");
|
||||
assert(lexer.lex_token() == Result(Result.Type.TOKEN, 0, 0, 0, Testparser.TOKEN___EOF));
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "aba";
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
|
||||
input = "abb";
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "a";
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == false);
|
||||
|
||||
input = "a b";
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
|
||||
input = "bb";
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "a 42";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "a\n123\na a";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context).row == 2);
|
||||
assert(p_position(&context).col == 3);
|
||||
assert(p_token(&context) == TOKEN_a);
|
||||
|
||||
input = "12";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context).row == 0);
|
||||
assert(p_position(&context).col == 0);
|
||||
assert(p_token(&context) == TOKEN_num);
|
||||
|
||||
input = "a 12\n\nab";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
|
||||
assert(p_position(&context).row == 2);
|
||||
assert(p_position(&context).col == 1);
|
||||
|
||||
input = "a 12\n\na\n\n77\na \xAA";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_DECODE_ERROR);
|
||||
assert(p_position(&context).row == 5);
|
||||
assert(p_position(&context).col == 4);
|
||||
|
||||
assert(strcmp(p_token_names[TOKEN_a], "a") == 0);
|
||||
assert(strcmp(p_token_names[TOKEN_num], "num") == 0);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "a 42";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "a\n123\na a";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context) == p_position_t(2, 3));
|
||||
assert(p_token(&context) == TOKEN_a);
|
||||
|
||||
input = "12";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context) == p_position_t(0, 0));
|
||||
assert(p_token(&context) == TOKEN_num);
|
||||
|
||||
input = "a 12\n\nab";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
|
||||
assert(p_position(&context) == p_position_t(2, 1));
|
||||
|
||||
input = "a 12\n\na\n\n77\na \xAA";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_DECODE_ERROR);
|
||||
assert(p_position(&context) == p_position_t(5, 4));
|
||||
|
||||
assert(p_token_names[TOKEN_a] == "a");
|
||||
assert(p_token_names[TOKEN_num] == "num");
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
size_t result;
|
||||
p_code_point_t code_point;
|
||||
uint8_t code_point_length;
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"5", 1u, &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == '5');
|
||||
assert(code_point_length == 1u);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"", 0u, &code_point, &code_point_length);
|
||||
assert(result == P_EOF);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"\xC2\xA9", 2u, &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == 0xA9u);
|
||||
assert(code_point_length == 2u);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xa1", 4u, &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == 0x1F9E1u);
|
||||
assert(code_point_length == 4u);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"\xf0\x9f\x27", 3u, &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"\xf0\x9f\xa7\xFF", 4u, &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
|
||||
result = p_decode_code_point((uint8_t const *)"\xfe", 1u, &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
|
||||
|
||||
p_token_info_t token_info;
|
||||
char const * input = "5 + 4 * \n677 + 567";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 0u);
|
||||
assert(token_info.position.col == 0u);
|
||||
assert(token_info.length == 1u);
|
||||
assert(token_info.token == TOKEN_int);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 0u);
|
||||
assert(token_info.position.col == 2u);
|
||||
assert(token_info.length == 1u);
|
||||
assert(token_info.token == TOKEN_plus);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 0u);
|
||||
assert(token_info.position.col == 4u);
|
||||
assert(token_info.length == 1u);
|
||||
assert(token_info.token == TOKEN_int);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 0u);
|
||||
assert(token_info.position.col == 6u);
|
||||
assert(token_info.length == 1u);
|
||||
assert(token_info.token == TOKEN_times);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 1u);
|
||||
assert(token_info.position.col == 0u);
|
||||
assert(token_info.length == 3u);
|
||||
assert(token_info.token == TOKEN_int);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 1u);
|
||||
assert(token_info.position.col == 4u);
|
||||
assert(token_info.length == 1u);
|
||||
assert(token_info.token == TOKEN_plus);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 1u);
|
||||
assert(token_info.position.col == 6u);
|
||||
assert(token_info.length == 3u);
|
||||
assert(token_info.token == TOKEN_int);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 1u);
|
||||
assert(token_info.position.col == 9u);
|
||||
assert(token_info.length == 0u);
|
||||
assert(token_info.token == TOKEN___EOF);
|
||||
|
||||
p_context_init(&context, (uint8_t const *)"", 0u);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info.position.row == 0u);
|
||||
assert(token_info.position.col == 0u);
|
||||
assert(token_info.length == 0u);
|
||||
assert(token_info.token == TOKEN___EOF);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
size_t result;
|
||||
p_code_point_t code_point;
|
||||
ubyte code_point_length;
|
||||
|
||||
result = p_decode_code_point("5", &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == '5');
|
||||
assert(code_point_length == 1u);
|
||||
|
||||
result = p_decode_code_point("", &code_point, &code_point_length);
|
||||
assert(result == P_EOF);
|
||||
|
||||
result = p_decode_code_point("\xC2\xA9", &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == 0xA9u);
|
||||
assert(code_point_length == 2u);
|
||||
|
||||
result = p_decode_code_point("\xf0\x9f\xa7\xa1", &code_point, &code_point_length);
|
||||
assert(result == P_SUCCESS);
|
||||
assert(code_point == 0x1F9E1u);
|
||||
assert(code_point_length == 4u);
|
||||
|
||||
result = p_decode_code_point("\xf0\x9f\x27", &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
|
||||
result = p_decode_code_point("\xf0\x9f\xa7\xFF", &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
|
||||
result = p_decode_code_point("\xfe", &code_point, &code_point_length);
|
||||
assert(result == P_DECODE_ERROR);
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
p_token_info_t token_info;
|
||||
string input = "5 + 4 * \n677 + 567";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(0, 0), 1, TOKEN_int));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(0, 2), 1, TOKEN_plus));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(0, 4), 1, TOKEN_int));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(0, 6), 1, TOKEN_times));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(1, 0), 3, TOKEN_int));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(1, 4), 1, TOKEN_plus));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(1, 6), 3, TOKEN_int));
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(1, 9), 0, TOKEN___EOF));
|
||||
|
||||
p_context_init(&context, "");
|
||||
assert(p_lex(&context, &token_info) == P_SUCCESS);
|
||||
assert(token_info == p_token_info_t(p_position_t(0, 0), 0, TOKEN___EOF));
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "identifier_123";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass1\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,8 +9,7 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = `identifier_123`;
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass1");
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "abc \"a string\" def";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass1\n");
|
||||
|
||||
input = "abc \"abc def\" def";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass2\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,13 +9,12 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = `abc "a string" def`;
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass1");
|
||||
|
||||
input = `abc "abc def" def`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass2");
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "x";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 1u);
|
||||
|
||||
input = "fabulous";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 8u);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,13 +9,12 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = `x`;
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 1u);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
assert(parser.result == 1u);
|
||||
|
||||
input = `fabulous`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 8u);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
assert(parser.result == 8u);
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "x";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
|
||||
|
||||
input = "123";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 123u);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = `x`;
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_UNEXPECTED_INPUT);
|
||||
|
||||
input = `123`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 123u);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "\a\b\t\n\v\f\rt";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "\a\b\t\n\v\f\rt";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#include "testparsermyp1.h"
|
||||
#include "testparsermyp2.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input1 = "a\n1";
|
||||
myp1_context_t context1;
|
||||
myp1_context_init(&context1, (uint8_t const *)input1, strlen(input1));
|
||||
assert(myp1_parse(&context1) == MYP1_SUCCESS);
|
||||
|
||||
char const * input2 = "bcb";
|
||||
myp2_context_t context2;
|
||||
myp2_context_init(&context2, (uint8_t const *)input2, strlen(input2));
|
||||
assert(myp2_parse(&context2) == MYP2_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import testparsermyp1;
|
||||
import testparsermyp2;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input1 = "a\n1";
|
||||
myp1_context_t context1;
|
||||
myp1_context_init(&context1, input1);
|
||||
assert(myp1_parse(&context1) == MYP1_SUCCESS);
|
||||
|
||||
string input2 = "bcb";
|
||||
myp2_context_t context2;
|
||||
myp2_context_init(&context2, input2);
|
||||
assert(myp2_parse(&context2) == MYP2_SUCCESS);
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "aba";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "abb";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "aba";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "abb";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context).row == 0);
|
||||
assert(p_position(&context).col == 1);
|
||||
assert(context.token == TOKEN___EOF);
|
||||
|
||||
input = "a b";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "bb";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_UNEXPECTED_TOKEN);
|
||||
assert(p_position(&context) == p_position_t(0, 1));
|
||||
assert(context.token == TOKEN___EOF);
|
||||
|
||||
input = "a b";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "bb";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "ab";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,7 +9,6 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = "ab";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include "json_types.h"
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "{}";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context)->id == JSON_OBJECT);
|
||||
|
||||
input = "[]";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context)->id == JSON_ARRAY);
|
||||
|
||||
input = "-45.6";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context)->id == JSON_NUMBER);
|
||||
assert(p_result(&context)->number == -45.6);
|
||||
|
||||
input = "2E-2";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context)->id == JSON_NUMBER);
|
||||
assert(p_result(&context)->number == 0.02);
|
||||
|
||||
input = "{\"hi\":true}";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
JSONValue * o = p_result(&context);
|
||||
assert(o->id == JSON_OBJECT);
|
||||
assert_eq(1, o->object.size);
|
||||
assert(strcmp(o->object.entries[0].name, "hi") == 0);
|
||||
assert(o->object.entries[0].value->id == JSON_TRUE);
|
||||
|
||||
input = "{\"ff\": false, \"nn\": null}";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
o = p_result(&context);
|
||||
assert(o->id == JSON_OBJECT);
|
||||
assert_eq(2, o->object.size);
|
||||
assert(strcmp(o->object.entries[0].name, "ff") == 0);
|
||||
assert(o->object.entries[0].value->id == JSON_FALSE);
|
||||
assert(strcmp(o->object.entries[1].name, "nn") == 0);
|
||||
assert(o->object.entries[1].value->id == JSON_NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
+24
-25
@@ -10,45 +10,44 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = ``;
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
|
||||
input = `{}`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONObject)p_result(&context));
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONObject)parser.result);
|
||||
|
||||
input = `[]`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONArray)p_result(&context));
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONArray)parser.result);
|
||||
|
||||
input = `-45.6`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONNumber)p_result(&context));
|
||||
assert((cast(JSONNumber)p_result(&context)).value == -45.6);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONNumber)parser.result);
|
||||
assert((cast(JSONNumber)parser.result).value == -45.6);
|
||||
|
||||
input = `2E-2`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONNumber)p_result(&context));
|
||||
assert((cast(JSONNumber)p_result(&context)).value == 0.02);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONNumber)parser.result);
|
||||
assert((cast(JSONNumber)parser.result).value == 0.02);
|
||||
|
||||
input = `{"hi":true}`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONObject)p_result(&context));
|
||||
JSONObject o = cast(JSONObject)p_result(&context);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONObject)parser.result);
|
||||
JSONObject o = cast(JSONObject)parser.result;
|
||||
assert(o.value["hi"]);
|
||||
assert(cast(JSONTrue)o.value["hi"]);
|
||||
|
||||
input = `{"ff": false, "nn": null}`;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(cast(JSONObject)p_result(&context));
|
||||
o = cast(JSONObject)p_result(&context);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse());
|
||||
assert(cast(JSONObject)parser.result);
|
||||
o = cast(JSONObject)parser.result;
|
||||
assert(o.value["ff"]);
|
||||
assert(cast(JSONFalse)o.value["ff"]);
|
||||
assert(o.value["nn"]);
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 1u);
|
||||
|
||||
input = "";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 0u);
|
||||
|
||||
input = "aaaaaaaaaaaaaaaa";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 16u);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,18 +9,17 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 1u);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
assert(parser.result == 1u);
|
||||
|
||||
input = "";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 0u);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
assert(parser.result == 0u);
|
||||
|
||||
input = "aaaaaaaaaaaaaaaa";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
assert(p_result(&context) == 16u);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
assert(parser.result == 16u);
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <stdio.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "abcdef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass1\n");
|
||||
|
||||
input = "defabcdef";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass2\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
+4
-5
@@ -9,13 +9,12 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = "abcdef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass1");
|
||||
|
||||
input = "defabcdef";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass2");
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "defghidef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,7 +9,6 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = "defghidef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "abcdef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass1\n");
|
||||
|
||||
input = "abcabcdef";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
printf("pass2\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -9,13 +9,12 @@ int main()
|
||||
unittest
|
||||
{
|
||||
string input = "abcdef";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
auto parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass1");
|
||||
|
||||
input = "abcabcdef";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
parser = new Testparser.Parser(input);
|
||||
assert(parser.parse() == true);
|
||||
writeln("pass2");
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "aacc";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "abc";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_USER_TERMINATED);
|
||||
assert(p_user_terminate_code(&context) == 4200);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "aacc";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "abc";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_USER_TERMINATED);
|
||||
assert(p_user_terminate_code(&context) == 4200);
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#include "testparser.h"
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int main()
|
||||
{
|
||||
char const * input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "b";
|
||||
p_context_init(&context, (uint8_t const *)input, strlen(input));
|
||||
assert(p_parse(&context) == P_USER_TERMINATED);
|
||||
assert(p_user_terminate_code(&context) == 8675309);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
import testparser;
|
||||
import std.stdio;
|
||||
|
||||
int main()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
unittest
|
||||
{
|
||||
string input = "a";
|
||||
p_context_t context;
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_SUCCESS);
|
||||
|
||||
input = "b";
|
||||
p_context_init(&context, input);
|
||||
assert(p_parse(&context) == P_USER_TERMINATED);
|
||||
assert(p_user_terminate_code(&context) == 8675309);
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
#include <stdio.h>
|
||||
#include <assert.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "testutils.h"
|
||||
|
||||
void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line)
|
||||
{
|
||||
if (expected != actual)
|
||||
{
|
||||
fprintf(stderr, "%s:%lu: expected %lu, got %lu\n", file, line, expected, actual);
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
|
||||
void str_init(str_t * str, char const * cs)
|
||||
{
|
||||
size_t length = strlen(cs);
|
||||
str->cs = malloc(length + 1u);
|
||||
strcpy(str->cs, cs);
|
||||
}
|
||||
|
||||
void str_append(str_t * str, char const * cs)
|
||||
{
|
||||
size_t length = strlen(str->cs);
|
||||
size_t length2 = strlen(cs);
|
||||
char * new_cs = malloc(length + length2 + 1u);
|
||||
memcpy(new_cs, str->cs, length);
|
||||
strcpy(&new_cs[length], cs);
|
||||
free(str->cs);
|
||||
str->cs = new_cs;
|
||||
}
|
||||
|
||||
void str_free(str_t * str)
|
||||
{
|
||||
free(str->cs);
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import std.stdio;
|
||||
|
||||
void assert_eq(T)(T expected, T actual, string file = __FILE__, size_t line = __LINE__)
|
||||
{
|
||||
if (expected != actual)
|
||||
{
|
||||
stderr.writeln(file, ":", line, ": expected ", expected, ", got ", actual);
|
||||
assert(false);
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
void assert_eq_size_t_i(size_t expected, size_t actual, char const * file, size_t line);
|
||||
|
||||
#define assert_eq(expected, actual) \
|
||||
assert_eq_size_t_i(expected, actual, __FILE__, __LINE__)
|
||||
|
||||
typedef struct
|
||||
{
|
||||
char * cs;
|
||||
} str_t;
|
||||
|
||||
void str_init(str_t * str, char const * cs);
|
||||
void str_append(str_t * str, char const * cs);
|
||||
void str_free(str_t * str);
|
||||
static inline char * str_cstr(str_t * str)
|
||||
{
|
||||
return str->cs;
|
||||
}
|
||||
Reference in new issue
Block a user