wip

CHANGELOG.md

@@ -1,9 +1,3 @@
-## v4.2.0
-
-### New Features
-
-- Add support for a custom lex function.
-
 ## v4.1.0
 
 ### New Features

Gemfile.lock

@@ -5,7 +5,7 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.2)
+    erb (6.0.1)
     psych (5.3.1)
       date
       stringio
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.8)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2026 Josh Holtrop
+Copyright (c) 2010-2024 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.d.erb

@@ -86,14 +86,6 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
-
-/** Parser value constructor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
-{
-    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
-}
-<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

assets/parser.h.erb

@@ -69,14 +69,6 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
-
-/** Parser value constructor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
-{
-    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
-}
-<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -316,61 +316,6 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
-##> `lex_fn` statement - specifying a custom lexer function
-
-Propane generates both a lexer and a parser.
-By default, the parser uses the generated `p_lex()` function directly to
-return information for the next lexed token from the input stream.
-
-However, the user can specify a custom lex function.
-This function may or may not use the Propane generated `p_lex()` function under
-the hood.
-For example, a token sequence could be injected or repeated from a previously
-saved macro definition.
-
-Example (C/C++):
-
-```
-<<
-static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
-{
-    static size_t count;
-    size_t result = P_SUCCESS;
-    if (count > 0)
-    {
-        out_token_info->token = TOKEN_a;
-        out_token_info->pvalue = p_value(count);
-        count--;
-    }
-    else
-    {
-        result = p_lex(context, out_token_info);
-        if (out_token_info->token == TOKEN_c)
-        {
-            count = 3;
-        }
-    }
-    return result;
-}
->>
-
-lex_fn mylexfn;
-```
-
-The `lex_fn` statement takes one argument specifying the name of the custom
-lexer function.
-The user must supply a value for the `token` field of the `p_token_info_t`
-output structure so that the parser knows what token was lexed.
-
-Additionally, if the parser user code makes use of the token's pvalue, then
-the lexer function must supply a value for the `pvalue` field of the
-`p_token_info_t` structure.
-The `p_value()` generated API function could be useful for specifying
-parser values to associate with the lexed token when tree generation is not
-enabled.
-When tree generation is enabled, the `pvalue` field can be set to an instance
-of whatever type the user has defined as the `ptype` type.
-
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -1062,17 +1007,6 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
-### `p_value_t`
-
-If tree generation mode is enabled, the `p_value_t` type is defined to be the
-type given to the `ptype` statement in the grammar file.
-
-If tree generation mode is not enabled, there could be more than one `ptype`
-given, so the `p_value_t` type is a union of all possible `ptype` types.
-In this case, the API functions `p_value()` and `p_value_XXX()` for each given
-`ptype` name `XXX` are generated to return `p_value_t` instances holding the
-corresponding `ptype`.
-
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1402,20 +1336,6 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
-### `p_value`
-
-When tree generation mode is not active, the `p_value_t` union can hold one of
-several different possible value types.
-
-The `p_value(v)` function returns an instance of the `p_value_t` with the
-default member set to the value of `v`.
-
-A `p_value_XXX(v)` function is set for each user-defined `ptype` name with
-the user-given name in place of the `XXX`.
-
-These functions are useful for custom lexer functions which need to return a
-parser value corresponding to a lexed token.
-
 ##> Data
 
 ### `p_token_names`

lib/propane/generator.rb

@@ -359,7 +359,7 @@ class Propane
     # @return [String]
     #   Lex function to use.
     def lex_fn
-      @grammar.lex_fn || "#{@grammar.prefix}lex"
+      @grammar.custom_lex_fn || "#{@grammar.prefix}lex"
     end
 
     # Get the parser value type for the start rule.

lib/propane/grammar.rb

@@ -6,7 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
-    attr_reader :lex_fn
+    attr_reader :custom_lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -70,7 +70,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
-      elsif parse_lex_fn!
+      elsif parse_custom_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -119,9 +119,9 @@ class Propane
       end
     end
 
-    def parse_lex_fn!
-      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
-        @lex_fn = md[1]
+    def parse_custom_lex_fn!
+      if md = consume!(/custom_lex_fn\b\s*(\w+)\s*;/)
+        @custom_lex_fn = $1
       end
     end
 

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.2.0"
+  VERSION = "4.1.0"
 end

spec/propane_spec.rb

@@ -1706,34 +1706,16 @@ EOF
 <<
 private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
 {
-    static size_t count;
-    size_t result = P_SUCCESS;
-    if (count > 0)
-    {
-        out_token_info.token = TOKEN_a;
-        out_token_info.pvalue = p_value(count);
-        count--;
-    }
-    else
-    {
-        result = p_lex(context, out_token_info);
-        if (out_token_info.token == TOKEN_c)
-        {
-            count = 3;
-        }
-    }
-    return result;
 }
 >>
 
-ptype size_t;
-lex_fn mylexfn;
+tree;
 
-token a << $$ = 7; >>
-token b << $$ = 8; >>
-token c << $$ = 9; >>
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
 Start -> << $$ = 0; >>
-Start -> Start ID << $$ = ($1 << 4) | $2; >>
+Start -> ID Start << $$ = ($1 << 4) | $2; >>
 ID -> a << $$ = $1; >>
 ID -> b << $$ = $1; >>
 ID -> c << $$ = $1; >>
@@ -1743,34 +1725,16 @@ EOF
 <<
 static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
 {
-    static size_t count;
-    size_t result = P_SUCCESS;
-    if (count > 0)
-    {
-        out_token_info->token = TOKEN_a;
-        out_token_info->pvalue = p_value(count);
-        count--;
-    }
-    else
-    {
-        result = p_lex(context, out_token_info);
-        if (out_token_info->token == TOKEN_c)
-        {
-            count = 3;
-        }
-    }
-    return result;
 }
 >>
 
-ptype size_t;
-lex_fn mylexfn;
+tree;
 
-token a << $$ = 7; >>
-token b << $$ = 8; >>
-token c << $$ = 9; >>
+token a << $$ = 1; >>
+token b << $$ = 2; >>
+token c << $$ = 3; >>
 Start -> << $$ = 0; >>
-Start -> Start ID << $$ = ($1 << 4) | $2; >>
+Start -> ID Start << $$ = ($1 << 4) | $2; >>
 ID -> a << $$ = $1; >>
 ID -> b << $$ = $1; >>
 ID -> c << $$ = $1; >>

spec/test_custom_lex_fn.c

@@ -1,15 +0,0 @@
-#include "testparser.h"
-#include "testutils.h"
-#include <string.h>
-
-int main()
-{
-    char const * input = "cbacba";
-    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
-    assert_eq(P_SUCCESS, p_parse(context));
-    size_t result = p_result(context);
-    assert_eq(0x932187932187, result);
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_custom_lex_fn.d

@@ -1,18 +0,0 @@
-import testparser;
-import std.stdio;
-import testutils;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    string input = "cbacba";
-    p_context_t * context = p_context_new(input);
-    assert_eq(P_SUCCESS, p_parse(context));
-    size_t result = p_result(context);
-    assert_eq(0x932187932187, result);
-    p_context_delete(context);
-}