add docs

CHANGELOG.md

@@ -1,34 +1,3 @@
-## v4.5.0
-
-### New Features
-
-- Add `noline` grammar statement to skip emitting `#line` directives
-- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
-
-### Fixes
-
-- Fix #line reset directives
-- Update keyword list in extra/vim/syntax/propane.vim
-- Fix propane.vim keyword detection
-
-## v4.4.0
-
-### New Features
-
-- Add p_value_get() / p_value_get_XXX() accessors
-
-## v4.3.0
-
-### New Features
-
-- Use #line for user code blocks to report input grammar position for errors.
-
-## v4.2.0
-
-### New Features
-
-- Add support for a custom lex function.
-
 ## v4.1.0
 
 ### New Features

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.4)
-    psych (5.4.0)
+    erb (6.0.1)
+    psych (5.3.1)
       date
       stringio
-    rake (13.4.2)
+    rake (13.3.1)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.8)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-  4.0.14
+   2.3.7

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2026 Josh Holtrop
+Copyright (c) 2010-2024 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.d.erb

@@ -94,14 +94,6 @@ public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "def
     return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
 }
 <%   end %>
-
-/** Parser value accessor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
-{
-    return pvalue.v_<%= name %>;
-}
-<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

assets/parser.h.erb

@@ -77,14 +77,6 @@ static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name
     return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
 }
 <%   end %>
-
-/** Parser value accessor(s). */
-<%   @grammar.ptypes.each do |name, typestring| %>
-static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
-{
-    return pvalue->v_<%= name %>;
-}
-<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -383,27 +383,6 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
-##> `noline` statement - disabling `#line` directives
-
-By default, Propane emits `#line` directives into the generated output around
-user code blocks.
-These directives instruct the compiler to report any warnings or errors in the
-user code using the file name and line number of the original grammar file,
-rather than the generated output file.
-This makes it easier to locate the source of a compiler diagnostic in the
-grammar file.
-
-The `noline` statement disables the emission of these `#line` directives.
-
-```
-noline;
-```
-
-When the `noline` statement is present, user code blocks are copied to the
-generated output without any surrounding `#line` directives.
-This can be useful when debugging the generated parser itself, or when the
-`#line` directives interfere with other tooling.
-
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -1103,9 +1082,6 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
-  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
-  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
-  `ptype` values.
 
 ### Tree Node Types
 
@@ -1428,35 +1404,18 @@ In this case, Propane will free a `Statement` tree structure returned by the
 
 ### `p_value`
 
-The `p_value(v)` function builds an instance of the `p_value_t` with the
+When tree generation mode is not active, the `p_value_t` union can hold one of
+several different possible value types.
+
+The `p_value(v)` function returns an instance of the `p_value_t` with the
 default member set to the value of `v`.
 
-A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
+A `p_value_XXX(v)` function is set for each user-defined `ptype` name with
 the user-given name in place of the `XXX`.
 
 These functions are useful for custom lexer functions which need to return a
 parser value corresponding to a lexed token.
 
-They are especially useful when tree generation mode is not active.
-In that case, the `p_value_t` union can hold one of several different possible
-value types.
-
-### `p_value_get`
-
-The `p_value_get()` accessor functions can be used to extract a user value
-from a `p_value_t` union.
-
-The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
-returns the value of its default member.
-
-A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
-name with the user-given name in place of the `XXX`, returning the value of the
-corresponding member.
-
-These functions are the counterpart to `p_value()` constructor functions and
-are useful for reading the parser value associated with a lexed token, for
-example `p_value_get(&token_info.pvalue)`.
-
 ##> Data
 
 ### `p_token_names`

extra/vim/syntax/propane.vim

@@ -8,11 +8,7 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
-    let b:propane_subtype = "d"
-  else
-    let b:propane_subtype = "cpp"
-  endif
+  let b:propane_subtype = "d"
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -24,19 +20,12 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-" Keywords that introduce a user-defined name. The name is consumed by
-" propaneName via nextgroup so a name matching a keyword (e.g. 'token start')
-" is not highlighted as a keyword. These must be a match (not syn keyword)
-" because a syn keyword always wins over a contained nextgroup match.
-syn match propaneNameDecl "\<\%(tokenid\|token\|lex_fn\|module\|start\|tree_prefix\|tree_suffix\)\>" nextgroup=propaneName skipwhite
-syn match propaneName "\<\h\w*\>" contained
-syn match propaneKeyword "\<\%(context_user_fields\|drop\|free_token_node\|noline\|on_token_node\|prefix\|ptype\|token_user_fields\|tree\)\>"
+syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 
 hi def link propaneComment Comment
 hi def link propaneKeyword Keyword
-hi def link propaneNameDecl Keyword
 hi def link propaneRegex String
 hi def link propaneOperator Operator
 hi def link propaneFieldOperator Operator

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file), input_file)
+        grammar = Grammar.new(File.read(input_file))
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -38,13 +38,7 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
-          if line == "#linereset\n"
-            %[#line #{i + 2} "#{output_file}"\n]
-          else
-            line
-          end
-        end.join
+        result = erb.result(binding.clone)
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end

lib/propane/grammar.rb

@@ -22,8 +22,7 @@ class Propane
     attr_reader :on_token_node
     attr_reader :token_user_fields
 
-    def initialize(input, filename)
-      @filename = filename
+    def initialize(input)
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -88,7 +87,6 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
-      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -306,14 +304,8 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code = code.chomp
-        unless @noline
-          if code.start_with?("\n")
-            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          else
-            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
-          end
-        end
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -331,13 +323,6 @@ class Propane
       end
     end
 
-    def parse_noline_statement!
-      if md = consume!(/noline\s*;/)
-        @noline = true
-        true
-      end
-    end
-
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -361,14 +346,9 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1].chomp
-        unless @noline
-          if code.start_with?("\n")
-            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
-          else
-            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
-          end
-        end
+        code = md[1]
+        code.sub!(/\A\n/, "")
+        code += "\n" unless code.end_with?("\n")
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.5.0"
+  VERSION = "4.1.0"
 end

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
+      expect(o.code).to eq "Code for the token\n"
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
+      expect(o.code).to eq "  a = 42;\n"
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
+      expect(o.code).to eq "  b = 0;\n"
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
+      expect(o.code).to eq "  a = b;\n  return c;\n"
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,42 +130,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
-    end
-
-    it "does not emit #line directives with noline statement" do
-      input = <<EOF
-noline;
-
-token code1 <<
-  a = b;
-  return c;
->>
-
-token code2 <<
-  writeln("Hello there");
->>
-
-tokenid token_with_no_pattern;
-
-prefix myparser_;
-EOF
-      grammar = Grammar.new(input, "test.propane")
-      expect(grammar.prefix).to eq "myparser_"
-
-      o = grammar.tokens.find {|token| token.name == "code1"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq %[\n  a = b;\n  return c;]
-
-      o = grammar.tokens.find {|token| token.name == "code2"}
-      expect(o).to_not be_nil
-
-      o = grammar.patterns.find {|pattern| pattern.token == o}
-      expect(o).to_not be_nil
-      expect(o.code).to eq %[\n  writeln("Hello there");]
+      expect(o.code).to eq %[  writeln("Hello there");\n]
     end
 
     it "supports mode labels" do
@@ -179,7 +144,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -232,7 +197,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input, "test.propane")
+      grammar = Grammar.new(input)
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar, "test.propane")
+  grammar = Propane::Grammar.new(grammar)
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -1781,39 +1781,6 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
-
-      it "provides accessor functions to extract user values from a parser value" do
-        if language == "d"
-          write_grammar <<EOF
-ptype int;
-ptype float = float;
-ptype string = string;
-
-drop /\\s+/;
-token num /\\d+/ << $$ = 42; >>
-token flt (float) /f/ << $$ = 1.5; >>
-token str (string) /s/ << $$ = "hello"; >>
-Start -> num flt str;
-EOF
-        else
-          write_grammar <<EOF
-ptype int;
-ptype float = float;
-ptype string = char *;
-
-drop /\\s+/;
-token num /\\d+/ << $$ = 42; >>
-token flt (float) /f/ << $$ = 1.5; >>
-token str (string) /s/ << $$ = (char *)"hello"; >>
-Start -> num flt str;
-EOF
-        end
-        run_propane(language: language)
-        compile("spec/test_value_accessors.#{language}", language: language)
-        results = run_test(language: language)
-        expect(results.stderr).to eq ""
-        expect(results.status).to eq 0
-      end
     end
   end
 end

spec/test_value_accessors.c

@@ -1,29 +0,0 @@
-#include "testparser.h"
-#include <assert.h>
-#include <string.h>
-
-int main()
-{
-    p_token_info_t token_info;
-    char const * input = "42 f s";
-    p_context_t * context;
-    context = p_context_new((uint8_t const *)input, strlen(input));
-
-    /* Default ptype value extracted with p_value_get(). */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_num);
-    assert(p_value_get(&token_info.pvalue) == 42);
-
-    /* Named ptype values extracted with p_value_get_XXX(). */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_flt);
-    assert(p_value_get_float(&token_info.pvalue) == 1.5);
-
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_str);
-    assert(strcmp(p_value_get_string(&token_info.pvalue), "hello") == 0);
-
-    p_context_delete(context);
-
-    return 0;
-}

spec/test_value_accessors.d

@@ -1,29 +0,0 @@
-import testparser;
-import std.stdio;
-
-int main()
-{
-    return 0;
-}
-
-unittest
-{
-    p_token_info_t token_info;
-    string input = "42 f s";
-    p_context_t * context;
-    context = p_context_new(input);
-
-    /* Default ptype value extracted with p_value_get(). */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_num);
-    assert(p_value_get(&token_info.pvalue) == 42);
-
-    /* Named ptype values extracted with p_value_get_XXX(). */
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_flt);
-    assert(p_value_get_float(&token_info.pvalue) == 1.5);
-
-    assert(p_lex(context, &token_info) == P_SUCCESS);
-    assert(token_info.token == TOKEN_str);
-    assert(p_value_get_string(&token_info.pvalue) == "hello");
-}