Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim

Allow user code to access specific pvalue values, especially when
multiple ptypes are in use.

CHANGELOG.md

@@ -1,3 +1,33 @@
+## v4.5.0
+
+### New Features
+
+- Add `noline` grammar statement to skip emitting `#line` directives
+- Attempt to autodetect target language (D/C++) in extra/vim/syntax/propane.vim
+
+### Fixes
+
+- Fix #line reset directives
+- Update keyword list in extra/vim/syntax/propane.vim
+
+## v4.4.0
+
+### New Features
+
+- Add p_value_get() / p_value_get_XXX() accessors
+
+## v4.3.0
+
+### New Features
+
+- Use #line for user code blocks to report input grammar position for errors.
+
+## v4.2.0
+
+### New Features
+
+- Add support for a custom lex function.
+
 ## v4.1.0
 
 ### New Features

Gemfile.lock

@@ -5,11 +5,11 @@ GEM
     date (3.5.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    erb (6.0.1)
-    psych (5.3.1)
+    erb (6.0.4)
+    psych (5.4.0)
       date
       stringio
-    rake (13.3.1)
+    rake (13.4.2)
     rdoc (7.2.0)
       erb
       psych (>= 4.0.0)
@@ -24,7 +24,7 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.7)
@@ -51,4 +51,4 @@ DEPENDENCIES
   syntax
 
 BUNDLED WITH
-   2.3.7
+  4.0.14

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2010-2024 Josh Holtrop
+Copyright (c) 2010-2026 Josh Holtrop
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

assets/parser.c.erb

@@ -981,7 +981,7 @@ static size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t start
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 result = lexer_result;

assets/parser.d.erb

@@ -86,6 +86,22 @@ public union <%= @grammar.prefix %>value_t
     <%= typestring %> v_<%= name %>;
 <%   end %>
 }
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(T)(T v)
+{
+    return <%= @grammar.prefix %>value_t(v_<%= name %>: v);
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+public <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t * pvalue)
+{
+    return pvalue.v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>
@@ -1037,7 +1053,7 @@ private size_t parse_from(<%= @grammar.prefix %>context_t * context, size_t star
     {
         if (token == INVALID_TOKEN_ID)
         {
-            size_t lexer_result = <%= @grammar.prefix %>lex(context, &token_info);
+            size_t lexer_result = <%= lex_fn %>(context, &token_info);
             if (lexer_result != P_SUCCESS)
             {
                 return lexer_result;

assets/parser.h.erb

@@ -69,6 +69,22 @@ typedef union
     <%= typestring %> v_<%= name %>;
 <%   end %>
 } <%= @grammar.prefix %>value_t;
+
+/** Parser value constructor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= @grammar.prefix %>value_t <%= @grammar.prefix %>value<%= name == "default" ? "" : "_#{name}" %>(<%= typestring %> v)
+{
+    return (<%= @grammar.prefix %>value_t){.v_<%= name %> = v};
+}
+<%   end %>
+
+/** Parser value accessor(s). */
+<%   @grammar.ptypes.each do |name, typestring| %>
+static inline <%= typestring %> <%= @grammar.prefix %>value_get<%= name == "default" ? "" : "_#{name}" %>(<%= @grammar.prefix %>value_t const * pvalue)
+{
+    return pvalue->v_<%= name %>;
+}
+<%   end %>
 <% end %>
 
 <% if @grammar.tree %>

doc/user_guide.md

@@ -316,6 +316,61 @@ free_token_node <<
 The `free_token_node` statement user code block is not emitted for D language
 since D has a garbage collector.
 
+##> `lex_fn` statement - specifying a custom lexer function
+
+Propane generates both a lexer and a parser.
+By default, the parser uses the generated `p_lex()` function directly to
+return information for the next lexed token from the input stream.
+
+However, the user can specify a custom lex function.
+This function may or may not use the Propane generated `p_lex()` function under
+the hood.
+For example, a token sequence could be injected or repeated from a previously
+saved macro definition.
+
+Example (C/C++):
+
+```
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+lex_fn mylexfn;
+```
+
+The `lex_fn` statement takes one argument specifying the name of the custom
+lexer function.
+The user must supply a value for the `token` field of the `p_token_info_t`
+output structure so that the parser knows what token was lexed.
+
+Additionally, if the parser user code makes use of the token's pvalue, then
+the lexer function must supply a value for the `pvalue` field of the
+`p_token_info_t` structure.
+The `p_value()` generated API function could be useful for specifying
+parser values to associate with the lexed token when tree generation is not
+enabled.
+When tree generation is enabled, the `pvalue` field can be set to an instance
+of whatever type the user has defined as the `ptype` type.
+
 ##> `module` statement - specifying the generated parser module name
 
 The `module` statement can be used to specify the module name for a generated
@@ -328,6 +383,27 @@ module proj.parser;
 If a module statement is not present, then the generated D module will not
 contain a module statement and the default module name will be used.
 
+##> `noline` statement - disabling `#line` directives
+
+By default, Propane emits `#line` directives into the generated output around
+user code blocks.
+These directives instruct the compiler to report any warnings or errors in the
+user code using the file name and line number of the original grammar file,
+rather than the generated output file.
+This makes it easier to locate the source of a compiler diagnostic in the
+grammar file.
+
+The `noline` statement disables the emission of these `#line` directives.
+
+```
+noline;
+```
+
+When the `noline` statement is present, user code blocks are copied to the
+generated output without any surrounding `#line` directives.
+This can be useful when debugging the generated parser itself, or when the
+`#line` directives interfere with other tooling.
+
 ##> `on_tree_node` statement -  custom initialization of a token tree node
 
 The `on_token_node` statement can be used to provide code that initializes
@@ -1007,6 +1083,17 @@ For C targets, the `p_position_t` structure can be checked for validity by
 calling `p_position_valid(pos)` where `pos` is a `p_position_t` structure
 instance.
 
+### `p_value_t`
+
+If tree generation mode is enabled, the `p_value_t` type is defined to be the
+type given to the `ptype` statement in the grammar file.
+
+If tree generation mode is not enabled, there could be more than one `ptype`
+given, so the `p_value_t` type is a union of all possible `ptype` types.
+In this case, the API functions `p_value()` and `p_value_XXX()` for each given
+`ptype` name `XXX` are generated to return `p_value_t` instances holding the
+corresponding `ptype`.
+
 ### `p_token_info_t`
 
 The `p_token_info_t` structure contains the following fields:
@@ -1016,6 +1103,9 @@ The `p_token_info_t` structure contains the following fields:
 * `length` (`size_t`) holds the number of input bytes used by the token.
 * `token` (`p_token_t`) holds the token ID of the lexed token
 * `pvalue` (`p_value_t`) holds the parser value associated with the token.
+  The actual user value can be extracted with `p_value_get(&token_info.pvalue)`
+  for the default value or `p_value_get_XXX(&token_info.pvalue)` for named
+  `ptype` values.
 
 ### Tree Node Types
 
@@ -1336,6 +1426,37 @@ p_tree_delete_Statement(statement_tree);
 In this case, Propane will free a `Statement` tree structure returned by the
 `p_parse_Statement(context)` function.
 
+### `p_value`
+
+The `p_value(v)` function builds an instance of the `p_value_t` with the
+default member set to the value of `v`.
+
+A `p_value_XXX(v)` function is defined for each user-defined `ptype` name with
+the user-given name in place of the `XXX`.
+
+These functions are useful for custom lexer functions which need to return a
+parser value corresponding to a lexed token.
+
+They are especially useful when tree generation mode is not active.
+In that case, the `p_value_t` union can hold one of several different possible
+value types.
+
+### `p_value_get`
+
+The `p_value_get()` accessor functions can be used to extract a user value
+from a `p_value_t` union.
+
+The `p_value_get(pvalue)` function accepts a pointer to a `p_value_t` and
+returns the value of its default member.
+
+A `p_value_get_XXX(pvalue)` function is generated for each user-defined `ptype`
+name with the user-given name in place of the `XXX`, returning the value of the
+corresponding member.
+
+These functions are the counterpart to `p_value()` constructor functions and
+are useful for reading the parser value associated with a lexed token, for
+example `p_value_get(&token_info.pvalue)`.
+
 ##> Data
 
 ### `p_token_names`

extra/vim/syntax/propane.vim

@@ -8,7 +8,11 @@ if exists("b:current_syntax")
 endif
 
 if !exists("b:propane_subtype")
-  let b:propane_subtype = "d"
+  if search('\<import\s\+\%(std\|core\)\.', 'nw') > 0
+    let b:propane_subtype = "d"
+  else
+    let b:propane_subtype = "cpp"
+  endif
 endif
 
 exe "syn include @propaneTarget syntax/".b:propane_subtype.".vim"
@@ -20,7 +24,7 @@ syn match propaneOperator "->"
 syn match propaneFieldAlias ":[a-zA-Z0-9_]\+" contains=propaneFieldOperator
 syn match propaneFieldOperator ":" contained
 syn match propaneOperator "?"
-syn keyword propaneKeyword drop free_token_node free_token_user_fields module prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
+syn keyword propaneKeyword context_user_fields drop free_token_node lex_fn module noline on_token_node prefix ptype start token token_user_fields tokenid tree tree_prefix tree_suffix
 
 syn region propaneRegex start="/" end="/" skip="\v\\\\|\\/"
 

lib/propane.rb

@@ -33,7 +33,7 @@ class Propane
 
     def run(input_file, output_file, log_file, options)
       begin
-        grammar = Grammar.new(File.read(input_file))
+        grammar = Grammar.new(File.read(input_file), input_file)
         generator = Generator.new(grammar, output_file, log_file, options)
         generator.generate
       rescue Error => e

lib/propane/generator.rb

@@ -38,7 +38,13 @@ class Propane
           output_file = @output_file
         end
         erb = ERB.new(template, trim_mode: "<>")
-        result = erb.result(binding.clone)
+        result = erb.result(binding.clone).lines.each_with_index.map do |line, i|
+          if line == "#linereset\n"
+            %[#line #{i + 2} "#{output_file}"\n]
+          else
+            line
+          end
+        end.join
         File.open(output_file, "wb") do |fh|
           fh.write(result)
         end
@@ -354,6 +360,14 @@ class Propane
       code
     end
 
+    # Get the lex function to use.
+    #
+    # @return [String]
+    #   Lex function to use.
+    def lex_fn
+      @grammar.lex_fn || "#{@grammar.prefix}lex"
+    end
+
     # Get the parser value type for the start rule.
     #
     # @return [Array<String>]

lib/propane/grammar.rb

@@ -6,6 +6,7 @@ class Propane
     IDENTIFIER_REGEX = /(?:[a-zA-Z]|_[a-zA-Z0-9])[a-zA-Z_0-9]*/
 
     attr_reader :context_user_fields
+    attr_reader :lex_fn
     attr_reader :tree
     attr_reader :tree_prefix
     attr_reader :tree_suffix
@@ -21,7 +22,8 @@ class Propane
     attr_reader :on_token_node
     attr_reader :token_user_fields
 
-    def initialize(input)
+    def initialize(input, filename)
+      @filename = filename
       @patterns = []
       @start_rules = []
       @tokens = []
@@ -69,6 +71,7 @@ class Propane
       elsif parse_comment_line!
       elsif @modeline.nil? && parse_mode_label!
       elsif parse_context_user_fields_statement!
+      elsif parse_lex_fn!
       elsif parse_tree_statement!
       elsif parse_tree_prefix_statement!
       elsif parse_tree_suffix_statement!
@@ -85,6 +88,7 @@ class Propane
       elsif parse_rule_statement!
       elsif parse_code_block_statement!
       elsif parse_prefix_statement!
+      elsif parse_noline_statement!
       else
         if @input.size > 25
           @input = @input.slice(0..20) + "..."
@@ -117,6 +121,12 @@ class Propane
       end
     end
 
+    def parse_lex_fn!
+      if md = consume!(/lex_fn\b\s*(\w+)\s*;/)
+        @lex_fn = md[1]
+      end
+    end
+
     def parse_tree_statement!
       if consume!(/tree\s*;/)
         @tree = true
@@ -296,8 +306,14 @@ class Propane
     def parse_code_block_statement!
       if md = consume!(/<<([a-z]*)(.*?)>>\n/m)
         name, code = md[1..2]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = code.chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         if @code_blocks[name]
           @code_blocks[name] += code
         else
@@ -315,6 +331,13 @@ class Propane
       end
     end
 
+    def parse_noline_statement!
+      if md = consume!(/noline\s*;/)
+        @noline = true
+        true
+      end
+    end
+
     def parse_pattern!
       if md = consume!(%r{/})
         pattern = ""
@@ -338,9 +361,14 @@ class Propane
 
     def parse_code_block!
       if md = consume!(/<<(.*?)>>\n/m)
-        code = md[1]
-        code.sub!(/\A\n/, "")
-        code += "\n" unless code.end_with?("\n")
+        code = md[1].chomp
+        unless @noline
+          if code.start_with?("\n")
+            code = %[#line #{@line_number + 1} "#{@filename}"#{code}\n#linereset\n]
+          else
+            code = %[#line #{@line_number} "#{@filename}"\n#{code}\n#linereset\n]
+          end
+        end
         code
       end
     end

lib/propane/version.rb

@@ -1,3 +1,3 @@
 class Propane
-  VERSION = "4.1.0"
+  VERSION = "4.4.0"
 end

spec/propane/grammar_spec.rb

@@ -28,7 +28,7 @@ B -> <<
   b = 0;
 >>
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.modulename).to eq "a.b"
       expect(grammar.ptype).to eq "XYZ  *"
       expect(grammar.ptypes).to eq("default" => "XYZ  *")
@@ -62,7 +62,7 @@ EOF
       expect(o).to_not be_nil
       expect(o.pattern).to eq "token_with_code"
       expect(o.line_number).to eq 11
-      expect(o.code).to eq "Code for the token\n"
+      expect(o.code).to eq %[#line 12 "test.propane"\nCode for the token\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "token_with_no_pattern"}
       expect(o).to_not be_nil
@@ -83,7 +83,7 @@ EOF
       expect(o.name).to eq "A"
       expect(o.components).to eq %w[B]
       expect(o.line_number).to eq 19
-      expect(o.code).to eq "  a = 42;\n"
+      expect(o.code).to eq %[#line 20 "test.propane"\n  a = 42;\n#linereset\n]
 
       o = grammar.rules[1]
       expect(o.name).to eq "B"
@@ -95,7 +95,7 @@ EOF
       expect(o.name).to eq "B"
       expect(o.components).to eq []
       expect(o.line_number).to eq 23
-      expect(o.code).to eq "  b = 0;\n"
+      expect(o.code).to eq %[#line 24 "test.propane"\n  b = 0;\n#linereset\n]
     end
 
     it "parses code segments with semicolons" do
@@ -113,7 +113,7 @@ tokenid token_with_no_pattern;
 
 prefix myparser_;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
       expect(grammar.prefix).to eq "myparser_"
 
       o = grammar.tokens.find {|token| token.name == "code1"}
@@ -122,7 +122,7 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq "  a = b;\n  return c;\n"
+      expect(o.code).to eq %[#line 2 "test.propane"\n  a = b;\n  return c;\n#linereset\n]
 
       o = grammar.tokens.find {|token| token.name == "code2"}
       expect(o).to_not be_nil
@@ -130,7 +130,42 @@ EOF
 
       o = grammar.patterns.find {|pattern| pattern.token == o}
       expect(o).to_not be_nil
-      expect(o.code).to eq %[  writeln("Hello there");\n]
+      expect(o.code).to eq %[#line 7 "test.propane"\n  writeln("Hello there");\n#linereset\n]
+    end
+
+    it "does not emit #line directives with noline statement" do
+      input = <<EOF
+noline;
+
+token code1 <<
+  a = b;
+  return c;
+>>
+
+token code2 <<
+  writeln("Hello there");
+>>
+
+tokenid token_with_no_pattern;
+
+prefix myparser_;
+EOF
+      grammar = Grammar.new(input, "test.propane")
+      expect(grammar.prefix).to eq "myparser_"
+
+      o = grammar.tokens.find {|token| token.name == "code1"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  a = b;\n  return c;]
+
+      o = grammar.tokens.find {|token| token.name == "code2"}
+      expect(o).to_not be_nil
+
+      o = grammar.patterns.find {|pattern| pattern.token == o}
+      expect(o).to_not be_nil
+      expect(o.code).to eq %[\n  writeln("Hello there");]
     end
 
     it "supports mode labels" do
@@ -144,7 +179,7 @@ m2: /bar/ <<
 drop /q/;
 m3: drop /r/;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "a"}
       expect(o).to_not be_nil
@@ -197,7 +232,7 @@ tokenid int(integer);
 Start (node) -> R;
 R -> abc int;
 EOF
-      grammar = Grammar.new(input)
+      grammar = Grammar.new(input, "test.propane")
 
       o = grammar.tokens.find {|token| token.name == "abc"}
       expect(o).to_not be_nil

spec/propane/lexer/dfa_spec.rb

@@ -51,7 +51,7 @@ class TestLexer
 end
 
 def run(grammar, input)
-  grammar = Propane::Grammar.new(grammar)
+  grammar = Propane::Grammar.new(grammar, "test.propane")
   token_dfa = Propane::Lexer::DFA.new(grammar.patterns)
   test_lexer = TestLexer.new(token_dfa)
   test_lexer.lex(input)

spec/propane_spec.rb

@@ -1699,6 +1699,121 @@ EOF
         results = run_test(language: language)
         expect(results.status).to eq 0
       end
+
+      it "allows a custom lex function" do
+        if language == "d"
+          write_grammar <<EOF
+<<
+private size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info.token = TOKEN_a;
+        out_token_info.pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info.token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        else
+          write_grammar <<EOF
+<<
+static size_t mylexfn(p_context_t * context, p_token_info_t * out_token_info)
+{
+    static size_t count;
+    size_t result = P_SUCCESS;
+    if (count > 0)
+    {
+        out_token_info->token = TOKEN_a;
+        out_token_info->pvalue = p_value(count);
+        count--;
+    }
+    else
+    {
+        result = p_lex(context, out_token_info);
+        if (out_token_info->token == TOKEN_c)
+        {
+            count = 3;
+        }
+    }
+    return result;
+}
+>>
+
+ptype size_t;
+lex_fn mylexfn;
+
+token a << $$ = 7; >>
+token b << $$ = 8; >>
+token c << $$ = 9; >>
+Start -> << $$ = 0; >>
+Start -> Start ID << $$ = ($1 << 4) | $2; >>
+ID -> a << $$ = $1; >>
+ID -> b << $$ = $1; >>
+ID -> c << $$ = $1; >>
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_custom_lex_fn.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.status).to eq 0
+      end
+
+      it "provides accessor functions to extract user values from a parser value" do
+        if language == "d"
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = string;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = "hello"; >>
+Start -> num flt str;
+EOF
+        else
+          write_grammar <<EOF
+ptype int;
+ptype float = float;
+ptype string = char *;
+
+drop /\\s+/;
+token num /\\d+/ << $$ = 42; >>
+token flt (float) /f/ << $$ = 1.5; >>
+token str (string) /s/ << $$ = (char *)"hello"; >>
+Start -> num flt str;
+EOF
+        end
+        run_propane(language: language)
+        compile("spec/test_value_accessors.#{language}", language: language)
+        results = run_test(language: language)
+        expect(results.stderr).to eq ""
+        expect(results.status).to eq 0
+      end
     end
   end
 end

spec/test_custom_lex_fn.c

@@ -0,0 +1,15 @@
+#include "testparser.h"
+#include "testutils.h"
+#include <string.h>
+
+int main()
+{
+    char const * input = "cbacba";
+    p_context_t * context = p_context_new((uint8_t const *)input, strlen(input));
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_custom_lex_fn.d

@@ -0,0 +1,18 @@
+import testparser;
+import std.stdio;
+import testutils;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    string input = "cbacba";
+    p_context_t * context = p_context_new(input);
+    assert_eq(P_SUCCESS, p_parse(context));
+    size_t result = p_result(context);
+    assert_eq(0x932187932187, result);
+    p_context_delete(context);
+}

spec/test_value_accessors.c

@@ -0,0 +1,29 @@
+#include "testparser.h"
+#include <assert.h>
+#include <string.h>
+
+int main()
+{
+    p_token_info_t token_info;
+    char const * input = "42 f s";
+    p_context_t * context;
+    context = p_context_new((uint8_t const *)input, strlen(input));
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(strcmp(p_value_get_string(&token_info.pvalue), "hello") == 0);
+
+    p_context_delete(context);
+
+    return 0;
+}

spec/test_value_accessors.d

@@ -0,0 +1,29 @@
+import testparser;
+import std.stdio;
+
+int main()
+{
+    return 0;
+}
+
+unittest
+{
+    p_token_info_t token_info;
+    string input = "42 f s";
+    p_context_t * context;
+    context = p_context_new(input);
+
+    /* Default ptype value extracted with p_value_get(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_num);
+    assert(p_value_get(&token_info.pvalue) == 42);
+
+    /* Named ptype values extracted with p_value_get_XXX(). */
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_flt);
+    assert(p_value_get_float(&token_info.pvalue) == 1.5);
+
+    assert(p_lex(context, &token_info) == P_SUCCESS);
+    assert(token_info.token == TOKEN_str);
+    assert(p_value_get_string(&token_info.pvalue) == "hello");
+}