assets/parser.d.erb

@@ -22,6 +22,8 @@ class <%= @classname %>
 <%   end %>
 <% end %>
         _TOKEN_COUNT = <%= @grammar.tokens.size %>,
+        _TOKEN_DECODE_ERROR = <%= TOKEN_DECODE_ERROR %>,
+        _TOKEN_DROP = <%= TOKEN_DROP %>,
     }
 
     struct Token
@@ -406,6 +408,7 @@ class <%= @classname %>
                 if (decoded.is_decode_error())
                 {
                     result.type = Result.Type.DECODE_ERROR;
+                    result.token = _TOKEN_DECODE_ERROR;
                     return;
                 }
                 bool lex_continue = false;

lib/propane.rb

@@ -25,6 +25,12 @@ require_relative "propane/version"
 
 class Propane
 
+  # Decoding error.
+  TOKEN_DECODE_ERROR = 0xFFFFFFFD
+
+  # Token ID for a "dropped" token.
+  TOKEN_DROP = 0xFFFFFFFE
+
   class Error < RuntimeError
   end