Detect other invalid UTF-8 encodings

assets/parser.d.erb

@@ -83,6 +83,10 @@ class <%= classname %>
                     code_point = c & 0x01u;
                     following_bytes = 5u;
                 }
+                else
+                {
+                    return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                }
                 if (input_length <= following_bytes)
                 {
                     return DecodedCodePoint(CODE_POINT_INVALID, 0u);
@@ -91,8 +95,12 @@ class <%= classname %>
                 while (following_bytes-- > 0u)
                 {
                     input++;
-                    code_point <<= 6u;
+                    ubyte b = *input;
-                    code_point |= *input & 0x3Fu;
+                    if ((b & 0xC0u) != 0u)
+                    {
+                        return DecodedCodePoint(CODE_POINT_INVALID, 0u);
+                    }
+                    code_point = (code_point << 6u) | b;
                 }
             }
             return DecodedCodePoint(code_point, code_point_length);

spec/test_d_lexer.d

@@ -40,11 +40,29 @@ unittest
     dcp = Testparser.Decoder.decode_code_point(input, input_length);
     assert(dcp == DCP(Testparser.Decoder.CODE_POINT_EOF, 0u));
 
-    inputstring = "\xf0\x9f\xa7\xa1";
+    inputstring = "\xf0\x1f\x27\x21";
     input = cast(const(ubyte) *)inputstring.ptr;
     input_length = inputstring.length;
     dcp = Testparser.Decoder.decode_code_point(input, input_length);
     assert(dcp == DCP(0x1F9E1, 4u));
+
+    inputstring = "\xf0\x1f\x27";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    inputstring = "\xf0\x1f\x27\xFF";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
+
+    inputstring = "\xfe";
+    input = cast(const(ubyte) *)inputstring.ptr;
+    input_length = inputstring.length;
+    dcp = Testparser.Decoder.decode_code_point(input, input_length);
+    assert(dcp == DCP(Testparser.Decoder.CODE_POINT_INVALID, 0u));
 }
 
 unittest