josh/jes
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix Encoding detection of UTF-16.

Browse Source
This commit is contained in:
Josh Holtrop 2020-11-15 22:06:17 -05:00
parent bc21e746fb
commit 97e01ab714
1 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/jes/core/encoding.d
View File

@ -46,11 +46,6 @@ struct Encoding
} }
/* No BOM found. Check heuristically. */ /* No BOM found. Check heuristically. */
if (validate_utf8(data, n))
{
return Encoding(UTF8);
}
if (check_utf16(data, n, true)) if (check_utf16(data, n, true))
{ {
return Encoding(UTF16_LE); return Encoding(UTF16_LE);
@ -61,6 +56,11 @@ struct Encoding
return Encoding(UTF16_BE); return Encoding(UTF16_BE);
} }
if (validate_utf8(data, n))
{
return Encoding(UTF8);
}
return Encoding(CP1252); return Encoding(CP1252);
} }
@ -136,7 +136,8 @@ struct Encoding
n_high_zeros++; n_high_zeros++;
} }
} }
return (20u * n_high_zeros / n) != 0u; /* 20% threshold of high zeroes required. */
return (40u * n_high_zeros / n) != 0u;
} }
unittest unittest