diff --git a/src/jes/core/encoding.d b/src/jes/core/encoding.d index 32a6a0d..230c438 100644 --- a/src/jes/core/encoding.d +++ b/src/jes/core/encoding.d @@ -138,4 +138,48 @@ struct Encoding } return (20u * n_high_zeros / n) != 0u; } + + unittest + { + import std.stdio; + import std.file; + const(ubyte)[] dat; + Encoding e; + size_t bom_size; + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-8.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF8); + assert(bom_size == 0u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-8-bom.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF8); + assert(bom_size == 3u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-16le.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF16_LE); + assert(bom_size == 2u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-16le-nobom.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF16_LE); + assert(bom_size == 0u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-16be.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF16_BE); + assert(bom_size == 2u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/utf-16be-nobom.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.UTF16_BE); + assert(bom_size == 0u); + + dat = cast(const(ubyte)[])std.file.read("test/files/encodings/cp-1252.txt"); + e = Encoding.detect_encoding(dat.ptr, dat.length, &bom_size); + assert(e == Encoding.CP1252); + assert(bom_size == 0u); + } };