| 1 | package encoding |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/binary" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "unicode/utf16" |
| 11 | |
| 12 | "golang.org/x/text/encoding/simplifiedchinese" |
| 13 | ) |
| 14 | |
| 15 | // --- Detect --- |
| 16 | |
| 17 | func TestDetectUTF8Plain(t *testing.T) { |
| 18 | enc, _ := Detect([]byte("hello world\n")) |
| 19 | if enc != UTF8 { |
| 20 | t.Errorf("got %v, want UTF8", enc) |
| 21 | } |
| 22 | } |
| 23 | |
| 24 | func TestDetectUTF8BOM(t *testing.T) { |
| 25 | in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("hello")...) |
| 26 | enc, _ := Detect(in) |
| 27 | if enc != UTF8BOM { |
| 28 | t.Errorf("got %v, want UTF8BOM", enc) |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | func TestDetectUTF16LE(t *testing.T) { |
| 33 | var b bytes.Buffer |
| 34 | b.Write([]byte{0xFF, 0xFE}) |
| 35 | for _, r := range utf16.Encode([]rune("hello")) { |
| 36 | _ = binary.Write(&b, binary.LittleEndian, r) |
| 37 | } |
| 38 | enc, _ := Detect(b.Bytes()) |
| 39 | if enc != UTF16LE { |
| 40 | t.Errorf("got %v, want UTF16LE", enc) |
| 41 | } |
| 42 | } |
| 43 | |
| 44 | func TestDetectUTF16BE(t *testing.T) { |
| 45 | var b bytes.Buffer |
| 46 | b.Write([]byte{0xFE, 0xFF}) |
| 47 | for _, r := range utf16.Encode([]rune("hello")) { |
| 48 | _ = binary.Write(&b, binary.BigEndian, r) |
| 49 | } |
| 50 | enc, _ := Detect(b.Bytes()) |
| 51 | if enc != UTF16BE { |
| 52 | t.Errorf("got %v, want UTF16BE", enc) |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | func TestDetectGB18030(t *testing.T) { |
| 57 | gb, err := simplifiedchinese.GB18030.NewEncoder().String("你好世界") |
| 58 | if err != nil { |
| 59 | t.Fatalf("encode: %v", err) |
| 60 | } |
| 61 | enc, _ := Detect([]byte(gb)) |
| 62 | if enc != GB18030 { |
| 63 | t.Errorf("got %v, want GB18030", enc) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | func TestDetectEmpty(t *testing.T) { |
| 68 | enc, _ := Detect(nil) |
| 69 | if enc != UTF8 { |
| 70 | t.Errorf("empty input: got %v, want UTF8", enc) |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | // --- Decode --- |
| 75 | |
| 76 | func TestDecodeUTF8(t *testing.T) { |
| 77 | in := []byte("hello\n世界") |
| 78 | out := Decode(in, UTF8) |
| 79 | if string(out) != "hello\n世界" { |
| 80 | t.Errorf("got %q", out) |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | func TestReadFileUTF8DecodesGB18030(t *testing.T) { |
| 85 | path := filepath.Join(t.TempDir(), "settings.json") |
| 86 | if err := os.WriteFile(path, Encode(`{"label":"中文"}`, GB18030), 0o644); err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | got, err := ReadFileUTF8(path) |
| 90 | if err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | if string(got) != `{"label":"中文"}` { |
| 94 | t.Fatalf("ReadFileUTF8 = %q", got) |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | func TestDecodeUTF8BOM(t *testing.T) { |
| 99 | in := append([]byte{0xEF, 0xBB, 0xBF}, []byte("hello")...) |
| 100 | out := Decode(in, UTF8BOM) |
| 101 | if string(out) != "hello" { |
| 102 | t.Errorf("got %q, want 'hello'", out) |
| 103 | } |
| 104 | if bytes.Contains(out, []byte{0xEF, 0xBB, 0xBF}) { |
| 105 | t.Error("BOM leaked into decoded output") |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | func TestDecodeUTF16LE(t *testing.T) { |
| 110 | var b bytes.Buffer |
| 111 | b.Write([]byte{0xFF, 0xFE}) |
| 112 | for _, r := range utf16.Encode([]rune("hello\nworld")) { |
| 113 | _ = binary.Write(&b, binary.LittleEndian, r) |
| 114 | } |
| 115 | out := Decode(b.Bytes(), UTF16LE) |
| 116 | if string(out) != "hello\nworld" { |
| 117 | t.Errorf("got %q", out) |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | func TestDecodeUTF16BE(t *testing.T) { |
| 122 | var b bytes.Buffer |
| 123 | b.Write([]byte{0xFE, 0xFF}) |
| 124 | for _, r := range utf16.Encode([]rune("hello\nworld")) { |
| 125 | _ = binary.Write(&b, binary.BigEndian, r) |
| 126 | } |
| 127 | out := Decode(b.Bytes(), UTF16BE) |
| 128 | if string(out) != "hello\nworld" { |
| 129 | t.Errorf("got %q", out) |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | func TestDecodeGB18030(t *testing.T) { |
| 134 | gb, _ := simplifiedchinese.GB18030.NewEncoder().String("你好世界\n第二行") |
| 135 | out := Decode([]byte(gb), GB18030) |
| 136 | if string(out) != "你好世界\n第二行" { |
| 137 | t.Errorf("got %q", out) |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | func TestDecodeLossyUTF8(t *testing.T) { |
| 142 | in := []byte{0xFF, 0xFE, 'a'} |
| 143 | out := Decode(in, LossyUTF8) |
| 144 | if !bytes.Equal(out, in) { |
| 145 | t.Errorf("LossyUTF8 should pass through, got %q", out) |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // --- Encode --- |
| 150 | |
| 151 | func TestEncodeUTF8(t *testing.T) { |
| 152 | out := Encode("hello", UTF8) |
| 153 | if string(out) != "hello" { |
| 154 | t.Errorf("got %q", out) |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | func TestEncodeUTF8BOM(t *testing.T) { |
| 159 | out := Encode("hello", UTF8BOM) |
| 160 | if !bytes.HasPrefix(out, []byte{0xEF, 0xBB, 0xBF}) { |
| 161 | t.Error("missing UTF-8 BOM prefix") |
| 162 | } |
| 163 | if string(out[3:]) != "hello" { |
| 164 | t.Errorf("body = %q", out[3:]) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | func TestEncodeUTF16LE(t *testing.T) { |
| 169 | out := Encode("hi", UTF16LE) |
| 170 | if len(out) < 2 || out[0] != 0xFF || out[1] != 0xFE { |
| 171 | t.Error("missing UTF-16LE BOM") |
| 172 | } |
| 173 | decoded := Decode(out, UTF16LE) |
| 174 | if string(decoded) != "hi" { |
| 175 | t.Errorf("round-trip failed: got %q", decoded) |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | func TestEncodeUTF16BE(t *testing.T) { |
| 180 | out := Encode("hi", UTF16BE) |
| 181 | if len(out) < 2 || out[0] != 0xFE || out[1] != 0xFF { |
| 182 | t.Error("missing UTF-16BE BOM") |
| 183 | } |
| 184 | decoded := Decode(out, UTF16BE) |
| 185 | if string(decoded) != "hi" { |
| 186 | t.Errorf("round-trip failed: got %q", decoded) |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | func TestEncodeGB18030(t *testing.T) { |
| 191 | out := Encode("你好", GB18030) |
| 192 | dec, _ := simplifiedchinese.GB18030.NewDecoder().Bytes(out) |
| 193 | if string(dec) != "你好" { |
| 194 | t.Errorf("round-trip failed: got %q", dec) |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | // --- Round-trip --- |
| 199 | |
| 200 | func TestRoundTripGB18030(t *testing.T) { |
| 201 | original := "你好世界\n第二行\n" |
| 202 | gb, _ := simplifiedchinese.GB18030.NewEncoder().String(original) |
| 203 | |
| 204 | enc, _ := Detect([]byte(gb)) |
| 205 | decoded := string(Decode([]byte(gb), enc)) |
| 206 | if decoded != original { |
| 207 | t.Fatalf("decode mismatch: %q", decoded) |
| 208 | } |
| 209 | |
| 210 | edited := strings.Replace(decoded, "第二行", "新的行", 1) |
| 211 | reencoded := Encode(edited, enc) |
| 212 | redecoded := string(Decode(reencoded, enc)) |
| 213 | if redecoded != edited { |
| 214 | t.Errorf("round-trip failed: got %q, want %q", redecoded, edited) |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | func TestRoundTripUTF16LE(t *testing.T) { |
| 219 | original := "hello\nworld\n" |
| 220 | encoded := Encode(original, UTF16LE) |
| 221 | enc, _ := Detect(encoded) |
| 222 | decoded := string(Decode(encoded, enc)) |
| 223 | if decoded != original { |
| 224 | t.Errorf("round-trip failed: got %q, want %q", decoded, original) |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | func TestRoundTripUTF8BOM(t *testing.T) { |
| 229 | original := "hello world\n" |
| 230 | encoded := Encode(original, UTF8BOM) |
| 231 | enc, _ := Detect(encoded) |
| 232 | decoded := string(Decode(encoded, enc)) |
| 233 | if decoded != original { |
| 234 | t.Errorf("round-trip failed: got %q, want %q", decoded, original) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // --- BOM-less UTF-16 --- |
| 239 | |
| 240 | func utf16NoBOM(t *testing.T, s string, order binary.ByteOrder) []byte { |
| 241 | t.Helper() |
| 242 | var b bytes.Buffer |
| 243 | for _, r := range utf16.Encode([]rune(s)) { |
| 244 | _ = binary.Write(&b, order, r) |
| 245 | } |
| 246 | return b.Bytes() |
| 247 | } |
| 248 | |
| 249 | func TestDetectUTF16LENoBOM(t *testing.T) { |
| 250 | in := utf16NoBOM(t, "// Created by 69431 on 2024/12/31\n#include \"x.h\"\n", binary.LittleEndian) |
| 251 | enc, _ := Detect(in) |
| 252 | if enc != UTF16LENoBOM { |
| 253 | t.Errorf("got %v, want UTF16LENoBOM", enc) |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | func TestDetectUTF16BENoBOM(t *testing.T) { |
| 258 | in := utf16NoBOM(t, "package main\nfunc main() {}\n", binary.BigEndian) |
| 259 | enc, _ := Detect(in) |
| 260 | if enc != UTF16BENoBOM { |
| 261 | t.Errorf("got %v, want UTF16BENoBOM", enc) |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | func TestDetectPlainUTF8NotUTF16(t *testing.T) { |
| 266 | enc, _ := Detect([]byte("the quick brown fox jumps over the lazy dog\n")) |
| 267 | if enc != UTF8 { |
| 268 | t.Errorf("ASCII text misdetected as %v", enc) |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | func TestDetectBinaryNotUTF16NoBOM(t *testing.T) { |
| 273 | // NULs on both parities — genuine binary must not look like BOM-less UTF-16. |
| 274 | bin := []byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0xFF, 0x00, 0x00, 0x03, 0x00, 0x00, 0x04, 0x00, 0x00, 0x05, 0x00, 0x00} |
| 275 | if k, ok := DetectUTF16NoBOM(bin); ok { |
| 276 | t.Errorf("binary misdetected as %v", k) |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | func TestDecodeUTF16LENoBOM(t *testing.T) { |
| 281 | in := utf16NoBOM(t, "hello\nworld", binary.LittleEndian) |
| 282 | if out := string(Decode(in, UTF16LENoBOM)); out != "hello\nworld" { |
| 283 | t.Errorf("got %q", out) |
| 284 | } |
| 285 | } |
| 286 | |
| 287 | func TestRoundTripUTF16LENoBOM(t *testing.T) { |
| 288 | original := "// c++ source\nint main() { return 0; }\n" |
| 289 | encoded := utf16NoBOM(t, original, binary.LittleEndian) |
| 290 | |
| 291 | enc, _ := Detect(encoded) |
| 292 | if enc != UTF16LENoBOM { |
| 293 | t.Fatalf("detect: got %v", enc) |
| 294 | } |
| 295 | decoded := string(Decode(encoded, enc)) |
| 296 | if decoded != original { |
| 297 | t.Fatalf("decode mismatch: %q", decoded) |
| 298 | } |
| 299 | edited := strings.Replace(decoded, "return 0", "return 1", 1) |
| 300 | reencoded := Encode(edited, enc) |
| 301 | if bytes.HasPrefix(reencoded, []byte{0xFF, 0xFE}) || bytes.HasPrefix(reencoded, []byte{0xFE, 0xFF}) { |
| 302 | t.Error("no-BOM re-encode leaked a BOM") |
| 303 | } |
| 304 | if redecoded := string(Decode(reencoded, enc)); redecoded != edited { |
| 305 | t.Errorf("round-trip failed: got %q, want %q", redecoded, edited) |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | // --- UTF-16 supplementary plane (surrogate pairs) --- |
| 310 | |
| 311 | func TestSurrogatePairRoundTrip(t *testing.T) { |
| 312 | // U+1F600 (😀) is in the supplementary plane and requires a surrogate pair. |
| 313 | original := "hello 😀 world" |
| 314 | encoded := Encode(original, UTF16LE) |
| 315 | decoded := string(Decode(encoded, UTF16LE)) |
| 316 | if decoded != original { |
| 317 | t.Errorf("surrogate pair round-trip failed: got %q, want %q", decoded, original) |
| 318 | } |
| 319 | } |
| 320 |