| 1 | package builtin |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "runtime" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | |
| 12 | "golang.org/x/text/encoding/simplifiedchinese" |
| 13 | ) |
| 14 | |
| 15 | // TestReadFileStreamsLargeGB18030 proves GB18030 content far past the 256KB |
| 16 | // detection sample still decodes correctly via the streaming read path. |
| 17 | func TestReadFileStreamsLargeGB18030(t *testing.T) { |
| 18 | dir := t.TempDir() |
| 19 | path := filepath.Join(dir, "big.gbk") |
| 20 | var sb strings.Builder |
| 21 | for i := 0; i < 20000; i++ { |
| 22 | sb.WriteString("第一行中文 line one 你好世界\n") |
| 23 | } |
| 24 | sb.WriteString("终点标记 THE-END\n") |
| 25 | enc, err := simplifiedchinese.GB18030.NewEncoder().String(sb.String()) |
| 26 | if err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | if err := os.WriteFile(path, []byte(enc), 0o644); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | args, _ := json.Marshal(map[string]any{"path": path, "offset": 19999, "limit": 2}) |
| 33 | out, err := readFile{}.Execute(context.Background(), args) |
| 34 | if err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | if !strings.Contains(out, "终点标记 THE-END") || !strings.Contains(out, "你好世界") { |
| 38 | t.Fatalf("deep GB18030 content not decoded correctly:\n%s", out) |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | // TestReadFileLargeBoundedMemory guards against re-slurping the whole file: a |
| 43 | // small read of a large file must allocate far less than the file size. |
| 44 | func TestReadFileLargeBoundedMemory(t *testing.T) { |
| 45 | dir := t.TempDir() |
| 46 | path := filepath.Join(dir, "big.txt") |
| 47 | var sb strings.Builder |
| 48 | for i := 0; i < 130000; i++ { // ~8 MB, no NUL |
| 49 | sb.WriteString("a line of perfectly ordinary text in a large utf-8 file\n") |
| 50 | } |
| 51 | if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | args, _ := json.Marshal(map[string]any{"path": path, "limit": 5}) |
| 55 | |
| 56 | runtime.GC() |
| 57 | var m0, m1 runtime.MemStats |
| 58 | runtime.ReadMemStats(&m0) |
| 59 | out, err := readFile{}.Execute(context.Background(), args) |
| 60 | runtime.ReadMemStats(&m1) |
| 61 | if err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | if alloc := m1.TotalAlloc - m0.TotalAlloc; alloc > 4<<20 { |
| 65 | t.Fatalf("read allocated %d bytes for a 5-line read of an ~8MB file — slurp regression", alloc) |
| 66 | } |
| 67 | if !strings.Contains(out, "1→a line") { |
| 68 | t.Fatalf("unexpected output: %q", out[:min(80, len(out))]) |
| 69 | } |
| 70 | } |
| 71 |