| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "os" |
| 5 | "path/filepath" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | ) |
| 9 | |
| 10 | // A failed task is only actionable if the record says what the agent actually |
| 11 | // produced. For an exploration task the grader's "want X, got Y" line is the |
| 12 | // entire finding — a bare false would leave every failure indistinguishable. |
| 13 | func TestGradeVerboseKeepsWhatTheGraderSaid(t *testing.T) { |
| 14 | requireRealBash(t) |
| 15 | taskDir := t.TempDir() |
| 16 | verify := "#!/usr/bin/env bash\necho \"answer.txt normalized to 'askrigg', want 'gorsefen'\" >&2\nexit 1\n" |
| 17 | if err := os.WriteFile(filepath.Join(taskDir, "verify.sh"), []byte(verify), 0o755); err != nil { |
| 18 | t.Fatal(err) |
| 19 | } |
| 20 | |
| 21 | passed, said := gradeVerbose(t.TempDir(), taskDir) |
| 22 | if passed { |
| 23 | t.Fatal("a grader exiting non-zero must not pass") |
| 24 | } |
| 25 | if !strings.Contains(said, "want 'gorsefen'") { |
| 26 | t.Fatalf("grader output lost: %q", said) |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | func TestGradeVerboseStaysQuietWhenTheTaskPasses(t *testing.T) { |
| 31 | requireRealBash(t) |
| 32 | taskDir := t.TempDir() |
| 33 | if err := os.WriteFile(filepath.Join(taskDir, "verify.sh"), []byte("#!/usr/bin/env bash\nexit 0\n"), 0o755); err != nil { |
| 34 | t.Fatal(err) |
| 35 | } |
| 36 | if passed, said := gradeVerbose(t.TempDir(), taskDir); !passed || said != "" { |
| 37 | t.Fatalf("clean grade = %v, %q", passed, said) |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | // The excerpt is bounded so a chatty grader cannot crowd out the report, and |
| 42 | // bounded on runes so a multi-byte log cannot be cut mid-character. |
| 43 | func TestUtf8PrefixBoundsOnRunes(t *testing.T) { |
| 44 | if got := utf8Prefix("short", 10); got != "short" { |
| 45 | t.Errorf("under the limit was altered: %q", got) |
| 46 | } |
| 47 | got := utf8Prefix("这是一个很长的输出", 4) |
| 48 | if []rune(got)[4] != '…' || len([]rune(got)) != 5 { |
| 49 | t.Errorf("utf8Prefix = %q, want 4 runes plus an ellipsis", got) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | func TestAppendNoteKeepsBothCauses(t *testing.T) { |
| 54 | if got := appendNote("", "grader: x"); got != "grader: x" { |
| 55 | t.Errorf("first note = %q", got) |
| 56 | } |
| 57 | // A run that both errored and failed grading must report each: the run |
| 58 | // error explains the grade, and dropping it hides why. |
| 59 | if got := appendNote("run: exit 1", "grader: want y"); !strings.Contains(got, "run: exit 1") || !strings.Contains(got, "grader: want y") { |
| 60 | t.Errorf("combined note lost a cause: %q", got) |
| 61 | } |
| 62 | } |
| 63 |