| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "reflect" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | |
| 9 | "reasonix/internal/ablation" |
| 10 | ) |
| 11 | |
| 12 | // The harness prints an unmangled local image key but pulls a mangled one; the |
| 13 | // two differ and only the mangled form exists in the registry. |
| 14 | func TestSwebenchImageManglesDoubleUnderscore(t *testing.T) { |
| 15 | got := swebenchImage("swebench", "psf__requests-2317") |
| 16 | want := "swebench/sweb.eval.x86_64.psf_1776_requests-2317:latest" |
| 17 | if got != want { |
| 18 | t.Fatalf("image = %q, want %q", got, want) |
| 19 | } |
| 20 | if got := swebenchImage("swebench", "pylint-dev__pylint-7080"); !strings.Contains(got, "pylint-dev_1776_pylint-7080") { |
| 21 | t.Fatalf("hyphenated org mangled wrong: %q", got) |
| 22 | } |
| 23 | } |
| 24 | |
| 25 | func TestTestbedShellActivatesTheInstanceCondaEnv(t *testing.T) { |
| 26 | got := testbedShell("git diff") |
| 27 | if len(got) != 3 || got[0] != "bash" || got[1] != "-lc" { |
| 28 | t.Fatalf("shell wrapper = %v", got) |
| 29 | } |
| 30 | for _, want := range []string{"/opt/miniconda3/bin/activate", "conda activate testbed", "cd /testbed", "git diff"} { |
| 31 | if !strings.Contains(got[2], want) { |
| 32 | t.Errorf("command missing %q: %s", want, got[2]) |
| 33 | } |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | func TestSwebenchPromptWithholdsTheAnswerKey(t *testing.T) { |
| 38 | prompt := swebenchPrompt(swebenchInstance{ |
| 39 | InstanceID: "psf__requests-2317", |
| 40 | Problem: "method = builtin_str(method) breaks binary strings", |
| 41 | }) |
| 42 | if !strings.Contains(prompt, "builtin_str(method)") { |
| 43 | t.Fatal("the issue text must reach the agent") |
| 44 | } |
| 45 | for _, leak := range []string{"FAIL_TO_PASS", "PASS_TO_PASS", "test_patch"} { |
| 46 | if strings.Contains(prompt, leak) { |
| 47 | t.Errorf("prompt leaks the answer key: %s", leak) |
| 48 | } |
| 49 | } |
| 50 | if !strings.Contains(prompt, "do not modify any test file") { |
| 51 | t.Error("the no-test-edit rule must be stated; the grader replaces test files anyway") |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | func TestEncodePredictionsWritesOneHarnessRecordPerLine(t *testing.T) { |
| 56 | out, err := encodePredictions("reasonix", map[string]string{ |
| 57 | "a__a-1": "diff --git a/x b/x\n", |
| 58 | "b__b-2": "diff --git a/y b/y\n", |
| 59 | }, []string{"a__a-1", "missing__missing-9", "b__b-2"}) |
| 60 | if err != nil { |
| 61 | t.Fatalf("encode: %v", err) |
| 62 | } |
| 63 | lines := strings.Split(strings.TrimSpace(out), "\n") |
| 64 | if len(lines) != 2 { |
| 65 | t.Fatalf("lines = %d, want 2 (an instance with no patch is skipped, not emitted empty)", len(lines)) |
| 66 | } |
| 67 | for _, want := range []string{`"instance_id":"a__a-1"`, `"model_name_or_path":"reasonix"`, `"model_patch":"diff --git a/x b/x\n"`} { |
| 68 | if !strings.Contains(lines[0], want) { |
| 69 | t.Errorf("first record missing %q: %s", want, lines[0]) |
| 70 | } |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | func TestGradedClassNeverGuessesForAnUnmentionedInstance(t *testing.T) { |
| 75 | report := swebenchReport{ |
| 76 | ResolvedIDs: []string{"a__a-1"}, |
| 77 | UnresolvedIDs: []string{"b__b-2"}, |
| 78 | ErrorIDs: []string{"c__c-3"}, |
| 79 | EmptyPatchIDs: []string{"d__d-4"}, |
| 80 | IncompleteIDs: []string{"f__f-6"}, |
| 81 | } |
| 82 | for id, want := range map[string]string{ |
| 83 | "a__a-1": "solved", |
| 84 | "b__b-2": "wrong_patch", |
| 85 | "c__c-3": "grader_error", |
| 86 | "d__d-4": "no_patch", |
| 87 | "f__f-6": "eval_timeout", |
| 88 | "e__e-5": "ungraded", |
| 89 | } { |
| 90 | if got := report.gradedClass(id); got != want { |
| 91 | t.Errorf("class(%s) = %q, want %q", id, got, want) |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // Captured from a real run on 2026-08-04: swebench 4.1.0, gold predictions, |
| 97 | // run_id goldpylint. Pins the field names and the report path we depend on. |
| 98 | func TestSwebenchReportParsesTheHarnessSummary(t *testing.T) { |
| 99 | const captured = `{"total_instances":1,"submitted_instances":500,"completed_instances":1, |
| 100 | "resolved_instances":1,"unresolved_instances":0,"empty_patch_instances":0,"error_instances":0, |
| 101 | "completed_ids":["pylint-dev__pylint-7080"],"incomplete_ids":[],"empty_patch_ids":[], |
| 102 | "resolved_ids":["pylint-dev__pylint-7080"],"unresolved_ids":[],"error_ids":[],"schema_version":2}` |
| 103 | |
| 104 | var report swebenchReport |
| 105 | if err := json.Unmarshal([]byte(captured), &report); err != nil { |
| 106 | t.Fatalf("unmarshal: %v", err) |
| 107 | } |
| 108 | if got := report.gradedClass("pylint-dev__pylint-7080"); got != "solved" { |
| 109 | t.Fatalf("gold patch classified as %q, want solved", got) |
| 110 | } |
| 111 | if got := swebenchReportPath("gold", "goldpylint"); got != "gold.goldpylint.json" { |
| 112 | t.Fatalf("report path = %q, want gold.goldpylint.json", got) |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | // Issue text arrives verbatim from GitHub and routinely contains quotes, |
| 117 | // backticks, $ and newlines. It becomes one argv element inside a `bash -lc` |
| 118 | // string, so a quoting slip would let a problem statement run commands. |
| 119 | func TestShellQuoteAllContainsHostileIssueText(t *testing.T) { |
| 120 | hostile := "it's broken; `rm -rf /`; $(whoami)\n\"quoted\" && echo pwned" |
| 121 | got := shellQuoteAll([]string{"run", hostile}) |
| 122 | if !strings.HasPrefix(got, "'run' '") || !strings.HasSuffix(got, "'") { |
| 123 | t.Fatalf("every element must be single-quoted: %s", got) |
| 124 | } |
| 125 | // Inside single quotes the shell expands nothing, so the only way out is an |
| 126 | // unescaped apostrophe: every one in the payload must have been rewritten. |
| 127 | if strings.Count(got, `'\''`) != strings.Count(hostile, "'") { |
| 128 | t.Fatalf("apostrophes not all escaped: %s", got) |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | func TestSwebenchAgentArgsKeepTheControlArmClean(t *testing.T) { |
| 133 | got := swebenchAgentArgs("/tmp/m.json", "e2e", benchmarkProfileBaseline, "auto", ablation.Set{}, 60, "fix it") |
| 134 | want := []string{"run", "--permission-mode=auto", "--metrics", "/tmp/m.json", "--model", "e2e", "--max-steps", "60", "fix it"} |
| 135 | if !reflect.DeepEqual(got, want) { |
| 136 | t.Fatalf("control args = %v, want %v", got, want) |
| 137 | } |
| 138 | ablated := swebenchAgentArgs("/tmp/m.json", "", benchmarkProfileBaseline, "auto", ablation.New(ablation.Evidence), 0, "fix it") |
| 139 | if !reflect.DeepEqual(ablated, []string{"run", "--permission-mode=auto", "--metrics", "/tmp/m.json", "--ablate", "evidence", "fix it"}) { |
| 140 | t.Fatalf("ablated args = %v", ablated) |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // The two postures must differ in exactly one argument. If anything else moved |
| 145 | // between arms, the published delta would not isolate the permission gate. |
| 146 | func TestPermissionPostureIsTheOnlyDifferenceBetweenArms(t *testing.T) { |
| 147 | a := swebenchAgentArgs("/m.json", "e2e", benchmarkProfileBaseline, "auto", ablation.Set{}, 60, "fix it") |
| 148 | b := swebenchAgentArgs("/m.json", "e2e", benchmarkProfileBaseline, "yolo", ablation.Set{}, 60, "fix it") |
| 149 | if len(a) != len(b) { |
| 150 | t.Fatalf("arms differ in argument count: %v vs %v", a, b) |
| 151 | } |
| 152 | diffs := 0 |
| 153 | for i := range a { |
| 154 | if a[i] != b[i] { |
| 155 | diffs++ |
| 156 | } |
| 157 | } |
| 158 | if diffs != 1 || a[1] != "--permission-mode=auto" || b[1] != "--permission-mode=bypassPermissions" { |
| 159 | t.Fatalf("arms must differ only in the posture flag: %v vs %v", a, b) |
| 160 | } |
| 161 | if _, err := permissionFlag("bypass"); err == nil { |
| 162 | t.Fatal("an unknown posture must fail loudly rather than silently running unattended") |
| 163 | } |
| 164 | } |
| 165 |