| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "reflect" |
| 5 | "testing" |
| 6 | |
| 7 | "reasonix/internal/ablation" |
| 8 | ) |
| 9 | |
| 10 | func TestAppendBenchmarkProfileArgsBaselineIsByteIdentical(t *testing.T) { |
| 11 | args := []string{"run", "fix the bug"} |
| 12 | if got := appendBenchmarkProfileArgs(args, benchmarkProfileBaseline); !reflect.DeepEqual(got, args) { |
| 13 | t.Fatalf("baseline args changed: %v", got) |
| 14 | } |
| 15 | } |
| 16 | |
| 17 | func TestAppendBenchmarkProfileArgsDeliveryUsesRealRuntimeProfile(t *testing.T) { |
| 18 | args := []string{"run"} |
| 19 | got := appendBenchmarkProfileArgs(args, benchmarkProfileDelivery) |
| 20 | want := []string{"run", "--profile", "delivery"} |
| 21 | if !reflect.DeepEqual(got, want) { |
| 22 | t.Fatalf("delivery args = %v, want %v", got, want) |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | func TestBuildRunTaskArgsEnablesUnattendedWorkspaceWrites(t *testing.T) { |
| 27 | got := buildRunTaskArgs("metrics.json", "e2e", benchmarkProfileDelivery, ablation.Set{}, 12, "fix it") |
| 28 | want := []string{ |
| 29 | "run", "--auto", "--metrics", "metrics.json", |
| 30 | "--model", "e2e", "--max-steps", "12", |
| 31 | "--profile", "delivery", "fix it", |
| 32 | } |
| 33 | if !reflect.DeepEqual(got, want) { |
| 34 | t.Fatalf("run task args = %v, want %v", got, want) |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | func TestBuildRunTaskArgsPassesTheAblationArmThrough(t *testing.T) { |
| 39 | got := buildRunTaskArgs("m.json", "", benchmarkProfileBaseline, ablation.New(ablation.Evidence, ablation.Planner), 0, "fix it") |
| 40 | want := []string{"run", "--auto", "--metrics", "m.json", "--ablate", "evidence,planner", "fix it"} |
| 41 | if !reflect.DeepEqual(got, want) { |
| 42 | t.Fatalf("ablated args = %v, want %v", got, want) |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | func TestDefaultSuiteBudgetCoversCurrentFiveTaskBaseline(t *testing.T) { |
| 47 | // The real-provider baseline exceeded 400k after only three successful |
| 48 | // tasks. Keep enough headroom to grade all five instead of silently skipping |
| 49 | // the final scenarios as normal model and cache usage varies. |
| 50 | if defaultSuiteTokenBudget < 800_000 { |
| 51 | t.Fatalf("default suite token budget = %d, want at least 800000", defaultSuiteTokenBudget) |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | func TestNormalizeBenchmarkProfile(t *testing.T) { |
| 56 | for _, input := range []string{"", "baseline", " BASELINE "} { |
| 57 | if got, err := normalizeBenchmarkProfile(input); err != nil || got != benchmarkProfileBaseline { |
| 58 | t.Fatalf("normalize(%q) = %q, %v", input, got, err) |
| 59 | } |
| 60 | } |
| 61 | if got, err := normalizeBenchmarkProfile("delivery"); err != nil || got != benchmarkProfileDelivery { |
| 62 | t.Fatalf("normalize(delivery) = %q, %v", got, err) |
| 63 | } |
| 64 | if _, err := normalizeBenchmarkProfile("fast"); err == nil { |
| 65 | t.Fatal("unknown profile should fail") |
| 66 | } |
| 67 | } |
| 68 |