| 1 | package doctor |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "reasonix/internal/agent" |
| 11 | ) |
| 12 | |
| 13 | func TestCollectQualityProducesPublicSafeSummary(t *testing.T) { |
| 14 | dir := t.TempDir() |
| 15 | path := filepath.Join(dir, "private-project-session.jsonl") |
| 16 | const secret = "private-source-and-prompt-secret" |
| 17 | messages := []string{ |
| 18 | `{"role":"system","content":"` + secret + `"}`, |
| 19 | `{"role":"user","content":"fix /private/example/repo and keep ` + secret + `"}`, |
| 20 | `{"role":"assistant","tool_calls":[{"id":"edit-1","name":"edit_file","arguments":"{\"path\":\"/private/example/repo/app.go\"}"}]}`, |
| 21 | `{"role":"tool","tool_call_id":"edit-1","name":"edit_file","content":"updated ` + secret + `"}`, |
| 22 | `{"role":"assistant","reasoning_content":"verify the change","tool_calls":[{"id":"test-1","name":"bash","arguments":"{\"command\":\"go test ./...\"}"}]}`, |
| 23 | `{"role":"tool","tool_call_id":"test-1","name":"bash","content":"ok"}`, |
| 24 | `{"role":"user","content":"<compaction-summary>\nprivate summary ` + secret + `"}`, |
| 25 | `{"role":"assistant","content":"done"}`, |
| 26 | } |
| 27 | if err := os.WriteFile(path, []byte(strings.Join(messages, "\n")+"\n"), 0o644); err != nil { |
| 28 | t.Fatal(err) |
| 29 | } |
| 30 | if err := agent.SaveBranchMeta(path, agent.BranchMeta{ |
| 31 | ID: "private-session-id", |
| 32 | Model: "corp.private.example/supersecret-model", |
| 33 | TokenMode: "economy", |
| 34 | Mode: "plan-yolo", |
| 35 | ToolApprovalMode: "yolo", |
| 36 | Goal: "finish " + secret, |
| 37 | Recovered: true, |
| 38 | }); err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | telemetry := `{ |
| 42 | "version":2, |
| 43 | "usage":{ |
| 44 | "promptTokens":1000, |
| 45 | "completionTokens":200, |
| 46 | "reasoningTokens":80, |
| 47 | "cacheHitTokens":750, |
| 48 | "cacheMissTokens":250, |
| 49 | "requestCount":5, |
| 50 | "sources":{ |
| 51 | "executor":{"requestCount":2}, |
| 52 | "planner":{"requestCount":1}, |
| 53 | "subagent":{"requestCount":1}, |
| 54 | "` + secret + `":{"requestCount":1} |
| 55 | } |
| 56 | } |
| 57 | }` |
| 58 | if err := os.WriteFile(path+".telemetry.json", []byte(telemetry), 0o644); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | |
| 62 | report, err := CollectQuality(QualityOptions{Version: "v-test", SessionRef: path}) |
| 63 | if err != nil { |
| 64 | t.Fatalf("CollectQuality: %v", err) |
| 65 | } |
| 66 | if report.Profile.ModelFamily != "custom/unknown" || report.Profile.RuntimeProfile != "economy" { |
| 67 | t.Fatalf("profile = %+v", report.Profile) |
| 68 | } |
| 69 | if report.Profile.CollaborationMode != "plan" || report.Profile.ToolApprovalMode != "yolo" || !report.Profile.GoalActive || !report.Profile.Recovered { |
| 70 | t.Fatalf("profile modes = %+v", report.Profile) |
| 71 | } |
| 72 | if report.Transcript.ToolCalls != 2 || report.Transcript.WriterCalls != 1 || report.Transcript.VerificationCalls != 1 || report.Transcript.CompactionSummaries != 1 { |
| 73 | t.Fatalf("transcript = %+v", report.Transcript) |
| 74 | } |
| 75 | if report.Transcript.ToolCallTurnsWithoutReasoning != 1 { |
| 76 | t.Fatalf("tool-call turns without reasoning = %d", report.Transcript.ToolCallTurnsWithoutReasoning) |
| 77 | } |
| 78 | if report.Usage.CacheHitPercent == nil || *report.Usage.CacheHitPercent != 75 { |
| 79 | t.Fatalf("usage = %+v", report.Usage) |
| 80 | } |
| 81 | if report.Signals.ExecutorRequests != 2 || report.Signals.PlannerRequests != 1 || report.Signals.SubagentRequests != 1 || report.Signals.OtherRequests != 1 { |
| 82 | t.Fatalf("signals = %+v", report.Signals) |
| 83 | } |
| 84 | |
| 85 | jsonData, err := json.Marshal(report) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | text := RenderQualityText(report) |
| 90 | for _, output := range []string{string(jsonData), text} { |
| 91 | for _, forbidden := range []string{secret, dir, "/private/example", "private-session-id", "supersecret-model", "corp.private.example"} { |
| 92 | if strings.Contains(output, forbidden) { |
| 93 | t.Fatalf("quality summary leaked %q:\n%s", forbidden, output) |
| 94 | } |
| 95 | } |
| 96 | } |
| 97 | if !strings.Contains(text, "privacy: transcript text, paths") { |
| 98 | t.Fatalf("text missing privacy statement:\n%s", text) |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | func TestCollectQualityWithoutTelemetryStaysUseful(t *testing.T) { |
| 103 | path := filepath.Join(t.TempDir(), "session.jsonl") |
| 104 | if err := os.WriteFile(path, []byte(`{"role":"user","content":"hello"}`+"\n"), 0o644); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | report, err := CollectQuality(QualityOptions{Version: "v-test", SessionRef: path}) |
| 108 | if err != nil { |
| 109 | t.Fatalf("CollectQuality: %v", err) |
| 110 | } |
| 111 | if report.Usage.Available { |
| 112 | t.Fatal("usage should be unavailable without desktop telemetry") |
| 113 | } |
| 114 | if len(report.Warnings) != 1 || !strings.Contains(report.Warnings[0], "telemetry") { |
| 115 | t.Fatalf("warnings = %v", report.Warnings) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | func TestPublicTokenModeUsesRuntimeProfileNames(t *testing.T) { |
| 120 | for input, want := range map[string]string{ |
| 121 | "": "balanced", |
| 122 | "full": "balanced", |
| 123 | "balanced": "balanced", |
| 124 | "economy": "economy", |
| 125 | "delivery": "delivery", |
| 126 | } { |
| 127 | if got := publicTokenMode(input); got != want { |
| 128 | t.Errorf("publicTokenMode(%q) = %q, want %q", input, got, want) |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 |