| 1 | package doctor |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "fmt" |
| 6 | "os" |
| 7 | "strings" |
| 8 | |
| 9 | "reasonix/internal/agent" |
| 10 | "reasonix/internal/provider" |
| 11 | ) |
| 12 | |
| 13 | // verificationCommandMarkers identify shell commands whose exit status verifies |
| 14 | // the turn's work (tests, vet, typecheck, lint, build). Matched as lowercase |
| 15 | // substrings of the bash tool's raw argument JSON. Inlined from the removed |
| 16 | // Memory v5 compiler so the quality report keeps its counting behavior. |
| 17 | var verificationCommandMarkers = []string{ |
| 18 | "go test", "go vet", "go build", "gofmt -l", "golangci-lint", |
| 19 | "npm test", "npm run test", "pnpm test", "pnpm run test", "yarn test", |
| 20 | "vitest", "jest", "npx tsc", "tsc ", "typecheck", "type-check", "check:css", |
| 21 | "eslint", "pytest", "ruff check", "cargo test", "cargo check", "cargo build", |
| 22 | "mvn test", "gradle test", "make test", "ctest", "phpunit", "rspec", |
| 23 | } |
| 24 | |
| 25 | // isVerificationToolCall reports whether a persisted tool call is a shell |
| 26 | // command whose exit status provides implementation evidence. It intentionally |
| 27 | // returns only a boolean so diagnostic callers never need to expose arguments. |
| 28 | func isVerificationToolCall(name, args string) bool { |
| 29 | if !strings.EqualFold(strings.TrimSpace(name), "bash") { |
| 30 | return false |
| 31 | } |
| 32 | args = strings.ToLower(args) |
| 33 | if args == "" { |
| 34 | return false |
| 35 | } |
| 36 | for _, marker := range verificationCommandMarkers { |
| 37 | if strings.Contains(args, marker) { |
| 38 | return true |
| 39 | } |
| 40 | } |
| 41 | return false |
| 42 | } |
| 43 | |
| 44 | const qualityReportSchemaVersion = 1 |
| 45 | |
| 46 | // QualityOptions selects one persisted session and produces a content-free |
| 47 | // diagnostic summary suitable for a public issue or discussion. |
| 48 | type QualityOptions struct { |
| 49 | Version string |
| 50 | SessionRef string |
| 51 | } |
| 52 | |
| 53 | type QualityReport struct { |
| 54 | SchemaVersion int `json:"schema_version"` |
| 55 | Version string `json:"version"` |
| 56 | Profile QualityProfile `json:"profile"` |
| 57 | Transcript QualityTranscript `json:"transcript"` |
| 58 | Usage QualityUsage `json:"usage"` |
| 59 | Signals QualitySignals `json:"signals"` |
| 60 | Warnings []string `json:"warnings,omitempty"` |
| 61 | } |
| 62 | |
| 63 | type QualityProfile struct { |
| 64 | ModelFamily string `json:"model_family"` |
| 65 | RuntimeProfile string `json:"runtime_profile"` |
| 66 | CollaborationMode string `json:"collaboration_mode"` |
| 67 | ToolApprovalMode string `json:"tool_approval_mode"` |
| 68 | GoalActive bool `json:"goal_active"` |
| 69 | Recovered bool `json:"recovered"` |
| 70 | } |
| 71 | |
| 72 | type QualityTranscript struct { |
| 73 | Messages int `json:"messages"` |
| 74 | UserMessages int `json:"user_messages"` |
| 75 | AssistantMessages int `json:"assistant_messages"` |
| 76 | ToolResults int `json:"tool_results"` |
| 77 | ToolCalls int `json:"tool_calls"` |
| 78 | WriterCalls int `json:"writer_calls"` |
| 79 | VerificationCalls int `json:"verification_calls"` |
| 80 | ToolCallTurnsWithoutReasoning int `json:"tool_call_turns_without_reasoning"` |
| 81 | CompactionSummaries int `json:"compaction_summaries"` |
| 82 | } |
| 83 | |
| 84 | type QualityUsage struct { |
| 85 | Available bool `json:"available"` |
| 86 | Requests int `json:"requests"` |
| 87 | PromptTokens int `json:"prompt_tokens"` |
| 88 | CompletionTokens int `json:"completion_tokens"` |
| 89 | ReasoningTokens int `json:"reasoning_tokens"` |
| 90 | CacheHitTokens int `json:"cache_hit_tokens"` |
| 91 | CacheMissTokens int `json:"cache_miss_tokens"` |
| 92 | Estimated bool `json:"estimated,omitempty"` |
| 93 | CacheHitPercent *int `json:"cache_hit_percent,omitempty"` |
| 94 | } |
| 95 | |
| 96 | type QualitySignals struct { |
| 97 | ExecutorRequests int `json:"executor_requests"` |
| 98 | PlannerRequests int `json:"planner_requests"` |
| 99 | SubagentRequests int `json:"subagent_requests"` |
| 100 | CompactionRequests int `json:"compaction_requests"` |
| 101 | ClassifierRequests int `json:"classifier_requests"` |
| 102 | OtherRequests int `json:"other_requests"` |
| 103 | } |
| 104 | |
| 105 | type qualityTelemetry struct { |
| 106 | Version int `json:"version"` |
| 107 | Usage struct { |
| 108 | PromptTokens int `json:"promptTokens"` |
| 109 | CompletionTokens int `json:"completionTokens"` |
| 110 | ReasoningTokens int `json:"reasoningTokens"` |
| 111 | CacheHitTokens int `json:"cacheHitTokens"` |
| 112 | CacheMissTokens int `json:"cacheMissTokens"` |
| 113 | Estimated bool `json:"estimated,omitempty"` |
| 114 | RequestCount int `json:"requestCount"` |
| 115 | Sources map[string]qualitySourceUsage `json:"sources"` |
| 116 | } `json:"usage"` |
| 117 | } |
| 118 | |
| 119 | type qualitySourceUsage struct { |
| 120 | RequestCount int `json:"requestCount"` |
| 121 | } |
| 122 | |
| 123 | // CollectQuality reads a session without returning any transcript text, path, |
| 124 | // identifier, tool argument/output, custom model name, or provider endpoint. |
| 125 | func CollectQuality(opts QualityOptions) (QualityReport, error) { |
| 126 | path, err := resolveSessionBundlePath(strings.TrimSpace(opts.SessionRef)) |
| 127 | if err != nil { |
| 128 | return QualityReport{}, err |
| 129 | } |
| 130 | session, err := agent.LoadSession(path) |
| 131 | if err != nil { |
| 132 | return QualityReport{}, err |
| 133 | } |
| 134 | meta, _, err := agent.LoadBranchMeta(path) |
| 135 | if err != nil { |
| 136 | return QualityReport{}, err |
| 137 | } |
| 138 | |
| 139 | report := QualityReport{ |
| 140 | SchemaVersion: qualityReportSchemaVersion, |
| 141 | Version: strings.TrimSpace(opts.Version), |
| 142 | Profile: QualityProfile{ |
| 143 | ModelFamily: publicModelFamily(meta.Model), |
| 144 | RuntimeProfile: publicTokenMode(meta.TokenMode), |
| 145 | CollaborationMode: publicCollaborationMode(meta.Mode, meta.Goal), |
| 146 | ToolApprovalMode: publicToolApprovalMode(meta.Mode, meta.ToolApprovalMode), |
| 147 | GoalActive: strings.TrimSpace(meta.Goal) != "", |
| 148 | Recovered: meta.Recovered, |
| 149 | }, |
| 150 | } |
| 151 | report.Transcript = summarizeQualityTranscript(session.Snapshot()) |
| 152 | if telemetry, ok := loadQualityTelemetry(path + ".telemetry.json"); ok { |
| 153 | report.Usage = summarizeQualityUsage(telemetry) |
| 154 | report.Signals = summarizeQualitySignals(telemetry.Usage.Sources, telemetry.Usage.RequestCount) |
| 155 | } else { |
| 156 | report.Warnings = append(report.Warnings, "desktop telemetry is unavailable for this session") |
| 157 | } |
| 158 | if report.Profile.ModelFamily == "deepseek" && report.Transcript.ToolCallTurnsWithoutReasoning > 0 { |
| 159 | report.Warnings = append(report.Warnings, "one or more DeepSeek tool-call turns have no persisted reasoning content") |
| 160 | } |
| 161 | if report.Transcript.WriterCalls > 0 && report.Transcript.VerificationCalls == 0 { |
| 162 | report.Warnings = append(report.Warnings, "writer tools were used without a persisted verification-shaped shell call") |
| 163 | } |
| 164 | return report, nil |
| 165 | } |
| 166 | |
| 167 | func summarizeQualityTranscript(messages []provider.Message) QualityTranscript { |
| 168 | var out QualityTranscript |
| 169 | out.Messages = len(messages) |
| 170 | for _, message := range messages { |
| 171 | switch message.Role { |
| 172 | case provider.RoleUser: |
| 173 | out.UserMessages++ |
| 174 | if agent.IsCompactionSummary(message) { |
| 175 | out.CompactionSummaries++ |
| 176 | } |
| 177 | case provider.RoleAssistant: |
| 178 | out.AssistantMessages++ |
| 179 | if len(message.ToolCalls) > 0 && strings.TrimSpace(message.ReasoningContent) == "" { |
| 180 | out.ToolCallTurnsWithoutReasoning++ |
| 181 | } |
| 182 | for _, call := range message.ToolCalls { |
| 183 | out.ToolCalls++ |
| 184 | if qualityWriterTool(call.Name) { |
| 185 | out.WriterCalls++ |
| 186 | } |
| 187 | if isVerificationToolCall(call.Name, call.Arguments) { |
| 188 | out.VerificationCalls++ |
| 189 | } |
| 190 | } |
| 191 | case provider.RoleTool: |
| 192 | out.ToolResults++ |
| 193 | } |
| 194 | } |
| 195 | return out |
| 196 | } |
| 197 | |
| 198 | func qualityWriterTool(name string) bool { |
| 199 | switch strings.ToLower(strings.TrimSpace(name)) { |
| 200 | case "edit_file", "multi_edit", "write_file", "move_file", "delete_range", "delete_symbol", "notebook_edit": |
| 201 | return true |
| 202 | default: |
| 203 | return false |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | func loadQualityTelemetry(path string) (qualityTelemetry, bool) { |
| 208 | data, err := os.ReadFile(path) |
| 209 | if err != nil { |
| 210 | return qualityTelemetry{}, false |
| 211 | } |
| 212 | var telemetry qualityTelemetry |
| 213 | if json.Unmarshal(data, &telemetry) != nil || telemetry.Version <= 0 { |
| 214 | return qualityTelemetry{}, false |
| 215 | } |
| 216 | return telemetry, true |
| 217 | } |
| 218 | |
| 219 | func summarizeQualityUsage(telemetry qualityTelemetry) QualityUsage { |
| 220 | usage := QualityUsage{ |
| 221 | Available: true, |
| 222 | Requests: telemetry.Usage.RequestCount, |
| 223 | PromptTokens: telemetry.Usage.PromptTokens, |
| 224 | CompletionTokens: telemetry.Usage.CompletionTokens, |
| 225 | ReasoningTokens: telemetry.Usage.ReasoningTokens, |
| 226 | CacheHitTokens: telemetry.Usage.CacheHitTokens, |
| 227 | CacheMissTokens: telemetry.Usage.CacheMissTokens, |
| 228 | Estimated: telemetry.Usage.Estimated, |
| 229 | } |
| 230 | if total := usage.CacheHitTokens + usage.CacheMissTokens; total > 0 { |
| 231 | percent := usage.CacheHitTokens * 100 / total |
| 232 | usage.CacheHitPercent = &percent |
| 233 | } |
| 234 | return usage |
| 235 | } |
| 236 | |
| 237 | func summarizeQualitySignals(sources map[string]qualitySourceUsage, total int) QualitySignals { |
| 238 | var out QualitySignals |
| 239 | for source, usage := range sources { |
| 240 | switch strings.ToLower(strings.TrimSpace(source)) { |
| 241 | case "", "executor": |
| 242 | out.ExecutorRequests += usage.RequestCount |
| 243 | case "planner": |
| 244 | out.PlannerRequests += usage.RequestCount |
| 245 | case "subagent": |
| 246 | out.SubagentRequests += usage.RequestCount |
| 247 | case "compaction": |
| 248 | out.CompactionRequests += usage.RequestCount |
| 249 | case "classifier": |
| 250 | out.ClassifierRequests += usage.RequestCount |
| 251 | default: |
| 252 | out.OtherRequests += usage.RequestCount |
| 253 | } |
| 254 | } |
| 255 | accounted := out.ExecutorRequests + out.PlannerRequests + out.SubagentRequests + out.CompactionRequests + out.ClassifierRequests + out.OtherRequests |
| 256 | if total > accounted { |
| 257 | out.OtherRequests += total - accounted |
| 258 | } |
| 259 | return out |
| 260 | } |
| 261 | |
| 262 | func publicModelFamily(model string) string { |
| 263 | model = strings.ToLower(strings.TrimSpace(model)) |
| 264 | switch { |
| 265 | case model == "": |
| 266 | return "unknown" |
| 267 | case strings.Contains(model, "deepseek"): |
| 268 | return "deepseek" |
| 269 | case strings.Contains(model, "claude"), strings.Contains(model, "anthropic"): |
| 270 | return "anthropic" |
| 271 | case strings.Contains(model, "openai"), strings.Contains(model, "gpt"), strings.Contains(model, "codex"): |
| 272 | return "openai" |
| 273 | case strings.Contains(model, "glm"), strings.Contains(model, "zhipu"): |
| 274 | return "zhipu/glm" |
| 275 | case strings.Contains(model, "kimi"), strings.Contains(model, "moonshot"): |
| 276 | return "kimi/moonshot" |
| 277 | case strings.Contains(model, "minimax"): |
| 278 | return "minimax" |
| 279 | case strings.Contains(model, "qwen"), strings.Contains(model, "dashscope"): |
| 280 | return "qwen" |
| 281 | case strings.Contains(model, "mimo"): |
| 282 | return "mimo" |
| 283 | case strings.Contains(model, "gemini"), strings.Contains(model, "google"): |
| 284 | return "google/gemini" |
| 285 | case strings.Contains(model, "ollama"): |
| 286 | return "ollama/local" |
| 287 | default: |
| 288 | return "custom/unknown" |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | func publicTokenMode(mode string) string { |
| 293 | switch strings.ToLower(strings.TrimSpace(mode)) { |
| 294 | case "economy": |
| 295 | return "economy" |
| 296 | case "delivery": |
| 297 | return "delivery" |
| 298 | default: |
| 299 | return "balanced" |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | func publicCollaborationMode(mode, goal string) string { |
| 304 | if strings.Contains(strings.ToLower(strings.TrimSpace(mode)), "plan") { |
| 305 | return "plan" |
| 306 | } |
| 307 | if strings.TrimSpace(goal) != "" { |
| 308 | return "goal" |
| 309 | } |
| 310 | return "normal" |
| 311 | } |
| 312 | |
| 313 | func publicToolApprovalMode(mode, approval string) string { |
| 314 | switch strings.ToLower(strings.TrimSpace(approval)) { |
| 315 | case "auto": |
| 316 | return "auto" |
| 317 | case "yolo": |
| 318 | return "yolo" |
| 319 | } |
| 320 | if strings.Contains(strings.ToLower(strings.TrimSpace(mode)), "yolo") { |
| 321 | return "yolo" |
| 322 | } |
| 323 | return "ask" |
| 324 | } |
| 325 | |
| 326 | func RenderQualityText(report QualityReport) string { |
| 327 | cache := "n/a" |
| 328 | if report.Usage.CacheHitPercent != nil { |
| 329 | cache = fmt.Sprintf("%d%%", *report.Usage.CacheHitPercent) |
| 330 | } |
| 331 | var out strings.Builder |
| 332 | fmt.Fprintf(&out, "Reasonix quality diagnostics\n") |
| 333 | fmt.Fprintf(&out, "- version: %s\n", valueOrUnknown(report.Version)) |
| 334 | fmt.Fprintf(&out, "- model family: %s\n", report.Profile.ModelFamily) |
| 335 | fmt.Fprintf(&out, "- profile: runtime=%s collaboration=%s approval=%s goal=%t recovered=%t\n", |
| 336 | report.Profile.RuntimeProfile, report.Profile.CollaborationMode, report.Profile.ToolApprovalMode, |
| 337 | report.Profile.GoalActive, report.Profile.Recovered) |
| 338 | fmt.Fprintf(&out, "- transcript: messages=%d user=%d assistant=%d tool_results=%d tool_calls=%d writers=%d verification=%d compaction_summaries=%d\n", |
| 339 | report.Transcript.Messages, report.Transcript.UserMessages, report.Transcript.AssistantMessages, |
| 340 | report.Transcript.ToolResults, report.Transcript.ToolCalls, report.Transcript.WriterCalls, |
| 341 | report.Transcript.VerificationCalls, report.Transcript.CompactionSummaries) |
| 342 | fmt.Fprintf(&out, "- reasoning: tool_call_turns_without_reasoning=%d\n", report.Transcript.ToolCallTurnsWithoutReasoning) |
| 343 | if report.Usage.Available { |
| 344 | fmt.Fprintf(&out, "- usage: requests=%d prompt=%d completion=%d reasoning=%d cache_hit=%s\n", |
| 345 | report.Usage.Requests, report.Usage.PromptTokens, report.Usage.CompletionTokens, |
| 346 | report.Usage.ReasoningTokens, cache) |
| 347 | fmt.Fprintf(&out, "- request sources: executor=%d planner=%d subagent=%d compaction=%d classifier=%d other=%d\n", |
| 348 | report.Signals.ExecutorRequests, report.Signals.PlannerRequests, report.Signals.SubagentRequests, |
| 349 | report.Signals.CompactionRequests, report.Signals.ClassifierRequests, report.Signals.OtherRequests) |
| 350 | } else { |
| 351 | fmt.Fprintf(&out, "- usage: unavailable\n") |
| 352 | } |
| 353 | for _, warning := range report.Warnings { |
| 354 | fmt.Fprintf(&out, "- warning: %s\n", warning) |
| 355 | } |
| 356 | fmt.Fprintf(&out, "- privacy: transcript text, paths, session identifiers, tool arguments/output, endpoints, and custom model names are omitted\n") |
| 357 | return out.String() |
| 358 | } |
| 359 | |
| 360 | func valueOrUnknown(value string) string { |
| 361 | if value = strings.TrimSpace(value); value != "" { |
| 362 | return value |
| 363 | } |
| 364 | return "unknown" |
| 365 | } |
| 366 |