| 1 | //go:build live |
| 2 | |
| 3 | package openai |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "reasonix/internal/provider" |
| 14 | ) |
| 15 | |
| 16 | // probeResult captures the cache-relevant numbers from one real completion. |
| 17 | type probeResult struct { |
| 18 | prompt, hit, miss, reasoning int |
| 19 | sawReasoning bool |
| 20 | reasoningText string |
| 21 | } |
| 22 | |
| 23 | // TestRealDeepSeekCacheProbe is a build-tagged, env-gated end-to-end probe |
| 24 | // against the live DeepSeek API. It answers, with real numbers: |
| 25 | // 1. does DeepSeek's auto cache actually serve reasonix's request shape, and how |
| 26 | // much does a repeated prefix hit; |
| 27 | // 2. does deepseek-v4-flash even return reasoning_content (i.e. is the round-trip |
| 28 | // amplifier real for this model); |
| 29 | // 3. how much does DeepSeek's required tool-call reasoning replay add to |
| 30 | // prompt_tokens, and does the replayed prefix still receive cache hits. |
| 31 | // |
| 32 | // Run with: set -a; source .env; set +a; go test -tags live ./internal/provider/openai/ -run TestRealDeepSeekCacheProbe -v -count=1 |
| 33 | func TestRealDeepSeekCacheProbe(t *testing.T) { |
| 34 | key := os.Getenv("DEEPSEEK_API_KEY") |
| 35 | if key == "" { |
| 36 | t.Skip("DEEPSEEK_API_KEY not set — skipping live probe") |
| 37 | } |
| 38 | |
| 39 | p, err := New(provider.Config{ |
| 40 | Name: "deepseek", |
| 41 | BaseURL: "https://api.deepseek.com", |
| 42 | Model: "deepseek-v4-flash", |
| 43 | APIKey: key, |
| 44 | Extra: map[string]any{"api_key_env": "DEEPSEEK_API_KEY"}, |
| 45 | }) |
| 46 | if err != nil { |
| 47 | t.Fatalf("New: %v", err) |
| 48 | } |
| 49 | |
| 50 | send := func(msgs []provider.Message) (probeResult, error) { |
| 51 | ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) |
| 52 | defer cancel() |
| 53 | ch, err := p.Stream(ctx, provider.Request{Messages: msgs, Temperature: provider.TemperaturePtr(0), MaxTokens: 16}) |
| 54 | if err != nil { |
| 55 | return probeResult{}, err |
| 56 | } |
| 57 | var res probeResult |
| 58 | var rb strings.Builder |
| 59 | for chunk := range ch { |
| 60 | switch chunk.Type { |
| 61 | case provider.ChunkReasoning: |
| 62 | res.sawReasoning = true |
| 63 | rb.WriteString(chunk.Text) |
| 64 | case provider.ChunkUsage: |
| 65 | if chunk.Usage != nil { |
| 66 | res.prompt = chunk.Usage.PromptTokens |
| 67 | res.hit = chunk.Usage.CacheHitTokens |
| 68 | res.miss = chunk.Usage.CacheMissTokens |
| 69 | res.reasoning = chunk.Usage.ReasoningTokens |
| 70 | } |
| 71 | case provider.ChunkError: |
| 72 | return res, chunk.Err |
| 73 | } |
| 74 | } |
| 75 | res.reasoningText = rb.String() |
| 76 | return res, nil |
| 77 | } |
| 78 | |
| 79 | rate := func(r probeResult) string { |
| 80 | denom := r.hit + r.miss |
| 81 | if denom == 0 { |
| 82 | denom = r.prompt |
| 83 | } |
| 84 | if denom == 0 { |
| 85 | return "n/a" |
| 86 | } |
| 87 | return formatPct(r.hit, denom) |
| 88 | } |
| 89 | |
| 90 | // A large, stable head so the repeated prefix comfortably exceeds DeepSeek's |
| 91 | // 64-token cache block granularity and a hit is unambiguous. |
| 92 | bigHead := "You are a coding agent. Follow these standing instructions precisely. " + |
| 93 | strings.Repeat("Keep the prefix identical across turns so the context cache can serve it. ", 60) |
| 94 | |
| 95 | // ---- Probe 1: does the cache serve a repeated prefix at all? ---- |
| 96 | base := []provider.Message{ |
| 97 | {Role: provider.RoleSystem, Content: bigHead}, |
| 98 | {Role: provider.RoleUser, Content: "Reply with the single word: ok."}, |
| 99 | } |
| 100 | p1a, err := send(base) |
| 101 | if err != nil { |
| 102 | t.Fatalf("probe1 first call: %v", err) |
| 103 | } |
| 104 | time.Sleep(3 * time.Second) // give the async cache a moment to populate |
| 105 | p1b, err := send(base) |
| 106 | if err != nil { |
| 107 | t.Fatalf("probe1 second call: %v", err) |
| 108 | } |
| 109 | t.Logf("==== Probe 1: cache on a repeated prefix ====") |
| 110 | t.Logf("call 1 (cold): prompt=%d hit=%d miss=%d rate=%s", p1a.prompt, p1a.hit, p1a.miss, rate(p1a)) |
| 111 | t.Logf("call 2 (warm): prompt=%d hit=%d miss=%d rate=%s", p1b.prompt, p1b.hit, p1b.miss, rate(p1b)) |
| 112 | if p1b.hit == 0 { |
| 113 | t.Logf("WARNING: warm call still shows 0 cache hit — either caching is off for this account/model, " + |
| 114 | "or the prefix is below the cacheable size") |
| 115 | } |
| 116 | |
| 117 | // ---- Probe 3 (cheap, do it early): does v4-flash emit reasoning_content? ---- |
| 118 | t.Logf("==== Probe 3: does deepseek-v4-flash return reasoning_content? ====") |
| 119 | t.Logf("saw reasoning chunks: %v reasoning_tokens reported: %d reasoning_text_len: %d", |
| 120 | p1a.sawReasoning, p1a.reasoning, len(p1a.reasoningText)) |
| 121 | if !p1a.sawReasoning && p1a.reasoning == 0 { |
| 122 | t.Logf("→ this v4-flash response did not contain reasoning_content; replay cost applies only to tool-call turns that actually carry provider-issued reasoning") |
| 123 | } |
| 124 | |
| 125 | // ---- Probe 2: cost/cache effect of required tool-call reasoning replay ---- |
| 126 | longReasoning := strings.Repeat("Let me think carefully about each requirement and weigh the trade-offs. ", 40) |
| 127 | histBase := func(withReasoning bool) []provider.Message { |
| 128 | asst := provider.Message{ |
| 129 | Role: provider.RoleAssistant, |
| 130 | Content: "", |
| 131 | ToolCalls: []provider.ToolCall{ |
| 132 | {ID: "call_1", Name: "read_file", Arguments: `{"path":"config.toml"}`}, |
| 133 | }, |
| 134 | } |
| 135 | if withReasoning { |
| 136 | asst.ReasoningContent = longReasoning |
| 137 | } |
| 138 | return []provider.Message{ |
| 139 | {Role: provider.RoleSystem, Content: bigHead}, |
| 140 | {Role: provider.RoleUser, Content: "Read the config and tell me the model."}, |
| 141 | asst, |
| 142 | {Role: provider.RoleTool, Content: "model = deepseek-v4-flash", ToolCallID: "call_1", Name: "read_file"}, |
| 143 | {Role: provider.RoleUser, Content: "Thanks. Now reply with the single word: ok."}, |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | withR := histBase(true) |
| 148 | noR := histBase(false) |
| 149 | |
| 150 | // DeepSeek thinking mode requires provider-issued reasoning_content to be |
| 151 | // replayed on assistant tool_calls turns. These histories intentionally |
| 152 | // produce different wire requests: withR carries the reasoning text, while |
| 153 | // noR exercises Reasonix's empty-key recovery fallback. Warm each prefix once, |
| 154 | // then measure the second (cache-eligible) call. |
| 155 | if _, err := send(withR); err != nil { |
| 156 | t.Fatalf("probe2 required reasoning replay: %v", err) |
| 157 | } else { |
| 158 | if _, err := send(noR); err != nil { |
| 159 | t.Fatalf("probe2 no-reasoning warm: %v", err) |
| 160 | } |
| 161 | time.Sleep(3 * time.Second) |
| 162 | p2withR, err := send(withR) |
| 163 | if err != nil { |
| 164 | t.Fatalf("probe2 with-reasoning measure: %v", err) |
| 165 | } |
| 166 | p2noR, err := send(noR) |
| 167 | if err != nil { |
| 168 | t.Fatalf("probe2 no-reasoning measure: %v", err) |
| 169 | } |
| 170 | t.Logf("==== Probe 2: reasoning_content round-trip on real cache ====") |
| 171 | t.Logf("REQUIRED replay: prompt=%d hit=%d miss=%d rate=%s", p2withR.prompt, p2withR.hit, p2withR.miss, rate(p2withR)) |
| 172 | t.Logf("EMPTY-key fallback: prompt=%d hit=%d miss=%d rate=%s", p2noR.prompt, p2noR.hit, p2noR.miss, rate(p2noR)) |
| 173 | t.Logf("prompt_tokens replay cost (required - empty fallback) = %d", p2withR.prompt-p2noR.prompt) |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | func formatPct(a, b int) string { |
| 178 | if b == 0 { |
| 179 | return "n/a" |
| 180 | } |
| 181 | return fmt.Sprintf("%d%%", a*100/b) |
| 182 | } |
| 183 |