| 1 | package boot |
| 2 | |
| 3 | // A truncating provider answers HTTP 200 with no error field, so the overflow |
| 4 | // ladder, which needs a parseable 400/413/422, never sees it. These assert the |
| 5 | // replacement signal through the real Build stack. |
| 6 | |
| 7 | import ( |
| 8 | "context" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "testing" |
| 12 | |
| 13 | "reasonix/internal/event" |
| 14 | "reasonix/internal/provider" |
| 15 | ) |
| 16 | |
| 17 | // truncatingProvider reports a fixed prompt size however much is sent, which is |
| 18 | // what a server serving a smaller window than it was asked for looks like. |
| 19 | type truncatingProvider struct { |
| 20 | mu sync.Mutex |
| 21 | promptTokens int |
| 22 | honest bool |
| 23 | calls int |
| 24 | } |
| 25 | |
| 26 | func (p *truncatingProvider) Name() string { return "boot-truncation-test" } |
| 27 | |
| 28 | func (p *truncatingProvider) Stream(_ context.Context, req provider.Request) (<-chan provider.Chunk, error) { |
| 29 | p.mu.Lock() |
| 30 | p.calls++ |
| 31 | reported := p.promptTokens |
| 32 | if p.honest { |
| 33 | chars := 0 |
| 34 | for _, m := range req.Messages { |
| 35 | chars += len(m.Content) |
| 36 | } |
| 37 | for _, tool := range req.Tools { |
| 38 | chars += len(tool.Name) + len(tool.Description) + len(tool.Parameters) |
| 39 | } |
| 40 | reported = max(1, chars/4) |
| 41 | } |
| 42 | p.mu.Unlock() |
| 43 | |
| 44 | chunks := []provider.Chunk{ |
| 45 | {Type: provider.ChunkText, Text: "ok"}, |
| 46 | {Type: provider.ChunkUsage, Usage: &provider.Usage{ |
| 47 | PromptTokens: reported, CompletionTokens: 1, |
| 48 | TotalTokens: reported + 1, RequestCount: 1, |
| 49 | }}, |
| 50 | {Type: provider.ChunkDone}, |
| 51 | } |
| 52 | ch := make(chan provider.Chunk, len(chunks)) |
| 53 | for _, c := range chunks { |
| 54 | ch <- c |
| 55 | } |
| 56 | close(ch) |
| 57 | return ch, nil |
| 58 | } |
| 59 | |
| 60 | type noticeSink struct { |
| 61 | mu sync.Mutex |
| 62 | notices []event.Event |
| 63 | } |
| 64 | |
| 65 | func (s *noticeSink) Emit(e event.Event) { |
| 66 | if e.Kind != event.Notice { |
| 67 | return |
| 68 | } |
| 69 | s.mu.Lock() |
| 70 | defer s.mu.Unlock() |
| 71 | s.notices = append(s.notices, e) |
| 72 | } |
| 73 | |
| 74 | func (s *noticeSink) truncationNotices() []event.Event { |
| 75 | s.mu.Lock() |
| 76 | defer s.mu.Unlock() |
| 77 | var out []event.Event |
| 78 | for _, e := range s.notices { |
| 79 | if e.Code == event.NoticeCodePromptTruncatedByServer { |
| 80 | out = append(out, e) |
| 81 | } |
| 82 | } |
| 83 | return out |
| 84 | } |
| 85 | |
| 86 | // truncationRun drives two turns with prompts big enough to clear the detector's |
| 87 | // minimum, and returns the notices the frontend sink actually received. |
| 88 | func truncationRun(t *testing.T, kind string, p *truncatingProvider) *noticeSink { |
| 89 | t.Helper() |
| 90 | isolateConfigHome(t) |
| 91 | dir := robustTempDir(t) |
| 92 | t.Chdir(dir) |
| 93 | |
| 94 | provider.Register(kind, func(provider.Config) (provider.Provider, error) { return p, nil }) |
| 95 | writeFile(t, dir, "reasonix.toml", ` |
| 96 | default_model = "test-model" |
| 97 | |
| 98 | [agent] |
| 99 | system_prompt = "BASE" |
| 100 | |
| 101 | [environment] |
| 102 | enabled = false |
| 103 | |
| 104 | [[providers]] |
| 105 | name = "test-model" |
| 106 | kind = "`+kind+`" |
| 107 | model = "x" |
| 108 | context_window = 200000 |
| 109 | `) |
| 110 | approveWorkspace(t, dir) |
| 111 | |
| 112 | sink := ¬iceSink{} |
| 113 | ctrl, err := Build(context.Background(), Options{Sink: sink}) |
| 114 | if err != nil { |
| 115 | t.Fatalf("Build: %v", err) |
| 116 | } |
| 117 | defer ctrl.Close() |
| 118 | |
| 119 | big := strings.Repeat("the quick brown fox jumps over the lazy dog. ", 1500) |
| 120 | for range 2 { |
| 121 | if err := ctrl.Run(context.Background(), big); err != nil { |
| 122 | t.Fatalf("Run: %v", err) |
| 123 | } |
| 124 | } |
| 125 | if p.calls == 0 { |
| 126 | t.Fatal("no request reached the provider boundary") |
| 127 | } |
| 128 | return sink |
| 129 | } |
| 130 | |
| 131 | // The defect: the user is told nothing while the model silently loses its tool |
| 132 | // definitions. One warning, and only one, however many turns run. |
| 133 | func TestEffectSilentTruncationWarnsExactlyOnce(t *testing.T) { |
| 134 | p := &truncatingProvider{promptTokens: 2051} |
| 135 | got := truncationRun(t, "truncation-effect-warn", p).truncationNotices() |
| 136 | if len(got) != 1 { |
| 137 | t.Fatalf("got %d truncation notices across two turns, want exactly 1", len(got)) |
| 138 | } |
| 139 | if got[0].Level != event.LevelWarn { |
| 140 | t.Errorf("notice level = %v, want LevelWarn: nothing was recovered", got[0].Level) |
| 141 | } |
| 142 | if !strings.Contains(got[0].Detail, "2051") { |
| 143 | t.Errorf("notice detail %q does not name the accepted prompt size", got[0].Detail) |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | // The no-false-positive guard: an honest provider must stay silent, or the |
| 148 | // warning is noise on every healthy session. |
| 149 | func TestEffectHonestProviderNeverWarns(t *testing.T) { |
| 150 | p := &truncatingProvider{honest: true} |
| 151 | if got := truncationRun(t, "truncation-effect-honest", p).truncationNotices(); len(got) != 0 { |
| 152 | t.Fatalf("an honest provider produced %d truncation notices: %+v", len(got), got) |
| 153 | } |
| 154 | } |
| 155 |