| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "database/sql" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "io" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "reflect" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "reasonix/internal/historywork" |
| 18 | "reasonix/internal/projectiondb" |
| 19 | "reasonix/internal/provider" |
| 20 | "reasonix/internal/store" |
| 21 | ) |
| 22 | |
| 23 | type cancelEventRead struct { |
| 24 | source io.Reader |
| 25 | cancel context.CancelFunc |
| 26 | reads int |
| 27 | } |
| 28 | |
| 29 | func stageEventDisplayMessages(ctx context.Context, db *sql.DB, decoder *json.Decoder) (int, error) { |
| 30 | scanner := &eventPagerScanner{ctx: ctx, db: db, decoder: decoder} |
| 31 | defer func() { |
| 32 | if scanner.tx != nil { |
| 33 | _ = scanner.tx.Rollback() |
| 34 | } |
| 35 | }() |
| 36 | if err := scanner.transaction(); err != nil { |
| 37 | return 0, err |
| 38 | } |
| 39 | if err := scanner.messages(); err != nil { |
| 40 | return scanner.progress.Pending, err |
| 41 | } |
| 42 | return scanner.progress.Pending, scanner.tx.Commit() |
| 43 | } |
| 44 | |
| 45 | func (r *cancelEventRead) Read(p []byte) (int, error) { |
| 46 | n, err := r.source.Read(p[:min(len(p), 512)]) |
| 47 | r.reads++ |
| 48 | if r.reads == 2 { |
| 49 | r.cancel() |
| 50 | } |
| 51 | return n, err |
| 52 | } |
| 53 | |
| 54 | func TestDisplayPagerSchemaOneCancellationDiscardsUncommittedBatch(t *testing.T) { |
| 55 | handle, err := projectiondb.Open(t.Context(), projectiondb.OpenOptions{Path: filepath.Join(t.TempDir(), "partial.sqlite"), Migrations: displayPagerMigrations, RequireDisk: true, MaxOpenConns: 1}) |
| 56 | if err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | defer handle.DB.Close() |
| 60 | if _, err := handle.DB.ExecContext(t.Context(), `CREATE TABLE event_pending(position INTEGER PRIMARY KEY,offset INTEGER NOT NULL,length INTEGER NOT NULL)`); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | array := "[" + strings.Repeat(`{"role":"user","content":"question"},`, 499) + `{"role":"user","content":"last"}]` |
| 64 | ctx, cancel := context.WithCancel(t.Context()) |
| 65 | defer cancel() |
| 66 | source := &cancelEventRead{source: strings.NewReader(array), cancel: cancel} |
| 67 | _, err = stageEventDisplayMessages(ctx, handle.DB, json.NewDecoder(&historywork.Reader{Context: ctx, Source: source})) |
| 68 | if !errors.Is(err, context.Canceled) || source.reads != 2 { |
| 69 | t.Fatalf("read checkpoint failed: calls=%d err=%v", source.reads, err) |
| 70 | } |
| 71 | var rows int |
| 72 | if err := handle.DB.QueryRowContext(t.Context(), `SELECT COUNT(*) FROM event_pending`).Scan(&rows); err != nil || rows != 0 { |
| 73 | t.Fatalf("canceled transaction leaked: %d %v", rows, err) |
| 74 | } |
| 75 | count, err := stageEventDisplayMessages(t.Context(), handle.DB, json.NewDecoder(strings.NewReader(array))) |
| 76 | if err != nil || count != 500 { |
| 77 | t.Fatalf("retry: %d %v", count, err) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func eventDisplayFixture(t *testing.T, text string) (string, string) { |
| 82 | t.Helper() |
| 83 | path := filepath.Join(t.TempDir(), "events.jsonl") |
| 84 | if err := os.WriteFile(path, []byte("{\"role\":\"user\",\"content\":\"stale checkpoint\"}\n"), 0600); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if err := os.WriteFile(store.SessionEventLog(path), []byte(text), 0600); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | return path, filepath.Join(t.TempDir(), "display.sqlite") |
| 91 | } |
| 92 | |
| 93 | func TestDisplayPagerSchemaOneMatchesReplayAndReusesCache(t *testing.T) { |
| 94 | var messages []provider.Message |
| 95 | for i := range 300 { |
| 96 | messages = append(messages, provider.Message{Role: provider.RoleUser, Content: fmt.Sprintf("question %d", i)}, provider.Message{Role: provider.RoleAssistant, Content: fmt.Sprintf("answer %d", i)}) |
| 97 | } |
| 98 | created := time.Unix(1700000000, 0).UTC() |
| 99 | records := []sessionEventRecord{ |
| 100 | {SchemaVersion: 1, Type: sessionEventTypeReplace, Messages: []provider.Message{{Role: provider.RoleUser, Content: "obsolete"}}}, |
| 101 | {SchemaVersion: 1, Type: sessionEventTypeAppend, MessageIndex: 1, Messages: []provider.Message{{Role: provider.RoleAssistant, Content: "obsolete answer"}}}, |
| 102 | {SchemaVersion: 1, Type: sessionEventTypeReplace, Messages: messages[:500]}, |
| 103 | {SchemaVersion: 1, Type: sessionEventTypeAppend, MessageIndex: 500, Messages: messages[500:], CreatedAt: created}, |
| 104 | } |
| 105 | var log strings.Builder |
| 106 | for _, record := range records { |
| 107 | if err := json.NewEncoder(&log).Encode(record); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | } |
| 111 | path, cache := eventDisplayFixture(t, log.String()) |
| 112 | checkpoint, _ := os.ReadFile(path) |
| 113 | p, err := OpenDisplayPager(t.Context(), path, cache) |
| 114 | if err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | if !p.SchemaOne || p.DAG || !p.Built || len(p.Header.Entries) != 0 || p.Header.MessageCount != 600 { |
| 118 | t.Fatalf("event projection: %+v", p.Header) |
| 119 | } |
| 120 | replay, err := replaySessionEventLog(store.SessionEventLog(path)) |
| 121 | if err != nil || replay.damaged { |
| 122 | t.Fatalf("fixture replay: %+v %v", replay, err) |
| 123 | } |
| 124 | var got []provider.Message |
| 125 | for lo := 0; lo < 600; lo += 32 { |
| 126 | page, err := p.EventMessages(lo, min(lo+32, 600)) |
| 127 | if err != nil { |
| 128 | t.Fatal(err) |
| 129 | } |
| 130 | got = append(got, page...) |
| 131 | } |
| 132 | for i := range replay.msgs { |
| 133 | if replay.msgs[i].CreatedAt <= 0 && !replay.times[i].IsZero() { |
| 134 | replay.msgs[i].CreatedAt = replay.times[i].UnixMilli() |
| 135 | } |
| 136 | } |
| 137 | if !reflect.DeepEqual(got, replay.msgs) { |
| 138 | t.Fatal("paged event messages differ from replace/append replay") |
| 139 | } |
| 140 | turns, err := p.TurnEntries(257, 3) |
| 141 | if err != nil || len(turns) != 3 || turns[0].Index != 512 || turns[0].AuthoredTurn != 257 { |
| 142 | t.Fatalf("event outline: %+v %v", turns, err) |
| 143 | } |
| 144 | if err := p.Close(); err != nil { |
| 145 | t.Fatal(err) |
| 146 | } |
| 147 | var meter historywork.Coordinator |
| 148 | reopened, err := OpenDisplayPager(meter.Context(t.Context()), path, cache) |
| 149 | if err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | defer reopened.Close() |
| 153 | if reopened.Built || !reopened.SchemaOne || meter.Diagnostics().InstrumentedReadBytes >= int64(len(log.String())) { |
| 154 | t.Fatal("cached reopen replayed the entire log") |
| 155 | } |
| 156 | after, _ := os.ReadFile(path) |
| 157 | events, _ := os.ReadFile(store.SessionEventLog(path)) |
| 158 | if string(checkpoint) != string(after) || log.String() != string(events) { |
| 159 | t.Fatal("cold event reading rewrote authoritative files") |
| 160 | } |
| 161 | if _, err := os.Stat(store.SessionDisplayIndex(path)); !os.IsNotExist(err) { |
| 162 | t.Fatal("event reading created a compatibility display sidecar") |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | func TestDisplayPagerSchemaOneAcceptsFieldOrderAndEmptyReplacement(t *testing.T) { |
| 167 | for _, suffix := range []string{"", `{"messages":null,"type":"replace","schema_version":1}`, `{"schema_version":1,"type":"replace"}`} { |
| 168 | t.Run(fmt.Sprint(len(suffix)), func(t *testing.T) { |
| 169 | // The header is beyond the fast probe. A valid field order must not |
| 170 | // cause a full-record allocation or fallback to the stale checkpoint. |
| 171 | text := `{"messages":[{"role":"user","content":"` + strings.Repeat("x", 10000) + `"}],"extra":{"array":[1,{"two":[true,null]}]},"type":"replace","schema_version":1}` + "\n" + suffix |
| 172 | path, cache := eventDisplayFixture(t, text) |
| 173 | p, err := OpenDisplayPager(t.Context(), path, cache) |
| 174 | if err != nil { |
| 175 | t.Fatal(err) |
| 176 | } |
| 177 | defer p.Close() |
| 178 | want := 1 |
| 179 | if suffix != "" { |
| 180 | want = 0 |
| 181 | } |
| 182 | if !p.SchemaOne || p.Header.MessageCount != want { |
| 183 | t.Fatalf("empty replacement: %+v", p.Header) |
| 184 | } |
| 185 | p.Close() |
| 186 | var meter historywork.Coordinator |
| 187 | reopened, err := OpenDisplayPager(meter.Context(t.Context()), path, cache) |
| 188 | if err != nil { |
| 189 | t.Fatal(err) |
| 190 | } |
| 191 | defer reopened.Close() |
| 192 | if reopened.Built || meter.Diagnostics().InstrumentedReadBytes != 0 { |
| 193 | t.Fatal("cached reopen rescanned a late schema header") |
| 194 | } |
| 195 | }) |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | func TestDisplayPagerSchemaOneDamagedTailNeverPublishesPrefix(t *testing.T) { |
| 200 | base := `{"schema_version":1,"type":"replace","messages":[{"role":"user","content":"one"}]}` + "\n" |
| 201 | for _, tail := range []string{ |
| 202 | `{"schema_version":1,"type":"append","message_index":1,"messages":[`, |
| 203 | `{"schema_version":1,"type":"append","message_index":7,"messages":[]}`, |
| 204 | `{"schema_version":1,"type":"rewrite","messages":[]}`, |
| 205 | `{"schema_version":4,"type":"replace","messages":[]}`, |
| 206 | } { |
| 207 | t.Run(tail, func(t *testing.T) { |
| 208 | path, cache := eventDisplayFixture(t, base) |
| 209 | p, err := OpenDisplayPager(t.Context(), path, cache) |
| 210 | if err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | p.Close() |
| 214 | if err := os.WriteFile(store.SessionEventLog(path), []byte(base+tail), 0600); err != nil { |
| 215 | t.Fatal(err) |
| 216 | } |
| 217 | if p, err := OpenDisplayPager(t.Context(), path, cache); !errors.Is(err, ErrSessionDisplayReadModelDamaged) { |
| 218 | if p != nil { |
| 219 | p.Close() |
| 220 | } |
| 221 | t.Fatalf("damaged prefix accepted: %v", err) |
| 222 | } |
| 223 | actual, _ := os.ReadFile(store.SessionEventLog(path)) |
| 224 | if string(actual) != base+tail { |
| 225 | t.Fatal("display preparation repaired the damaged source") |
| 226 | } |
| 227 | }) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | func TestDisplayPagerNewWriterCannotBeHiddenByFreshSidecar(t *testing.T) { |
| 232 | path := filepath.Join(t.TempDir(), "future.jsonl") |
| 233 | session := NewSession("system") |
| 234 | session.Add(provider.Message{Role: provider.RoleUser, Content: "old"}) |
| 235 | if err := session.SaveSnapshot(path); err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | if err := os.WriteFile(store.SessionEventLog(path), []byte("{\"schema_version\":99,\"type\":\"replace\",\"messages\":[]}\n"), 0600); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | // Satisfy the old sidecar timestamp gate deliberately: a newer authority |
| 242 | // must still win over offsets that only describe the previous checkpoint. |
| 243 | stamp := SessionContentModTime(path).Add(time.Second) |
| 244 | if err := os.Chtimes(store.SessionDisplayIndex(path), stamp, stamp); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | if p, err := OpenDisplayPager(t.Context(), path, filepath.Join(t.TempDir(), "cache.sqlite")); !errors.Is(err, ErrDisplayFormatUnsupported) { |
| 248 | if p != nil { |
| 249 | p.Close() |
| 250 | } |
| 251 | t.Fatalf("future log ignored: %v", err) |
| 252 | } |
| 253 | } |
| 254 |