| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log/slog" |
| 8 | "strings" |
| 9 | "sync" |
| 10 | "time" |
| 11 | |
| 12 | "reasonix/internal/agent" |
| 13 | "reasonix/internal/control" |
| 14 | "reasonix/internal/secrets" |
| 15 | ) |
| 16 | |
| 17 | // deferredRebuildRetryInterval is how often the retry loop probes a held |
| 18 | // session lease. Package-level so tests can shorten it. |
| 19 | var deferredRebuildRetryInterval = 2 * time.Second |
| 20 | |
| 21 | const deferredStartupBuildLabel = "__startup__" |
| 22 | |
| 23 | // deferredRuntimeReloadLabel marks a queued ReloadRuntime in the pending map. |
| 24 | // Like the startup label it is not a user setting name; the retry loop routes |
| 25 | // it to the boot.Rebuild reload path instead of a settings rebuild. |
| 26 | const deferredRuntimeReloadLabel = "__reload__" |
| 27 | |
| 28 | // deferredRebuildState tracks tabs whose settings were saved to disk but whose |
| 29 | // runtime could not refresh, plus tabs whose initial startup failed, because |
| 30 | // the session lease was held by another Reasonix process. A single background |
| 31 | // loop probes the lease and replays the rebuild once the other side releases |
| 32 | // it. The loop only runs after enableDeferredRebuildRetry (the wails startup |
| 33 | // hook); tests that never call it get the pending bookkeeping without a |
| 34 | // background goroutine. |
| 35 | type deferredRebuildState struct { |
| 36 | mu sync.Mutex |
| 37 | pending map[string]string // tab ID -> setting label for notices |
| 38 | enabled bool |
| 39 | running bool |
| 40 | stopped bool |
| 41 | stop chan struct{} |
| 42 | } |
| 43 | |
| 44 | // enableDeferredRebuildRetry arms the retry loop; called from startup. |
| 45 | func (a *App) enableDeferredRebuildRetry() { |
| 46 | d := &a.deferredRebuild |
| 47 | d.mu.Lock() |
| 48 | defer d.mu.Unlock() |
| 49 | d.enabled = true |
| 50 | a.startDeferredRebuildLoopLocked() |
| 51 | } |
| 52 | |
| 53 | // startDeferredRebuildLoopLocked starts the loop when it is armed, idle, and |
| 54 | // has work. Callers must hold d.mu. |
| 55 | func (a *App) startDeferredRebuildLoopLocked() { |
| 56 | d := &a.deferredRebuild |
| 57 | if !d.enabled || d.running || d.stopped || len(d.pending) == 0 { |
| 58 | return |
| 59 | } |
| 60 | d.running = true |
| 61 | if d.stop == nil { |
| 62 | d.stop = make(chan struct{}) |
| 63 | } |
| 64 | go a.deferredRebuildLoop(d.stop) |
| 65 | } |
| 66 | |
| 67 | // scheduleDeferredRebuild records that tabID needs a runtime refresh for |
| 68 | // setting and starts the retry loop if it is not running yet. Repeated calls |
| 69 | // for the same tab collapse into one retry carrying the latest label. |
| 70 | func (a *App) scheduleDeferredRebuild(tabID, setting string) { |
| 71 | tabID = strings.TrimSpace(tabID) |
| 72 | if tabID == "" { |
| 73 | return |
| 74 | } |
| 75 | d := &a.deferredRebuild |
| 76 | d.mu.Lock() |
| 77 | defer d.mu.Unlock() |
| 78 | if d.stopped { |
| 79 | return |
| 80 | } |
| 81 | if d.pending == nil { |
| 82 | d.pending = map[string]string{} |
| 83 | } |
| 84 | d.pending[tabID] = setting |
| 85 | a.startDeferredRebuildLoopLocked() |
| 86 | } |
| 87 | |
| 88 | func (a *App) scheduleDeferredStartupBuild(tabID string) { |
| 89 | a.scheduleDeferredRebuild(tabID, deferredStartupBuildLabel) |
| 90 | } |
| 91 | |
| 92 | func isDeferredStartupBuild(setting string) bool { |
| 93 | return setting == deferredStartupBuildLabel |
| 94 | } |
| 95 | |
| 96 | func isDeferredRuntimeReload(setting string) bool { |
| 97 | return setting == deferredRuntimeReloadLabel |
| 98 | } |
| 99 | |
| 100 | func (a *App) clearDeferredRebuild(tabID string) { |
| 101 | d := &a.deferredRebuild |
| 102 | d.mu.Lock() |
| 103 | delete(d.pending, tabID) |
| 104 | d.mu.Unlock() |
| 105 | } |
| 106 | |
| 107 | func (a *App) deferredRebuildPending(tabID string) bool { |
| 108 | d := &a.deferredRebuild |
| 109 | d.mu.Lock() |
| 110 | defer d.mu.Unlock() |
| 111 | _, ok := d.pending[tabID] |
| 112 | return ok |
| 113 | } |
| 114 | |
| 115 | // stopDeferredRebuildRetry permanently stops the retry loop; used on shutdown |
| 116 | // and by tests. |
| 117 | func (a *App) stopDeferredRebuildRetry() { |
| 118 | d := &a.deferredRebuild |
| 119 | d.mu.Lock() |
| 120 | defer d.mu.Unlock() |
| 121 | if d.stopped { |
| 122 | return |
| 123 | } |
| 124 | d.stopped = true |
| 125 | if d.stop != nil { |
| 126 | close(d.stop) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | func (a *App) deferredRebuildLoop(stop <-chan struct{}) { |
| 131 | ticker := time.NewTicker(deferredRebuildRetryInterval) |
| 132 | defer ticker.Stop() |
| 133 | for { |
| 134 | select { |
| 135 | case <-stop: |
| 136 | d := &a.deferredRebuild |
| 137 | d.mu.Lock() |
| 138 | d.running = false |
| 139 | d.mu.Unlock() |
| 140 | return |
| 141 | case <-ticker.C: |
| 142 | } |
| 143 | if a.deferredRebuildTickDone() { |
| 144 | return |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // deferredRebuildTickDone runs one retry pass and reports true when the loop |
| 150 | // should exit because nothing is pending anymore. |
| 151 | func (a *App) deferredRebuildTickDone() bool { |
| 152 | return a.deferredRebuildTick(true) |
| 153 | } |
| 154 | |
| 155 | func (a *App) deferredRebuildTick(markIdle bool) bool { |
| 156 | d := &a.deferredRebuild |
| 157 | d.mu.Lock() |
| 158 | if d.stopped || len(d.pending) == 0 { |
| 159 | if markIdle { |
| 160 | d.running = false |
| 161 | } |
| 162 | d.mu.Unlock() |
| 163 | return true |
| 164 | } |
| 165 | pending := make(map[string]string, len(d.pending)) |
| 166 | for id, setting := range d.pending { |
| 167 | pending[id] = setting |
| 168 | } |
| 169 | d.mu.Unlock() |
| 170 | |
| 171 | for tabID, setting := range pending { |
| 172 | a.retryDeferredRebuild(tabID, setting) |
| 173 | } |
| 174 | return false |
| 175 | } |
| 176 | |
| 177 | func (a *App) kickDeferredRebuildRetry() { |
| 178 | if a.ctx == nil { |
| 179 | return |
| 180 | } |
| 181 | a.goSafe("deferredRebuildKick", func() { |
| 182 | _ = a.deferredRebuildTick(false) |
| 183 | }) |
| 184 | } |
| 185 | |
| 186 | func (a *App) retryDeferredRebuild(tabID, setting string) { |
| 187 | if a.ctx == nil { |
| 188 | return |
| 189 | } |
| 190 | tab := a.tabByID(tabID) |
| 191 | if tab == nil || tab.ID != tabID { |
| 192 | // The tab is gone; nothing left to refresh. |
| 193 | a.clearDeferredRebuild(tabID) |
| 194 | return |
| 195 | } |
| 196 | if isDeferredStartupBuild(setting) { |
| 197 | a.retryDeferredStartupBuild(tabID, tab) |
| 198 | return |
| 199 | } |
| 200 | if isDeferredRuntimeReload(setting) { |
| 201 | a.retryDeferredRuntimeReload(tabID, tab) |
| 202 | return |
| 203 | } |
| 204 | // Hold the rebuild mutex across probe + rebuild: the probe briefly acquires |
| 205 | // the session lease, and a concurrent manual rebuild's ensureSessionLease |
| 206 | // would see that probe as "held by another runtime" and spuriously defer. |
| 207 | a.runtimeRebuildMu.Lock() |
| 208 | defer a.runtimeRebuildMu.Unlock() |
| 209 | // rebuildSettingLocked refreshes the active tab only. Wait until the user |
| 210 | // is back on this tab so we refresh the runtime the pending setting was |
| 211 | // meant for, not whichever tab happens to be focused. |
| 212 | if a.activeTab() != tab { |
| 213 | return |
| 214 | } |
| 215 | ctrl := a.controllerForTab(tab) |
| 216 | if ctrl == nil { |
| 217 | // Mid-(re)build on another path (provider retarget, workspace repair); |
| 218 | // racing a second build+swap against it is what this loop must avoid. |
| 219 | return |
| 220 | } |
| 221 | if controllerHasActiveRuntimeWork(ctrl) { |
| 222 | return |
| 223 | } |
| 224 | if !a.deferredRebuildLeaseLooksFree(tab) { |
| 225 | return |
| 226 | } |
| 227 | err := a.rebuildSettingLocked(setting) |
| 228 | if err == nil { |
| 229 | // rebuildSettingLocked already cleared the pending entry for the tab it |
| 230 | // refreshed; just announce it. |
| 231 | a.noticeForTab(tabID, fmt.Sprintf("%s applied: session refreshed after the lease was released", setting)) |
| 232 | return |
| 233 | } |
| 234 | if errors.Is(err, agent.ErrSessionLeaseHeld) { |
| 235 | return // grabbed back before we could rebuild; keep waiting |
| 236 | } |
| 237 | var busy *rebuildBusyError |
| 238 | if errors.As(err, &busy) { |
| 239 | return // a turn started meanwhile; retry once it finishes |
| 240 | } |
| 241 | // Anything else will not resolve by waiting; give up loudly instead of |
| 242 | // retrying forever. |
| 243 | a.clearDeferredRebuild(tabID) |
| 244 | slog.Warn("desktop: deferred settings rebuild failed", "setting", setting, "tab", tabID, "err", err) |
| 245 | a.warnForTab(tabID, fmt.Sprintf("%s was saved but the session could not refresh: %s", setting, err.Error())) |
| 246 | } |
| 247 | |
| 248 | // retryDeferredRuntimeReload drives one queued ReloadRuntime pass. The |
| 249 | // probing contract mirrors retryDeferredRebuild: wait for the tab to be |
| 250 | // active and idle and for its lease to look free, then run the boot.Rebuild |
| 251 | // reload; busy/lease answers keep waiting, anything else gives up loudly. |
| 252 | func (a *App) retryDeferredRuntimeReload(tabID string, tab *WorkspaceTab) { |
| 253 | // Hold the rebuild mutex across probe + reload: the probe briefly |
| 254 | // acquires the session lease, and a concurrent rebuild's ensure lease |
| 255 | // would read that probe as "held by another runtime" and spuriously |
| 256 | // defer (same contract as retryDeferredRebuild). |
| 257 | a.runtimeRebuildMu.Lock() |
| 258 | defer a.runtimeRebuildMu.Unlock() |
| 259 | // The reload shares rebuildSettingTurnLocked, whose bookkeeping is written |
| 260 | // for the active tab; wait until the user is back on this tab rather than |
| 261 | // refreshing whichever tab happens to be focused. |
| 262 | if a.activeTab() != tab { |
| 263 | return |
| 264 | } |
| 265 | ctrl := a.controllerForTab(tab) |
| 266 | if ctrl == nil { |
| 267 | // Mid-(re)build on another path; racing a second build+swap against it |
| 268 | // is what this loop must avoid. |
| 269 | return |
| 270 | } |
| 271 | if controllerHasActiveRuntimeWork(ctrl) { |
| 272 | return |
| 273 | } |
| 274 | if !a.deferredRebuildLeaseLooksFree(tab) { |
| 275 | return |
| 276 | } |
| 277 | err := a.reloadRuntimeTurnLocked(tab) |
| 278 | if err == nil { |
| 279 | // rebuildSettingTurnLocked already cleared the pending entry for the |
| 280 | // tab it refreshed; just announce it. |
| 281 | a.noticeForTab(tabID, "runtime reloaded after the session went idle") |
| 282 | return |
| 283 | } |
| 284 | if errors.Is(err, agent.ErrSessionLeaseHeld) { |
| 285 | return // grabbed back before we could reload; keep waiting |
| 286 | } |
| 287 | var busy *rebuildBusyError |
| 288 | if errors.As(err, &busy) { |
| 289 | return // a turn started meanwhile; retry once it finishes |
| 290 | } |
| 291 | // Anything else will not resolve by waiting; give up loudly instead of |
| 292 | // retrying forever. The error may come from provider/config plumbing and |
| 293 | // carry credential-shaped values (passwords, resolved API keys) — the |
| 294 | // tested helper redacts before the text reaches logs or the frontend. |
| 295 | a.clearDeferredRebuild(tabID) |
| 296 | failure := deferredReloadFailedText(err) |
| 297 | slog.Warn("desktop: "+failure, "tab", tabID) |
| 298 | a.warnForTab(tabID, failure) |
| 299 | } |
| 300 | |
| 301 | // deferredReloadFailedText is the failure line for an unrecoverable deferred |
| 302 | // reload — the single formatter both the log and the tab warning use, so a |
| 303 | // credential-shaped error can never reach either sink unredacted. |
| 304 | func deferredReloadFailedText(err error) string { |
| 305 | return "runtime reload failed: " + secrets.RedactCredentials(err.Error()) |
| 306 | } |
| 307 | |
| 308 | func (a *App) retryDeferredStartupBuild(tabID string, tab *WorkspaceTab) { |
| 309 | a.runtimeRebuildMu.Lock() |
| 310 | defer a.runtimeRebuildMu.Unlock() |
| 311 | if !a.tabHasRetryableStartupLeaseError(tab) { |
| 312 | a.clearDeferredRebuild(tabID) |
| 313 | return |
| 314 | } |
| 315 | a.mu.RLock() |
| 316 | path := strings.TrimSpace(tab.SessionPath) |
| 317 | a.mu.RUnlock() |
| 318 | if path != "" && a.attachExistingSessionRuntime(tab, path, a.ctx) { |
| 319 | a.clearDeferredRebuild(tabID) |
| 320 | return |
| 321 | } |
| 322 | if !a.deferredRebuildLeaseLooksFree(tab) { |
| 323 | return |
| 324 | } |
| 325 | err := a.rebuildStartupTabLocked(tab) |
| 326 | if err == nil { |
| 327 | a.clearDeferredRebuild(tabID) |
| 328 | return |
| 329 | } |
| 330 | if errors.Is(err, agent.ErrSessionLeaseHeld) { |
| 331 | return |
| 332 | } |
| 333 | a.clearDeferredRebuild(tabID) |
| 334 | slog.Warn("desktop: deferred session startup failed", "tab", tabID, "err", err) |
| 335 | } |
| 336 | |
| 337 | func (a *App) tabHasRetryableStartupLeaseError(tab *WorkspaceTab) bool { |
| 338 | if tab == nil { |
| 339 | return false |
| 340 | } |
| 341 | a.mu.RLock() |
| 342 | defer a.mu.RUnlock() |
| 343 | return a.tabs[tab.ID] == tab && !tab.removed && tab.Ctrl == nil && tab.StartupErrLeaseHeld |
| 344 | } |
| 345 | |
| 346 | func (a *App) rebuildStartupTabLocked(tab *WorkspaceTab) error { |
| 347 | buildCtx, cancel := context.WithCancel(a.bootContext()) |
| 348 | a.mu.Lock() |
| 349 | if tab == nil || a.tabs[tab.ID] != tab || tab.removed { |
| 350 | a.mu.Unlock() |
| 351 | cancel() |
| 352 | return nil |
| 353 | } |
| 354 | if tab.Ctrl != nil { |
| 355 | a.mu.Unlock() |
| 356 | cancel() |
| 357 | return nil |
| 358 | } |
| 359 | if !tab.StartupErrLeaseHeld { |
| 360 | a.mu.Unlock() |
| 361 | cancel() |
| 362 | return nil |
| 363 | } |
| 364 | tab.buildGeneration++ |
| 365 | generation := tab.buildGeneration |
| 366 | if tab.buildCancel != nil { |
| 367 | tab.buildCancel() |
| 368 | } |
| 369 | tab.buildCancel = cancel |
| 370 | tab.Ready = false |
| 371 | clearTabStartupError(tab) |
| 372 | a.setSessionRuntimePhaseLocked(tab, sessionRuntimeStarting, nil) |
| 373 | tab.ActivityStatus = "" |
| 374 | if tab.sink == nil { |
| 375 | tab.sink = &tabEventSink{tabID: tab.ID, app: a, ctx: a.ctx} |
| 376 | } |
| 377 | a.saveTabsLocked() |
| 378 | a.mu.Unlock() |
| 379 | |
| 380 | a.buildTabControllerWithContext(tab, loadedTabSession{}, buildCtx, generation, cancel) |
| 381 | |
| 382 | a.mu.RLock() |
| 383 | stillCurrent := false |
| 384 | var ctrl control.SessionAPI |
| 385 | startupErr := "" |
| 386 | leaseHeld := false |
| 387 | if tab != nil { |
| 388 | stillCurrent = a.tabs[tab.ID] == tab && !tab.removed |
| 389 | ctrl = tab.Ctrl |
| 390 | startupErr = tab.StartupErr |
| 391 | leaseHeld = tab.StartupErrLeaseHeld |
| 392 | } |
| 393 | a.mu.RUnlock() |
| 394 | if !stillCurrent || ctrl != nil { |
| 395 | return nil |
| 396 | } |
| 397 | if leaseHeld { |
| 398 | return agent.ErrSessionLeaseHeld |
| 399 | } |
| 400 | if strings.TrimSpace(startupErr) != "" { |
| 401 | return fmt.Errorf("session startup: %s", startupErr) |
| 402 | } |
| 403 | return fmt.Errorf("session startup: controller was not built") |
| 404 | } |
| 405 | |
| 406 | func (a *App) tryRecoverStartupLeaseHeldTab(tab *WorkspaceTab) bool { |
| 407 | if a.ctx == nil || !a.tabHasRetryableStartupLeaseError(tab) { |
| 408 | return false |
| 409 | } |
| 410 | a.runtimeRebuildMu.Lock() |
| 411 | defer a.runtimeRebuildMu.Unlock() |
| 412 | if !a.tabHasRetryableStartupLeaseError(tab) { |
| 413 | return a.controllerForTab(tab) != nil |
| 414 | } |
| 415 | if !a.deferredRebuildLeaseLooksFree(tab) { |
| 416 | return false |
| 417 | } |
| 418 | err := a.rebuildStartupTabLocked(tab) |
| 419 | if err == nil { |
| 420 | a.clearDeferredRebuild(tab.ID) |
| 421 | return a.controllerForTab(tab) != nil |
| 422 | } |
| 423 | if errors.Is(err, agent.ErrSessionLeaseHeld) { |
| 424 | a.scheduleDeferredStartupBuild(tab.ID) |
| 425 | } else { |
| 426 | a.clearDeferredRebuild(tab.ID) |
| 427 | } |
| 428 | return false |
| 429 | } |
| 430 | |
| 431 | // deferredRebuildLeaseLooksFree cheaply probes whether the tab's session lease |
| 432 | // could be acquired right now, without touching tab.sessionLease (only the |
| 433 | // serialized rebuild paths may mutate that). The probe path can lag the |
| 434 | // reconciled path the rebuild will use; a stale answer either re-defers on the |
| 435 | // next tick or lets the rebuild fail back into the pending set, so a mismatch |
| 436 | // only delays the retry. |
| 437 | func (a *App) deferredRebuildLeaseLooksFree(tab *WorkspaceTab) bool { |
| 438 | a.mu.RLock() |
| 439 | ctrl := tab.Ctrl |
| 440 | path := strings.TrimSpace(tab.SessionPath) |
| 441 | a.mu.RUnlock() |
| 442 | if ctrl != nil { |
| 443 | if p := strings.TrimSpace(ctrl.SessionPath()); p != "" { |
| 444 | path = p |
| 445 | } |
| 446 | } |
| 447 | if path == "" { |
| 448 | return true // nothing to probe; let the rebuild decide |
| 449 | } |
| 450 | key := sessionRuntimeKey(path) |
| 451 | a.mu.RLock() |
| 452 | rt := a.runtimeBySessionKey[key] |
| 453 | ownedByTab := rt != nil && rt.Owner == tab |
| 454 | ownedByOther := rt != nil && rt.Owner != nil && rt.Owner != tab && a.runtimeOwnerLiveLocked(rt) |
| 455 | a.mu.RUnlock() |
| 456 | if ownedByOther { |
| 457 | return false |
| 458 | } |
| 459 | if ownedByTab && tab.sessionLeaseRuntimeKey() == key { |
| 460 | return true |
| 461 | } |
| 462 | lease, err := agent.TryAcquireSessionLease(key) |
| 463 | if err != nil { |
| 464 | if sameCurrentProcessLease(err) { |
| 465 | // The registry ruled out a live sibling owner above, so this is an |
| 466 | // orphaned current-process lease. Let the rebuild helper reclaim it |
| 467 | // under runtimeRebuildMu instead of looping against our own marker. |
| 468 | return true |
| 469 | } |
| 470 | return !errors.Is(err, agent.ErrSessionLeaseHeld) |
| 471 | } |
| 472 | lease.Release() |
| 473 | return true |
| 474 | } |
| 475 |