| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log/slog" |
| 8 | "sort" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "time" |
| 12 | |
| 13 | "reasonix/internal/config" |
| 14 | "reasonix/internal/control" |
| 15 | "reasonix/internal/repair" |
| 16 | "reasonix/internal/stats" |
| 17 | ) |
| 18 | |
| 19 | const ( |
| 20 | shutdownReasonUserQuit = "user_quit" |
| 21 | shutdownReasonUpdateRestart = "update_restart" |
| 22 | shutdownReasonStartupFailure = "startup_failure" |
| 23 | shutdownReasonConnectionLost = "connection_lost" |
| 24 | shutdownReasonSystemSignal = "system_signal" |
| 25 | ) |
| 26 | |
| 27 | type shutdownRequest struct { |
| 28 | RequestID string `json:"requestId"` |
| 29 | Reason string `json:"reason"` |
| 30 | } |
| 31 | |
| 32 | type shutdownStatus struct { |
| 33 | RequestID string `json:"requestId"` |
| 34 | Reason string `json:"reason"` |
| 35 | Phase string `json:"phase"` |
| 36 | Outcome string `json:"outcome"` |
| 37 | Completed bool `json:"completed"` |
| 38 | Retryable bool `json:"retryable"` |
| 39 | ErrorCode string `json:"errorCode,omitempty"` |
| 40 | Error string `json:"error,omitempty"` |
| 41 | UpdatedAt string `json:"updatedAt"` |
| 42 | } |
| 43 | |
| 44 | type desktopShutdownItem struct { |
| 45 | tab *WorkspaceTab |
| 46 | ctrl control.SessionAPI |
| 47 | readOnly bool |
| 48 | } |
| 49 | |
| 50 | // desktopShutdownCoordinator owns the one process shutdown transaction. It |
| 51 | // retains successful save and close steps so a retry resumes at the failed |
| 52 | // step instead of closing an already released session a second time. |
| 53 | type desktopShutdownCoordinator struct { |
| 54 | mu sync.Mutex |
| 55 | running bool |
| 56 | done chan struct{} |
| 57 | status shutdownStatus |
| 58 | frozen bool |
| 59 | items []desktopShutdownItem |
| 60 | saved map[string]bool |
| 61 | finished map[string]bool |
| 62 | } |
| 63 | |
| 64 | type shutdownStepError struct { |
| 65 | code string |
| 66 | err error |
| 67 | } |
| 68 | |
| 69 | func (e *shutdownStepError) Error() string { return e.err.Error() } |
| 70 | func (e *shutdownStepError) Unwrap() error { return e.err } |
| 71 | |
| 72 | func normalizeShutdownReason(reason string) string { |
| 73 | switch strings.TrimSpace(reason) { |
| 74 | case shutdownReasonUserQuit, shutdownReasonUpdateRestart, shutdownReasonStartupFailure, |
| 75 | shutdownReasonConnectionLost, shutdownReasonSystemSignal: |
| 76 | return strings.TrimSpace(reason) |
| 77 | default: |
| 78 | return shutdownReasonConnectionLost |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | func (a *App) shutdownState() *desktopShutdownCoordinator { |
| 83 | a.shutdownMu.Lock() |
| 84 | defer a.shutdownMu.Unlock() |
| 85 | if a.shutdownCoordinator == nil { |
| 86 | a.shutdownCoordinator = &desktopShutdownCoordinator{ |
| 87 | saved: map[string]bool{}, |
| 88 | finished: map[string]bool{}, |
| 89 | status: shutdownStatus{Phase: "idle", Outcome: "not_started"}, |
| 90 | } |
| 91 | } |
| 92 | return a.shutdownCoordinator |
| 93 | } |
| 94 | |
| 95 | func (a *App) shutdownStatus(requestID string) shutdownStatus { |
| 96 | c := a.shutdownState() |
| 97 | c.mu.Lock() |
| 98 | defer c.mu.Unlock() |
| 99 | status := c.status |
| 100 | if requestID != "" && status.RequestID != "" && requestID != status.RequestID { |
| 101 | return shutdownStatus{ |
| 102 | RequestID: requestID, Phase: "idle", Outcome: "not_started", Retryable: true, |
| 103 | ErrorCode: "shutdown_request_not_found", |
| 104 | } |
| 105 | } |
| 106 | return status |
| 107 | } |
| 108 | |
| 109 | func (a *App) requestShutdown(ctx context.Context, request shutdownRequest) (shutdownStatus, error) { |
| 110 | request.RequestID = strings.TrimSpace(request.RequestID) |
| 111 | if request.RequestID == "" { |
| 112 | return shutdownStatus{}, &shutdownStepError{code: "invalid_request", err: errors.New("shutdown requestId is required")} |
| 113 | } |
| 114 | request.Reason = normalizeShutdownReason(request.Reason) |
| 115 | c := a.shutdownState() |
| 116 | |
| 117 | c.mu.Lock() |
| 118 | if c.status.Completed { |
| 119 | status := c.status |
| 120 | c.mu.Unlock() |
| 121 | return status, nil |
| 122 | } |
| 123 | if c.running { |
| 124 | done := c.done |
| 125 | c.mu.Unlock() |
| 126 | select { |
| 127 | case <-done: |
| 128 | status := a.shutdownStatus("") |
| 129 | if status.Outcome == "failed" { |
| 130 | return status, errors.New(status.Error) |
| 131 | } |
| 132 | return status, nil |
| 133 | case <-ctx.Done(): |
| 134 | return a.shutdownStatus(""), ctx.Err() |
| 135 | } |
| 136 | } |
| 137 | // A retry resumes the original transaction. Later EOF, signal, or RPC |
| 138 | // requests must not rewrite the trigger that started it. |
| 139 | if c.status.RequestID != "" { |
| 140 | request.RequestID = c.status.RequestID |
| 141 | request.Reason = c.status.Reason |
| 142 | } |
| 143 | c.running = true |
| 144 | c.done = make(chan struct{}) |
| 145 | c.status = shutdownStatus{ |
| 146 | RequestID: request.RequestID, |
| 147 | Reason: request.Reason, |
| 148 | Phase: "preparing", Outcome: "in_progress", |
| 149 | UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano), |
| 150 | } |
| 151 | done := c.done |
| 152 | c.mu.Unlock() |
| 153 | |
| 154 | err := a.runShutdown(c) |
| 155 | if err != nil { |
| 156 | status := a.shutdownStatus("") |
| 157 | a.lifecycle.tracker.markShutdown(status.Reason, status.Phase, "failed") |
| 158 | } |
| 159 | c.mu.Lock() |
| 160 | if err != nil { |
| 161 | var step *shutdownStepError |
| 162 | code := "shutdown_failed" |
| 163 | if errors.As(err, &step) { |
| 164 | code = step.code |
| 165 | } |
| 166 | c.status.Outcome = "failed" |
| 167 | c.status.Retryable = true |
| 168 | c.status.ErrorCode = code |
| 169 | c.status.Error = err.Error() |
| 170 | c.status.Completed = false |
| 171 | } else { |
| 172 | c.status.Phase = "completed" |
| 173 | c.status.Outcome = "success" |
| 174 | c.status.Completed = true |
| 175 | c.status.Retryable = false |
| 176 | c.status.ErrorCode = "" |
| 177 | c.status.Error = "" |
| 178 | } |
| 179 | c.status.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) |
| 180 | c.running = false |
| 181 | close(done) |
| 182 | status := c.status |
| 183 | c.mu.Unlock() |
| 184 | return status, err |
| 185 | } |
| 186 | |
| 187 | func (c *desktopShutdownCoordinator) setPhase(phase string) { |
| 188 | c.mu.Lock() |
| 189 | c.status.Phase = phase |
| 190 | c.status.Outcome = "in_progress" |
| 191 | c.status.ErrorCode = "" |
| 192 | c.status.Error = "" |
| 193 | c.status.Retryable = false |
| 194 | c.status.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) |
| 195 | c.mu.Unlock() |
| 196 | } |
| 197 | |
| 198 | func (a *App) runShutdown(c *desktopShutdownCoordinator) (err error) { |
| 199 | defer func() { |
| 200 | if recovered := recover(); recovered != nil { |
| 201 | err = &shutdownStepError{code: "cleanup_panic", err: fmt.Errorf("shutdown cleanup panic: %v", recovered)} |
| 202 | } |
| 203 | }() |
| 204 | |
| 205 | c.mu.Lock() |
| 206 | frozen := c.frozen |
| 207 | reason := c.status.Reason |
| 208 | c.mu.Unlock() |
| 209 | if !frozen { |
| 210 | c.setPhase("cancelling_background") |
| 211 | a.lifecycle.tracker.markShutdown(reason, "cancelling_background", "in_progress") |
| 212 | a.manualCreationMu.Lock() |
| 213 | a.shuttingDown.Store(true) |
| 214 | m := a.manualCreations |
| 215 | a.manualCreationMu.Unlock() |
| 216 | if m != nil { |
| 217 | m.StopAdmission() |
| 218 | } |
| 219 | c.runStep("cancel-session-navigation", a.cancelSessionNavigation) |
| 220 | c.runStep("cancel-tab-builds", a.cancelAllTabBuilds) |
| 221 | if err := c.runErrorStep("manual-session-creation", a.stopManualCreations); err != nil { |
| 222 | return &shutdownStepError{code: "manual_creation_stop_timeout", err: err} |
| 223 | } |
| 224 | c.runStep("cancel-session-exports", a.cancelSessionExports) |
| 225 | c.runStep("runtime-projections", a.flushRuntimeProjections) |
| 226 | c.runStep("tab-layout", a.flushTabLayoutWrites) |
| 227 | c.runStep("historical-imports", a.stopHistoricalImports) |
| 228 | c.runStep("session-catalog", func() { a.stopSessionCatalog(250 * time.Millisecond) }) |
| 229 | c.mu.Lock() |
| 230 | c.frozen = true |
| 231 | c.mu.Unlock() |
| 232 | } |
| 233 | |
| 234 | // Use the normal runtime lock order and never hold App.mu while invoking a |
| 235 | // controller. This prevents callback re-entry deadlocks during snapshots. |
| 236 | c.setPhase("waiting_runtime_rebuild") |
| 237 | a.lifecycle.tracker.markShutdown(reason, "waiting_runtime_rebuild", "in_progress") |
| 238 | a.runtimeRebuildMu.Lock() |
| 239 | defer a.runtimeRebuildMu.Unlock() |
| 240 | c.setPhase("waiting_runtime_admission") |
| 241 | a.lifecycle.tracker.markShutdown(reason, "waiting_runtime_admission", "in_progress") |
| 242 | a.runtimeAdmissionMu.Lock() |
| 243 | defer a.runtimeAdmissionMu.Unlock() |
| 244 | |
| 245 | c.mu.Lock() |
| 246 | needsItems := c.items == nil |
| 247 | c.mu.Unlock() |
| 248 | if needsItems { |
| 249 | a.mu.RLock() |
| 250 | tabs := a.runtimeTabsLocked() |
| 251 | items := make([]desktopShutdownItem, 0, len(tabs)) |
| 252 | for _, tab := range tabs { |
| 253 | if tab.Ctrl != nil { |
| 254 | items = append(items, desktopShutdownItem{tab: tab, ctrl: tab.Ctrl, readOnly: tab.ReadOnly}) |
| 255 | } |
| 256 | } |
| 257 | a.mu.RUnlock() |
| 258 | sort.Slice(items, func(i, j int) bool { return items[i].tab.ID < items[j].tab.ID }) |
| 259 | c.mu.Lock() |
| 260 | c.items = items |
| 261 | c.mu.Unlock() |
| 262 | } |
| 263 | c.mu.Lock() |
| 264 | items := append([]desktopShutdownItem(nil), c.items...) |
| 265 | reason = c.status.Reason |
| 266 | c.mu.Unlock() |
| 267 | |
| 268 | c.setPhase("saving") |
| 269 | a.lifecycle.tracker.markShutdown(reason, "saving", "in_progress") |
| 270 | for _, item := range items { |
| 271 | if item.readOnly || historicalPreview(item.ctrl) { |
| 272 | continue |
| 273 | } |
| 274 | c.mu.Lock() |
| 275 | saved := c.saved[item.tab.ID] |
| 276 | c.mu.Unlock() |
| 277 | if saved { |
| 278 | continue |
| 279 | } |
| 280 | if err := item.ctrl.SnapshotForShutdown(); err != nil { |
| 281 | a.lifecycle.tracker.markShutdown(reason, "saving", "failed") |
| 282 | return &shutdownStepError{code: "session_save_failed", err: fmt.Errorf("save session %s: %w", item.tab.ID, err)} |
| 283 | } |
| 284 | c.mu.Lock() |
| 285 | c.saved[item.tab.ID] = true |
| 286 | c.mu.Unlock() |
| 287 | } |
| 288 | |
| 289 | c.setPhase("closing") |
| 290 | a.lifecycle.tracker.markShutdown(reason, "closing", "in_progress") |
| 291 | if err := a.shutdownBody(c, items); err != nil { |
| 292 | return err |
| 293 | } |
| 294 | a.lifecycle.tracker.markShutdown(reason, "completed", "success") |
| 295 | if reason == shutdownReasonUserQuit || reason == shutdownReasonUpdateRestart { |
| 296 | a.lifecycle.tracker.clean() |
| 297 | } |
| 298 | return nil |
| 299 | } |
| 300 | |
| 301 | // completeDesktopShutdown remains the small panic-preserving primitive used by |
| 302 | // lifecycle compatibility tests and callers outside the coordinated App path. |
| 303 | func completeDesktopShutdown(tracker *desktopLifecycleTracker, body func()) { |
| 304 | tracker.stopWriter() |
| 305 | tracker.mark("shutting_down") |
| 306 | body() |
| 307 | tracker.clean() |
| 308 | } |
| 309 | |
| 310 | func (a *App) shutdownBody(c *desktopShutdownCoordinator, items []desktopShutdownItem) error { |
| 311 | if a.sessionUI != nil { |
| 312 | if err := c.runErrorStep("session-ui", a.sessionUI.Close); err != nil { |
| 313 | return &shutdownStepError{code: "session_ui_close_failed", err: err} |
| 314 | } |
| 315 | } |
| 316 | if a.desktopDrafts != nil { |
| 317 | if err := c.runErrorStep("desktop-drafts", a.desktopDrafts.Close); err != nil { |
| 318 | return &shutdownStepError{code: "draft_close_failed", err: err} |
| 319 | } |
| 320 | } |
| 321 | c.runStep("workspace-preview", a.stopWorkspacePreviewOrigin) |
| 322 | if a.desktopShell.coordinator != nil { |
| 323 | c.runStep("shell-coordinator", a.desktopShell.coordinator.stop) |
| 324 | } |
| 325 | if a.workspaceHub != nil { |
| 326 | c.runStep("workspace-hub", a.workspaceHub.close) |
| 327 | } |
| 328 | c.runStep("remote-windows", a.closeAllRemoteWindows) |
| 329 | c.runStep("deferred-rebuild", a.stopDeferredRebuildRetry) |
| 330 | c.runStep("takeover-mirrors-stop", a.stopTakeoverMirrors) |
| 331 | if a.heartbeat != nil { |
| 332 | c.runStep("heartbeat", a.heartbeat.Stop) |
| 333 | } |
| 334 | c.runStep("bot-runtime", a.stopBotRuntime) |
| 335 | c.runStep("bot-bridge", a.stopBotBridge) |
| 336 | c.runStep("remote-runtime", a.stopRemoteRuntime) |
| 337 | c.runStep("tray", a.stopTray) |
| 338 | if a.terminals != nil { |
| 339 | c.runStep("terminals", a.terminals.closeAll) |
| 340 | } |
| 341 | c.runStep("window-state", a.saveWindowStateSync) |
| 342 | |
| 343 | for _, item := range items { |
| 344 | c.runStep("session:"+item.tab.ID, func() { |
| 345 | item.ctrl.Close() |
| 346 | if !a.returnTakeoverLeaseForShutdown(item.tab) { |
| 347 | item.tab.releaseSessionLease() |
| 348 | } |
| 349 | a.mu.Lock() |
| 350 | a.releaseSessionRuntimeLocked(item.tab) |
| 351 | a.mu.Unlock() |
| 352 | }) |
| 353 | } |
| 354 | c.runStep("takeover-mirrors-end", a.endTakeoverMirrors) |
| 355 | c.runStep("update-health", func() { |
| 356 | if !a.startupReady.Load() { |
| 357 | return |
| 358 | } |
| 359 | if err := a.commitPendingUpdateHealth(); err != nil { |
| 360 | slog.Warn("desktop: commit healthy update during shutdown", "err", err) |
| 361 | } |
| 362 | if archived, err := archiveSupersededPendingUpdateAfterReady(); err != nil { |
| 363 | slog.Warn("desktop: retire superseded update during shutdown", "err", err) |
| 364 | } else if archived { |
| 365 | slog.Info("desktop: archived superseded update transaction during shutdown") |
| 366 | } |
| 367 | _ = repair.RecordHealthyConfig(version) |
| 368 | }) |
| 369 | c.runStep("shared-hosts", a.closeAllSharedHosts) |
| 370 | c.runStep("derived-state", func() { |
| 371 | flushCtx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) |
| 372 | defer cancel() |
| 373 | if err := stats.Flush(flushCtx, config.StatsDir()); err != nil { |
| 374 | slog.Warn("desktop: flush shutdown stats", "err", err) |
| 375 | } |
| 376 | if err := flushDesktopDerivedCatalogs(flushCtx); err != nil { |
| 377 | slog.Warn("desktop: flush derived catalogs", "err", err) |
| 378 | } |
| 379 | }) |
| 380 | if a.topicState != nil { |
| 381 | c.runStep("topic-state", a.topicState.close) |
| 382 | } |
| 383 | if err := c.runErrorStep("session-services", a.closeSessionServicesResult); err != nil { |
| 384 | return &shutdownStepError{code: "session_service_close_failed", err: err} |
| 385 | } |
| 386 | // Last: every controller that subscribed to the host watcher is gone, and |
| 387 | // boot.Build leaves a caller-owned service alone, so this is the only close. |
| 388 | c.runStep("skill-watch-service", a.closeSharedSkillWatchService) |
| 389 | return nil |
| 390 | } |
| 391 |