| 1 | //! Turn context and tracking. |
| 2 | //! |
| 3 | //! A "turn" is one user message and the resulting AI response, |
| 4 | //! including any tool calls that occur. |
| 5 | //! |
| 6 | //! ## Snapshot lifecycle hooks |
| 7 | //! |
| 8 | //! [`restore_point_snapshot`] (`pre-turn:`, `tool:`, `post-tool:`) and |
| 9 | //! [`post_turn_snapshot`] book-end a turn and its tool calls by |
| 10 | //! taking a workspace-level snapshot into a side git repo (see |
| 11 | //! `crate::snapshot`). They are intentionally non-blocking and |
| 12 | //! non-fatal: any IO error is logged at WARN and swallowed so a busted |
| 13 | //! filesystem or missing `git` binary never derails the agent loop. |
| 14 | //! `/restore N` and the `revert_turn` tool both consume these |
| 15 | //! snapshots. |
| 16 | |
| 17 | use crate::core::events::TurnRoute; |
| 18 | use crate::snapshot::{SnapshotRepo, TakenSnapshot}; |
| 19 | use codewhale_models::Usage; |
| 20 | use std::path::Path; |
| 21 | use std::time::{Duration, Instant}; |
| 22 | |
| 23 | /// Which configured limit governs a turn's step budget (#5994). |
| 24 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 25 | pub enum StepBudgetSource { |
| 26 | /// The ordinary interactive ceiling (`max_steps`). |
| 27 | Interactive, |
| 28 | /// The goal-turn allowance (`[goal] max_steps`). |
| 29 | Goal, |
| 30 | } |
| 31 | |
| 32 | impl StepBudgetSource { |
| 33 | /// The configuration key named in soft-landing and exhaustion notices. |
| 34 | #[must_use] |
| 35 | pub const fn key_label(self) -> &'static str { |
| 36 | match self { |
| 37 | Self::Interactive => "max_steps", |
| 38 | Self::Goal => "[goal] max_steps", |
| 39 | } |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | /// Where a turn's user message sat in the session right after it was added: |
| 44 | /// the session length and messages revision then. Together they identify |
| 45 | /// that message; a later append or rewrite changes one of them (#6566). |
| 46 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 47 | pub struct UnansweredUserMessage { |
| 48 | pub len: usize, |
| 49 | pub revision: u64, |
| 50 | } |
| 51 | |
| 52 | /// Context for a single turn (user message + AI response). |
| 53 | #[derive(Debug)] |
| 54 | pub struct TurnContext { |
| 55 | pub max_output_tokens: Option<std::num::NonZeroU32>, |
| 56 | /// Turn ID |
| 57 | pub id: String, |
| 58 | |
| 59 | /// When the turn started |
| 60 | pub started_at: Instant, |
| 61 | |
| 62 | /// Current step in the turn (tool call iteration) |
| 63 | pub step: u32, |
| 64 | |
| 65 | /// Configured steps, or `u32::MAX` for no limit. Use `step_limit` for |
| 66 | /// budget decisions; the counter saturates without stopping an uncapped turn. |
| 67 | pub max_steps: u32, |
| 68 | |
| 69 | /// Which configured limit `max_steps` came from. |
| 70 | pub budget_source: StepBudgetSource, |
| 71 | |
| 72 | /// The turn's step budget was exhausted and the bounded final report was |
| 73 | /// granted (#5994). Set by the turn loop; the cross-turn goal fence reads |
| 74 | /// it so an exhausted goal pauses instead of re-arming. |
| 75 | pub budget_exhausted_final_report: bool, |
| 76 | /// This turn's user message while no model request has been accepted |
| 77 | /// yet. A credential rejection while it is still the last message takes |
| 78 | /// it back out of the session, so a retry after fixing the key is not a |
| 79 | /// duplicate (#6566). Cleared once a request is accepted: from then on a |
| 80 | /// model has seen the message. |
| 81 | pub unanswered_user_message: Option<UnansweredUserMessage>, |
| 82 | |
| 83 | pub(crate) stop_diagnostics: crate::tool_inspection::TurnStopDiagnostics, |
| 84 | pub(crate) last_request_snapshot: Option<crate::tool_inspection::ToolInspectionSnapshot>, |
| 85 | |
| 86 | /// Number of tool calls made in this turn. |
| 87 | |
| 88 | /// Usage for this turn |
| 89 | pub usage: Usage, |
| 90 | |
| 91 | /// Subset of `usage` served by the parent turn's frozen route. Programmatic |
| 92 | /// reviewer/RLM calls remain in the total above but are billed only from |
| 93 | /// their own routed receipts. |
| 94 | pub parent_route_usage: Usage, |
| 95 | |
| 96 | /// Provider calls whose usage became ambiguous after dispatch (for |
| 97 | /// example an RLM timeout). A non-zero value makes cost coverage |
| 98 | /// explicitly incomplete instead of inventing a zero-usage receipt. |
| 99 | pub routed_usage_dropped_records: u64, |
| 100 | |
| 101 | /// Input tokens reported for the most recent parent-route model request. |
| 102 | /// This is deliberately separate from `usage`, which accumulates every |
| 103 | /// parent step and programmatic child call for billing. |
| 104 | pub(crate) latest_parent_input_tokens: Option<u32>, |
| 105 | |
| 106 | /// `session.messages.len()` at the parent request whose billed prompt is |
| 107 | /// in `latest_parent_input_tokens`. Tool results appended after that |
| 108 | /// request are not in the bill; GrokBuild's pre-sampling gate adds a |
| 109 | /// byte-estimate of that suffix so auto-compact can fire mid-turn. |
| 110 | pub(crate) messages_len_at_last_parent_prompt: Option<usize>, |
| 111 | |
| 112 | /// One-shot latch: an automatic-compaction refusal has already been |
| 113 | /// surfaced this turn. Pressure is re-checked every step, and repeating |
| 114 | /// the same refusal on each of a long turn's steps would be noise. |
| 115 | pub(crate) compaction_refusal_notified: bool, |
| 116 | |
| 117 | /// Route facts resolved for this turn but not timestamped until the first |
| 118 | /// provider request is actually dispatched. |
| 119 | pub(crate) pending_route: Option<TurnRoute>, |
| 120 | |
| 121 | /// The provider's answer when it rejected an emergency context-recovery |
| 122 | /// request (capability, auth, unreachable). The turn then fails on that |
| 123 | /// cause, not on the context budget the recovery was trying to fix. |
| 124 | pub(crate) context_recovery_rejection: Option<anyhow::Error>, |
| 125 | } |
| 126 | |
| 127 | impl TurnContext { |
| 128 | /// Create a new turn context |
| 129 | pub fn new(max_steps: u32) -> Self { |
| 130 | Self::with_budget_source(max_steps, StepBudgetSource::Interactive) |
| 131 | } |
| 132 | |
| 133 | /// Create a turn context with an explicit budget provenance (#5994). |
| 134 | pub fn with_budget_source(max_steps: u32, budget_source: StepBudgetSource) -> Self { |
| 135 | Self { |
| 136 | max_output_tokens: None, |
| 137 | id: uuid::Uuid::new_v4().to_string(), |
| 138 | started_at: Instant::now(), |
| 139 | step: 0, |
| 140 | max_steps, |
| 141 | budget_source, |
| 142 | budget_exhausted_final_report: false, |
| 143 | unanswered_user_message: None, |
| 144 | stop_diagnostics: crate::tool_inspection::TurnStopDiagnostics { |
| 145 | effective_max_steps: (max_steps != u32::MAX).then_some(max_steps), |
| 146 | step_budget_source: budget_source.key_label(), |
| 147 | ..Default::default() |
| 148 | }, |
| 149 | last_request_snapshot: None, |
| 150 | usage: Usage { |
| 151 | input_tokens: 0, |
| 152 | output_tokens: 0, |
| 153 | ..Usage::default() |
| 154 | }, |
| 155 | parent_route_usage: Usage::default(), |
| 156 | routed_usage_dropped_records: 0, |
| 157 | latest_parent_input_tokens: None, |
| 158 | messages_len_at_last_parent_prompt: None, |
| 159 | compaction_refusal_notified: false, |
| 160 | pending_route: None, |
| 161 | context_recovery_rejection: None, |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | /// Increment the step counter |
| 166 | pub fn next_step(&mut self) -> bool { |
| 167 | self.step = self.step.saturating_add(1); |
| 168 | self.step_limit().is_none_or(|limit| self.step <= limit) |
| 169 | } |
| 170 | |
| 171 | /// A resolved integer default means no ceiling, including at counter |
| 172 | /// saturation. Explicit positive configuration is clamped before here. |
| 173 | #[must_use] |
| 174 | pub fn step_limit(&self) -> Option<u32> { |
| 175 | (self.max_steps != u32::MAX).then_some(self.max_steps) |
| 176 | } |
| 177 | |
| 178 | /// Check if the turn has reached max steps |
| 179 | pub fn at_max_steps(&self) -> bool { |
| 180 | self.step_limit().is_some_and(|limit| self.step >= limit) |
| 181 | } |
| 182 | |
| 183 | /// Model steps consumed so far (for soft-landing and reporting). |
| 184 | #[must_use] |
| 185 | pub fn steps_used(&self) -> u32 { |
| 186 | self.step |
| 187 | } |
| 188 | |
| 189 | /// Get the elapsed time |
| 190 | pub fn elapsed(&self) -> Duration { |
| 191 | self.started_at.elapsed() |
| 192 | } |
| 193 | |
| 194 | /// Complete the existing request projection with observed turn-exit facts. |
| 195 | /// A turn that never prepared a request has no request snapshot to publish. |
| 196 | pub(crate) fn terminal_request_snapshot( |
| 197 | &mut self, |
| 198 | status: super::events::TurnOutcomeStatus, |
| 199 | ) -> Option<crate::tool_inspection::ToolInspectionSnapshot> { |
| 200 | use crate::tool_inspection::TurnStopReason; |
| 201 | self.stop_diagnostics.status = Some(status); |
| 202 | self.stop_diagnostics.model_step_index = self.step; |
| 203 | self.stop_diagnostics.final_report_requested |= self.budget_exhausted_final_report; |
| 204 | self.stop_diagnostics.last_reported_input_tokens = self.latest_parent_input_tokens; |
| 205 | match status { |
| 206 | super::events::TurnOutcomeStatus::Interrupted => { |
| 207 | self.stop_diagnostics.reason = Some(TurnStopReason::Interrupted); |
| 208 | } |
| 209 | super::events::TurnOutcomeStatus::Failed if self.stop_diagnostics.reason.is_none() => { |
| 210 | self.stop_diagnostics.reason = Some(TurnStopReason::Failed); |
| 211 | } |
| 212 | _ => {} |
| 213 | } |
| 214 | let mut snapshot = self.last_request_snapshot.take()?; |
| 215 | snapshot.terminal = Some(self.stop_diagnostics.clone()); |
| 216 | Some(snapshot) |
| 217 | } |
| 218 | |
| 219 | /// Add usage from an API response |
| 220 | pub fn add_usage(&mut self, usage: &Usage) { |
| 221 | add_usage_to(&mut self.usage, usage); |
| 222 | } |
| 223 | |
| 224 | /// Record one parent-route response for both billing and live-context |
| 225 | /// pressure. Child-model usage must call [`Self::add_usage`] directly so |
| 226 | /// it cannot masquerade as the parent request's context size. |
| 227 | pub fn add_parent_usage(&mut self, usage: &Usage) { |
| 228 | self.latest_parent_input_tokens = (usage.input_tokens > 0).then_some(usage.input_tokens); |
| 229 | self.add_usage(usage); |
| 230 | add_usage_to(&mut self.parent_route_usage, usage); |
| 231 | } |
| 232 | |
| 233 | pub fn add_routed_usage_dropped_records(&mut self, dropped_records: u64) { |
| 234 | self.routed_usage_dropped_records = self |
| 235 | .routed_usage_dropped_records |
| 236 | .saturating_add(dropped_records); |
| 237 | } |
| 238 | |
| 239 | /// Add programmatic child-call usage to the authoritative total and |
| 240 | /// return the same batch aggregate for telemetry emission. |
| 241 | pub fn add_routed_usages<'a>(&mut self, usages: impl IntoIterator<Item = &'a Usage>) -> Usage { |
| 242 | let mut aggregate = Usage::default(); |
| 243 | for usage in usages { |
| 244 | self.add_usage(usage); |
| 245 | add_usage_to(&mut aggregate, usage); |
| 246 | } |
| 247 | aggregate |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | pub(crate) fn add_usage_to(total: &mut Usage, delta: &Usage) { |
| 252 | total.input_tokens = total.input_tokens.saturating_add(delta.input_tokens); |
| 253 | total.output_tokens = total.output_tokens.saturating_add(delta.output_tokens); |
| 254 | total.prompt_cache_hit_tokens = |
| 255 | add_optional_usage(total.prompt_cache_hit_tokens, delta.prompt_cache_hit_tokens); |
| 256 | total.prompt_cache_miss_tokens = add_optional_usage( |
| 257 | total.prompt_cache_miss_tokens, |
| 258 | delta.prompt_cache_miss_tokens, |
| 259 | ); |
| 260 | total.prompt_cache_write_tokens = add_optional_usage( |
| 261 | total.prompt_cache_write_tokens, |
| 262 | delta.prompt_cache_write_tokens, |
| 263 | ); |
| 264 | total.reasoning_tokens = add_optional_usage(total.reasoning_tokens, delta.reasoning_tokens); |
| 265 | total.reasoning_replay_tokens = |
| 266 | add_optional_usage(total.reasoning_replay_tokens, delta.reasoning_replay_tokens); |
| 267 | if let Some(delta) = delta.server_tool_use.as_ref() { |
| 268 | let server_total = total.server_tool_use.get_or_insert_default(); |
| 269 | server_total.code_execution_requests = add_optional_usage( |
| 270 | server_total.code_execution_requests, |
| 271 | delta.code_execution_requests, |
| 272 | ); |
| 273 | server_total.tool_search_requests = add_optional_usage( |
| 274 | server_total.tool_search_requests, |
| 275 | delta.tool_search_requests, |
| 276 | ); |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | impl TurnContext { |
| 281 | /// Billed prompt the compaction gate should honor: this turn's latest |
| 282 | /// parent request, else the session-carried receipt from the previous |
| 283 | /// turn. A fresh `TurnContext` starts empty, so without the session |
| 284 | /// fallback an 842k DeepSeek bill dies at the turn boundary and the |
| 285 | /// next send never auto-compacts (#5577). |
| 286 | #[must_use] |
| 287 | pub(crate) fn billed_input_tokens_for_compaction( |
| 288 | &self, |
| 289 | session_billed: Option<u32>, |
| 290 | ) -> Option<u64> { |
| 291 | self.latest_parent_input_tokens |
| 292 | .or(session_billed) |
| 293 | .map(u64::from) |
| 294 | } |
| 295 | |
| 296 | /// Record how long the transcript was when the latest parent prompt was |
| 297 | /// billed. Call immediately after `add_parent_usage`, before this |
| 298 | /// response's assistant/tool messages are appended. |
| 299 | pub(crate) fn note_parent_prompt_len(&mut self, message_count: usize) { |
| 300 | self.messages_len_at_last_parent_prompt = Some(message_count); |
| 301 | } |
| 302 | |
| 303 | /// Live context for the auto-compact gate: last billed prompt plus a |
| 304 | /// /4 estimate of messages appended since that prompt (tool results, |
| 305 | /// the assistant reply that will be replayed on the next request). |
| 306 | /// |
| 307 | /// `max(billed, estimate(full list))` hides mid-turn growth when the |
| 308 | /// estimator undercounts the whole transcript below the last bill — |
| 309 | /// which is why auto-compact never fired even with the UI meter above |
| 310 | /// 80%. GrokBuild's `check_auto_compact_needed` uses the same split: |
| 311 | /// exact prior count + byte-estimate of items since last response. |
| 312 | #[must_use] |
| 313 | pub(crate) fn live_input_tokens_for_compaction( |
| 314 | &self, |
| 315 | messages: &[codewhale_models::Message], |
| 316 | system_prompt: Option<&codewhale_models::SystemPrompt>, |
| 317 | session_billed: Option<u32>, |
| 318 | ) -> Option<u64> { |
| 319 | let billed = self.billed_input_tokens_for_compaction(session_billed); |
| 320 | let suffix_start = self |
| 321 | .messages_len_at_last_parent_prompt |
| 322 | .unwrap_or(messages.len()) |
| 323 | .min(messages.len()); |
| 324 | let suffix = &messages[suffix_start..]; |
| 325 | let growth = if suffix.is_empty() { |
| 326 | 0 |
| 327 | } else { |
| 328 | u64::try_from(crate::compaction::estimate_input_tokens_for_pressure( |
| 329 | suffix, None, |
| 330 | )) |
| 331 | .unwrap_or(u64::MAX) |
| 332 | }; |
| 333 | let estimated = u64::try_from(crate::compaction::estimate_input_tokens_for_pressure( |
| 334 | messages, |
| 335 | system_prompt, |
| 336 | )) |
| 337 | .unwrap_or(u64::MAX); |
| 338 | let live = estimated.max(billed.unwrap_or(0).saturating_add(growth)); |
| 339 | (live > 0).then_some(live) |
| 340 | } |
| 341 | |
| 342 | /// Drop the turn-local billed receipt after history is rewritten so the |
| 343 | /// next step cannot compact again on the pre-compaction prompt. |
| 344 | pub(crate) fn clear_parent_input_tokens(&mut self) { |
| 345 | self.latest_parent_input_tokens = None; |
| 346 | self.messages_len_at_last_parent_prompt = None; |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | fn add_optional_usage(total: Option<u32>, delta: Option<u32>) -> Option<u32> { |
| 351 | match (total, delta) { |
| 352 | (Some(total), Some(delta)) => Some(total.saturating_add(delta)), |
| 353 | (None, Some(delta)) => Some(delta), |
| 354 | (Some(total), None) => Some(total), |
| 355 | (None, None) => None, |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | #[cfg(test)] |
| 360 | mod usage_tests { |
| 361 | use super::*; |
| 362 | use codewhale_models::ServerToolUsage; |
| 363 | |
| 364 | #[test] |
| 365 | fn add_usage_preserves_replay_and_saturates_server_tool_counters() { |
| 366 | let mut turn = TurnContext::new(2); |
| 367 | turn.add_usage(&Usage { |
| 368 | reasoning_replay_tokens: Some(u32::MAX - 1), |
| 369 | server_tool_use: Some(ServerToolUsage { |
| 370 | code_execution_requests: Some(u32::MAX), |
| 371 | tool_search_requests: Some(2), |
| 372 | }), |
| 373 | ..Usage::default() |
| 374 | }); |
| 375 | turn.add_usage(&Usage { |
| 376 | reasoning_replay_tokens: Some(9), |
| 377 | server_tool_use: Some(ServerToolUsage { |
| 378 | code_execution_requests: Some(1), |
| 379 | tool_search_requests: Some(3), |
| 380 | }), |
| 381 | ..Usage::default() |
| 382 | }); |
| 383 | |
| 384 | assert_eq!(turn.usage.reasoning_replay_tokens, Some(u32::MAX)); |
| 385 | let server = turn.usage.server_tool_use.expect("server tool usage"); |
| 386 | assert_eq!(server.code_execution_requests, Some(u32::MAX)); |
| 387 | assert_eq!(server.tool_search_requests, Some(5)); |
| 388 | } |
| 389 | |
| 390 | fn below_threshold(messages: &[codewhale_models::Message], turn: &TurnContext) -> bool { |
| 391 | let config = crate::compaction::CompactionConfig { |
| 392 | enabled: true, |
| 393 | token_threshold: 100_000, |
| 394 | ..Default::default() |
| 395 | }; |
| 396 | !crate::compaction::compaction_pressure_reached_with_billed( |
| 397 | messages, |
| 398 | None, |
| 399 | &config, |
| 400 | turn.latest_parent_input_tokens.map(u64::from), |
| 401 | ) |
| 402 | } |
| 403 | |
| 404 | #[test] |
| 405 | fn cumulative_low_context_parent_steps_cannot_trigger_compaction() { |
| 406 | let mut turn = TurnContext::new(4); |
| 407 | turn.add_parent_usage(&Usage { |
| 408 | input_tokens: 60_000, |
| 409 | ..Usage::default() |
| 410 | }); |
| 411 | turn.add_parent_usage(&Usage { |
| 412 | input_tokens: 70_000, |
| 413 | ..Usage::default() |
| 414 | }); |
| 415 | |
| 416 | assert_eq!(turn.usage.input_tokens, 130_000); |
| 417 | assert_eq!(turn.latest_parent_input_tokens, Some(70_000)); |
| 418 | assert!(below_threshold(&[], &turn)); |
| 419 | } |
| 420 | |
| 421 | #[test] |
| 422 | fn child_usage_cannot_replace_parent_context_pressure() { |
| 423 | let mut turn = TurnContext::new(4); |
| 424 | turn.add_parent_usage(&Usage { |
| 425 | input_tokens: 70_000, |
| 426 | ..Usage::default() |
| 427 | }); |
| 428 | turn.add_usage(&Usage { |
| 429 | input_tokens: 250_000, |
| 430 | ..Usage::default() |
| 431 | }); |
| 432 | |
| 433 | assert_eq!(turn.usage.input_tokens, 320_000); |
| 434 | assert_eq!(turn.latest_parent_input_tokens, Some(70_000)); |
| 435 | assert!(below_threshold(&[], &turn)); |
| 436 | } |
| 437 | |
| 438 | #[test] |
| 439 | fn fresh_turn_inherits_session_billed_prompt_for_compaction() { |
| 440 | let turn = TurnContext::new(4); |
| 441 | assert_eq!(turn.latest_parent_input_tokens, None); |
| 442 | assert_eq!( |
| 443 | turn.billed_input_tokens_for_compaction(Some(842_000)), |
| 444 | Some(842_000) |
| 445 | ); |
| 446 | assert_eq!(turn.billed_input_tokens_for_compaction(None), None); |
| 447 | } |
| 448 | |
| 449 | #[test] |
| 450 | fn live_turn_billed_outranks_stale_session_billed() { |
| 451 | let mut turn = TurnContext::new(4); |
| 452 | turn.add_parent_usage(&Usage { |
| 453 | input_tokens: 12_000, |
| 454 | ..Usage::default() |
| 455 | }); |
| 456 | assert_eq!( |
| 457 | turn.billed_input_tokens_for_compaction(Some(842_000)), |
| 458 | Some(12_000) |
| 459 | ); |
| 460 | turn.clear_parent_input_tokens(); |
| 461 | assert_eq!(turn.latest_parent_input_tokens, None); |
| 462 | assert_eq!(turn.messages_len_at_last_parent_prompt, None); |
| 463 | assert_eq!( |
| 464 | turn.billed_input_tokens_for_compaction(Some(842_000)), |
| 465 | Some(842_000) |
| 466 | ); |
| 467 | } |
| 468 | |
| 469 | #[test] |
| 470 | fn live_compaction_tokens_include_tool_results_after_the_billed_prompt() { |
| 471 | // GrokBuild/Codex: last billed prompt + items since that request. |
| 472 | // A 70k bill plus a large tool result must cross an 80k trigger even |
| 473 | // when the full-list /4 estimate stays below the bill (the failure |
| 474 | // mode that kept auto-compact from firing mid-turn above 80%). |
| 475 | let mut turn = TurnContext::new(4); |
| 476 | turn.add_parent_usage(&Usage { |
| 477 | input_tokens: 70_000, |
| 478 | ..Usage::default() |
| 479 | }); |
| 480 | let prompt = vec![codewhale_models::Message { |
| 481 | role: codewhale_models::Role::User, |
| 482 | content: vec![codewhale_models::ContentBlock::Text { |
| 483 | text: "do the work".to_string(), |
| 484 | cache_control: None, |
| 485 | }], |
| 486 | }]; |
| 487 | turn.note_parent_prompt_len(prompt.len()); |
| 488 | |
| 489 | let mut with_tool = prompt; |
| 490 | with_tool.push(codewhale_models::Message { |
| 491 | role: codewhale_models::Role::User, |
| 492 | content: vec![codewhale_models::ContentBlock::ToolResult { |
| 493 | execution_id: None, |
| 494 | tool_use_id: "call-1".to_string(), |
| 495 | content: "x".repeat(80_000), |
| 496 | is_error: None, |
| 497 | content_blocks: None, |
| 498 | }], |
| 499 | }); |
| 500 | |
| 501 | let config = crate::compaction::CompactionConfig { |
| 502 | enabled: true, |
| 503 | token_threshold: 80_000, |
| 504 | ..Default::default() |
| 505 | }; |
| 506 | assert!( |
| 507 | !crate::compaction::compaction_pressure_reached_with_billed( |
| 508 | &with_tool, |
| 509 | None, |
| 510 | &config, |
| 511 | turn.billed_input_tokens_for_compaction(None), |
| 512 | ), |
| 513 | "stale billed prompt alone must not be the live gate" |
| 514 | ); |
| 515 | let live = turn |
| 516 | .live_input_tokens_for_compaction(&with_tool, None, None) |
| 517 | .expect("live tokens"); |
| 518 | assert!( |
| 519 | live >= 80_000, |
| 520 | "tool-result suffix must lift live tokens over the trigger, got {live}" |
| 521 | ); |
| 522 | assert!(crate::compaction::compaction_pressure_reached_with_billed( |
| 523 | &with_tool, |
| 524 | None, |
| 525 | &config, |
| 526 | Some(live), |
| 527 | )); |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | /// Maximum characters of the user prompt snippet to embed in a snapshot |
| 532 | /// label. Longer prompts are truncated with an ellipsis. |
| 533 | const USER_PROMPT_LABEL_MAX: usize = 100; |
| 534 | |
| 535 | /// Format a snapshot label that includes the user prompt for readability |
| 536 | /// in `/restore` listings. |
| 537 | /// |
| 538 | /// Takes the first line of the prompt (up to `USER_PROMPT_LABEL_MAX` |
| 539 | /// characters) and appends it to the traditional `type:seq` label so |
| 540 | /// users can identify which turn each snapshot belongs to. |
| 541 | pub(crate) fn format_snapshot_label( |
| 542 | prefix: &str, |
| 543 | turn_seq: u64, |
| 544 | user_prompt: Option<&str>, |
| 545 | ) -> String { |
| 546 | let base = format!("{prefix}:{turn_seq}"); |
| 547 | match user_prompt { |
| 548 | None | Some("") => base, |
| 549 | Some(prompt) => match snapshot_label_prompt_snippet(prompt) { |
| 550 | None => base, |
| 551 | Some(snippet) => format!("{base}: {snippet}"), |
| 552 | }, |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | /// The exact prompt snippet [`format_snapshot_label`] embeds after `type:seq`. |
| 557 | /// |
| 558 | /// Read surfaces that want to correlate a recorded prompt back to a restore |
| 559 | /// point must go through this function rather than re-deriving the truncation, |
| 560 | /// so the reader and the writer can never disagree about what a label means. |
| 561 | /// Returns `None` when the prompt contributes no snippet at all. |
| 562 | pub(crate) fn snapshot_label_prompt_snippet(prompt: &str) -> Option<String> { |
| 563 | if prompt.is_empty() { |
| 564 | return None; |
| 565 | } |
| 566 | let first_line = prompt.lines().next().unwrap_or(""); |
| 567 | let truncated: String = first_line.chars().take(USER_PROMPT_LABEL_MAX).collect(); |
| 568 | if truncated.chars().count() < first_line.chars().count() { |
| 569 | Some(format!("{truncated}…")) |
| 570 | } else { |
| 571 | Some(truncated) |
| 572 | } |
| 573 | } |
| 574 | |
| 575 | /// A snapshot label parsed back into its parts. |
| 576 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 577 | pub(crate) struct ParsedSnapshotLabel { |
| 578 | /// `pre-turn`, `post-turn`, `tool`, or whatever prefix produced it. |
| 579 | pub kind: String, |
| 580 | /// The turn sequence for `pre-turn`/`post-turn` labels. `tool` labels |
| 581 | /// carry a call id rather than a sequence, so this stays `None` for them. |
| 582 | pub seq: Option<u64>, |
| 583 | /// The embedded prompt snippet, exactly as |
| 584 | /// [`snapshot_label_prompt_snippet`] produced it. |
| 585 | pub prompt_snippet: Option<String>, |
| 586 | } |
| 587 | |
| 588 | /// Parse a label produced by [`format_snapshot_label`]. |
| 589 | /// |
| 590 | /// This is deliberately total: an unrecognized label still yields a record with |
| 591 | /// the raw text as `kind`, because a read surface must describe what is really |
| 592 | /// stored rather than silently dropping rows it does not recognize. |
| 593 | pub(crate) fn parse_snapshot_label(label: &str) -> ParsedSnapshotLabel { |
| 594 | let (head, snippet) = match label.split_once(": ") { |
| 595 | Some((head, rest)) => (head, Some(rest.to_string())), |
| 596 | None => (label, None), |
| 597 | }; |
| 598 | match head.split_once(':') { |
| 599 | Some((kind, seq)) => ParsedSnapshotLabel { |
| 600 | kind: kind.to_string(), |
| 601 | seq: seq.parse::<u64>().ok(), |
| 602 | prompt_snippet: snippet, |
| 603 | }, |
| 604 | None => ParsedSnapshotLabel { |
| 605 | kind: head.to_string(), |
| 606 | seq: None, |
| 607 | prompt_snippet: snippet, |
| 608 | }, |
| 609 | } |
| 610 | } |
| 611 | |
| 612 | /// Take a `pre-turn:<seq>` workspace snapshot. |
| 613 | /// |
| 614 | /// `cap_bytes` is the workspace-size ceiling that gates first-init |
| 615 | /// (passed through to [`SnapshotRepo::open_or_init_with_cap`]); pass |
| 616 | /// `0` to disable the cap. |
| 617 | /// `user_prompt` is an optional snippet of the user's message for this |
| 618 | /// turn, embedded in the snapshot label so `/restore` listings are |
| 619 | /// human-readable. |
| 620 | /// |
| 621 | /// Returns the snapshot (commit and tree) on success, `None` on any error. |
| 622 | /// Errors are logged at WARN; the turn loop must not block on this. The |
| 623 | /// engine takes its pre-turn snapshots through [`restore_point_snapshot`]. |
| 624 | #[cfg(test)] |
| 625 | pub fn pre_turn_snapshot( |
| 626 | workspace: &Path, |
| 627 | turn_seq: u64, |
| 628 | cap_bytes: u64, |
| 629 | user_prompt: Option<&str>, |
| 630 | session_id: Option<&str>, |
| 631 | ) -> Option<TakenSnapshot> { |
| 632 | snapshot_with_label( |
| 633 | workspace, |
| 634 | &format_snapshot_label("pre-turn", turn_seq, user_prompt), |
| 635 | cap_bytes, |
| 636 | session_id, |
| 637 | ) |
| 638 | } |
| 639 | |
| 640 | /// Take a workspace snapshot under `label` for the running turn, and report |
| 641 | /// which paths changed since the turn's previous snapshot `since` (a tree or |
| 642 | /// commit id). |
| 643 | /// |
| 644 | /// The comparison runs before the count prune that follows every snapshot, |
| 645 | /// so `since` is still in the store; `None` for the changed paths means there |
| 646 | /// was no `since` or it could not be compared. `None` on any error, logged |
| 647 | /// at WARN; the turn loop must not block on it. |
| 648 | pub fn restore_point_snapshot( |
| 649 | workspace: &Path, |
| 650 | label: &str, |
| 651 | cap_bytes: u64, |
| 652 | session_id: Option<&str>, |
| 653 | since: Option<&crate::snapshot::SnapshotId>, |
| 654 | ) -> Option<(TakenSnapshot, Option<Vec<std::path::PathBuf>>)> { |
| 655 | snapshot_with_label_since(workspace, label, cap_bytes, session_id, since) |
| 656 | } |
| 657 | |
| 658 | /// Take a `post-turn:<seq>` workspace snapshot. Same failure model as |
| 659 | /// [`restore_point_snapshot`]. |
| 660 | pub fn post_turn_snapshot( |
| 661 | workspace: &Path, |
| 662 | turn_seq: u64, |
| 663 | cap_bytes: u64, |
| 664 | user_prompt: Option<&str>, |
| 665 | session_id: Option<&str>, |
| 666 | ) -> Option<TakenSnapshot> { |
| 667 | snapshot_with_label( |
| 668 | workspace, |
| 669 | &format_snapshot_label("post-turn", turn_seq, user_prompt), |
| 670 | cap_bytes, |
| 671 | session_id, |
| 672 | ) |
| 673 | } |
| 674 | |
| 675 | fn snapshot_with_label( |
| 676 | workspace: &Path, |
| 677 | label: &str, |
| 678 | cap_bytes: u64, |
| 679 | session_id: Option<&str>, |
| 680 | ) -> Option<TakenSnapshot> { |
| 681 | snapshot_with_label_since(workspace, label, cap_bytes, session_id, None).map(|(taken, _)| taken) |
| 682 | } |
| 683 | |
| 684 | fn snapshot_with_label_since( |
| 685 | workspace: &Path, |
| 686 | label: &str, |
| 687 | cap_bytes: u64, |
| 688 | session_id: Option<&str>, |
| 689 | since: Option<&crate::snapshot::SnapshotId>, |
| 690 | ) -> Option<(TakenSnapshot, Option<Vec<std::path::PathBuf>>)> { |
| 691 | match SnapshotRepo::open_or_init_with_cap(workspace, cap_bytes) { |
| 692 | Ok(repo) => { |
| 693 | // Undo that silently stops working is the failure this guards |
| 694 | // (B2): a repaired history and a failing snapshot both reach the |
| 695 | // user through the same notice as the gates, never only a log. |
| 696 | let taken = repo.repair_broken_head().and_then(|repaired| { |
| 697 | repo.take_snapshot(label, session_id) |
| 698 | .map(|taken| (taken, repaired)) |
| 699 | }); |
| 700 | let (id, repaired) = match taken { |
| 701 | Ok((taken, repaired)) => (Some(taken), repaired), |
| 702 | Err(e) => { |
| 703 | tracing::warn!(target: "snapshot", "snapshot '{label}' failed: {e}"); |
| 704 | record_snapshot_notice( |
| 705 | workspace, |
| 706 | session_id, |
| 707 | SnapshotsDisabledScope::Failing, |
| 708 | snapshot_failure_detail(&e), |
| 709 | ); |
| 710 | return None; |
| 711 | } |
| 712 | }; |
| 713 | clear_snapshots_disabled_status(workspace, session_id); |
| 714 | if repaired { |
| 715 | record_snapshot_notice( |
| 716 | workspace, |
| 717 | session_id, |
| 718 | SnapshotsDisabledScope::HistoryRepaired, |
| 719 | String::new(), |
| 720 | ); |
| 721 | } |
| 722 | // What changed since the turn's previous snapshot, compared |
| 723 | // before the prune below can drop that snapshot. |
| 724 | let changed = match (&id, since) { |
| 725 | (Some(taken), Some(since)) if since.as_str() == taken.tree.as_str() => { |
| 726 | Some(Vec::new()) |
| 727 | } |
| 728 | (Some(taken), Some(since)) => { |
| 729 | match repo.changed_paths_between(since, &taken.tree) { |
| 730 | Ok(paths) => Some(paths), |
| 731 | Err(e) => { |
| 732 | tracing::warn!( |
| 733 | target: "snapshot", |
| 734 | "comparing snapshot '{label}' with the turn's previous one failed: {e}" |
| 735 | ); |
| 736 | None |
| 737 | } |
| 738 | } |
| 739 | } |
| 740 | _ => None, |
| 741 | }; |
| 742 | // Prune oldest snapshots to cap disk usage (#1112). |
| 743 | if let Err(e) = repo.prune_keep_last_n_batched(max_snapshots_for(workspace)) { |
| 744 | tracing::warn!(target: "snapshot", "snapshot prune failed: {e}"); |
| 745 | } |
| 746 | id.map(|taken| (taken, changed)) |
| 747 | } |
| 748 | Err(e) => { |
| 749 | // The first gated failure belongs to this session, even when other |
| 750 | // sessions use the same workspace in this process (#5930). |
| 751 | if maybe_notify_snapshots_disabled_once(workspace, session_id, cap_bytes, &e) { |
| 752 | tracing::warn!(target: "snapshot", session_id, "snapshot repo init failed: {e}"); |
| 753 | } else { |
| 754 | tracing::debug!(target: "snapshot", "snapshot repo init still failing: {e}"); |
| 755 | } |
| 756 | None |
| 757 | } |
| 758 | } |
| 759 | } |
| 760 | |
| 761 | /// The count cap [`SnapshotRepo::prune_keep_last_n`] applies after each |
| 762 | /// snapshot: [`crate::snapshot::DEFAULT_MAX_SNAPSHOTS`], which tests can |
| 763 | /// lower per workspace to exercise pruning without taking fifty snapshots. |
| 764 | fn max_snapshots_for(workspace: &Path) -> usize { |
| 765 | #[cfg(test)] |
| 766 | if let Some(max) = test_max_snapshots::get(workspace) { |
| 767 | return max; |
| 768 | } |
| 769 | let _ = workspace; |
| 770 | crate::snapshot::DEFAULT_MAX_SNAPSHOTS |
| 771 | } |
| 772 | |
| 773 | #[cfg(test)] |
| 774 | pub(crate) mod test_max_snapshots { |
| 775 | use std::collections::HashMap; |
| 776 | use std::path::{Path, PathBuf}; |
| 777 | |
| 778 | static OVERRIDES: parking_lot::Mutex<Option<HashMap<PathBuf, usize>>> = |
| 779 | parking_lot::Mutex::new(None); |
| 780 | |
| 781 | fn key(workspace: &Path) -> PathBuf { |
| 782 | workspace |
| 783 | .canonicalize() |
| 784 | .unwrap_or_else(|_| workspace.to_path_buf()) |
| 785 | } |
| 786 | |
| 787 | /// Prune `workspace`'s snapshots to `max` (plus as many turn |
| 788 | /// boundaries) after each snapshot, for the rest of the test process. |
| 789 | pub(crate) fn set(workspace: &Path, max: usize) { |
| 790 | OVERRIDES |
| 791 | .lock() |
| 792 | .get_or_insert_with(HashMap::new) |
| 793 | .insert(key(workspace), max); |
| 794 | } |
| 795 | |
| 796 | pub(super) fn get(workspace: &Path) -> Option<usize> { |
| 797 | OVERRIDES |
| 798 | .lock() |
| 799 | .as_ref() |
| 800 | .and_then(|overrides| overrides.get(&key(workspace)).copied()) |
| 801 | } |
| 802 | } |
| 803 | |
| 804 | /// Which gate turned snapshots off. Each variant selects its own consequence |
| 805 | /// and recovery copy: only [`Self::WorkspaceTooLarge`] is lifted by |
| 806 | /// [`SNAPSHOTS_CAP_CONFIG_KEY`], so the other two must never advertise it. |
| 807 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 808 | pub enum SnapshotsDisabledScope { |
| 809 | /// Snapshot-eligible content exceeds `[snapshots] max_workspace_gb`. |
| 810 | WorkspaceTooLarge, |
| 811 | /// The bounded walk hit the entry ceiling. Raising (or zeroing) the GB cap |
| 812 | /// does not lift this bound. |
| 813 | TooManyFiles, |
| 814 | /// Home, filesystem root, or a top-level home folder: refused for safety, |
| 815 | /// and no config value changes that. |
| 816 | UnsafeLocation, |
| 817 | /// The side repo's HEAD named a missing commit; history was restarted, so |
| 818 | /// earlier restore points are gone although new turns are protected. |
| 819 | HistoryRepaired, |
| 820 | /// Snapshots open but fail (a real git or disk error, not a gate): undo |
| 821 | /// cannot restore the turns taken since. `limit` carries the error. |
| 822 | Failing, |
| 823 | } |
| 824 | |
| 825 | /// Snapshot availability observed for a session and its workspace. Delivering |
| 826 | /// the notice does not erase the status: `/status` can still explain why undo |
| 827 | /// is unavailable after the transient toast has expired (#5930). |
| 828 | /// |
| 829 | /// The notice carries the gate, not prose: every surface renders exactly one |
| 830 | /// localized line from it, so the workspace, the limit, and the recovery are |
| 831 | /// each stated once (#6042). |
| 832 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 833 | pub struct SnapshotsDisabledNotice { |
| 834 | pub workspace: String, |
| 835 | pub scope: SnapshotsDisabledScope, |
| 836 | /// Preformatted limit for the scope that names one (`2.0 GB`, `200000`), |
| 837 | /// or the failure detail for [`SnapshotsDisabledScope::Failing`]. Empty |
| 838 | /// for scopes whose message names neither. |
| 839 | pub limit: String, |
| 840 | } |
| 841 | |
| 842 | impl SnapshotsDisabledNotice { |
| 843 | fn message_id(&self) -> codewhale_localization::MessageId { |
| 844 | use codewhale_localization::MessageId; |
| 845 | match self.scope { |
| 846 | SnapshotsDisabledScope::WorkspaceTooLarge => MessageId::SnapshotsDisabledTooLarge, |
| 847 | SnapshotsDisabledScope::TooManyFiles => MessageId::SnapshotsDisabledTooManyFiles, |
| 848 | SnapshotsDisabledScope::UnsafeLocation => MessageId::SnapshotsDisabledUnsafeLocation, |
| 849 | SnapshotsDisabledScope::HistoryRepaired => MessageId::SnapshotsHistoryRepaired, |
| 850 | SnapshotsDisabledScope::Failing => MessageId::SnapshotsFailing, |
| 851 | } |
| 852 | } |
| 853 | |
| 854 | /// The single user-facing line: what is off, for which workspace, why, and |
| 855 | /// the recovery that actually applies to this gate. |
| 856 | pub fn localize(&self, locale: codewhale_localization::Locale) -> String { |
| 857 | codewhale_localization::tr(locale, self.message_id()) |
| 858 | .replace("{workspace}", &self.workspace) |
| 859 | .replace("{limit}", &self.limit) |
| 860 | .replace("{config_key}", SNAPSHOTS_CAP_CONFIG_KEY) |
| 861 | } |
| 862 | } |
| 863 | |
| 864 | /// The config key that lifts the size gate. Named only by the size-gate |
| 865 | /// notice: it is not a remedy for the entry ceiling or the safety refusal. |
| 866 | pub const SNAPSHOTS_CAP_CONFIG_KEY: &str = "[snapshots] max_workspace_gb"; |
| 867 | |
| 868 | /// Human-readable byte cap for the size-gate notice. Keeps small test caps |
| 869 | /// from rendering as a misleading `0 GB`. |
| 870 | fn format_cap_bytes(bytes: u64) -> String { |
| 871 | const KIB: f64 = 1024.0; |
| 872 | let value = bytes as f64; |
| 873 | if value >= KIB.powi(3) { |
| 874 | format!("{:.1} GB", value / KIB.powi(3)) |
| 875 | } else if value >= KIB.powi(2) { |
| 876 | format!("{:.1} MB", value / KIB.powi(2)) |
| 877 | } else if value >= KIB { |
| 878 | format!("{:.1} KB", value / KIB) |
| 879 | } else { |
| 880 | format!("{bytes} bytes") |
| 881 | } |
| 882 | } |
| 883 | |
| 884 | type SnapshotNoticeKey = (std::path::PathBuf, Option<String>); |
| 885 | |
| 886 | #[derive(Default)] |
| 887 | struct SnapshotNoticeState { |
| 888 | warned: bool, |
| 889 | pending: bool, |
| 890 | disabled: Option<SnapshotsDisabledNotice>, |
| 891 | } |
| 892 | |
| 893 | fn snapshot_notices() |
| 894 | -> &'static std::sync::Mutex<std::collections::HashMap<SnapshotNoticeKey, SnapshotNoticeState>> { |
| 895 | static NOTICES: std::sync::OnceLock< |
| 896 | std::sync::Mutex<std::collections::HashMap<SnapshotNoticeKey, SnapshotNoticeState>>, |
| 897 | > = std::sync::OnceLock::new(); |
| 898 | NOTICES.get_or_init(Default::default) |
| 899 | } |
| 900 | |
| 901 | fn snapshot_notice_key(workspace: &Path, session_id: Option<&str>) -> SnapshotNoticeKey { |
| 902 | (workspace.to_path_buf(), session_id.map(str::to_owned)) |
| 903 | } |
| 904 | |
| 905 | /// Take only this session's pending delivery. Other sessions in the same |
| 906 | /// workspace keep their own notice; the observed disabled status remains. |
| 907 | pub fn take_snapshots_disabled_notices( |
| 908 | workspace: &Path, |
| 909 | session_id: Option<&str>, |
| 910 | ) -> Vec<SnapshotsDisabledNotice> { |
| 911 | let mut states = snapshot_notices() |
| 912 | .lock() |
| 913 | .unwrap_or_else(std::sync::PoisonError::into_inner); |
| 914 | let Some(state) = states.get_mut(&snapshot_notice_key(workspace, session_id)) else { |
| 915 | return Vec::new(); |
| 916 | }; |
| 917 | if !std::mem::take(&mut state.pending) { |
| 918 | return Vec::new(); |
| 919 | } |
| 920 | state.disabled.iter().cloned().collect() |
| 921 | } |
| 922 | |
| 923 | /// Non-consuming availability projection for the current session's status. |
| 924 | pub fn snapshots_disabled_status( |
| 925 | workspace: &Path, |
| 926 | session_id: Option<&str>, |
| 927 | ) -> Option<SnapshotsDisabledNotice> { |
| 928 | snapshot_notices() |
| 929 | .lock() |
| 930 | .unwrap_or_else(std::sync::PoisonError::into_inner) |
| 931 | .get(&snapshot_notice_key(workspace, session_id)) |
| 932 | .and_then(|state| state.disabled.clone()) |
| 933 | } |
| 934 | |
| 935 | fn clear_snapshots_disabled_status(workspace: &Path, session_id: Option<&str>) { |
| 936 | if let Some(state) = snapshot_notices() |
| 937 | .lock() |
| 938 | .unwrap_or_else(std::sync::PoisonError::into_inner) |
| 939 | .get_mut(&snapshot_notice_key(workspace, session_id)) |
| 940 | { |
| 941 | state.disabled = None; |
| 942 | state.pending = false; |
| 943 | } |
| 944 | } |
| 945 | |
| 946 | // Keep stderr for headless sessions. The TUI receives the same notice via the |
| 947 | // existing Engine event, and `/status` reads the retained observation. |
| 948 | // Production snapshot callers always supply the current Engine session id; |
| 949 | // callers without one retain the legacy workspace scope. |
| 950 | #[allow(clippy::print_stderr)] |
| 951 | fn maybe_notify_snapshots_disabled_once( |
| 952 | workspace: &Path, |
| 953 | session_id: Option<&str>, |
| 954 | cap_bytes: u64, |
| 955 | error: &std::io::Error, |
| 956 | ) -> bool { |
| 957 | let message = error.to_string(); |
| 958 | // The gate markers are declared by the snapshot policy that produces them, |
| 959 | // so this stays one classifier rather than a second copy of the rules. |
| 960 | let scope = if message.contains(crate::snapshot::GATE_TOO_LARGE_MARKER) { |
| 961 | SnapshotsDisabledScope::WorkspaceTooLarge |
| 962 | } else if message.contains(crate::snapshot::GATE_TOO_MANY_ENTRIES_MARKER) { |
| 963 | SnapshotsDisabledScope::TooManyFiles |
| 964 | } else if message.contains(crate::snapshot::GATE_UNSAFE_LOCATION_MARKER) { |
| 965 | SnapshotsDisabledScope::UnsafeLocation |
| 966 | } else { |
| 967 | // A real snapshot/data-loss error, not a gate: say snapshots are |
| 968 | // failing and why, never a "snapshots are off" gate notice. |
| 969 | return record_snapshot_notice( |
| 970 | workspace, |
| 971 | session_id, |
| 972 | SnapshotsDisabledScope::Failing, |
| 973 | snapshot_failure_detail(error), |
| 974 | ); |
| 975 | }; |
| 976 | let limit = match scope { |
| 977 | SnapshotsDisabledScope::WorkspaceTooLarge => format_cap_bytes(cap_bytes), |
| 978 | SnapshotsDisabledScope::TooManyFiles => crate::snapshot::SIZE_WALK_MAX_ENTRIES.to_string(), |
| 979 | _ => String::new(), |
| 980 | }; |
| 981 | record_snapshot_notice(workspace, session_id, scope, limit) |
| 982 | } |
| 983 | |
| 984 | /// One line of a snapshot error for the notice: the first line, bounded. |
| 985 | fn snapshot_failure_detail(error: &std::io::Error) -> String { |
| 986 | const MAX_CHARS: usize = 200; |
| 987 | let text = error.to_string(); |
| 988 | let line = text.lines().next().unwrap_or_default().trim(); |
| 989 | if line.chars().count() > MAX_CHARS { |
| 990 | let cut: String = line.chars().take(MAX_CHARS).collect(); |
| 991 | format!("{cut}…") |
| 992 | } else { |
| 993 | line.to_string() |
| 994 | } |
| 995 | } |
| 996 | |
| 997 | /// Record the session's snapshot availability and, the first time in this |
| 998 | /// session, queue its notice for the TUI and print it for headless runs. |
| 999 | /// Returns whether this call delivered the notice. |
| 1000 | #[allow(clippy::print_stderr)] |
| 1001 | fn record_snapshot_notice( |
| 1002 | workspace: &Path, |
| 1003 | session_id: Option<&str>, |
| 1004 | scope: SnapshotsDisabledScope, |
| 1005 | limit: String, |
| 1006 | ) -> bool { |
| 1007 | let notice = SnapshotsDisabledNotice { |
| 1008 | workspace: workspace.to_string_lossy().into_owned(), |
| 1009 | scope, |
| 1010 | limit, |
| 1011 | }; |
| 1012 | let mut states = snapshot_notices() |
| 1013 | .lock() |
| 1014 | .unwrap_or_else(std::sync::PoisonError::into_inner); |
| 1015 | let state = states |
| 1016 | .entry(snapshot_notice_key(workspace, session_id)) |
| 1017 | .or_default(); |
| 1018 | state.disabled = Some(notice.clone()); |
| 1019 | if std::mem::replace(&mut state.warned, true) { |
| 1020 | return false; |
| 1021 | } |
| 1022 | state.pending = true; |
| 1023 | drop(states); |
| 1024 | // Headless stderr has no session locale to resolve; English is the pack |
| 1025 | // this path has always printed. The TUI and `/status` localize properly. |
| 1026 | eprintln!( |
| 1027 | "warning: {}", |
| 1028 | notice.localize(codewhale_localization::Locale::En) |
| 1029 | ); |
| 1030 | true |
| 1031 | } |
| 1032 | |
| 1033 | #[cfg(test)] |
| 1034 | mod snapshot_notice_tests { |
| 1035 | use super::*; |
| 1036 | use std::sync::{ |
| 1037 | Arc, |
| 1038 | atomic::{AtomicUsize, Ordering}, |
| 1039 | }; |
| 1040 | use tracing_subscriber::prelude::*; |
| 1041 | |
| 1042 | #[derive(Clone, Default)] |
| 1043 | struct SnapshotWarnings(Arc<AtomicUsize>); |
| 1044 | |
| 1045 | impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for SnapshotWarnings { |
| 1046 | fn on_event( |
| 1047 | &self, |
| 1048 | event: &tracing::Event<'_>, |
| 1049 | _context: tracing_subscriber::layer::Context<'_, S>, |
| 1050 | ) { |
| 1051 | if event.metadata().target() == "snapshot" |
| 1052 | && *event.metadata().level() == tracing::Level::WARN |
| 1053 | { |
| 1054 | self.0.fetch_add(1, Ordering::SeqCst); |
| 1055 | } |
| 1056 | } |
| 1057 | } |
| 1058 | |
| 1059 | #[test] |
| 1060 | fn oversized_workspace_warns_once_per_session_and_retains_status_after_delivery() { |
| 1061 | let _env = crate::test_support::lock_test_env(); |
| 1062 | let root = tempfile::tempdir().unwrap(); |
| 1063 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 1064 | let _user_home = crate::test_support::EnvVarGuard::set("HOME", root.path()); |
| 1065 | let _user_profile = crate::test_support::EnvVarGuard::set("USERPROFILE", root.path()); |
| 1066 | let workspace = root.path().join("workspace"); |
| 1067 | std::fs::create_dir(&workspace).unwrap(); |
| 1068 | std::fs::write(workspace.join("large.txt"), vec![b'x'; 4096]).unwrap(); |
| 1069 | let warnings = SnapshotWarnings::default(); |
| 1070 | let subscriber = tracing_subscriber::registry().with(warnings.clone()); |
| 1071 | tracing::subscriber::with_default(subscriber, || { |
| 1072 | for session in ["session-a", "session-b"] { |
| 1073 | for turn in 1..=3 { |
| 1074 | assert!( |
| 1075 | pre_turn_snapshot(&workspace, turn, 1024, None, Some(session)).is_none() |
| 1076 | ); |
| 1077 | assert!( |
| 1078 | post_turn_snapshot(&workspace, turn, 1024, None, Some(session)).is_none() |
| 1079 | ); |
| 1080 | } |
| 1081 | } |
| 1082 | }); |
| 1083 | assert_eq!( |
| 1084 | warnings.0.load(Ordering::SeqCst), |
| 1085 | 2, |
| 1086 | "exactly one real WARN for each session" |
| 1087 | ); |
| 1088 | for session in ["session-b", "session-a"] { |
| 1089 | let notices = take_snapshots_disabled_notices(&workspace, Some(session)); |
| 1090 | assert_eq!(notices.len(), 1, "each session receives its own notice"); |
| 1091 | assert_eq!(notices[0].scope, SnapshotsDisabledScope::WorkspaceTooLarge); |
| 1092 | let line = notices[0].localize(codewhale_localization::Locale::En); |
| 1093 | assert_eq!(line.lines().count(), 1, "one line, not a stacked notice"); |
| 1094 | assert_eq!( |
| 1095 | line.matches(&workspace.display().to_string()).count(), |
| 1096 | 1, |
| 1097 | "the workspace is named exactly once: {line}" |
| 1098 | ); |
| 1099 | assert_eq!( |
| 1100 | line.matches(SNAPSHOTS_CAP_CONFIG_KEY).count(), |
| 1101 | 1, |
| 1102 | "the remedy is stated exactly once: {line}" |
| 1103 | ); |
| 1104 | assert!(line.contains("1.0 KB"), "the tripped cap is named: {line}"); |
| 1105 | assert!(take_snapshots_disabled_notices(&workspace, Some(session)).is_empty()); |
| 1106 | assert_eq!( |
| 1107 | snapshots_disabled_status(&workspace, Some(session)), |
| 1108 | notices.first().cloned(), |
| 1109 | "delivery must not erase /status" |
| 1110 | ); |
| 1111 | } |
| 1112 | assert!(snapshots_disabled_status(&workspace, Some("session-c")).is_none()); |
| 1113 | assert!(snapshots_disabled_status(&root.path().join("other"), Some("session-a")).is_none()); |
| 1114 | } |
| 1115 | |
| 1116 | /// The quiet case: a workspace under the cap snapshots and says nothing. |
| 1117 | #[test] |
| 1118 | fn small_workspace_snapshots_with_no_notice_at_all() { |
| 1119 | let _env = crate::test_support::lock_test_env(); |
| 1120 | let root = tempfile::tempdir().unwrap(); |
| 1121 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 1122 | let _user_home = crate::test_support::EnvVarGuard::set("HOME", root.path()); |
| 1123 | let _user_profile = crate::test_support::EnvVarGuard::set("USERPROFILE", root.path()); |
| 1124 | let workspace = root.path().join("workspace"); |
| 1125 | std::fs::create_dir(&workspace).unwrap(); |
| 1126 | std::fs::write(workspace.join("small.txt"), b"tiny").unwrap(); |
| 1127 | assert!(pre_turn_snapshot(&workspace, 1, 1024 * 1024, None, Some("session")).is_some()); |
| 1128 | assert!(snapshots_disabled_status(&workspace, Some("session")).is_none()); |
| 1129 | assert!(take_snapshots_disabled_notices(&workspace, Some("session")).is_empty()); |
| 1130 | } |
| 1131 | |
| 1132 | #[test] |
| 1133 | fn successful_snapshot_clears_disabled_status_and_pending_notice() { |
| 1134 | let _env = crate::test_support::lock_test_env(); |
| 1135 | let root = tempfile::tempdir().unwrap(); |
| 1136 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 1137 | let _user_home = crate::test_support::EnvVarGuard::set("HOME", root.path()); |
| 1138 | let _user_profile = crate::test_support::EnvVarGuard::set("USERPROFILE", root.path()); |
| 1139 | let workspace = root.path().join("workspace"); |
| 1140 | std::fs::create_dir(&workspace).unwrap(); |
| 1141 | std::fs::write(workspace.join("large.txt"), vec![b'x'; 4096]).unwrap(); |
| 1142 | assert!(pre_turn_snapshot(&workspace, 1, 1024, None, Some("session")).is_none()); |
| 1143 | assert!(snapshots_disabled_status(&workspace, Some("session")).is_some()); |
| 1144 | assert!(pre_turn_snapshot(&workspace, 2, 0, None, Some("session")).is_some()); |
| 1145 | assert!(snapshots_disabled_status(&workspace, Some("session")).is_none()); |
| 1146 | assert!(take_snapshots_disabled_notices(&workspace, Some("session")).is_empty()); |
| 1147 | } |
| 1148 | |
| 1149 | /// A real error is not a gate: the user is told snapshots are failing |
| 1150 | /// and why, never that they are "off" behind a limit. |
| 1151 | #[test] |
| 1152 | fn unrelated_snapshot_errors_say_snapshots_are_failing() { |
| 1153 | let workspace = tempfile::tempdir().unwrap(); |
| 1154 | let error = std::io::Error::other("disk full"); |
| 1155 | assert!(maybe_notify_snapshots_disabled_once( |
| 1156 | workspace.path(), |
| 1157 | Some("session"), |
| 1158 | 1024, |
| 1159 | &error |
| 1160 | )); |
| 1161 | let notices = take_snapshots_disabled_notices(workspace.path(), Some("session")); |
| 1162 | assert_eq!(notices.len(), 1); |
| 1163 | assert_eq!(notices[0].scope, SnapshotsDisabledScope::Failing); |
| 1164 | let line = notices[0].localize(codewhale_localization::Locale::En); |
| 1165 | assert!( |
| 1166 | line.contains("failing") && line.contains("disk full"), |
| 1167 | "{line}" |
| 1168 | ); |
| 1169 | assert!(!line.contains(SNAPSHOTS_CAP_CONFIG_KEY), "{line}"); |
| 1170 | assert!(snapshots_disabled_status(workspace.path(), Some("session")).is_some()); |
| 1171 | assert!( |
| 1172 | !maybe_notify_snapshots_disabled_once(workspace.path(), Some("session"), 1024, &error), |
| 1173 | "told once per session" |
| 1174 | ); |
| 1175 | } |
| 1176 | |
| 1177 | /// B2: a side repo whose HEAD names a missing commit is repaired on the |
| 1178 | /// next turn snapshot, and the user is told earlier restore points are |
| 1179 | /// gone instead of /undo silently dying. |
| 1180 | #[test] |
| 1181 | fn broken_snapshot_history_is_repaired_and_the_user_is_told() { |
| 1182 | let _env = crate::test_support::lock_test_env(); |
| 1183 | let root = tempfile::tempdir().unwrap(); |
| 1184 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 1185 | let _user_home = crate::test_support::EnvVarGuard::set("HOME", root.path()); |
| 1186 | let _user_profile = crate::test_support::EnvVarGuard::set("USERPROFILE", root.path()); |
| 1187 | let workspace = root.path().join("workspace"); |
| 1188 | std::fs::create_dir(&workspace).unwrap(); |
| 1189 | std::fs::write(workspace.join("a.txt"), b"alpha").unwrap(); |
| 1190 | assert!(pre_turn_snapshot(&workspace, 1, 0, None, Some("session")).is_some()); |
| 1191 | let repo = SnapshotRepo::open_or_init(&workspace).unwrap(); |
| 1192 | repo.point_head_at_missing_commit_for_test(); |
| 1193 | // With a reflog the repair recovers the last good commit silently |
| 1194 | // (restore points survive); without one history restarts, which is |
| 1195 | // the case the user must be told about. |
| 1196 | std::fs::remove_dir_all(repo.git_dir().join("logs")).expect("drop reflogs"); |
| 1197 | |
| 1198 | std::fs::write(workspace.join("a.txt"), b"beta").unwrap(); |
| 1199 | assert!( |
| 1200 | post_turn_snapshot(&workspace, 1, 0, None, Some("session")).is_some(), |
| 1201 | "the snapshot succeeds after the repair" |
| 1202 | ); |
| 1203 | let notices = take_snapshots_disabled_notices(&workspace, Some("session")); |
| 1204 | assert_eq!(notices.len(), 1, "{notices:?}"); |
| 1205 | assert_eq!(notices[0].scope, SnapshotsDisabledScope::HistoryRepaired); |
| 1206 | let line = notices[0].localize(codewhale_localization::Locale::En); |
| 1207 | assert!(line.contains("restarted"), "{line}"); |
| 1208 | assert!(line.contains(&workspace.display().to_string()), "{line}"); |
| 1209 | assert!(pre_turn_snapshot(&workspace, 2, 0, None, Some("session")).is_some()); |
| 1210 | assert!( |
| 1211 | take_snapshots_disabled_notices(&workspace, Some("session")).is_empty(), |
| 1212 | "a healthy history says nothing more" |
| 1213 | ); |
| 1214 | } |
| 1215 | |
| 1216 | /// Every gate must state a recovery that actually lifts *that* gate. The |
| 1217 | /// entry ceiling and the home/root refusal are not raised by the GB cap, |
| 1218 | /// so naming it there is the unhelpful follow-up this packet removes. |
| 1219 | #[test] |
| 1220 | fn each_gate_gets_its_own_accurate_recovery() { |
| 1221 | let workspace = tempfile::tempdir().unwrap(); |
| 1222 | for (gate_message, scope, cap_bytes) in [ |
| 1223 | ( |
| 1224 | format!( |
| 1225 | "{}: over 2 bytes in x", |
| 1226 | crate::snapshot::GATE_TOO_MANY_ENTRIES_MARKER |
| 1227 | ), |
| 1228 | SnapshotsDisabledScope::TooManyFiles, |
| 1229 | 0, |
| 1230 | ), |
| 1231 | ( |
| 1232 | format!( |
| 1233 | "{} for home directory: x", |
| 1234 | crate::snapshot::GATE_UNSAFE_LOCATION_MARKER |
| 1235 | ), |
| 1236 | SnapshotsDisabledScope::UnsafeLocation, |
| 1237 | 2 * 1024 * 1024 * 1024, |
| 1238 | ), |
| 1239 | ] { |
| 1240 | let session = format!("{scope:?}"); |
| 1241 | let error = std::io::Error::new(std::io::ErrorKind::InvalidInput, gate_message); |
| 1242 | assert!(maybe_notify_snapshots_disabled_once( |
| 1243 | workspace.path(), |
| 1244 | Some(&session), |
| 1245 | cap_bytes, |
| 1246 | &error |
| 1247 | )); |
| 1248 | let notice = snapshots_disabled_status(workspace.path(), Some(&session)) |
| 1249 | .expect("gated error must be retained for /status"); |
| 1250 | assert_eq!(notice.scope, scope); |
| 1251 | let line = notice.localize(codewhale_localization::Locale::En); |
| 1252 | assert_eq!(line.lines().count(), 1, "one line, not a stacked notice"); |
| 1253 | assert!( |
| 1254 | !line.contains(SNAPSHOTS_CAP_CONFIG_KEY), |
| 1255 | "{scope:?} must not advertise a config key that cannot lift it: {line}" |
| 1256 | ); |
| 1257 | assert!(line.contains("/undo"), "the consequence is named: {line}"); |
| 1258 | if scope == SnapshotsDisabledScope::TooManyFiles { |
| 1259 | // The `{limit}` this notice carries is the entry ceiling. |
| 1260 | // Nothing else asserts it reaches the user, so a dropped |
| 1261 | // placeholder would render "more than files" silently. |
| 1262 | assert!( |
| 1263 | line.contains(&crate::snapshot::SIZE_WALK_MAX_ENTRIES.to_string()), |
| 1264 | "the entry ceiling must be stated, not left as a blank limit: {line}" |
| 1265 | ); |
| 1266 | } |
| 1267 | } |
| 1268 | } |
| 1269 | |
| 1270 | #[test] |
| 1271 | fn oversize_notice_names_the_cap_and_only_then_the_config_key() { |
| 1272 | let workspace = tempfile::tempdir().unwrap(); |
| 1273 | let error = std::io::Error::new( |
| 1274 | std::io::ErrorKind::InvalidInput, |
| 1275 | format!( |
| 1276 | "{}: over x bytes in y", |
| 1277 | crate::snapshot::GATE_TOO_LARGE_MARKER |
| 1278 | ), |
| 1279 | ); |
| 1280 | assert!(maybe_notify_snapshots_disabled_once( |
| 1281 | workspace.path(), |
| 1282 | Some("session"), |
| 1283 | 2 * 1024 * 1024 * 1024, |
| 1284 | &error |
| 1285 | )); |
| 1286 | let notice = |
| 1287 | snapshots_disabled_status(workspace.path(), Some("session")).expect("retained"); |
| 1288 | let line = notice.localize(codewhale_localization::Locale::En); |
| 1289 | assert!(line.contains("2.0 GB"), "{line}"); |
| 1290 | assert!(line.contains(SNAPSHOTS_CAP_CONFIG_KEY), "{line}"); |
| 1291 | } |
| 1292 | } |
| 1293 | |
| 1294 | #[cfg(test)] |
| 1295 | mod snapshot_label_tests { |
| 1296 | use super::*; |
| 1297 | |
| 1298 | #[test] |
| 1299 | fn label_writer_and_parser_agree_on_prompt_snippet() { |
| 1300 | let prompt = "rename the widget\nsecond line is dropped"; |
| 1301 | let label = format_snapshot_label("pre-turn", 7, Some(prompt)); |
| 1302 | assert_eq!(label, "pre-turn:7: rename the widget"); |
| 1303 | |
| 1304 | let parsed = parse_snapshot_label(&label); |
| 1305 | assert_eq!(parsed.kind, "pre-turn"); |
| 1306 | assert_eq!(parsed.seq, Some(7)); |
| 1307 | assert_eq!( |
| 1308 | parsed.prompt_snippet.as_deref(), |
| 1309 | snapshot_label_prompt_snippet(prompt).as_deref(), |
| 1310 | "a reader must recover exactly the snippet the writer embedded" |
| 1311 | ); |
| 1312 | } |
| 1313 | |
| 1314 | #[test] |
| 1315 | fn truncated_prompt_round_trips_with_its_ellipsis() { |
| 1316 | let prompt = "x".repeat(USER_PROMPT_LABEL_MAX + 25); |
| 1317 | let label = format_snapshot_label("post-turn", 2, Some(&prompt)); |
| 1318 | let parsed = parse_snapshot_label(&label); |
| 1319 | let snippet = parsed.prompt_snippet.expect("snippet"); |
| 1320 | assert!(snippet.ends_with('…')); |
| 1321 | assert_eq!(snippet.chars().count(), USER_PROMPT_LABEL_MAX + 1); |
| 1322 | assert_eq!( |
| 1323 | Some(snippet), |
| 1324 | snapshot_label_prompt_snippet(&prompt), |
| 1325 | "truncated snippets must also round-trip" |
| 1326 | ); |
| 1327 | } |
| 1328 | |
| 1329 | #[test] |
| 1330 | fn labels_without_a_prompt_parse_without_inventing_one() { |
| 1331 | let label = format_snapshot_label("pre-turn", 3, None); |
| 1332 | assert_eq!(label, "pre-turn:3"); |
| 1333 | let parsed = parse_snapshot_label(&label); |
| 1334 | assert_eq!(parsed.kind, "pre-turn"); |
| 1335 | assert_eq!(parsed.seq, Some(3)); |
| 1336 | assert_eq!(parsed.prompt_snippet, None); |
| 1337 | } |
| 1338 | |
| 1339 | #[test] |
| 1340 | fn tool_labels_carry_a_call_id_not_a_sequence() { |
| 1341 | let label = format!("tool:{}", "call_abc123"); |
| 1342 | let parsed = parse_snapshot_label(&label); |
| 1343 | assert_eq!(parsed.kind, "tool"); |
| 1344 | assert_eq!(parsed.seq, None, "a call id is not a turn sequence"); |
| 1345 | assert_eq!(parsed.prompt_snippet, None); |
| 1346 | } |
| 1347 | |
| 1348 | #[test] |
| 1349 | fn unrecognized_labels_are_reported_rather_than_dropped() { |
| 1350 | let parsed = parse_snapshot_label("manual checkpoint"); |
| 1351 | assert_eq!(parsed.kind, "manual checkpoint"); |
| 1352 | assert_eq!(parsed.seq, None); |
| 1353 | assert_eq!(parsed.prompt_snippet, None); |
| 1354 | } |
| 1355 | |
| 1356 | #[test] |
| 1357 | fn empty_prompt_contributes_no_snippet() { |
| 1358 | assert_eq!(snapshot_label_prompt_snippet(""), None); |
| 1359 | assert_eq!(format_snapshot_label("pre-turn", 1, Some("")), "pre-turn:1"); |
| 1360 | } |
| 1361 | } |
| 1362 |