| 1 | //! Find and scrub credentials that older builds stored in saved sessions. |
| 2 | //! |
| 3 | //! Tool output is redacted as it enters the transcript now (B1), so new |
| 4 | //! session files never hold a live token. Sessions written before that fix |
| 5 | //! can: the redaction used to run only when a request was built, while the |
| 6 | //! transcript on disk kept the raw tool output (a `cat ~/.codex/auth.json` |
| 7 | //! result, a printed bearer token). Nothing rewrites those files on its own — |
| 8 | //! `codewhale doctor` reports them and `codewhale sessions scrub-secrets` |
| 9 | //! rewrites them on request. |
| 10 | //! |
| 11 | //! Only transcript tool results and Runtime tool receipts (including event |
| 12 | //! copies) are touched. Cleanup masks credential shapes and currently configured |
| 13 | //! secret values; bare secrets absent from current configuration cannot be recovered. |
| 14 | //! Runtime rewrites require the exclusive process lease, and session rewrites |
| 15 | //! the session's live lease; busy stores and sessions open in an interactive |
| 16 | //! surface are reported and left untouched while the remaining files are |
| 17 | //! scrubbed. |
| 18 | |
| 19 | use std::io; |
| 20 | use std::path::{Path, PathBuf}; |
| 21 | |
| 22 | use serde_json::Value; |
| 23 | |
| 24 | /// The documented command that rewrites affected sessions. |
| 25 | pub(crate) const SCRUB_COMMAND: &str = "codewhale sessions scrub-secrets"; |
| 26 | |
| 27 | /// What a scan found (and, when applied, rewrote). |
| 28 | #[derive(Debug, Default, Clone, PartialEq, Eq)] |
| 29 | pub(crate) struct ScrubReport { |
| 30 | /// Session, checkpoint, and Runtime receipt files examined. |
| 31 | pub files_scanned: usize, |
| 32 | /// Files holding at least one unredacted credential in tool output. |
| 33 | pub flagged_files: Vec<PathBuf>, |
| 34 | /// Tool-result text fields that contained a credential. |
| 35 | pub flagged_tool_results: usize, |
| 36 | /// Files the scan could not read or parse, left untouched. |
| 37 | pub unreadable: Vec<PathBuf>, |
| 38 | /// Credential-bearing files left untouched because their Runtime store is |
| 39 | /// live or their session is open in an interactive surface. |
| 40 | pub busy: Vec<PathBuf>, |
| 41 | } |
| 42 | |
| 43 | /// Session/checkpoint JSON and Runtime items/events, newest first. Resolve the |
| 44 | /// user-selected roots (including the fixed checkpoints root), but never follow |
| 45 | /// symlink entries discovered inside those roots. Enumeration failures are partial |
| 46 | /// coverage, not a reason to discard the files that can still be scrubbed. |
| 47 | pub(crate) fn session_files( |
| 48 | sessions_dir: &Path, |
| 49 | standalone_runtime: &Path, |
| 50 | unreadable: &mut Vec<PathBuf>, |
| 51 | ) -> Vec<PathBuf> { |
| 52 | let sessions_root = canonical_store_root(sessions_dir, unreadable); |
| 53 | let mut dirs = Vec::new(); |
| 54 | let mut runtime_roots = Vec::new(); |
| 55 | if let Some(root) = canonical_store_root(standalone_runtime, unreadable) { |
| 56 | runtime_roots.push(root); |
| 57 | } |
| 58 | if let Some(root) = sessions_root { |
| 59 | if let Some(checkpoints) = canonical_store_root(&root.join("checkpoints"), unreadable) { |
| 60 | dirs.push(checkpoints); |
| 61 | } |
| 62 | // Pre-import transcript copies hold the same tool output. Session |
| 63 | // deletion does not follow a linked archive directory, so neither does |
| 64 | // the scrub: it is reported as not covered instead. |
| 65 | let archive = root.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR); |
| 66 | match std::fs::symlink_metadata(&archive) { |
| 67 | Ok(metadata) if metadata.is_dir() => dirs.push(archive), |
| 68 | Err(error) if error.kind() == io::ErrorKind::NotFound => {} |
| 69 | _ => unreadable.push(archive), |
| 70 | } |
| 71 | for session in directory_entries(&root, unreadable) { |
| 72 | if !entry_has_type(&session, true, unreadable) { |
| 73 | continue; |
| 74 | } |
| 75 | for runtime in directory_entries(&session.path(), unreadable) { |
| 76 | let name = runtime.file_name(); |
| 77 | let name = name.to_string_lossy(); |
| 78 | if (name == "runtime" || name.starts_with("runtime-recovered-")) |
| 79 | && entry_has_type(&runtime, true, unreadable) |
| 80 | { |
| 81 | runtime_roots.push(runtime.path()); |
| 82 | } |
| 83 | } |
| 84 | } |
| 85 | dirs.push(root); |
| 86 | } |
| 87 | for root in runtime_roots { |
| 88 | dirs.push(root.join("items")); |
| 89 | dirs.push(root.join("events")); |
| 90 | } |
| 91 | let mut files = Vec::new(); |
| 92 | for dir in dirs { |
| 93 | for entry in directory_entries(&dir, unreadable) { |
| 94 | let path = entry.path(); |
| 95 | if !entry_has_type(&entry, false, unreadable) |
| 96 | || !matches!( |
| 97 | path.extension().and_then(|ext| ext.to_str()), |
| 98 | Some("json" | "jsonl") |
| 99 | ) |
| 100 | { |
| 101 | continue; |
| 102 | } |
| 103 | match entry.metadata() { |
| 104 | Ok(meta) => files.push((meta.modified().unwrap_or(std::time::UNIX_EPOCH), path)), |
| 105 | Err(_) => unreadable.push(path), |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | files.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.cmp(&b.1))); |
| 110 | files.dedup_by(|a, b| a.1 == b.1); |
| 111 | unreadable.sort(); |
| 112 | unreadable.dedup(); |
| 113 | files.into_iter().map(|(_, path)| path).collect() |
| 114 | } |
| 115 | |
| 116 | fn canonical_store_root(dir: &Path, unreadable: &mut Vec<PathBuf>) -> Option<PathBuf> { |
| 117 | match std::fs::canonicalize(dir) { |
| 118 | Ok(root) => Some(root), |
| 119 | Err(error) if error.kind() == io::ErrorKind::NotFound => None, |
| 120 | Err(_) => { |
| 121 | unreadable.push(dir.to_path_buf()); |
| 122 | None |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | fn entry_has_type( |
| 128 | entry: &std::fs::DirEntry, |
| 129 | directory: bool, |
| 130 | unreadable: &mut Vec<PathBuf>, |
| 131 | ) -> bool { |
| 132 | match entry.file_type() { |
| 133 | Ok(kind) => { |
| 134 | if directory { |
| 135 | kind.is_dir() |
| 136 | } else { |
| 137 | kind.is_file() |
| 138 | } |
| 139 | } |
| 140 | Err(_) => { |
| 141 | unreadable.push(entry.path()); |
| 142 | false |
| 143 | } |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | fn directory_entries(dir: &Path, unreadable: &mut Vec<PathBuf>) -> Vec<std::fs::DirEntry> { |
| 148 | let entries = match std::fs::symlink_metadata(dir) { |
| 149 | Err(error) if error.kind() == io::ErrorKind::NotFound => return Vec::new(), |
| 150 | Ok(meta) if meta.is_dir() => std::fs::read_dir(dir).ok(), |
| 151 | _ => None, |
| 152 | }; |
| 153 | let Some(entries) = entries else { |
| 154 | unreadable.push(dir.to_path_buf()); |
| 155 | return Vec::new(); |
| 156 | }; |
| 157 | entries |
| 158 | .filter_map(|entry| match entry { |
| 159 | Ok(entry) => Some(entry), |
| 160 | Err(_) => { |
| 161 | unreadable.push(dir.to_path_buf()); |
| 162 | None |
| 163 | } |
| 164 | }) |
| 165 | .collect() |
| 166 | } |
| 167 | |
| 168 | /// Reserve an independent budget for transcripts/checkpoints and Runtime receipts, |
| 169 | /// so one busy Runtime cannot evict every saved conversation from doctor's scan. |
| 170 | pub(crate) fn doctor_files(files: Vec<PathBuf>, per_category: usize) -> Vec<PathBuf> { |
| 171 | let mut counts = [0, 0]; |
| 172 | files |
| 173 | .into_iter() |
| 174 | .filter(|path| { |
| 175 | let count = &mut counts[usize::from(runtime_root(path).is_some())]; |
| 176 | *count += 1; |
| 177 | *count <= per_category |
| 178 | }) |
| 179 | .collect() |
| 180 | } |
| 181 | |
| 182 | /// Called once per command from its blocking worker. This is read-only, including |
| 183 | /// active-key resolution; the resulting values must never appear in a report. |
| 184 | pub(crate) fn configured_secrets( |
| 185 | config_path: Option<PathBuf>, |
| 186 | profile: Option<&str>, |
| 187 | ) -> anyhow::Result<Vec<String>> { |
| 188 | let config = crate::config::Config::load(config_path, profile)?; |
| 189 | let active_key = config.active_route_api_key_read_only().unwrap_or_default(); |
| 190 | Ok(crate::client::configured_model_bound_secret_values( |
| 191 | &config, |
| 192 | &active_key, |
| 193 | )) |
| 194 | } |
| 195 | |
| 196 | /// Whether `path` sits directly in `manager`'s pre-import archive directory |
| 197 | /// (never a linked one; see [`session_files`]). |
| 198 | fn is_import_archive(path: &Path, manager: &crate::session_manager::SessionManager) -> bool { |
| 199 | let archive = manager |
| 200 | .sessions_dir() |
| 201 | .join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR); |
| 202 | std::fs::symlink_metadata(&archive).is_ok_and(|metadata| metadata.is_dir()) |
| 203 | && std::fs::canonicalize(&archive) |
| 204 | .is_ok_and(|archive| path.parent() == Some(archive.as_path())) |
| 205 | } |
| 206 | |
| 207 | fn runtime_root(path: &Path) -> Option<&Path> { |
| 208 | let parent = path.parent()?; |
| 209 | matches!(parent.file_name()?.to_str()?, "items" | "events") |
| 210 | .then(|| parent.parent()) |
| 211 | .flatten() |
| 212 | } |
| 213 | |
| 214 | /// Scan `files`; with `apply`, rewrite each affected file atomically with its |
| 215 | /// tool-result credentials masked. Each rewrite re-reads its file under the |
| 216 | /// same per-session lock every session save takes, so a live session saving |
| 217 | /// at the same moment is never overwritten with older content. |
| 218 | pub(crate) fn scrub_files( |
| 219 | files: &[PathBuf], |
| 220 | apply: Option<&crate::session_manager::SessionManager>, |
| 221 | secrets: &[String], |
| 222 | ) -> io::Result<ScrubReport> { |
| 223 | let mut report = ScrubReport::default(); |
| 224 | for path in files { |
| 225 | report.files_scanned += 1; |
| 226 | // The scan is read-only; its counts are what the report prints. |
| 227 | let scan = scrub_file(path, false, secrets)?; |
| 228 | let scan = match (apply, scan) { |
| 229 | (Some(_), FileScan::Dirty(_)) if runtime_root(path).is_some() => { |
| 230 | let _lease = match crate::runtime_threads::RuntimeProcessOwnerLock::acquire( |
| 231 | runtime_root(path).unwrap(), |
| 232 | ) { |
| 233 | Ok(lease) => lease, |
| 234 | Err(error) |
| 235 | if error |
| 236 | .downcast_ref::<io::Error>() |
| 237 | .is_some_and(|error| error.kind() == io::ErrorKind::WouldBlock) => |
| 238 | { |
| 239 | report.busy.push(path.clone()); |
| 240 | continue; |
| 241 | } |
| 242 | Err(error) => return Err(io::Error::other(error)), |
| 243 | }; |
| 244 | scrub_file(path, true, secrets)? |
| 245 | } |
| 246 | (Some(manager), FileScan::Dirty(redacted)) => { |
| 247 | // `<id>.json` and `checkpoints/<id>.json` share the id's lock. |
| 248 | // The rewrite re-reads the file under that lock, so it masks |
| 249 | // whatever the file holds at that moment. |
| 250 | let session_id = path.file_stem().and_then(|stem| stem.to_str()); |
| 251 | match session_id.map(|id| { |
| 252 | manager |
| 253 | .with_session_file_lock(id, || scrub_file(path, true, secrets).map(|_| ())) |
| 254 | }) { |
| 255 | Some(Ok(Some(()))) => FileScan::Dirty(redacted), |
| 256 | // A pre-import copy an earlier build left behind when it |
| 257 | // deleted the session. Finish that deletion: rewriting it |
| 258 | // would keep a deleted transcript, and could put back a |
| 259 | // copy a concurrent delete had just removed. |
| 260 | Some(Ok(None)) if is_import_archive(path, manager) => { |
| 261 | match std::fs::remove_file(path) { |
| 262 | Ok(()) => {} |
| 263 | Err(error) if error.kind() == io::ErrorKind::NotFound => {} |
| 264 | Err(error) => return Err(error), |
| 265 | } |
| 266 | FileScan::Dirty(redacted) |
| 267 | } |
| 268 | // A deleted session is not resurrected by a rewrite. |
| 269 | Some(Ok(None)) => FileScan::Clean, |
| 270 | // No lockable session id: leave the file untouched. |
| 271 | None => FileScan::Unreadable, |
| 272 | Some(Err(error)) if error.kind() == io::ErrorKind::InvalidInput => { |
| 273 | FileScan::Unreadable |
| 274 | } |
| 275 | // Open in an interactive session, whose next autosave |
| 276 | // would put the credential back: report and leave it. |
| 277 | Some(Err(error)) if error.kind() == io::ErrorKind::ResourceBusy => { |
| 278 | report.busy.push(path.clone()); |
| 279 | continue; |
| 280 | } |
| 281 | Some(Err(error)) => return Err(error), |
| 282 | } |
| 283 | } |
| 284 | (_, scan) => scan, |
| 285 | }; |
| 286 | match scan { |
| 287 | FileScan::Unreadable => report.unreadable.push(path.clone()), |
| 288 | FileScan::Clean => {} |
| 289 | FileScan::Dirty(redacted) => { |
| 290 | report.flagged_tool_results += redacted; |
| 291 | report.flagged_files.push(path.clone()); |
| 292 | } |
| 293 | } |
| 294 | } |
| 295 | Ok(report) |
| 296 | } |
| 297 | |
| 298 | enum FileScan { |
| 299 | Unreadable, |
| 300 | Clean, |
| 301 | Dirty(usize), |
| 302 | } |
| 303 | |
| 304 | fn scrub_file(path: &Path, apply: bool, secrets: &[String]) -> io::Result<FileScan> { |
| 305 | let Ok(raw) = std::fs::read(path) else { |
| 306 | return Ok(FileScan::Unreadable); |
| 307 | }; |
| 308 | let jsonl = path.extension().and_then(|ext| ext.to_str()) == Some("jsonl"); |
| 309 | let parsed = if jsonl { |
| 310 | serde_json::Deserializer::from_slice(&raw) |
| 311 | .into_iter::<Value>() |
| 312 | .collect::<Result<Vec<_>, _>>() |
| 313 | } else { |
| 314 | serde_json::from_slice::<Value>(&raw).map(|value| vec![value]) |
| 315 | }; |
| 316 | let Ok(mut values) = parsed else { |
| 317 | return Ok(FileScan::Unreadable); |
| 318 | }; |
| 319 | let redacted = values |
| 320 | .iter_mut() |
| 321 | .map(|value| scrub_value(value, secrets)) |
| 322 | .sum(); |
| 323 | if redacted == 0 { |
| 324 | return Ok(FileScan::Clean); |
| 325 | } |
| 326 | if apply { |
| 327 | let mut bytes = Vec::new(); |
| 328 | for value in values { |
| 329 | if jsonl { |
| 330 | serde_json::to_writer(&mut bytes, &value).map_err(io::Error::other)?; |
| 331 | bytes.push(b'\n'); |
| 332 | } else { |
| 333 | serde_json::to_writer_pretty(&mut bytes, &value).map_err(io::Error::other)?; |
| 334 | } |
| 335 | } |
| 336 | codewhale_config::persistence::atomic_write(path, &bytes).map_err(io::Error::other)?; |
| 337 | } |
| 338 | Ok(FileScan::Dirty(redacted)) |
| 339 | } |
| 340 | |
| 341 | /// Mask credentials in every `tool_result` text field below `value`. |
| 342 | /// Returns how many fields changed. |
| 343 | fn scrub_value(value: &mut Value, secrets: &[String]) -> usize { |
| 344 | match value { |
| 345 | Value::Object(map) => { |
| 346 | let mut changed = 0; |
| 347 | if map.get("type").and_then(Value::as_str) == Some("tool_result") { |
| 348 | if let Some(Value::String(content)) = map.get_mut("content") { |
| 349 | changed += usize::from(scrub_text(content, secrets)); |
| 350 | } |
| 351 | if let Some(Value::Array(blocks)) = map.get_mut("content_blocks") { |
| 352 | for block in blocks { |
| 353 | if block.get("type").and_then(Value::as_str) == Some("text") |
| 354 | && let Some(Value::String(text)) = block.get_mut("text") |
| 355 | { |
| 356 | changed += usize::from(scrub_text(text, secrets)); |
| 357 | } |
| 358 | } |
| 359 | } |
| 360 | } |
| 361 | if map.get("kind").and_then(Value::as_str) == Some("tool_call") { |
| 362 | for key in ["summary", "detail", "metadata"] { |
| 363 | if let Some(value) = map.get_mut(key) { |
| 364 | let redacted = crate::client::redact_json_model_bound_text(value, secrets); |
| 365 | if *value != redacted { |
| 366 | *value = redacted; |
| 367 | changed += 1; |
| 368 | } |
| 369 | } |
| 370 | } |
| 371 | } |
| 372 | // Strings below are only rewritten through a tool_result or receipt above, |
| 373 | // so walking the rest (including this object's own fields) never |
| 374 | // double-counts. |
| 375 | changed |
| 376 | + map |
| 377 | .values_mut() |
| 378 | .map(|value| scrub_value(value, secrets)) |
| 379 | .sum::<usize>() |
| 380 | } |
| 381 | Value::Array(items) => items |
| 382 | .iter_mut() |
| 383 | .map(|value| scrub_value(value, secrets)) |
| 384 | .sum(), |
| 385 | _ => 0, |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | fn scrub_text(text: &mut String, secrets: &[String]) -> bool { |
| 390 | let redacted = crate::client::redact_model_bound_text(text, secrets); |
| 391 | if redacted == *text { |
| 392 | return false; |
| 393 | } |
| 394 | *text = redacted; |
| 395 | true |
| 396 | } |
| 397 | |
| 398 | #[cfg(test)] |
| 399 | mod tests { |
| 400 | use super::*; |
| 401 | use serde_json::json; |
| 402 | |
| 403 | const TOKEN: &str = "sk-ant-oat01-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789abcdefghij"; |
| 404 | |
| 405 | fn session_with_tool_output(output: &str) -> Value { |
| 406 | json!({ |
| 407 | "schema_version": 1, |
| 408 | "metadata": {"id": "s1"}, |
| 409 | "messages": [ |
| 410 | {"role": "user", "content": [{"type": "text", "text": "show auth"}]}, |
| 411 | {"role": "user", "content": [{ |
| 412 | "type": "tool_result", |
| 413 | "tool_use_id": "call-1", |
| 414 | "content": output, |
| 415 | "content_blocks": [{"type": "text", "text": output}] |
| 416 | }]} |
| 417 | ], |
| 418 | "journal": {"entries": [{"message": {"role": "user", "content": [{ |
| 419 | "type": "tool_result", "tool_use_id": "call-1", "content": output |
| 420 | }]}}]} |
| 421 | }) |
| 422 | } |
| 423 | |
| 424 | #[test] |
| 425 | fn scan_reports_and_scrub_masks_stored_tool_output_tokens() { |
| 426 | let dir = tempfile::tempdir().expect("tempdir"); |
| 427 | let output = format!("{{\"access_token\": \"{TOKEN}\"}}"); |
| 428 | let dirty = dir.path().join("dirty.json"); |
| 429 | std::fs::write(&dirty, session_with_tool_output(&output).to_string()).unwrap(); |
| 430 | let clean = dir.path().join("clean.json"); |
| 431 | std::fs::write( |
| 432 | &clean, |
| 433 | session_with_tool_output("nothing secret").to_string(), |
| 434 | ) |
| 435 | .unwrap(); |
| 436 | std::fs::create_dir_all(dir.path().join("checkpoints")).unwrap(); |
| 437 | let checkpoint = dir.path().join("checkpoints").join("dirty.json"); |
| 438 | std::fs::write(&checkpoint, session_with_tool_output(&output).to_string()).unwrap(); |
| 439 | std::fs::write(dir.path().join("broken.json"), "{not json").unwrap(); |
| 440 | |
| 441 | let files = session_files(dir.path(), &dir.path().join("standalone"), &mut Vec::new()); |
| 442 | assert_eq!(files.len(), 4, "{files:?}"); |
| 443 | |
| 444 | let report = scrub_files(&files, None, &[]).expect("scan"); |
| 445 | assert_eq!(report.files_scanned, 4); |
| 446 | assert_eq!(report.flagged_files.len(), 2, "{report:?}"); |
| 447 | assert_eq!(report.flagged_tool_results, 6); |
| 448 | assert_eq!( |
| 449 | report.unreadable, |
| 450 | vec![dir.path().join("broken.json").canonicalize().unwrap()] |
| 451 | ); |
| 452 | assert!( |
| 453 | std::fs::read_to_string(&dirty).unwrap().contains(TOKEN), |
| 454 | "a scan must not rewrite anything" |
| 455 | ); |
| 456 | |
| 457 | let manager = |
| 458 | crate::session_manager::SessionManager::new(dir.path().to_path_buf()).expect("manager"); |
| 459 | let applied = scrub_files(&files, Some(&manager), &[]).expect("scrub"); |
| 460 | assert_eq!(applied.flagged_files.len(), 2); |
| 461 | assert_eq!( |
| 462 | applied.unreadable, |
| 463 | vec![dir.path().join("broken.json").canonicalize().unwrap()] |
| 464 | ); |
| 465 | for path in [&dirty, &checkpoint] { |
| 466 | let text = std::fs::read_to_string(path).unwrap(); |
| 467 | assert!(!text.contains(TOKEN), "{text}"); |
| 468 | let value: Value = serde_json::from_str(&text).expect("still valid JSON"); |
| 469 | assert_eq!(value["metadata"]["id"], "s1"); |
| 470 | assert_eq!(value["messages"][0]["content"][0]["text"], "show auth"); |
| 471 | } |
| 472 | assert_eq!( |
| 473 | scrub_files(&files, None, &[]) |
| 474 | .expect("rescan") |
| 475 | .flagged_files, |
| 476 | Vec::<PathBuf>::new(), |
| 477 | "a scrubbed store scans clean" |
| 478 | ); |
| 479 | } |
| 480 | |
| 481 | /// The pre-import copy of a transcript holds the same tool output. It is |
| 482 | /// scrubbed with the sessions. A copy an earlier build left behind for a |
| 483 | /// session it deleted is removed, finishing that deletion. |
| 484 | #[test] |
| 485 | fn work_graph_import_archive_copies_are_scrubbed() { |
| 486 | let dir = tempfile::tempdir().expect("tempdir"); |
| 487 | let output = format!("{{\"access_token\": \"{TOKEN}\"}}"); |
| 488 | let manager = |
| 489 | crate::session_manager::SessionManager::new(dir.path().to_path_buf()).expect("manager"); |
| 490 | std::fs::write( |
| 491 | dir.path().join("gone.json"), |
| 492 | session_with_tool_output("nothing").to_string(), |
| 493 | ) |
| 494 | .unwrap(); |
| 495 | manager.delete_session("gone").expect("delete"); |
| 496 | let archive_dir = dir |
| 497 | .path() |
| 498 | .join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR); |
| 499 | std::fs::create_dir_all(&archive_dir).unwrap(); |
| 500 | let kept = archive_dir.join("kept.json"); |
| 501 | let orphan = archive_dir.join("gone.json"); |
| 502 | for path in [&kept, &orphan] { |
| 503 | std::fs::write(path, session_with_tool_output(&output).to_string()).unwrap(); |
| 504 | } |
| 505 | |
| 506 | let files = session_files(dir.path(), &dir.path().join("standalone"), &mut Vec::new()); |
| 507 | let report = scrub_files(&files, Some(&manager), &[]).expect("scrub"); |
| 508 | |
| 509 | assert_eq!(report.flagged_files.len(), 2, "{report:?}"); |
| 510 | let text = std::fs::read_to_string(&kept).unwrap(); |
| 511 | assert!(!text.contains(TOKEN), "{text}"); |
| 512 | assert!(!orphan.exists(), "the deleted session's copy is removed"); |
| 513 | } |
| 514 | |
| 515 | /// A linked archive directory is not followed, matching session deletion. |
| 516 | #[cfg(unix)] |
| 517 | #[test] |
| 518 | fn a_linked_import_archive_is_reported_not_followed() { |
| 519 | let dir = tempfile::tempdir().expect("tempdir"); |
| 520 | let sessions = dir.path().join("sessions"); |
| 521 | let elsewhere = dir.path().join("elsewhere"); |
| 522 | std::fs::create_dir_all(&sessions).unwrap(); |
| 523 | std::fs::create_dir_all(&elsewhere).unwrap(); |
| 524 | let outside = elsewhere.join("gone.json"); |
| 525 | let output = format!("{{\"access_token\": \"{TOKEN}\"}}"); |
| 526 | std::fs::write(&outside, session_with_tool_output(&output).to_string()).unwrap(); |
| 527 | std::os::unix::fs::symlink( |
| 528 | &elsewhere, |
| 529 | sessions.join(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR), |
| 530 | ) |
| 531 | .unwrap(); |
| 532 | |
| 533 | let mut unreadable = Vec::new(); |
| 534 | let files = session_files(&sessions, &dir.path().join("standalone"), &mut unreadable); |
| 535 | |
| 536 | assert!( |
| 537 | !files.iter().any(|file| file.ends_with("gone.json")), |
| 538 | "{files:?}" |
| 539 | ); |
| 540 | assert_eq!(unreadable.len(), 1, "{unreadable:?}"); |
| 541 | assert!(unreadable[0].ends_with(crate::session_manager::WORK_GRAPH_IMPORT_ARCHIVE_DIR)); |
| 542 | assert!(std::fs::read_to_string(&outside).unwrap().contains(TOKEN)); |
| 543 | } |
| 544 | |
| 545 | /// An interactive session holds the conversation in memory; its next |
| 546 | /// autosave would put a masked credential back. Such a session is |
| 547 | /// reported busy and left byte-for-byte, like a live Runtime store. |
| 548 | #[test] |
| 549 | fn a_session_open_in_an_interactive_surface_is_reported_busy() { |
| 550 | let dir = tempfile::tempdir().expect("tempdir"); |
| 551 | let output = format!("{{\"access_token\": \"{TOKEN}\"}}"); |
| 552 | let open = dir.path().join("open-elsewhere.json"); |
| 553 | std::fs::write(&open, session_with_tool_output(&output).to_string()).unwrap(); |
| 554 | let before = std::fs::read(&open).unwrap(); |
| 555 | let manager = |
| 556 | crate::session_manager::SessionManager::new(dir.path().to_path_buf()).expect("manager"); |
| 557 | let _held = manager.hold_live_lease_elsewhere("open-elsewhere"); |
| 558 | |
| 559 | let files = session_files(dir.path(), &dir.path().join("standalone"), &mut Vec::new()); |
| 560 | let report = scrub_files(&files, Some(&manager), &[]).expect("scrub"); |
| 561 | assert_eq!( |
| 562 | report.busy, |
| 563 | vec![open.canonicalize().unwrap()], |
| 564 | "{report:?}" |
| 565 | ); |
| 566 | assert!(report.flagged_files.is_empty(), "{report:?}"); |
| 567 | assert_eq!(std::fs::read(&open).unwrap(), before); |
| 568 | } |
| 569 | |
| 570 | #[test] |
| 571 | fn busy_runtime_does_not_abort_scrub() { |
| 572 | let dir = tempfile::tempdir().unwrap(); |
| 573 | let sessions = dir.path().join("sessions"); |
| 574 | let standalone = dir.path().join("standalone"); |
| 575 | let roots = [sessions.join("s1/runtime"), standalone.clone()]; |
| 576 | let item = json!({"id":"item1", "turn_id":"turn1", "kind":"tool_call", "status":"completed", "summary": TOKEN, "detail": TOKEN, "metadata":{"stdout_summary":TOKEN, "exit_code":0}}); |
| 577 | let event = json!({"seq": 7, "payload":{"item":item}}); |
| 578 | for root in &roots { |
| 579 | std::fs::create_dir_all(root.join("items")).unwrap(); |
| 580 | std::fs::create_dir_all(root.join("events")).unwrap(); |
| 581 | std::fs::write(root.join("items/item1.json"), item.to_string()).unwrap(); |
| 582 | std::fs::write(root.join("events/thread1.jsonl"), format!("{event}\n")).unwrap(); |
| 583 | } |
| 584 | let files = session_files(&sessions, &standalone, &mut Vec::new()); |
| 585 | let report = scrub_files(&files, None, &[]).unwrap(); |
| 586 | assert_eq!(report.files_scanned, 4); |
| 587 | assert_eq!(report.flagged_files.len(), 4); |
| 588 | assert_eq!(report.flagged_tool_results, 12); |
| 589 | let transcript = sessions.join("s1.json"); |
| 590 | std::fs::write(&transcript, session_with_tool_output(TOKEN).to_string()).unwrap(); |
| 591 | let manager = crate::session_manager::SessionManager::new(sessions).unwrap(); |
| 592 | let lease = crate::runtime_threads::RuntimeProcessOwnerLock::acquire(&standalone).unwrap(); |
| 593 | let standalone_file = standalone.join("items/item1.json"); |
| 594 | let report = scrub_files( |
| 595 | &[standalone_file.clone(), transcript.clone()], |
| 596 | Some(&manager), |
| 597 | &[], |
| 598 | ) |
| 599 | .unwrap(); |
| 600 | assert_eq!(report.busy, vec![standalone_file.clone()]); |
| 601 | assert_eq!(report.flagged_files, vec![transcript.clone()]); |
| 602 | assert_eq!(report.files_scanned, 2); |
| 603 | assert!( |
| 604 | !std::fs::read_to_string(&transcript) |
| 605 | .unwrap() |
| 606 | .contains(TOKEN) |
| 607 | ); |
| 608 | assert!( |
| 609 | std::fs::read_to_string(&standalone_file) |
| 610 | .unwrap() |
| 611 | .contains(TOKEN) |
| 612 | ); |
| 613 | drop(lease); |
| 614 | // Non-contention failures still propagate, rather than masquerading as busy. |
| 615 | let lock_path = standalone.join("runtime-process.owner.lock"); |
| 616 | std::fs::remove_file(&lock_path).unwrap(); |
| 617 | std::fs::create_dir(&lock_path).unwrap(); |
| 618 | assert!(scrub_files(std::slice::from_ref(&standalone_file), Some(&manager), &[]).is_err()); |
| 619 | std::fs::remove_dir(&lock_path).unwrap(); |
| 620 | scrub_files(&files, Some(&manager), &[]).unwrap(); |
| 621 | assert!( |
| 622 | scrub_files(&files, None, &[]) |
| 623 | .unwrap() |
| 624 | .flagged_files |
| 625 | .is_empty() |
| 626 | ); |
| 627 | for path in &files { |
| 628 | let raw = std::fs::read_to_string(path).unwrap(); |
| 629 | assert!(!raw.contains(TOKEN)); |
| 630 | let value: Value = serde_json::from_str(&raw).unwrap(); |
| 631 | let item = if runtime_root(path).unwrap().join("items") == path.parent().unwrap() { |
| 632 | &value |
| 633 | } else { |
| 634 | &value["payload"]["item"] |
| 635 | }; |
| 636 | assert_eq!(item["metadata"]["exit_code"], 0); |
| 637 | assert_eq!(item["turn_id"], "turn1"); |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | #[cfg(unix)] |
| 642 | #[test] |
| 643 | fn symlinked_roots_scan_without_following_entries() { |
| 644 | use std::os::unix::fs::symlink; |
| 645 | let tmp = tempfile::tempdir().unwrap(); |
| 646 | let root = tmp.path().canonicalize().unwrap(); |
| 647 | let sessions = root.join("sessions-real"); |
| 648 | let checkpoints = root.join("checkpoints-real"); |
| 649 | let runtime = root.join("runtime-real"); |
| 650 | for dir in [&sessions, &checkpoints, &runtime.join("items")] { |
| 651 | std::fs::create_dir_all(dir).unwrap(); |
| 652 | } |
| 653 | let transcript = sessions.join("s1.json"); |
| 654 | let checkpoint = checkpoints.join("s1.json"); |
| 655 | let item = runtime.join("items/item.json"); |
| 656 | for path in [&transcript, &checkpoint, &item] { |
| 657 | std::fs::write(path, session_with_tool_output(TOKEN).to_string()).unwrap(); |
| 658 | } |
| 659 | symlink(&sessions, root.join("sessions")).unwrap(); |
| 660 | symlink(&checkpoints, sessions.join("checkpoints")).unwrap(); |
| 661 | symlink(&runtime, root.join("runtime")).unwrap(); |
| 662 | // Discovered file and session-directory links must not expand the scan. |
| 663 | symlink(&transcript, sessions.join("outside.json")).unwrap(); |
| 664 | symlink(&runtime, sessions.join("outside-session")).unwrap(); |
| 665 | std::fs::create_dir(sessions.join("s1")).unwrap(); |
| 666 | symlink(&runtime, sessions.join("s1/runtime")).unwrap(); |
| 667 | let mut unreadable = Vec::new(); |
| 668 | let files = session_files( |
| 669 | &root.join("sessions"), |
| 670 | &root.join("runtime"), |
| 671 | &mut unreadable, |
| 672 | ); |
| 673 | assert!(unreadable.is_empty(), "{unreadable:?}"); |
| 674 | assert_eq!(files.len(), 3); |
| 675 | for path in [&transcript, &checkpoint, &item] { |
| 676 | assert!(files.contains(path)); |
| 677 | } |
| 678 | let manager = crate::session_manager::SessionManager::new(root.join("sessions")).unwrap(); |
| 679 | assert_eq!( |
| 680 | scrub_files(&files, Some(&manager), &[]) |
| 681 | .unwrap() |
| 682 | .flagged_files |
| 683 | .len(), |
| 684 | 3 |
| 685 | ); |
| 686 | assert!( |
| 687 | scrub_files(&files, None, &[]) |
| 688 | .unwrap() |
| 689 | .flagged_files |
| 690 | .is_empty() |
| 691 | ); |
| 692 | } |
| 693 | |
| 694 | #[cfg(unix)] |
| 695 | #[test] |
| 696 | fn unreadable_directory_keeps_partial_scan() { |
| 697 | use std::os::unix::fs::PermissionsExt; |
| 698 | let tmp = tempfile::tempdir().unwrap(); |
| 699 | let root = tmp.path().canonicalize().unwrap(); |
| 700 | let blocked = root.join("blocked-session"); |
| 701 | std::fs::create_dir(&blocked).unwrap(); |
| 702 | let transcript = root.join("s1.json"); |
| 703 | std::fs::write(&transcript, session_with_tool_output(TOKEN).to_string()).unwrap(); |
| 704 | std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o000)).unwrap(); |
| 705 | let mut unreadable = Vec::new(); |
| 706 | let files = session_files(&root, &root.join("absent-runtime"), &mut unreadable); |
| 707 | std::fs::set_permissions(&blocked, std::fs::Permissions::from_mode(0o700)).unwrap(); |
| 708 | assert_eq!(unreadable, vec![blocked]); |
| 709 | assert_eq!(files, vec![transcript]); |
| 710 | let mut report = scrub_files(&files, None, &[]).unwrap(); |
| 711 | report.unreadable = unreadable; |
| 712 | assert_eq!(report.flagged_files.len(), 1); |
| 713 | let summary = crate::doctor_stored_secrets_summary(&report, files.len()); |
| 714 | assert!(summary.contains("scan incomplete") && summary.contains("blocked-session")); |
| 715 | assert!(!summary.contains('✓')); |
| 716 | } |
| 717 | |
| 718 | #[test] |
| 719 | fn configured_bare_secret_is_scrubbed() { |
| 720 | use crate::test_support::{EnvVarGuard, lock_test_env}; |
| 721 | let _env = lock_test_env(); |
| 722 | let tmp = tempfile::tempdir().unwrap(); |
| 723 | let _home = EnvVarGuard::set("CODEWHALE_HOME", tmp.path()); |
| 724 | let _user_home = EnvVarGuard::set("HOME", tmp.path()); |
| 725 | let _profile = EnvVarGuard::set("USERPROFILE", tmp.path()); |
| 726 | let config = tmp.path().join("config.toml"); |
| 727 | let _config = EnvVarGuard::set("CODEWHALE_CONFIG_PATH", &config); |
| 728 | let secret = "abcdefghijklmnopqrst"; |
| 729 | std::fs::write(&config, format!("sandbox_api_key = \"{secret}\"\n")).unwrap(); |
| 730 | let secrets = configured_secrets(Some(config), None).unwrap(); |
| 731 | assert!(secrets.iter().any(|value| value == secret)); |
| 732 | let sessions = tmp.path().join("sessions"); |
| 733 | let runtime = sessions.join("s1/runtime"); |
| 734 | std::fs::create_dir_all(runtime.join("items")).unwrap(); |
| 735 | std::fs::create_dir_all(runtime.join("events")).unwrap(); |
| 736 | let transcript = sessions.join("s1.json"); |
| 737 | let item = runtime.join("items/item.json"); |
| 738 | let event = runtime.join("events/thread.jsonl"); |
| 739 | let receipt = json!({"kind":"tool_call", "summary":secret, "detail":secret, "metadata":{"stdout_summary":secret, "exit_code":0}}); |
| 740 | std::fs::write(&transcript, session_with_tool_output(secret).to_string()).unwrap(); |
| 741 | std::fs::write(&item, receipt.to_string()).unwrap(); |
| 742 | std::fs::write(&event, format!("{}\n", json!({"payload":{"item":receipt}}))).unwrap(); |
| 743 | let files = vec![transcript, item, event]; |
| 744 | assert!( |
| 745 | scrub_files(&files, None, &[]) |
| 746 | .unwrap() |
| 747 | .flagged_files |
| 748 | .is_empty() |
| 749 | ); |
| 750 | let report = scrub_files(&files, None, &secrets).unwrap(); |
| 751 | assert_eq!(report.flagged_files.len(), 3); |
| 752 | assert_eq!(report.flagged_tool_results, 9); |
| 753 | let manager = crate::session_manager::SessionManager::new(sessions).unwrap(); |
| 754 | scrub_files(&files, Some(&manager), &secrets).unwrap(); |
| 755 | for path in files { |
| 756 | let raw = std::fs::read_to_string(path).unwrap(); |
| 757 | assert!(!raw.contains(secret)); |
| 758 | assert!(raw.contains(codewhale_config::persistence::REDACTED)); |
| 759 | } |
| 760 | } |
| 761 | |
| 762 | #[test] |
| 763 | fn doctor_reserves_transcript_scan_budget() { |
| 764 | // Already ordered newest-first: an active store has over 50 receipts, |
| 765 | // all newer than the credential-bearing transcript and checkpoint. |
| 766 | let mut files: Vec<PathBuf> = (0..60) |
| 767 | .map(|i| PathBuf::from(format!("s1/runtime/items/{i}.json"))) |
| 768 | .collect(); |
| 769 | let tmp = tempfile::tempdir().unwrap(); |
| 770 | let transcript = tmp.path().join("s1.json"); |
| 771 | let checkpoint = tmp.path().join("checkpoints/s1.json"); |
| 772 | std::fs::create_dir_all(checkpoint.parent().unwrap()).unwrap(); |
| 773 | for path in [&transcript, &checkpoint] { |
| 774 | std::fs::write(path, session_with_tool_output(TOKEN).to_string()).unwrap(); |
| 775 | files.push(path.clone()); |
| 776 | } |
| 777 | let checked = doctor_files(files, crate::DOCTOR_SECRET_SCAN_FILES); |
| 778 | assert_eq!(checked.len(), 52); |
| 779 | assert!(checked.contains(&transcript) && checked.contains(&checkpoint)); |
| 780 | let report = scrub_files(&checked, None, &[]).unwrap(); |
| 781 | assert_eq!(report.flagged_files, vec![transcript, checkpoint]); |
| 782 | let summary = crate::doctor_stored_secrets_summary(&report, 62); |
| 783 | assert!(summary.contains("per category") && summary.contains("sessions/checkpoints")); |
| 784 | } |
| 785 | |
| 786 | #[test] |
| 787 | fn non_tool_result_text_is_left_alone() { |
| 788 | let mut value = json!({ |
| 789 | "messages": [{"role": "user", "content": [{"type": "text", "text": TOKEN}]}] |
| 790 | }); |
| 791 | assert_eq!(scrub_value(&mut value, &[]), 0); |
| 792 | assert_eq!(value["messages"][0]["content"][0]["text"], TOKEN); |
| 793 | } |
| 794 | } |
| 795 |