| 1 | //! Pure output-sanitization primitives shared by portable command helpers |
| 2 | //! (FEAT-025 D4). |
| 3 | //! |
| 4 | //! These helpers previously lived in the TUI command, client, and OSC8 |
| 5 | //! modules. Relocating the pure algorithms here gives `/export` and |
| 6 | //! `/structcopy` exactly one implementation with no TUI, client, or |
| 7 | //! configuration dependency. TUI callers delegate back to these functions so |
| 8 | //! behavior cannot drift. |
| 9 | |
| 10 | use std::sync::OnceLock; |
| 11 | |
| 12 | use regex::Regex; |
| 13 | use serde_json::Value; |
| 14 | |
| 15 | use crate::redact::redact_secrets; |
| 16 | |
| 17 | /// Strip ANSI/OSC/control sequences from `s` into `out`. |
| 18 | /// |
| 19 | /// Handles CSI (`ESC [ … final`), OSC (`ESC ] … BEL` or `ESC \`), DCS, SOS, |
| 20 | /// PM, APC, and standalone two-byte ESC sequences. OSC 8 hyperlink wrappers |
| 21 | /// (`ESC ] 8 ; … BEL` / `ESC \`) are stripped along with the rest. |
| 22 | pub fn strip_ansi_into(s: &str, out: &mut String) { |
| 23 | strip_ansi_impl(s, out, false); |
| 24 | } |
| 25 | |
| 26 | /// Drop control bytes from a streaming text chunk, keeping printable |
| 27 | /// characters, `\n` and `\t`. The terminal UI applies it to every streamed |
| 28 | /// chunk before rendering; notification payloads apply it after stripping |
| 29 | /// whole escape sequences. |
| 30 | #[must_use] |
| 31 | pub fn sanitize_stream_chunk(chunk: &str) -> String { |
| 32 | chunk |
| 33 | .chars() |
| 34 | .filter(|c| *c == '\n' || *c == '\t' || !c.is_control()) |
| 35 | .collect() |
| 36 | } |
| 37 | |
| 38 | /// Like [`strip_ansi_into`], but SGR sequences (`ESC [ … m`: colour, bold, |
| 39 | /// underline, reset) pass through untouched so a renderer that understands |
| 40 | /// them can paint the output as the tool emitted it. Everything else — OSC |
| 41 | /// (including OSC 8 hyperlink wrappers), cursor movement, DCS, lone control |
| 42 | /// bytes — is still removed; only the styling survives. |
| 43 | pub fn strip_ansi_keep_sgr_into(s: &str, out: &mut String) { |
| 44 | strip_ansi_impl(s, out, true); |
| 45 | } |
| 46 | |
| 47 | /// Length in bytes of the UTF-8 sequence that starts with `lead`. Falls back |
| 48 | /// to `1` for continuation bytes / invalid leads so callers always make |
| 49 | /// forward progress. |
| 50 | pub fn utf8_seq_len(lead: u8) -> usize { |
| 51 | if lead < 0xc0 { |
| 52 | 1 |
| 53 | } else if lead < 0xe0 { |
| 54 | 2 |
| 55 | } else if lead < 0xf0 { |
| 56 | 3 |
| 57 | } else { |
| 58 | 4 |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | fn strip_ansi_impl(s: &str, out: &mut String, keep_sgr: bool) { |
| 63 | let bytes = s.as_bytes(); |
| 64 | let mut i = 0; |
| 65 | while i < bytes.len() { |
| 66 | if bytes[i] == 0x1b && i + 1 < bytes.len() { |
| 67 | let next = bytes[i + 1]; |
| 68 | match next { |
| 69 | // CSI: ESC [ ... <final byte 0x40..=0x7E> |
| 70 | b'[' => { |
| 71 | let mut j = i + 2; |
| 72 | let mut final_byte = 0u8; |
| 73 | while j < bytes.len() { |
| 74 | let b = bytes[j]; |
| 75 | if (0x40..=0x7e).contains(&b) { |
| 76 | final_byte = b; |
| 77 | j += 1; |
| 78 | break; |
| 79 | } |
| 80 | j += 1; |
| 81 | } |
| 82 | if keep_sgr |
| 83 | && final_byte == b'm' |
| 84 | && let Ok(seq) = std::str::from_utf8(&bytes[i..j]) |
| 85 | { |
| 86 | out.push_str(seq); |
| 87 | } |
| 88 | i = j; |
| 89 | continue; |
| 90 | } |
| 91 | // OSC / DCS / SOS / PM / APC: ESC ] | P | X | ^ | _ ... ST(ESC \) or BEL |
| 92 | b']' | b'P' | b'X' | b'^' | b'_' => { |
| 93 | let mut j = i + 2; |
| 94 | while j < bytes.len() { |
| 95 | if bytes[j] == 0x07 { |
| 96 | j += 1; |
| 97 | break; |
| 98 | } |
| 99 | if bytes[j] == 0x1b && j + 1 < bytes.len() && bytes[j + 1] == b'\\' { |
| 100 | j += 2; |
| 101 | break; |
| 102 | } |
| 103 | j += 1; |
| 104 | } |
| 105 | i = j; |
| 106 | continue; |
| 107 | } |
| 108 | // Standalone two-byte ESC sequence (RIS, charset selection, etc.) |
| 109 | _ => { |
| 110 | i += 2; |
| 111 | continue; |
| 112 | } |
| 113 | } |
| 114 | } |
| 115 | // Strip lone control bytes that ratatui would otherwise drop (and which |
| 116 | // mean nothing in transcript output) but keep \n, \r, \t as legitimate |
| 117 | // formatting. |
| 118 | let b = bytes[i]; |
| 119 | if b < 0x80 { |
| 120 | if b < 0x20 && b != b'\n' && b != b'\r' && b != b'\t' { |
| 121 | i += 1; |
| 122 | continue; |
| 123 | } |
| 124 | out.push(b as char); |
| 125 | i += 1; |
| 126 | } else { |
| 127 | // UTF-8 multi-byte sequence: copy the whole code point intact. |
| 128 | // Pushing `b as char` would mis-decode it as Latin-1 and mangle |
| 129 | // non-ASCII text (CJK, accented Latin, emoji, …). |
| 130 | let len = utf8_seq_len(b); |
| 131 | let end = (i + len).min(bytes.len()); |
| 132 | if let Ok(chunk) = std::str::from_utf8(&bytes[i..end]) { |
| 133 | out.push_str(chunk); |
| 134 | } |
| 135 | i = end; |
| 136 | } |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | /// Mask credentials in a URL so it can appear in output or a report. |
| 141 | /// |
| 142 | /// Userinfo is replaced with `***` and query values under sensitive keys are |
| 143 | /// masked. A URL that does not parse is returned unchanged. |
| 144 | pub fn redact_url_for_display(url: &str) -> String { |
| 145 | let Ok(mut parsed) = url::Url::parse(url) else { |
| 146 | return url.to_string(); |
| 147 | }; |
| 148 | if !parsed.username().is_empty() || parsed.password().is_some() { |
| 149 | let _ = parsed.set_username("***"); |
| 150 | let _ = parsed.set_password(Some("***")); |
| 151 | } |
| 152 | if parsed.query().is_none() { |
| 153 | return parsed.to_string(); |
| 154 | } |
| 155 | let pairs: Vec<(String, String)> = parsed |
| 156 | .query_pairs() |
| 157 | .map(|(key, value)| { |
| 158 | let value = if is_sensitive_url_query_key(&key) { |
| 159 | "***".to_string() |
| 160 | } else { |
| 161 | value.into_owned() |
| 162 | }; |
| 163 | (key.into_owned(), value) |
| 164 | }) |
| 165 | .collect(); |
| 166 | parsed.set_query(None); |
| 167 | let mut query = parsed.query_pairs_mut(); |
| 168 | for (key, value) in pairs { |
| 169 | query.append_pair(&key, &value); |
| 170 | } |
| 171 | drop(query); |
| 172 | parsed.to_string() |
| 173 | } |
| 174 | |
| 175 | fn is_sensitive_url_query_key(key: &str) -> bool { |
| 176 | is_sensitive_key_name(key) |
| 177 | } |
| 178 | |
| 179 | /// True when `role` names an internal, non-user-visible message role. |
| 180 | pub fn is_internal_role(role: &str) -> bool { |
| 181 | matches!( |
| 182 | role.trim().to_ascii_lowercase().as_str(), |
| 183 | "system" | "developer" | "internal" |
| 184 | ) |
| 185 | } |
| 186 | |
| 187 | /// Credential names shared by config, URL and JSON output checks. |
| 188 | const SENSITIVE_KEY_NAMES: &[&str] = &[ |
| 189 | "access_key", |
| 190 | "access_token", |
| 191 | "api_key", |
| 192 | "api_keys", |
| 193 | "apikey", |
| 194 | "auth_token", |
| 195 | "authorization", |
| 196 | "bearer", |
| 197 | "client_secret", |
| 198 | "cookie", |
| 199 | "credential", |
| 200 | "credentials", |
| 201 | "id_token", |
| 202 | "passwd", |
| 203 | "password", |
| 204 | "passwords", |
| 205 | "private_key", |
| 206 | "proxy_authorization", |
| 207 | "refresh_token", |
| 208 | "secret", |
| 209 | "secrets", |
| 210 | "session_key", |
| 211 | "set_cookie", |
| 212 | "sas", |
| 213 | "token", |
| 214 | "tokens", |
| 215 | ]; |
| 216 | |
| 217 | /// Use the existing text redactor's identifier normalization, including |
| 218 | /// camelCase and acronym boundaries, instead of maintaining another parser. |
| 219 | #[must_use] |
| 220 | pub fn normalize_key(key: &str) -> String { |
| 221 | crate::redact::normalize_sensitive_key(key) |
| 222 | } |
| 223 | |
| 224 | fn key_spellings(key: &str) -> [String; 2] { |
| 225 | [normalize_key(key), normalize_key(&key.to_ascii_lowercase())] |
| 226 | } |
| 227 | |
| 228 | /// Precise credential check for settings, headers and flags. Usage settings |
| 229 | /// such as max_tokens and token_budget remain visible. |
| 230 | #[must_use] |
| 231 | pub fn is_sensitive_key_name(key: &str) -> bool { |
| 232 | key_spellings(key).iter().any(|normalized| { |
| 233 | SENSITIVE_KEY_NAMES.contains(&normalized.as_str()) |
| 234 | || ["_authorization", "_cookie", "_passwd", "_password", "_secret", "_token"] |
| 235 | .iter().any(|suffix| normalized.ends_with(suffix)) |
| 236 | // Retain config's subscription-key and custom key protection. |
| 237 | || (normalized.ends_with("_key") |
| 238 | && !matches!(normalized.as_str(), "public_key" | "endpoint_key")) |
| 239 | }) |
| 240 | } |
| 241 | |
| 242 | /// Broad export check; model-bound text keeps its existing credential-shaped |
| 243 | /// policy in redact.rs. One shared normalization handles every spelling. |
| 244 | #[must_use] |
| 245 | pub fn is_sensitive_key(key: &str) -> bool { |
| 246 | is_sensitive_key_name(key) |
| 247 | || key_spellings(key).iter().any(|normalized| { |
| 248 | [ |
| 249 | "api_key", |
| 250 | "apikey", |
| 251 | "secret", |
| 252 | "token", |
| 253 | "password", |
| 254 | "passwd", |
| 255 | "authorization", |
| 256 | "access_key", |
| 257 | "client_secret", |
| 258 | "private_key", |
| 259 | "cookie", |
| 260 | "session_key", |
| 261 | ] |
| 262 | .iter() |
| 263 | .any(|hint| normalized.contains(hint)) |
| 264 | }) |
| 265 | } |
| 266 | |
| 267 | /// Sanitize arbitrary text for safe export output. |
| 268 | /// |
| 269 | /// Strips ANSI/control bytes, normalizes newlines, then applies private-key, |
| 270 | /// bearer-token, JWT, URL-credential, and keyed-secret redaction in the exact |
| 271 | /// established order. |
| 272 | pub fn sanitize_text(input: &str) -> String { |
| 273 | let mut visible = String::with_capacity(input.len()); |
| 274 | strip_ansi_into(input, &mut visible); |
| 275 | let visible = visible.replace("\r\n", "\n").replace('\r', "\n"); |
| 276 | let visible: String = visible |
| 277 | .chars() |
| 278 | .filter(|ch| *ch == '\n' || *ch == '\t' || !ch.is_control()) |
| 279 | .collect(); |
| 280 | let private_keys = private_key_regex().replace_all(&visible, "[redacted private key]"); |
| 281 | let bearer = bearer_regex().replace_all(&private_keys, "Bearer [redacted]"); |
| 282 | let jwt = jwt_regex().replace_all(&bearer, "[redacted token]"); |
| 283 | let urls = url_regex().replace_all(&jwt, |captures: ®ex::Captures<'_>| { |
| 284 | redact_url_match(captures.get(0).map_or("", |value| value.as_str())) |
| 285 | }); |
| 286 | redact_secrets(&urls) |
| 287 | } |
| 288 | |
| 289 | /// Whether an arbitrary value carries material the shared redactor masks. |
| 290 | /// Formatting-only changes (URL canonicalization, newlines, ANSI) do not make |
| 291 | /// a harmless config value a credential. |
| 292 | #[must_use] |
| 293 | pub fn contains_secret(input: &str) -> bool { |
| 294 | let mut visible = String::with_capacity(input.len()); |
| 295 | strip_ansi_into(input, &mut visible); |
| 296 | private_key_regex().is_match(&visible) |
| 297 | || bearer_regex().is_match(&visible) |
| 298 | || jwt_regex().is_match(&visible) |
| 299 | || redact_secrets(&visible) != visible |
| 300 | || url_regex().find_iter(&visible).any(|matched| { |
| 301 | let raw = matched.as_str().trim_end_matches(['.', ',', ';', '!']); |
| 302 | url::Url::parse(raw).is_ok_and(|url| { |
| 303 | !url.username().is_empty() |
| 304 | || url.password().is_some() |
| 305 | || url |
| 306 | .query_pairs() |
| 307 | .any(|(key, _)| is_sensitive_url_query_key(&key)) |
| 308 | }) |
| 309 | }) |
| 310 | } |
| 311 | |
| 312 | /// Recursively redact a JSON value. |
| 313 | /// |
| 314 | /// A value under a sensitive key is replaced wholesale; other strings pass |
| 315 | /// through [`sanitize_text`], and arrays/objects are traversed in place. |
| 316 | pub fn redact_json(value: &mut Value, key: Option<&str>) { |
| 317 | if key.is_some_and(is_sensitive_key) { |
| 318 | *value = Value::String("[redacted]".to_string()); |
| 319 | return; |
| 320 | } |
| 321 | match value { |
| 322 | Value::String(text) => *text = sanitize_text(text), |
| 323 | Value::Array(items) => { |
| 324 | for item in items { |
| 325 | redact_json(item, None); |
| 326 | } |
| 327 | } |
| 328 | Value::Object(map) => { |
| 329 | for (key, value) in map { |
| 330 | redact_json(value, Some(key)); |
| 331 | } |
| 332 | } |
| 333 | Value::Null | Value::Bool(_) | Value::Number(_) => {} |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | /// Collapse whitespace and neutralize inline backticks for a single-line |
| 338 | /// export field. |
| 339 | pub fn inline_text(input: &str) -> String { |
| 340 | sanitize_text(input) |
| 341 | .split_whitespace() |
| 342 | .collect::<Vec<_>>() |
| 343 | .join(" ") |
| 344 | .replace('`', "'") |
| 345 | } |
| 346 | |
| 347 | fn redact_url_match(raw: &str) -> String { |
| 348 | let trimmed = raw.trim_end_matches(['.', ',', ';', '!']); |
| 349 | let suffix = &raw[trimmed.len()..]; |
| 350 | format!("{}{}", redact_url_for_display(trimmed), suffix) |
| 351 | } |
| 352 | |
| 353 | fn private_key_regex() -> &'static Regex { |
| 354 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 355 | RE.get_or_init(|| { |
| 356 | Regex::new( |
| 357 | r"(?is)-----BEGIN [^-\r\n]*PRIVATE KEY-----.*?-----END [^-\r\n]*PRIVATE KEY-----", |
| 358 | ) |
| 359 | .expect("private-key redaction regex") |
| 360 | }) |
| 361 | } |
| 362 | |
| 363 | fn bearer_regex() -> &'static Regex { |
| 364 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 365 | RE.get_or_init(|| { |
| 366 | Regex::new(r"(?i)\bbearer\s+[a-z0-9._~+/=-]{6,}").expect("bearer redaction regex") |
| 367 | }) |
| 368 | } |
| 369 | |
| 370 | fn jwt_regex() -> &'static Regex { |
| 371 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 372 | RE.get_or_init(|| { |
| 373 | Regex::new(r"\beyJ[a-zA-Z0-9_-]{5,}\.[a-zA-Z0-9_-]{5,}(?:\.[a-zA-Z0-9_-]{5,})?\b") |
| 374 | .expect("JWT redaction regex") |
| 375 | }) |
| 376 | } |
| 377 | |
| 378 | fn url_regex() -> &'static Regex { |
| 379 | static RE: OnceLock<Regex> = OnceLock::new(); |
| 380 | RE.get_or_init(|| { |
| 381 | Regex::new(r#"https?://[^\s<>\"'`\]\[\)\(\}\{]+"#).expect("URL redaction regex") |
| 382 | }) |
| 383 | } |
| 384 | |
| 385 | #[cfg(test)] |
| 386 | mod tests { |
| 387 | use super::*; |
| 388 | |
| 389 | #[test] |
| 390 | fn shared_vocabulary_redacts_camel_case_and_keeps_safe_key_names() { |
| 391 | for key in [ |
| 392 | "accessToken", |
| 393 | "clientSecret", |
| 394 | "privateKey", |
| 395 | "refreshToken", |
| 396 | "APIKey", |
| 397 | "X-API-Key", |
| 398 | "sessionKey", |
| 399 | "oauth2Token", |
| 400 | "aUtHoRiZaTiOn", |
| 401 | "Ocp-Apim-Subscription-Key", |
| 402 | "Set-Cookie", |
| 403 | "sas", |
| 404 | ] { |
| 405 | assert!(is_sensitive_key_name(key), "{key}"); |
| 406 | let mut value = serde_json::json!({key: "s10-synthetic-value"}); |
| 407 | redact_json(&mut value, None); |
| 408 | assert!(!value.to_string().contains("s10-synthetic-value"), "{key}"); |
| 409 | } |
| 410 | for key in [ |
| 411 | "max_tokens", |
| 412 | "token_budget", |
| 413 | "maxTokens", |
| 414 | "api_key_source", |
| 415 | "authMode", |
| 416 | "publicKey", |
| 417 | "endpoint_key", |
| 418 | "monkey", |
| 419 | ] { |
| 420 | assert!(!is_sensitive_key_name(key), "{key}"); |
| 421 | } |
| 422 | assert_eq!(normalize_key("APIKey"), "api_key"); |
| 423 | } |
| 424 | |
| 425 | #[test] |
| 426 | fn shared_secret_detection_preserves_formatting_only_config_values() { |
| 427 | for safe in [ |
| 428 | "https://api.example.com", |
| 429 | "https://api.example.com/v1?team=core", |
| 430 | "https://api.example.com/?tokenBudget=5", |
| 431 | "normal text", |
| 432 | "max tokens = 8192", |
| 433 | "tokenCount = 4", |
| 434 | ] { |
| 435 | assert!(!contains_secret(safe), "safe value classified as secret"); |
| 436 | } |
| 437 | for secret in [ |
| 438 | "clientSecret=opaque-value", |
| 439 | "https://alice:opaque-value@example.com", |
| 440 | "https://example.com/?privateKey=opaque-value", |
| 441 | "https://example.com/?sas=opaque-value", |
| 442 | ] { |
| 443 | assert!(contains_secret(secret), "credential not classified"); |
| 444 | } |
| 445 | assert_ne!( |
| 446 | sanitize_text("https://api.example.com"), |
| 447 | "https://api.example.com" |
| 448 | ); |
| 449 | } |
| 450 | |
| 451 | #[test] |
| 452 | fn strip_ansi_removes_control_sequences_but_keeps_text() { |
| 453 | let mut out = String::new(); |
| 454 | strip_ansi_into("a\u{1b}[31mred\u{1b}[0m b", &mut out); |
| 455 | assert_eq!(out, "ared b"); |
| 456 | } |
| 457 | |
| 458 | #[test] |
| 459 | fn url_redaction_masks_userinfo_and_sensitive_query_values() { |
| 460 | assert_eq!( |
| 461 | redact_url_for_display( |
| 462 | "https://alice:password@example.com/path?token=very-secret&ok=1" |
| 463 | ), |
| 464 | "https://***:***@example.com/path?token=***&ok=1" |
| 465 | ); |
| 466 | } |
| 467 | |
| 468 | #[test] |
| 469 | fn sensitive_keys_normalize_separators_and_quotes() { |
| 470 | assert!(is_sensitive_key("API-KEY")); |
| 471 | assert!(is_sensitive_key("\"client secret\"")); |
| 472 | assert!(!is_sensitive_key("monkey")); |
| 473 | } |
| 474 | |
| 475 | #[test] |
| 476 | fn sanitize_text_redacts_private_keys_bearer_and_jwt() { |
| 477 | // Assemble the PEM markers and the provider-token prefix at runtime so |
| 478 | // this source file never contains a literal private-key header (or a |
| 479 | // literal token) for a secret scanner to match. The runtime strings are |
| 480 | // identical to the real shapes, and this mirrors the convention the |
| 481 | // pre-move test used in `config::persistence` - moving that test into |
| 482 | // this crate silently dropped it, which is what GitGuardian caught. |
| 483 | let begin = ["-----BEGIN RSA", " PRIVATE KEY-----"].concat(); |
| 484 | let end = ["-----END RSA", " PRIVATE KEY-----"].concat(); |
| 485 | let bearer = format!("{} abcdefghijklmnop", "Bearer"); |
| 486 | let opaque = ["sk-", "abcdef1234567890"].concat(); |
| 487 | let text = format!("{begin}\nMII\n{end}\nAuthorization: {bearer}\n{opaque}"); |
| 488 | |
| 489 | let out = sanitize_text(&text); |
| 490 | assert!(!out.contains("MII"), "{out}"); |
| 491 | assert!(!out.contains("abcdefghijklmnop"), "{out}"); |
| 492 | assert!(out.contains("[redacted]"), "{out}"); |
| 493 | } |
| 494 | } |
| 495 |