| 1 | //! Which (route, model) pairs this person actually uses (#6533). |
| 2 | //! |
| 3 | //! Replaces the additive `settings.toml [enabled_models]` store as the |
| 4 | //! `/model` picker's notion of "models I use". That store was appended to on |
| 5 | //! every switch by older builds and never pruned, so the default picker view |
| 6 | //! reflected history nobody curated. This index is derived from data that |
| 7 | //! already exists — saved session metadata (`model_provider[_id]`, `model`, |
| 8 | //! `updated_at`) and the redacted `cost.route_receipts` each session keeps — |
| 9 | //! and adds no new store. |
| 10 | //! |
| 11 | //! Known limitations: |
| 12 | //! - Resolution is per session, not per turn: a session's `updated_at` is the |
| 13 | //! last-used time for every route it touched, and a route counts once per |
| 14 | //! session however many turns it ran. |
| 15 | //! - In-session use is recorded on route switches only (see |
| 16 | //! [`RouteUsageIndex::record`]); turns on the startup route reach the index |
| 17 | //! through the saved session on the next start. |
| 18 | //! - Receipts are redacted strings; a model id containing characters the |
| 19 | //! receipt formatter replaces is keyed by its redacted spelling. |
| 20 | |
| 21 | use std::collections::{BTreeSet, HashMap, HashSet}; |
| 22 | use std::sync::{Arc, RwLock}; |
| 23 | |
| 24 | use chrono::{DateTime, Duration, Utc}; |
| 25 | |
| 26 | use crate::session_manager::SessionMetadata; |
| 27 | |
| 28 | /// Sessions older than this contribute nothing. |
| 29 | pub(crate) const WINDOW_DAYS: i64 = 30; |
| 30 | /// Score halves every week. |
| 31 | const HALF_LIFE_DAYS: f64 = 7.0; |
| 32 | |
| 33 | #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] |
| 34 | struct RouteKey { |
| 35 | /// Exact persistence identity (`deepseek`, or a named custom route's |
| 36 | /// `[providers.<name>]` key). Case is preserved: case-distinct custom |
| 37 | /// tables are distinct routes. |
| 38 | identity: String, |
| 39 | /// Lower-cased model id. |
| 40 | model: String, |
| 41 | } |
| 42 | |
| 43 | #[derive(Debug, Clone, PartialEq)] |
| 44 | pub(crate) struct RouteUsage { |
| 45 | pub identity: String, |
| 46 | /// Model id as first seen (display spelling). |
| 47 | pub model: String, |
| 48 | pub last_used: DateTime<Utc>, |
| 49 | pub sessions: u32, |
| 50 | } |
| 51 | |
| 52 | #[derive(Debug, Clone, Default)] |
| 53 | pub(crate) struct RouteUsageIndex { |
| 54 | entries: HashMap<RouteKey, RouteUsage>, |
| 55 | /// Routes already counted by [`Self::record`] in this live session, so |
| 56 | /// switching A → B → A counts A once. |
| 57 | live: HashSet<RouteKey>, |
| 58 | } |
| 59 | |
| 60 | /// Shared between the startup builder thread and the UI. |
| 61 | pub(crate) type SharedRouteUsage = Arc<RwLock<RouteUsageIndex>>; |
| 62 | |
| 63 | impl RouteUsageIndex { |
| 64 | /// Build from saved session metadata. Archived sessions and sessions not |
| 65 | /// updated within [`WINDOW_DAYS`] of `now` are skipped. |
| 66 | pub(crate) fn from_sessions(sessions: &[SessionMetadata], now: DateTime<Utc>) -> Self { |
| 67 | let cutoff = now - Duration::days(WINDOW_DAYS); |
| 68 | let mut index = Self::default(); |
| 69 | for session in sessions { |
| 70 | if session.archived || session.updated_at < cutoff { |
| 71 | continue; |
| 72 | } |
| 73 | let mut routes: BTreeSet<(String, String)> = BTreeSet::new(); |
| 74 | let identity = session |
| 75 | .model_provider_id |
| 76 | .as_deref() |
| 77 | .map(str::trim) |
| 78 | .filter(|id| !id.is_empty()) |
| 79 | .unwrap_or(session.model_provider.trim()); |
| 80 | routes.insert((identity.to_string(), session.model.trim().to_string())); |
| 81 | for receipt in &session.cost.route_receipts { |
| 82 | if let Some(route) = parse_route_receipt(receipt) { |
| 83 | routes.insert(route); |
| 84 | } |
| 85 | } |
| 86 | // One session counts once per route, however many receipts or |
| 87 | // spellings name it. |
| 88 | let mut seen = BTreeSet::new(); |
| 89 | for (identity, model) in routes { |
| 90 | if seen.insert(key(&identity, &model)) { |
| 91 | index.add(&identity, &model, session.updated_at, 1); |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | index |
| 96 | } |
| 97 | |
| 98 | fn add(&mut self, identity: &str, model: &str, at: DateTime<Utc>, sessions: u32) { |
| 99 | let identity = identity.trim(); |
| 100 | let model = model.trim(); |
| 101 | if identity.is_empty() || model.is_empty() || model.eq_ignore_ascii_case("auto") { |
| 102 | return; |
| 103 | } |
| 104 | let entry = self |
| 105 | .entries |
| 106 | .entry(key(identity, model)) |
| 107 | .or_insert_with(|| RouteUsage { |
| 108 | identity: identity.to_string(), |
| 109 | model: model.to_string(), |
| 110 | last_used: at, |
| 111 | sessions: 0, |
| 112 | }); |
| 113 | entry.sessions = entry.sessions.saturating_add(sessions); |
| 114 | if at > entry.last_used { |
| 115 | entry.last_used = at; |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | /// Record in-session use of a route (a committed switch). The first record |
| 120 | /// of a route counts as one more session so a route picked today outranks |
| 121 | /// one last used a month ago; later records of the same route in this |
| 122 | /// session only refresh its last-used time. |
| 123 | pub(crate) fn record(&mut self, identity: &str, model: &str, at: DateTime<Utc>) { |
| 124 | let first_this_session = self.live.insert(key(identity, model)); |
| 125 | self.add(identity, model, at, u32::from(first_this_session)); |
| 126 | } |
| 127 | |
| 128 | /// Fold a freshly built index into this one, keeping in-session records |
| 129 | /// made while the builder was still reading. |
| 130 | pub(crate) fn merge(&mut self, other: Self) { |
| 131 | for usage in other.entries.into_values() { |
| 132 | self.add( |
| 133 | &usage.identity, |
| 134 | &usage.model, |
| 135 | usage.last_used, |
| 136 | usage.sessions, |
| 137 | ); |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | /// `sessions × 0.5^(age_days / 7)`; zero outside the window. |
| 142 | #[cfg(test)] |
| 143 | pub(crate) fn score(&self, identity: &str, model: &str, now: DateTime<Utc>) -> f64 { |
| 144 | self.entries |
| 145 | .get(&key(identity, model)) |
| 146 | .map_or(0.0, |usage| decayed_score(usage, now)) |
| 147 | } |
| 148 | |
| 149 | /// Routes used within the window, highest score first. Ties break on the |
| 150 | /// more recent use, then identity/model for a stable order. |
| 151 | pub(crate) fn ranked(&self, now: DateTime<Utc>) -> Vec<&RouteUsage> { |
| 152 | let cutoff = now - Duration::days(WINDOW_DAYS); |
| 153 | let mut ranked: Vec<_> = self |
| 154 | .entries |
| 155 | .values() |
| 156 | .filter(|usage| usage.last_used >= cutoff) |
| 157 | .collect(); |
| 158 | ranked.sort_by(|a, b| { |
| 159 | decayed_score(b, now) |
| 160 | .total_cmp(&decayed_score(a, now)) |
| 161 | .then_with(|| b.last_used.cmp(&a.last_used)) |
| 162 | .then_with(|| a.identity.cmp(&b.identity)) |
| 163 | .then_with(|| a.model.cmp(&b.model)) |
| 164 | }); |
| 165 | ranked |
| 166 | } |
| 167 | |
| 168 | /// Model ids used on exactly this route within the window. |
| 169 | pub(crate) fn models_for_identity(&self, identity: &str, now: DateTime<Utc>) -> Vec<String> { |
| 170 | self.ranked(now) |
| 171 | .into_iter() |
| 172 | .filter(|usage| usage.identity == identity) |
| 173 | .map(|usage| usage.model.clone()) |
| 174 | .collect() |
| 175 | } |
| 176 | |
| 177 | pub(crate) fn identity_used(&self, identity: &str, now: DateTime<Utc>) -> bool { |
| 178 | let cutoff = now - Duration::days(WINDOW_DAYS); |
| 179 | self.entries |
| 180 | .values() |
| 181 | .any(|usage| usage.identity == identity && usage.last_used >= cutoff) |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | fn key(identity: &str, model: &str) -> RouteKey { |
| 186 | RouteKey { |
| 187 | identity: identity.trim().to_string(), |
| 188 | model: model.trim().to_ascii_lowercase(), |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | fn decayed_score(usage: &RouteUsage, now: DateTime<Utc>) -> f64 { |
| 193 | let age = now.signed_duration_since(usage.last_used); |
| 194 | if age > Duration::days(WINDOW_DAYS) { |
| 195 | return 0.0; |
| 196 | } |
| 197 | let age_days = (age.num_seconds().max(0) as f64) / 86_400.0; |
| 198 | f64::from(usage.sessions) * 0.5_f64.powf(age_days / HALF_LIFE_DAYS) |
| 199 | } |
| 200 | |
| 201 | /// `(identity, model)` from one `cost_status::route_receipt` string. The |
| 202 | /// identity falls back to the provider kind when the receipt carries `-`. |
| 203 | fn parse_route_receipt(receipt: &str) -> Option<(String, String)> { |
| 204 | let mut provider = None; |
| 205 | let mut identity = None; |
| 206 | let mut model = None; |
| 207 | for field in receipt.split_whitespace() { |
| 208 | if let Some(value) = field.strip_prefix("provider=") { |
| 209 | provider = Some(value); |
| 210 | } else if let Some(value) = field.strip_prefix("identity=") { |
| 211 | identity = Some(value); |
| 212 | } else if let Some(value) = field.strip_prefix("model=") { |
| 213 | model = Some(value); |
| 214 | } |
| 215 | } |
| 216 | let identity = identity |
| 217 | .filter(|id| *id != "-" && !id.is_empty()) |
| 218 | .or(provider)?; |
| 219 | let model = model.filter(|model| *model != "-" && !model.is_empty())?; |
| 220 | Some((identity.to_string(), model.to_string())) |
| 221 | } |
| 222 | |
| 223 | /// Build the index from the saved sessions on a dedicated thread and merge it |
| 224 | /// into `shared` when done. Listing reads only each file's metadata prefix, |
| 225 | /// but hundreds of files are still too slow for the UI thread. |
| 226 | pub(crate) fn spawn_build(shared: SharedRouteUsage) { |
| 227 | // Unit tests build `App`s by the hundred; they seed usage explicitly |
| 228 | // rather than racing a reader of the (hermetic) sessions directory. |
| 229 | if cfg!(test) { |
| 230 | return; |
| 231 | } |
| 232 | let spawned = std::thread::Builder::new() |
| 233 | .name("model-relevance".to_string()) |
| 234 | .spawn(move || { |
| 235 | let Ok(manager) = crate::session_manager::SessionManager::default_location() else { |
| 236 | return; |
| 237 | }; |
| 238 | let Ok(sessions) = manager.list_sessions() else { |
| 239 | return; |
| 240 | }; |
| 241 | let built = RouteUsageIndex::from_sessions(&sessions, Utc::now()); |
| 242 | if let Ok(mut index) = shared.write() { |
| 243 | index.merge(built); |
| 244 | } |
| 245 | }); |
| 246 | if let Err(error) = spawned { |
| 247 | tracing::warn!("model relevance index not built: {error}"); |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | #[cfg(test)] |
| 252 | mod tests { |
| 253 | use super::*; |
| 254 | use crate::session_manager::SessionMetadata; |
| 255 | |
| 256 | fn session( |
| 257 | provider: &str, |
| 258 | provider_id: Option<&str>, |
| 259 | model: &str, |
| 260 | age_days: i64, |
| 261 | now: DateTime<Utc>, |
| 262 | ) -> SessionMetadata { |
| 263 | let mut metadata: SessionMetadata = serde_json::from_value(serde_json::json!({ |
| 264 | "id": "s", |
| 265 | "title": "t", |
| 266 | "created_at": now, |
| 267 | "updated_at": now, |
| 268 | "message_count": 1, |
| 269 | "total_tokens": 0, |
| 270 | "model": model, |
| 271 | "model_provider": provider, |
| 272 | "workspace": "/tmp", |
| 273 | })) |
| 274 | .expect("session metadata"); |
| 275 | metadata.model_provider_id = provider_id.map(str::to_string); |
| 276 | metadata.updated_at = now - Duration::days(age_days); |
| 277 | metadata |
| 278 | } |
| 279 | |
| 280 | #[test] |
| 281 | fn recency_and_session_count_rank_with_a_one_week_half_life() { |
| 282 | let now = Utc::now(); |
| 283 | let sessions = vec![ |
| 284 | // Used a lot, but three weeks ago: 4 × 0.125 = 0.5. |
| 285 | session("zai", None, "GLM-5.3", 21, now), |
| 286 | session("zai", None, "GLM-5.3", 21, now), |
| 287 | session("zai", None, "GLM-5.3", 21, now), |
| 288 | session("zai", None, "glm-5.3", 21, now), |
| 289 | // Once, today: 1.0. |
| 290 | session("xai", None, "grok-4.7", 0, now), |
| 291 | // Outside the window: nothing. |
| 292 | session("zai", None, "GLM-5.2", 31, now), |
| 293 | ]; |
| 294 | let index = RouteUsageIndex::from_sessions(&sessions, now); |
| 295 | let ranked: Vec<_> = index |
| 296 | .ranked(now) |
| 297 | .iter() |
| 298 | .map(|usage| { |
| 299 | ( |
| 300 | usage.identity.as_str(), |
| 301 | usage.model.as_str(), |
| 302 | usage.sessions, |
| 303 | ) |
| 304 | }) |
| 305 | .collect(); |
| 306 | assert_eq!( |
| 307 | ranked, |
| 308 | vec![("xai", "grok-4.7", 1), ("zai", "GLM-5.3", 4)], |
| 309 | "case-insensitive model ids merge; stale sessions drop out" |
| 310 | ); |
| 311 | assert!((index.score("zai", "glm-5.3", now) - 0.5).abs() < 1e-9); |
| 312 | assert_eq!(index.score("zai", "GLM-5.2", now), 0.0); |
| 313 | assert!(!index.identity_used("openrouter", now)); |
| 314 | } |
| 315 | |
| 316 | #[test] |
| 317 | fn archived_sessions_are_excluded() { |
| 318 | let now = Utc::now(); |
| 319 | let mut archived = session("openrouter", None, "stealth/ox-alpha", 0, now); |
| 320 | archived.archived = true; |
| 321 | let index = RouteUsageIndex::from_sessions(&[archived], now); |
| 322 | assert!(index.ranked(now).is_empty()); |
| 323 | } |
| 324 | |
| 325 | #[test] |
| 326 | fn named_custom_routes_with_the_same_model_stay_distinct() { |
| 327 | let now = Utc::now(); |
| 328 | let sessions = vec![ |
| 329 | session("custom", Some("command_code"), "deepseek/v4", 0, now), |
| 330 | session("custom", Some("other_code"), "deepseek/v4", 2, now), |
| 331 | session("custom", Some("Other_Code"), "deepseek/v4", 3, now), |
| 332 | ]; |
| 333 | let index = RouteUsageIndex::from_sessions(&sessions, now); |
| 334 | assert_eq!(index.ranked(now).len(), 3); |
| 335 | assert_eq!( |
| 336 | index.models_for_identity("command_code", now), |
| 337 | ["deepseek/v4"] |
| 338 | ); |
| 339 | assert!(index.identity_used("Other_Code", now)); |
| 340 | assert!(!index.identity_used("custom", now)); |
| 341 | } |
| 342 | |
| 343 | #[test] |
| 344 | fn route_receipts_add_background_routes_once_per_session() { |
| 345 | let now = Utc::now(); |
| 346 | let mut metadata = session("deepseek", None, "deepseek-flash", 1, now); |
| 347 | for receipt in [ |
| 348 | "provider=xai identity=- model=grok-4.6 surface=api endpoint_fp=a billing_mode=metered currency=USD", |
| 349 | "provider=xai identity=xai model=grok-4.6 surface=api endpoint_fp=b billing_mode=metered currency=USD", |
| 350 | "provider=custom identity=command_code model=deepseek/v4 surface=api endpoint_fp=c billing_mode=metered currency=USD", |
| 351 | "provider=deepseek identity=deepseek model=deepseek-flash surface=api endpoint_fp=d billing_mode=metered currency=USD", |
| 352 | ] { |
| 353 | metadata.cost.route_receipts.insert(receipt.to_string()); |
| 354 | } |
| 355 | let index = RouteUsageIndex::from_sessions(&[metadata], now); |
| 356 | let sessions = |identity: &str, model: &str| { |
| 357 | index |
| 358 | .ranked(now) |
| 359 | .into_iter() |
| 360 | .find(|usage| usage.identity == identity && usage.model.eq_ignore_ascii_case(model)) |
| 361 | .map(|usage| usage.sessions) |
| 362 | }; |
| 363 | assert_eq!(sessions("xai", "grok-4.6"), Some(1)); |
| 364 | assert_eq!(sessions("command_code", "deepseek/v4"), Some(1)); |
| 365 | assert_eq!(sessions("deepseek", "deepseek-flash"), Some(1)); |
| 366 | } |
| 367 | |
| 368 | #[test] |
| 369 | fn in_session_record_outranks_old_use_and_survives_a_late_build() { |
| 370 | let now = Utc::now(); |
| 371 | let mut live = RouteUsageIndex::default(); |
| 372 | live.record("stepfun", "step-5-preview", now); |
| 373 | let built = RouteUsageIndex::from_sessions( |
| 374 | &[session("deepseek", None, "deepseek-v4-pro", 14, now)], |
| 375 | now, |
| 376 | ); |
| 377 | live.merge(built); |
| 378 | let order: Vec<_> = live |
| 379 | .ranked(now) |
| 380 | .iter() |
| 381 | .map(|usage| usage.model.clone()) |
| 382 | .collect(); |
| 383 | assert_eq!(order, ["step-5-preview", "deepseek-v4-pro"]); |
| 384 | } |
| 385 | |
| 386 | #[test] |
| 387 | fn switching_back_and_forth_counts_each_route_once_per_session() { |
| 388 | let now = Utc::now(); |
| 389 | let mut live = RouteUsageIndex::default(); |
| 390 | live.record("deepseek", "deepseek-v4-pro", now - Duration::minutes(5)); |
| 391 | live.record("xai", "grok-4.7", now - Duration::minutes(4)); |
| 392 | live.record("deepseek", "DeepSeek-V4-Pro", now - Duration::minutes(3)); |
| 393 | live.record("deepseek", "deepseek-v4-pro", now); |
| 394 | let pro = live |
| 395 | .ranked(now) |
| 396 | .into_iter() |
| 397 | .find(|usage| usage.identity == "deepseek") |
| 398 | .expect("deepseek route recorded"); |
| 399 | assert_eq!(pro.sessions, 1, "re-selecting a route is not a new session"); |
| 400 | assert_eq!(pro.last_used, now, "re-selecting refreshes last use"); |
| 401 | } |
| 402 | } |
| 403 |