| 1 | //! Token / cache / cost scorecard (#3388). |
| 2 | //! |
| 3 | //! A release-gate view of an agent run's token economics: per-turn input / |
| 4 | //! output / cache-read tokens and cost, aggregate totals + cache-hit ratio, and |
| 5 | //! regression detection against a committed baseline. This is the measurement |
| 6 | //! layer the "token, cache, and context discipline" EPIC asks for — it makes a |
| 7 | //! cost/token regression visible instead of silently shipping. |
| 8 | //! |
| 9 | //! The core here is pure and offline: it turns already-recorded per-turn |
| 10 | //! [`Usage`] (captured on every turn, persisted in `TurnRecord`) into a |
| 11 | //! scorecard, reusing the existing pricing layer rather than reinventing cost |
| 12 | //! math. The `scorecard` subcommand is a thin I/O wrapper over this module. |
| 13 | |
| 14 | use chrono::{DateTime, Utc}; |
| 15 | use serde::{Deserialize, Serialize}; |
| 16 | |
| 17 | use crate::config::ApiProvider; |
| 18 | #[cfg(test)] |
| 19 | use crate::config::{DEEPSEEK_ALIAS_REPLACEMENT, DEEPSEEK_ALIAS_RETIREMENT_UTC}; |
| 20 | use crate::models::Usage; |
| 21 | use crate::pricing::{ |
| 22 | CostEstimate, TurnCostAudit, audit_turn_cost_for_route_at, token_usage_for_pricing, |
| 23 | }; |
| 24 | |
| 25 | /// One turn's normalized token economics. |
| 26 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] |
| 27 | pub struct TurnScore { |
| 28 | pub turn_id: String, |
| 29 | /// Timestamp used for historical/time-window pricing. `None` means the |
| 30 | /// recorder did not preserve when the turn occurred. |
| 31 | #[serde(default, skip_serializing_if = "Option::is_none")] |
| 32 | pub created_at: Option<DateTime<Utc>>, |
| 33 | /// Effective provider recorded for this turn. `None` means legacy or |
| 34 | /// otherwise unknown provenance, so cost must remain unpriced. |
| 35 | #[serde(default)] |
| 36 | pub provider: Option<String>, |
| 37 | /// Non-secret discriminator when one provider/model pair spans multiple |
| 38 | /// billing systems. Missing provenance keeps ambiguous routes unpriced. |
| 39 | #[serde(default, skip_serializing_if = "Option::is_none")] |
| 40 | pub billing_surface: Option<String>, |
| 41 | pub model: String, |
| 42 | /// Non-cached (billable) input tokens. |
| 43 | pub input_tokens: u64, |
| 44 | /// Output tokens, including reasoning output. |
| 45 | pub output_tokens: u64, |
| 46 | /// Cache-read (cache-hit) input tokens. |
| 47 | pub cache_read_tokens: u64, |
| 48 | /// Cache-write (cache-creation) input tokens. Billed at a premium on the |
| 49 | /// providers that publish one, so it is audited as its own class rather |
| 50 | /// than folded into input. Defaults to 0 for legacy records. |
| 51 | #[serde(default)] |
| 52 | pub cache_write_tokens: u64, |
| 53 | /// Reasoning tokens reported for the turn. **Informational only** — every |
| 54 | /// provider counts these inside `output_tokens`, so adding them here would |
| 55 | /// double-bill. Kept so a reasoning-heavy run can still be inspected. |
| 56 | #[serde(default)] |
| 57 | pub reasoning_tokens: u64, |
| 58 | pub cost_usd: f64, |
| 59 | pub cost_cny: f64, |
| 60 | /// True when provider provenance is missing/unknown or no authoritative USD |
| 61 | /// pricing row exists: numeric cost stays 0 for compatibility, while this |
| 62 | /// flag prevents it from being represented as a real zero-dollar charge. |
| 63 | pub cost_unpriced: bool, |
| 64 | /// Same availability marker for CNY. Most catalog offerings publish only |
| 65 | /// USD, so their CNY value is unavailable rather than a real zero. |
| 66 | #[serde(default)] |
| 67 | pub cost_cny_unpriced: bool, |
| 68 | /// Why USD cost is unavailable, when it is (`no_pricing_row`, |
| 69 | /// `missing_class_price`, `not_money_metered`, …). `None` for priced turns. |
| 70 | #[serde(default, skip_serializing_if = "Option::is_none")] |
| 71 | pub cost_unpriced_reason: Option<String>, |
| 72 | /// Token classes this turn used that carry no published price. Non-empty |
| 73 | /// means the estimate failed closed on purpose. |
| 74 | #[serde(default, skip_serializing_if = "Vec::is_empty")] |
| 75 | pub unpriced_classes: Vec<String>, |
| 76 | /// Provenance of the pricing row that was applied or attempted |
| 77 | /// (`models_dev_bundled`, `provider_live`, `provider_docs`, |
| 78 | /// `user_override`). `None` when no row was found at all. |
| 79 | #[serde(default, skip_serializing_if = "Option::is_none")] |
| 80 | pub pricing_provenance: Option<String>, |
| 81 | /// Live-pricing downgrade receipt: the live catalog row for this route could |
| 82 | /// not be verified (stale, or fetched from a different endpoint), so the |
| 83 | /// bundled published rates were used. Present even on priced turns, because |
| 84 | /// it explains *which* row the number came from. |
| 85 | #[serde(default, skip_serializing_if = "Option::is_none")] |
| 86 | pub live_pricing_defect: Option<String>, |
| 87 | /// Whether this turn is inside the money-metered coverage denominator. |
| 88 | /// |
| 89 | /// False only for routes exactly identified as non-metered. Serialized so a |
| 90 | /// re-read scorecard can reproduce the coverage split without re-deriving it |
| 91 | /// from `provider` + `billing_surface`, which are also preserved above. |
| 92 | #[serde(default)] |
| 93 | pub money_metered: bool, |
| 94 | } |
| 95 | |
| 96 | /// Aggregate metrics for a run. Serializes/deserializes as the baseline file. |
| 97 | #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] |
| 98 | pub struct ScorecardMetrics { |
| 99 | pub turns: usize, |
| 100 | /// Turns whose route meters money, or whose billing basis could not be |
| 101 | /// established. This — not `turns` — is the denominator the USD total is |
| 102 | /// meant to cover: a local or subscription turn owes no dollars, so counting |
| 103 | /// it would understate coverage, while an unknown-basis turn must stay in |
| 104 | /// (#4318). Defaults to zero so existing baseline JSON stays readable. |
| 105 | #[serde(default)] |
| 106 | pub money_metered_turns: usize, |
| 107 | /// Money-metered turns that could not be priced authoritatively in USD. |
| 108 | /// Defaults to zero so existing baseline JSON remains readable. |
| 109 | #[serde(default)] |
| 110 | pub unpriced_turns: usize, |
| 111 | /// Turns without authoritative CNY pricing. |
| 112 | #[serde(default)] |
| 113 | pub cny_unpriced_turns: usize, |
| 114 | /// Whether every turn contributed authoritative USD pricing. Legacy |
| 115 | /// baselines lack this field and therefore default to `false`, preventing |
| 116 | /// comparisons against totals that may have been inferred from model ids |
| 117 | /// alone. |
| 118 | #[serde(default)] |
| 119 | pub cost_complete: bool, |
| 120 | /// Whether every turn contributed authoritative CNY pricing. |
| 121 | #[serde(default)] |
| 122 | pub cny_cost_complete: bool, |
| 123 | /// Token classes used somewhere in the run that had no published price, in |
| 124 | /// stable order. Non-empty means `cost_complete` is false *because* of a |
| 125 | /// class-level pricing gap, not merely an unknown route. |
| 126 | #[serde(default)] |
| 127 | pub unpriced_classes: Vec<String>, |
| 128 | pub total_input_tokens: u64, |
| 129 | pub total_output_tokens: u64, |
| 130 | pub total_cache_read_tokens: u64, |
| 131 | /// Cache-write (cache-creation) tokens across the run. Defaults to zero so |
| 132 | /// existing baseline JSON stays readable. |
| 133 | #[serde(default)] |
| 134 | pub total_cache_write_tokens: u64, |
| 135 | /// Reasoning tokens across the run. Informational: already inside |
| 136 | /// `total_output_tokens`, never added to it. |
| 137 | #[serde(default)] |
| 138 | pub total_reasoning_tokens: u64, |
| 139 | pub total_cost_usd: f64, |
| 140 | pub total_cost_cny: f64, |
| 141 | /// `cache_read / (input + cache_read)`; `0.0` when there are no input |
| 142 | /// tokens. Higher is better (more of the prompt was served from cache). |
| 143 | pub cache_hit_ratio: f64, |
| 144 | } |
| 145 | |
| 146 | /// A metric that grew beyond the allowed threshold versus the baseline. |
| 147 | #[derive(Debug, Clone, Serialize, PartialEq)] |
| 148 | pub struct Regression { |
| 149 | pub metric: String, |
| 150 | pub baseline: f64, |
| 151 | pub current: f64, |
| 152 | /// Percent increase over baseline. `f64::INFINITY` when baseline was 0. |
| 153 | pub pct_increase: f64, |
| 154 | } |
| 155 | |
| 156 | fn cacheable_token_total(input: u64, cache_read: u64, cache_write: u64) -> u64 { |
| 157 | input.saturating_add(cache_read).saturating_add(cache_write) |
| 158 | } |
| 159 | |
| 160 | /// Full scorecard: per-turn breakdown plus aggregates. |
| 161 | #[derive(Debug, Clone, Serialize)] |
| 162 | pub struct Scorecard { |
| 163 | pub per_turn: Vec<TurnScore>, |
| 164 | pub metrics: ScorecardMetrics, |
| 165 | } |
| 166 | |
| 167 | /// One row of input to the scorecard: a turn id, the model that served it, and |
| 168 | /// the turn's recorded usage. |
| 169 | /// |
| 170 | /// `billing_surface` is explicit and has no default. The scorecard has two |
| 171 | /// entry modes, and they must agree: if this fixture mode could silently supply |
| 172 | /// an official first-party surface, every scorecard test would be asserting |
| 173 | /// against a route classification that `from_recorded_turns` never invents, and |
| 174 | /// the fail-closed path would go unexercised in the mode the tests use. |
| 175 | #[cfg(test)] |
| 176 | pub struct TurnInput<'a> { |
| 177 | pub turn_id: String, |
| 178 | pub created_at: Option<&'a DateTime<Utc>>, |
| 179 | pub provider: Option<&'a str>, |
| 180 | /// The route's recorded billing surface, or `None` when the recording did |
| 181 | /// not establish one. `None` must price exactly as it does for a recorded |
| 182 | /// turn: unknown, never official. |
| 183 | pub billing_surface: Option<&'a str>, |
| 184 | pub model: String, |
| 185 | pub usage: &'a Usage, |
| 186 | } |
| 187 | |
| 188 | #[derive(Debug, Clone, Copy)] |
| 189 | struct ScorecardTurnRef<'a> { |
| 190 | turn_id: &'a str, |
| 191 | created_at: Option<&'a DateTime<Utc>>, |
| 192 | provider: Option<&'a str>, |
| 193 | billing_surface: Option<&'a str>, |
| 194 | model: &'a str, |
| 195 | usage: &'a Usage, |
| 196 | } |
| 197 | |
| 198 | /// A recorded turn as read from a scorecard input file (a JSON array of these). |
| 199 | /// The base shape matches the per-turn data a `TurnEnd` hook emits. Recorders |
| 200 | /// and persisted runtime exports can add `provider` / `effective_provider` plus |
| 201 | /// non-secret billing-surface provenance. Legacy model-only recordings remain |
| 202 | /// readable but deliberately unpriced. |
| 203 | #[derive(Debug, Clone, Deserialize)] |
| 204 | pub struct RecordedTurn { |
| 205 | #[serde(default, alias = "id")] |
| 206 | pub turn_id: String, |
| 207 | #[serde(default)] |
| 208 | pub created_at: Option<DateTime<Utc>>, |
| 209 | /// New `turn_end` hooks mark shell-only lifecycle records false so the |
| 210 | /// model-cost scorecard can ignore them. Missing stays compatible with |
| 211 | /// legacy hook rows and persisted runtime turns, which are model-backed. |
| 212 | #[serde(default)] |
| 213 | pub model_backed: Option<bool>, |
| 214 | #[serde(default, alias = "effective_provider")] |
| 215 | pub provider: Option<String>, |
| 216 | #[serde(default, alias = "effective_billing_surface")] |
| 217 | pub billing_surface: Option<String>, |
| 218 | #[serde(default, alias = "effective_model")] |
| 219 | pub model: String, |
| 220 | #[serde(default)] |
| 221 | pub usage: Option<Usage>, |
| 222 | } |
| 223 | |
| 224 | impl RecordedTurn { |
| 225 | #[must_use] |
| 226 | pub fn contributes_to_scorecard(&self) -> bool { |
| 227 | self.model_backed.unwrap_or(true) && self.usage.is_some() && !self.model.trim().is_empty() |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | #[derive(Debug, Clone, Default)] |
| 232 | struct AvailableCost { |
| 233 | usd: Option<f64>, |
| 234 | cny: Option<f64>, |
| 235 | unpriced_reason: Option<String>, |
| 236 | unpriced_classes: Vec<String>, |
| 237 | provenance: Option<String>, |
| 238 | /// Live-pricing downgrade receipt, when the row used was a bundled fallback |
| 239 | /// for an unverifiable live row. |
| 240 | live_pricing_defect: Option<String>, |
| 241 | /// Whether this turn belongs in the money-metered coverage denominator. |
| 242 | /// False only for routes *exactly* identified as non-metered. |
| 243 | counts_toward_money_coverage: bool, |
| 244 | } |
| 245 | |
| 246 | impl AvailableCost { |
| 247 | /// Legacy/unknown provenance: no route to price against at all. |
| 248 | /// |
| 249 | /// This still counts toward money coverage. A recording whose provider text |
| 250 | /// CodeWhale cannot parse is a turn whose spend is unknown, not a turn that |
| 251 | /// cost nothing — excusing it would let a legacy input file report a complete |
| 252 | /// total (#4318). |
| 253 | fn unknown_route() -> Self { |
| 254 | Self { |
| 255 | unpriced_reason: Some("unknown_route".to_string()), |
| 256 | counts_toward_money_coverage: true, |
| 257 | ..Self::default() |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | /// Fails closed with an explicit reason, still inside money coverage. |
| 262 | fn failed_closed(reason: &str) -> Self { |
| 263 | Self { |
| 264 | unpriced_reason: Some(reason.to_string()), |
| 265 | counts_toward_money_coverage: true, |
| 266 | ..Self::default() |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | fn from_audit(audit: &TurnCostAudit) -> Self { |
| 271 | Self { |
| 272 | usd: audit |
| 273 | .estimate |
| 274 | .and_then(|cost| audit.usd_priced.then_some(cost.usd)), |
| 275 | cny: audit |
| 276 | .estimate |
| 277 | .and_then(|cost| audit.cny_priced.then_some(cost.cny)), |
| 278 | unpriced_reason: audit |
| 279 | .unpriced_reason |
| 280 | .map(|reason| reason.label().to_string()), |
| 281 | unpriced_classes: audit |
| 282 | .unpriced_classes |
| 283 | .iter() |
| 284 | .map(|class| class.label().to_string()) |
| 285 | .collect(), |
| 286 | provenance: audit |
| 287 | .provenance |
| 288 | .as_ref() |
| 289 | .map(|provenance| provenance.label().to_string()), |
| 290 | live_pricing_defect: audit |
| 291 | .live_pricing_defect |
| 292 | .as_ref() |
| 293 | .map(|defect| defect.label().to_string()), |
| 294 | counts_toward_money_coverage: audit.counts_toward_money_coverage(), |
| 295 | } |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | fn provider_scoped_cost( |
| 300 | provider: ApiProvider, |
| 301 | model: &str, |
| 302 | usage: &Usage, |
| 303 | created_at: Option<&DateTime<Utc>>, |
| 304 | billing_surface: Option<&str>, |
| 305 | ) -> AvailableCost { |
| 306 | // These provider identities are themselves exact billing provenance and |
| 307 | // override stale/junk recorded surfaces: they cannot become PAYG merely |
| 308 | // because an older recorder wrote a bogus endpoint classification. |
| 309 | let intrinsic_surface = match provider { |
| 310 | ApiProvider::Ollama | ApiProvider::Sglang | ApiProvider::Vllm => { |
| 311 | Some(crate::pricing::LOCAL_BILLING_SURFACE) |
| 312 | } |
| 313 | ApiProvider::OpenaiCodex | ApiProvider::OpencodeGo => { |
| 314 | Some(crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE) |
| 315 | } |
| 316 | _ => None, |
| 317 | }; |
| 318 | let billing_surface = intrinsic_surface.or(billing_surface); |
| 319 | // Every provider that supports both PAYG and plan/OAuth routes needs the |
| 320 | // recorded surface to choose between them. A model id or provider name is |
| 321 | // not sufficient evidence in an offline scorecard. |
| 322 | if billing_surface.is_none() |
| 323 | && matches!( |
| 324 | provider, |
| 325 | ApiProvider::Zai |
| 326 | | ApiProvider::Moonshot |
| 327 | | ApiProvider::Anthropic |
| 328 | | ApiProvider::XiaomiMimo |
| 329 | | ApiProvider::Xai |
| 330 | | ApiProvider::Minimax |
| 331 | | ApiProvider::MinimaxAnthropic |
| 332 | | ApiProvider::Stepfun |
| 333 | | ApiProvider::Custom |
| 334 | ) |
| 335 | { |
| 336 | return AvailableCost::failed_closed("missing_billing_surface"); |
| 337 | } |
| 338 | let direct_deepseek = matches!( |
| 339 | provider, |
| 340 | ApiProvider::Deepseek | ApiProvider::DeepseekCN | ApiProvider::DeepseekAnthropic |
| 341 | ); |
| 342 | let normalized_model = model.trim(); |
| 343 | let model_lower = normalized_model.to_ascii_lowercase(); |
| 344 | let needs_recorded_time = (direct_deepseek |
| 345 | && matches!(model_lower.as_str(), "deepseek-chat" | "deepseek-reasoner")) |
| 346 | || (provider == ApiProvider::Anthropic && model_lower == "claude-sonnet-5"); |
| 347 | let recorded_at = match (created_at, needs_recorded_time) { |
| 348 | (Some(recorded_at), _) => recorded_at.to_owned(), |
| 349 | // A time-windowed rate without a recorded time cannot be resolved to a |
| 350 | // single price; fail closed rather than guess a window. |
| 351 | (None, true) => return AvailableCost::failed_closed("missing_recorded_time"), |
| 352 | (None, false) => Utc::now(), |
| 353 | }; |
| 354 | |
| 355 | // The billing surface recorded with the turn is authoritative over any |
| 356 | // provider-level assumption, and it now covers every classification a route |
| 357 | // can carry — Z.ai Coding Plan, Kimi Code, MiniMax Token Plan, MiMo token |
| 358 | // plan, OAuth brokers, local runtimes, aggregators, first-party PAYG, and |
| 359 | // "unclassified" — not just StepFun's two surfaces (#4318). |
| 360 | match crate::pricing::endpoint_metering_for_billing_surface(billing_surface) { |
| 361 | // Exactly identified as non-metered: no dollar figure is owed, and the |
| 362 | // turn leaves the money-coverage denominator. |
| 363 | crate::pricing::EndpointMetering::ExactSubscription |
| 364 | | crate::pricing::EndpointMetering::LocalNoBill => { |
| 365 | return AvailableCost { |
| 366 | unpriced_reason: Some( |
| 367 | crate::pricing::UnpricedReason::NotMoneyMetered |
| 368 | .label() |
| 369 | .to_string(), |
| 370 | ), |
| 371 | counts_toward_money_coverage: false, |
| 372 | ..AvailableCost::default() |
| 373 | }; |
| 374 | } |
| 375 | // A recorded surface CodeWhale cannot place must not inherit the |
| 376 | // provider's default rates. |
| 377 | crate::pricing::EndpointMetering::Unknown if billing_surface.is_some() => { |
| 378 | return AvailableCost::failed_closed( |
| 379 | crate::pricing::UnpricedReason::UnknownBillingBasis.label(), |
| 380 | ); |
| 381 | } |
| 382 | crate::pricing::EndpointMetering::Unknown | crate::pricing::EndpointMetering::Money => {} |
| 383 | } |
| 384 | |
| 385 | // The pricing layer owns the exact provider/model catalog gate, explicit |
| 386 | // first-party hand-price allowlist, cache-class completeness checks, and |
| 387 | // endpoint-derived billing surfaces. Keeping one route-aware path prevents |
| 388 | // the scorecard from drifting back to model-only pricing. |
| 389 | let audit = audit_turn_cost_for_route_at( |
| 390 | provider, |
| 391 | normalized_model, |
| 392 | billing_surface, |
| 393 | usage, |
| 394 | recorded_at, |
| 395 | ); |
| 396 | AvailableCost::from_audit(&audit) |
| 397 | } |
| 398 | |
| 399 | impl Scorecard { |
| 400 | /// Build a scorecard from recorded per-turn usage. Pure + offline; cost is |
| 401 | /// computed via the shared pricing layer (`None` pricing → unpriced, 0 cost). |
| 402 | #[must_use] |
| 403 | #[cfg(test)] |
| 404 | pub fn from_turns(turns: &[TurnInput<'_>]) -> Self { |
| 405 | Self::from_turn_refs(turns.iter().map(|turn| ScorecardTurnRef { |
| 406 | turn_id: &turn.turn_id, |
| 407 | created_at: turn.created_at, |
| 408 | provider: turn.provider, |
| 409 | billing_surface: turn.billing_surface, |
| 410 | model: &turn.model, |
| 411 | usage: turn.usage, |
| 412 | })) |
| 413 | } |
| 414 | |
| 415 | /// Build directly from hook/runtime records, retaining billing provenance |
| 416 | /// while excluding explicitly non-model lifecycle rows. |
| 417 | #[must_use] |
| 418 | pub fn from_recorded_turns(turns: &[RecordedTurn]) -> Self { |
| 419 | Self::from_turn_refs(turns.iter().filter_map(|turn| { |
| 420 | if !turn.contributes_to_scorecard() { |
| 421 | return None; |
| 422 | } |
| 423 | let usage = turn.usage.as_ref()?; |
| 424 | Some(ScorecardTurnRef { |
| 425 | turn_id: &turn.turn_id, |
| 426 | created_at: turn.created_at.as_ref(), |
| 427 | provider: turn.provider.as_deref(), |
| 428 | billing_surface: turn.billing_surface.as_deref(), |
| 429 | model: &turn.model, |
| 430 | usage, |
| 431 | }) |
| 432 | })) |
| 433 | } |
| 434 | |
| 435 | fn from_turn_refs<'a>(turns: impl IntoIterator<Item = ScorecardTurnRef<'a>>) -> Self { |
| 436 | let turns = turns.into_iter(); |
| 437 | let mut per_turn = Vec::with_capacity(turns.size_hint().0); |
| 438 | let mut metrics = ScorecardMetrics::default(); |
| 439 | let mut unpriced_classes = std::collections::BTreeSet::new(); |
| 440 | |
| 441 | for turn in turns { |
| 442 | // Normalize provider usage into canonical billable classes once. |
| 443 | let classes = token_usage_for_pricing(turn.usage); |
| 444 | let provider = turn |
| 445 | .provider |
| 446 | .map(str::trim) |
| 447 | .filter(|value| !value.is_empty()); |
| 448 | let cost = provider.and_then(ApiProvider::parse).map_or_else( |
| 449 | AvailableCost::unknown_route, |
| 450 | |provider| { |
| 451 | provider_scoped_cost( |
| 452 | provider, |
| 453 | turn.model, |
| 454 | turn.usage, |
| 455 | turn.created_at, |
| 456 | turn.billing_surface, |
| 457 | ) |
| 458 | }, |
| 459 | ); |
| 460 | let cost_unpriced = cost.usd.is_none(); |
| 461 | let cost_cny_unpriced = cost.cny.is_none(); |
| 462 | let cost_usd = cost.usd.unwrap_or(0.0); |
| 463 | let cost_cny = cost.cny.unwrap_or(0.0); |
| 464 | let reasoning_tokens = u64::from(turn.usage.reasoning_tokens.unwrap_or(0)); |
| 465 | unpriced_classes.extend(cost.unpriced_classes.iter().cloned()); |
| 466 | |
| 467 | metrics.turns = metrics.turns.saturating_add(1); |
| 468 | // Only money-metered turns can make a dollar total incomplete. A |
| 469 | // local or plan turn is not an unpriced dollar; an *unknown* one is. |
| 470 | if cost.counts_toward_money_coverage { |
| 471 | metrics.money_metered_turns = metrics.money_metered_turns.saturating_add(1); |
| 472 | metrics.unpriced_turns = metrics |
| 473 | .unpriced_turns |
| 474 | .saturating_add(usize::from(cost_unpriced)); |
| 475 | metrics.cny_unpriced_turns = metrics |
| 476 | .cny_unpriced_turns |
| 477 | .saturating_add(usize::from(cost_cny_unpriced)); |
| 478 | } |
| 479 | metrics.total_input_tokens = metrics.total_input_tokens.saturating_add(classes.input); |
| 480 | metrics.total_output_tokens = |
| 481 | metrics.total_output_tokens.saturating_add(classes.output); |
| 482 | metrics.total_cache_read_tokens = metrics |
| 483 | .total_cache_read_tokens |
| 484 | .saturating_add(classes.cache_read); |
| 485 | metrics.total_cache_write_tokens = metrics |
| 486 | .total_cache_write_tokens |
| 487 | .saturating_add(classes.cache_write); |
| 488 | metrics.total_reasoning_tokens = metrics |
| 489 | .total_reasoning_tokens |
| 490 | .saturating_add(reasoning_tokens); |
| 491 | metrics.total_cost_usd = CostEstimate::usd_only(metrics.total_cost_usd) |
| 492 | .saturating_add(CostEstimate::usd_only(cost_usd)) |
| 493 | .usd; |
| 494 | metrics.total_cost_cny = CostEstimate { |
| 495 | usd: 0.0, |
| 496 | cny: metrics.total_cost_cny, |
| 497 | } |
| 498 | .saturating_add(CostEstimate { |
| 499 | usd: 0.0, |
| 500 | cny: cost_cny, |
| 501 | }) |
| 502 | .cny; |
| 503 | |
| 504 | per_turn.push(TurnScore { |
| 505 | turn_id: turn.turn_id.to_string(), |
| 506 | created_at: turn.created_at.cloned(), |
| 507 | provider: provider.map(str::to_string), |
| 508 | billing_surface: turn.billing_surface.map(str::to_string), |
| 509 | model: turn.model.to_string(), |
| 510 | input_tokens: classes.input, |
| 511 | output_tokens: classes.output, |
| 512 | cache_read_tokens: classes.cache_read, |
| 513 | cache_write_tokens: classes.cache_write, |
| 514 | reasoning_tokens, |
| 515 | cost_usd, |
| 516 | cost_cny, |
| 517 | cost_unpriced, |
| 518 | cost_cny_unpriced, |
| 519 | cost_unpriced_reason: cost.unpriced_reason, |
| 520 | unpriced_classes: cost.unpriced_classes, |
| 521 | pricing_provenance: cost.provenance, |
| 522 | live_pricing_defect: cost.live_pricing_defect, |
| 523 | money_metered: cost.counts_toward_money_coverage, |
| 524 | }); |
| 525 | } |
| 526 | metrics.unpriced_classes = unpriced_classes.into_iter().collect(); |
| 527 | |
| 528 | // Canonical denominator: hit / (non-cached input + hit + write). |
| 529 | // `total_input_tokens` here is already the *non-cached* input, because |
| 530 | // `token_usage_for_pricing` splits hits and writes out of the reported |
| 531 | // prompt total. Cache-write tokens were previously missing from the |
| 532 | // denominator, which reported a better hit ratio on precisely the turns |
| 533 | // that paid to populate the cache (#4318). Write stays a separate |
| 534 | // reported total so the premium is not hidden inside the ratio. |
| 535 | let cacheable = cacheable_token_total( |
| 536 | metrics.total_input_tokens, |
| 537 | metrics.total_cache_read_tokens, |
| 538 | metrics.total_cache_write_tokens, |
| 539 | ); |
| 540 | metrics.cache_hit_ratio = if cacheable > 0 { |
| 541 | metrics.total_cache_read_tokens as f64 / cacheable as f64 |
| 542 | } else { |
| 543 | 0.0 |
| 544 | }; |
| 545 | metrics.cost_complete = metrics.unpriced_turns == 0; |
| 546 | metrics.cny_cost_complete = metrics.cny_unpriced_turns == 0; |
| 547 | |
| 548 | Self { per_turn, metrics } |
| 549 | } |
| 550 | |
| 551 | /// Render a compact human-readable summary (used for non-JSON output). |
| 552 | #[must_use] |
| 553 | pub fn to_summary(&self) -> String { |
| 554 | let m = &self.metrics; |
| 555 | let mut out = String::new(); |
| 556 | out.push_str("Token / cache / cost scorecard\n"); |
| 557 | out.push_str(&format!( |
| 558 | "turns: {} money_metered_turns: {}\n", |
| 559 | m.turns, m.money_metered_turns |
| 560 | )); |
| 561 | out.push_str(&format!( |
| 562 | "input_tokens: {} output_tokens: {} cache_read_tokens: {} cache_write_tokens: {}\n", |
| 563 | m.total_input_tokens, |
| 564 | m.total_output_tokens, |
| 565 | m.total_cache_read_tokens, |
| 566 | m.total_cache_write_tokens |
| 567 | )); |
| 568 | out.push_str(&format!( |
| 569 | "reasoning_tokens: {} (informational; already inside output_tokens)\n", |
| 570 | m.total_reasoning_tokens |
| 571 | )); |
| 572 | out.push_str(&format!( |
| 573 | "cache_hit_ratio: {:.1}%\n", |
| 574 | m.cache_hit_ratio * 100.0 |
| 575 | )); |
| 576 | append_currency_summary( |
| 577 | &mut out, |
| 578 | "cost_usd", |
| 579 | "priced_cost_subtotal_usd", |
| 580 | "$", |
| 581 | m.total_cost_usd, |
| 582 | m.unpriced_turns, |
| 583 | // Coverage is reported against the money-metered turns, not every |
| 584 | // turn: a local or plan turn owes no dollars, so including it in the |
| 585 | // denominator would understate how complete the figure is. |
| 586 | m.money_metered_turns, |
| 587 | ); |
| 588 | append_currency_summary( |
| 589 | &mut out, |
| 590 | "cost_cny", |
| 591 | "priced_cost_subtotal_cny", |
| 592 | "¥", |
| 593 | m.total_cost_cny, |
| 594 | m.cny_unpriced_turns, |
| 595 | m.money_metered_turns, |
| 596 | ); |
| 597 | if m.unpriced_turns > 0 { |
| 598 | out.push_str(&format!( |
| 599 | "note: {} turn(s) had missing/unknown provider provenance or no authoritative USD pricing row; their USD cost is unavailable and excluded.\n", |
| 600 | m.unpriced_turns |
| 601 | )); |
| 602 | } |
| 603 | if m.cny_unpriced_turns > 0 { |
| 604 | out.push_str(&format!( |
| 605 | "note: {} turn(s) had no authoritative CNY pricing row; their CNY cost is unavailable and excluded.\n", |
| 606 | m.cny_unpriced_turns |
| 607 | )); |
| 608 | } |
| 609 | if !m.unpriced_classes.is_empty() { |
| 610 | out.push_str(&format!( |
| 611 | "note: token class(es) with no published price on a used route: {}. Those turns fail closed rather than under-report.\n", |
| 612 | m.unpriced_classes.join(", ") |
| 613 | )); |
| 614 | } |
| 615 | out |
| 616 | } |
| 617 | } |
| 618 | |
| 619 | fn append_currency_summary( |
| 620 | out: &mut String, |
| 621 | complete_label: &str, |
| 622 | subtotal_label: &str, |
| 623 | symbol: &str, |
| 624 | total: f64, |
| 625 | unpriced_turns: usize, |
| 626 | turns: usize, |
| 627 | ) { |
| 628 | if unpriced_turns == 0 { |
| 629 | out.push_str(&format!("{complete_label}: {symbol}{total:.4}\n")); |
| 630 | } else if unpriced_turns == turns { |
| 631 | out.push_str(&format!("{complete_label}: unavailable\n")); |
| 632 | } else { |
| 633 | out.push_str(&format!("{subtotal_label}: {symbol}{total:.4}\n")); |
| 634 | } |
| 635 | } |
| 636 | |
| 637 | impl ScorecardMetrics { |
| 638 | /// Flag metrics that grew more than `threshold_pct` over `baseline`. Cost |
| 639 | /// and token counts are "lower is better", so only *increases* are |
| 640 | /// regressions. (Cache-hit ratio is the opposite, reported separately.) |
| 641 | #[must_use] |
| 642 | pub fn regressions_against( |
| 643 | &self, |
| 644 | baseline: &ScorecardMetrics, |
| 645 | threshold_pct: f64, |
| 646 | ) -> Vec<Regression> { |
| 647 | let mut out = Vec::new(); |
| 648 | // A partial/unknown subtotal is not comparable to a complete baseline, |
| 649 | // but losing completeness is itself a regression. Otherwise removing |
| 650 | // provider provenance could turn real spend into a smaller subtotal |
| 651 | // and silently bypass the release gate. |
| 652 | if baseline.cost_complete && !self.cost_complete { |
| 653 | out.push(Regression { |
| 654 | metric: "cost_completeness_drop".to_string(), |
| 655 | baseline: 1.0, |
| 656 | current: 0.0, |
| 657 | pct_increase: 100.0, |
| 658 | }); |
| 659 | } else if self.cost_complete && baseline.cost_complete { |
| 660 | push_regression( |
| 661 | &mut out, |
| 662 | "total_cost_usd", |
| 663 | baseline.total_cost_usd, |
| 664 | self.total_cost_usd, |
| 665 | threshold_pct, |
| 666 | ); |
| 667 | } |
| 668 | if baseline.cny_cost_complete && !self.cny_cost_complete { |
| 669 | out.push(Regression { |
| 670 | metric: "cny_cost_completeness_drop".to_string(), |
| 671 | baseline: 1.0, |
| 672 | current: 0.0, |
| 673 | pct_increase: 100.0, |
| 674 | }); |
| 675 | } else if self.cny_cost_complete && baseline.cny_cost_complete { |
| 676 | push_regression( |
| 677 | &mut out, |
| 678 | "total_cost_cny", |
| 679 | baseline.total_cost_cny, |
| 680 | self.total_cost_cny, |
| 681 | threshold_pct, |
| 682 | ); |
| 683 | } |
| 684 | push_regression( |
| 685 | &mut out, |
| 686 | "total_input_tokens", |
| 687 | baseline.total_input_tokens as f64, |
| 688 | self.total_input_tokens as f64, |
| 689 | threshold_pct, |
| 690 | ); |
| 691 | push_regression( |
| 692 | &mut out, |
| 693 | "total_output_tokens", |
| 694 | baseline.total_output_tokens as f64, |
| 695 | self.total_output_tokens as f64, |
| 696 | threshold_pct, |
| 697 | ); |
| 698 | // Cache-hit ratio regresses when it *drops*; express the drop as a |
| 699 | // positive percentage so it reads like the others. |
| 700 | if baseline.cache_hit_ratio > 0.0 { |
| 701 | let drop_pct = (baseline.cache_hit_ratio - self.cache_hit_ratio) |
| 702 | / baseline.cache_hit_ratio |
| 703 | * 100.0; |
| 704 | if drop_pct > threshold_pct { |
| 705 | out.push(Regression { |
| 706 | metric: "cache_hit_ratio_drop".to_string(), |
| 707 | baseline: baseline.cache_hit_ratio, |
| 708 | current: self.cache_hit_ratio, |
| 709 | pct_increase: drop_pct, |
| 710 | }); |
| 711 | } |
| 712 | } |
| 713 | out |
| 714 | } |
| 715 | } |
| 716 | |
| 717 | fn push_regression( |
| 718 | out: &mut Vec<Regression>, |
| 719 | metric: &str, |
| 720 | base: f64, |
| 721 | cur: f64, |
| 722 | threshold_pct: f64, |
| 723 | ) { |
| 724 | if base > 0.0 { |
| 725 | let pct = (cur - base) / base * 100.0; |
| 726 | if pct > threshold_pct { |
| 727 | out.push(Regression { |
| 728 | metric: metric.to_string(), |
| 729 | baseline: base, |
| 730 | current: cur, |
| 731 | pct_increase: pct, |
| 732 | }); |
| 733 | } |
| 734 | } else if cur > 0.0 { |
| 735 | out.push(Regression { |
| 736 | metric: metric.to_string(), |
| 737 | baseline: base, |
| 738 | current: cur, |
| 739 | pct_increase: f64::INFINITY, |
| 740 | }); |
| 741 | } |
| 742 | } |
| 743 | |
| 744 | #[cfg(test)] |
| 745 | mod tests { |
| 746 | use super::*; |
| 747 | |
| 748 | fn usage(input: u32, output: u32, cache_hit: u32) -> Usage { |
| 749 | Usage { |
| 750 | input_tokens: input, |
| 751 | output_tokens: output, |
| 752 | prompt_cache_hit_tokens: Some(cache_hit), |
| 753 | ..Default::default() |
| 754 | } |
| 755 | } |
| 756 | |
| 757 | /// The scorecard has two entry modes and they must classify identically. |
| 758 | /// |
| 759 | /// `from_turns` is the fixture mode used by this test module; |
| 760 | /// `from_recorded_turns` is the mode that reads real recordings. The |
| 761 | /// fixture mode used to inject `first-party-payg` for every row, which |
| 762 | /// meant the whole suite was asserting against a route classification the |
| 763 | /// real mode never produces — the fail-closed path was untested precisely |
| 764 | /// where it mattered. Neither mode may invent an official surface. |
| 765 | #[test] |
| 766 | fn both_scorecard_entry_modes_agree_on_an_unestablished_billing_surface() { |
| 767 | let sample = usage(10_000, 1_000, 0); |
| 768 | |
| 769 | let fixture = Scorecard::from_turns(&[TurnInput { |
| 770 | turn_id: "t1".into(), |
| 771 | created_at: None, |
| 772 | provider: Some("anthropic"), |
| 773 | billing_surface: None, |
| 774 | model: "claude-haiku-4-5".into(), |
| 775 | usage: &sample, |
| 776 | }]); |
| 777 | let recorded = Scorecard::from_recorded_turns(&[RecordedTurn { |
| 778 | turn_id: "t1".to_string(), |
| 779 | created_at: None, |
| 780 | model_backed: None, |
| 781 | provider: Some("anthropic".to_string()), |
| 782 | billing_surface: None, |
| 783 | model: "claude-haiku-4-5".to_string(), |
| 784 | usage: Some(sample.clone()), |
| 785 | }]); |
| 786 | |
| 787 | assert_eq!(fixture.per_turn, recorded.per_turn); |
| 788 | assert_eq!(fixture.metrics, recorded.metrics); |
| 789 | assert!( |
| 790 | fixture.per_turn[0].cost_unpriced, |
| 791 | "a route with no established surface must not be priced" |
| 792 | ); |
| 793 | assert_eq!(fixture.per_turn[0].cost_usd, 0.0); |
| 794 | assert!(!fixture.metrics.cost_complete); |
| 795 | assert_eq!(fixture.metrics.money_metered_turns, 1); |
| 796 | assert_eq!(fixture.metrics.unpriced_turns, 1); |
| 797 | let summary = fixture.to_summary(); |
| 798 | assert!(summary.contains("cost_usd: unavailable"), "{summary}"); |
| 799 | assert!(summary.contains("cost_cny: unavailable"), "{summary}"); |
| 800 | assert!( |
| 801 | !summary.contains('$') && !summary.contains('¥'), |
| 802 | "an unpriced-only run must name no amount at all: {summary}" |
| 803 | ); |
| 804 | |
| 805 | // With the surface actually established, both modes price it — and |
| 806 | // still agree. |
| 807 | let priced_fixture = Scorecard::from_turns(&[TurnInput { |
| 808 | turn_id: "t1".into(), |
| 809 | created_at: None, |
| 810 | provider: Some("anthropic"), |
| 811 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 812 | model: "claude-haiku-4-5".into(), |
| 813 | usage: &sample, |
| 814 | }]); |
| 815 | let priced_recorded = Scorecard::from_recorded_turns(&[RecordedTurn { |
| 816 | turn_id: "t1".to_string(), |
| 817 | created_at: None, |
| 818 | model_backed: None, |
| 819 | provider: Some("anthropic".to_string()), |
| 820 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE.to_string()), |
| 821 | model: "claude-haiku-4-5".to_string(), |
| 822 | usage: Some(sample), |
| 823 | }]); |
| 824 | assert_eq!(priced_fixture.per_turn, priced_recorded.per_turn); |
| 825 | assert!(!priced_fixture.per_turn[0].cost_unpriced); |
| 826 | assert!(priced_fixture.metrics.cost_complete); |
| 827 | } |
| 828 | |
| 829 | #[test] |
| 830 | fn dual_mode_routes_require_surface_but_intrinsic_routes_override_junk() { |
| 831 | let usage = usage(10_000, 1_000, 0); |
| 832 | for (provider, model) in [ |
| 833 | (ApiProvider::Anthropic, "claude-haiku-4-5"), |
| 834 | (ApiProvider::Moonshot, "kimi-k2.7-code"), |
| 835 | (ApiProvider::Zai, "glm-5.2"), |
| 836 | (ApiProvider::Minimax, "minimax-m3"), |
| 837 | ] { |
| 838 | let cost = provider_scoped_cost(provider, model, &usage, None, None); |
| 839 | assert_eq!( |
| 840 | cost.unpriced_reason.as_deref(), |
| 841 | Some("missing_billing_surface"), |
| 842 | "{provider:?}" |
| 843 | ); |
| 844 | assert!(cost.usd.is_none(), "{provider:?}"); |
| 845 | } |
| 846 | |
| 847 | for provider in [ApiProvider::OpenaiCodex, ApiProvider::OpencodeGo] { |
| 848 | let cost = provider_scoped_cost( |
| 849 | provider, |
| 850 | "gpt-5.5", |
| 851 | &usage, |
| 852 | None, |
| 853 | Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 854 | ); |
| 855 | assert_eq!(cost.unpriced_reason.as_deref(), Some("not_money_metered")); |
| 856 | assert!(!cost.counts_toward_money_coverage); |
| 857 | } |
| 858 | let local = provider_scoped_cost( |
| 859 | ApiProvider::Ollama, |
| 860 | "llama3.2", |
| 861 | &usage, |
| 862 | None, |
| 863 | Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE), |
| 864 | ); |
| 865 | assert_eq!(local.unpriced_reason.as_deref(), Some("not_money_metered")); |
| 866 | assert!(!local.counts_toward_money_coverage); |
| 867 | } |
| 868 | |
| 869 | fn cache_write_usage(input: u32, output: u32, cache_hit: u32, cache_write: u32) -> Usage { |
| 870 | Usage { |
| 871 | input_tokens: input, |
| 872 | output_tokens: output, |
| 873 | prompt_cache_hit_tokens: Some(cache_hit), |
| 874 | prompt_cache_write_tokens: Some(cache_write), |
| 875 | reasoning_tokens: Some(output / 2), |
| 876 | ..Default::default() |
| 877 | } |
| 878 | } |
| 879 | |
| 880 | /// A mixed-route run: one fully-priced cache-write turn, one turn whose |
| 881 | /// route publishes no cache-write rate, and one non-metered OAuth turn. |
| 882 | /// The priced subtotal stays honest, `cost_complete` fails closed, and the |
| 883 | /// audit names the class and provenance behind each gap. |
| 884 | #[test] |
| 885 | fn mixed_route_run_audits_cache_write_classes_and_fails_closed() { |
| 886 | // 1M input of which 200k is a cache read, 100k is a cache write. |
| 887 | let priced = cache_write_usage(1_000_000, 100_000, 200_000, 100_000); |
| 888 | let unpriced_write = cache_write_usage(1_000_000, 100_000, 200_000, 100_000); |
| 889 | let oauth = cache_write_usage(1_000_000, 100_000, 200_000, 100_000); |
| 890 | let turns = [ |
| 891 | TurnInput { |
| 892 | turn_id: "anthropic".into(), |
| 893 | created_at: None, |
| 894 | provider: Some("anthropic"), |
| 895 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 896 | model: "claude-haiku-4-5".into(), |
| 897 | usage: &priced, |
| 898 | }, |
| 899 | TurnInput { |
| 900 | turn_id: "moonshot".into(), |
| 901 | created_at: None, |
| 902 | provider: Some("moonshot"), |
| 903 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 904 | model: "kimi-k2.7-code".into(), |
| 905 | usage: &unpriced_write, |
| 906 | }, |
| 907 | TurnInput { |
| 908 | turn_id: "oauth".into(), |
| 909 | created_at: None, |
| 910 | provider: Some("openai-codex"), |
| 911 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 912 | model: "gpt-5.5".into(), |
| 913 | usage: &oauth, |
| 914 | }, |
| 915 | ]; |
| 916 | |
| 917 | let card = Scorecard::from_turns(&turns); |
| 918 | |
| 919 | // Cache-write tokens are their own audited class on every turn. |
| 920 | for turn in &card.per_turn { |
| 921 | assert_eq!(turn.cache_write_tokens, 100_000, "{}", turn.turn_id); |
| 922 | assert_eq!(turn.input_tokens, 700_000, "{}", turn.turn_id); |
| 923 | assert_eq!(turn.cache_read_tokens, 200_000, "{}", turn.turn_id); |
| 924 | // Reasoning stays informational: never added to billable output. |
| 925 | assert_eq!(turn.output_tokens, 100_000, "{}", turn.turn_id); |
| 926 | assert_eq!(turn.reasoning_tokens, 50_000, "{}", turn.turn_id); |
| 927 | } |
| 928 | assert_eq!(card.metrics.total_cache_write_tokens, 300_000); |
| 929 | assert_eq!(card.metrics.total_output_tokens, 300_000); |
| 930 | assert_eq!(card.metrics.total_reasoning_tokens, 150_000); |
| 931 | |
| 932 | // Anthropic publishes a 1.25/M cache-write rate, so the write premium |
| 933 | // is billed rather than silently charged at the input rate. |
| 934 | let anthropic = &card.per_turn[0]; |
| 935 | assert!(!anthropic.cost_unpriced); |
| 936 | assert_eq!(anthropic.cost_unpriced_reason, None); |
| 937 | assert!(anthropic.unpriced_classes.is_empty()); |
| 938 | // Provenance is recorded (bundled snapshot offline, live after a |
| 939 | // catalog refresh); the point is that it is never absent for a |
| 940 | // priced turn. |
| 941 | assert!(anthropic.pricing_provenance.is_some()); |
| 942 | let expected = 0.7 * 1.0 + 0.1 * 5.0 + 0.2 * 0.1 + 0.1 * 1.25; |
| 943 | assert!((anthropic.cost_usd - expected).abs() < 1e-9); |
| 944 | |
| 945 | // Moonshot's row has no published cache-write rate: the whole turn |
| 946 | // fails closed instead of under-reporting the write tokens. |
| 947 | let moonshot = &card.per_turn[1]; |
| 948 | assert!(moonshot.cost_unpriced); |
| 949 | assert_eq!(moonshot.cost_usd, 0.0); |
| 950 | assert_eq!( |
| 951 | moonshot.cost_unpriced_reason.as_deref(), |
| 952 | Some("missing_class_price") |
| 953 | ); |
| 954 | assert_eq!(moonshot.unpriced_classes, vec!["cache_write".to_string()]); |
| 955 | assert!(moonshot.pricing_provenance.is_some()); |
| 956 | |
| 957 | // A subscription route is not "free"; it is not money-metered. |
| 958 | let oauth = &card.per_turn[2]; |
| 959 | assert!(oauth.cost_unpriced); |
| 960 | assert_eq!( |
| 961 | oauth.cost_unpriced_reason.as_deref(), |
| 962 | Some("not_money_metered") |
| 963 | ); |
| 964 | assert!(oauth.unpriced_classes.is_empty()); |
| 965 | assert_eq!(oauth.pricing_provenance, None); |
| 966 | |
| 967 | // Aggregates stay honest about what the number covers. Two of the three |
| 968 | // turns owe money (Anthropic and Moonshot); the OAuth turn does not, so |
| 969 | // it is outside the denominator rather than counted as an unpriced dollar. |
| 970 | assert_eq!(card.metrics.turns, 3); |
| 971 | assert_eq!(card.metrics.money_metered_turns, 2); |
| 972 | assert_eq!(card.metrics.unpriced_turns, 1); |
| 973 | assert!(!card.metrics.cost_complete); |
| 974 | assert_eq!( |
| 975 | card.metrics.unpriced_classes, |
| 976 | vec!["cache_write".to_string()] |
| 977 | ); |
| 978 | assert!((card.metrics.total_cost_usd - expected).abs() < 1e-9); |
| 979 | assert!(card.per_turn[0].money_metered); |
| 980 | assert!(card.per_turn[1].money_metered); |
| 981 | assert!(!card.per_turn[2].money_metered); |
| 982 | |
| 983 | let summary = card.to_summary(); |
| 984 | assert!(summary.contains("priced_cost_subtotal_usd")); |
| 985 | assert!(summary.contains("cache_write_tokens: 300000")); |
| 986 | assert!(summary.contains("no published price")); |
| 987 | // Coverage reads against the money-metered turns, not all three. |
| 988 | assert!(summary.contains("money_metered_turns: 2"), "{summary}"); |
| 989 | |
| 990 | let json = serde_json::to_value(&card).expect("serialize scorecard"); |
| 991 | assert_eq!(json["per_turn"][1]["unpriced_classes"][0], "cache_write"); |
| 992 | assert_eq!(json["metrics"]["total_cache_write_tokens"], 300_000); |
| 993 | assert_eq!(json["metrics"]["cost_complete"], false); |
| 994 | assert_eq!(json["metrics"]["money_metered_turns"], 2); |
| 995 | // Route identity survives serialization, so a re-read scorecard can be |
| 996 | // re-explained without the original input file. |
| 997 | assert_eq!(json["per_turn"][1]["provider"], "moonshot"); |
| 998 | assert_eq!(json["per_turn"][2]["money_metered"], false); |
| 999 | } |
| 1000 | |
| 1001 | /// Legacy baselines and per-turn records that predate the class audit must |
| 1002 | /// still deserialize; the new fields default rather than fail. |
| 1003 | #[test] |
| 1004 | fn legacy_turn_score_json_defaults_the_new_audit_fields() { |
| 1005 | let score: TurnScore = serde_json::from_value(serde_json::json!({ |
| 1006 | "turn_id": "t1", |
| 1007 | "model": "gpt-5.5", |
| 1008 | "input_tokens": 10, |
| 1009 | "output_tokens": 5, |
| 1010 | "cache_read_tokens": 0, |
| 1011 | "cost_usd": 0.1, |
| 1012 | "cost_cny": 0.0, |
| 1013 | "cost_unpriced": false |
| 1014 | })) |
| 1015 | .expect("legacy per-turn record stays readable"); |
| 1016 | assert_eq!(score.cache_write_tokens, 0); |
| 1017 | assert_eq!(score.reasoning_tokens, 0); |
| 1018 | assert!(score.unpriced_classes.is_empty()); |
| 1019 | assert_eq!(score.pricing_provenance, None); |
| 1020 | assert_eq!(score.cost_unpriced_reason, None); |
| 1021 | assert_eq!(score.live_pricing_defect, None); |
| 1022 | // A legacy row carries no coverage evidence, so `money_metered` defaults |
| 1023 | // to false rather than asserting the row was inside a complete total. |
| 1024 | assert!(!score.money_metered); |
| 1025 | |
| 1026 | // Legacy aggregate baselines stay readable too, defaulting the new |
| 1027 | // coverage denominator rather than failing the parse. |
| 1028 | let metrics: ScorecardMetrics = serde_json::from_value(serde_json::json!({ |
| 1029 | "turns": 3, |
| 1030 | "total_input_tokens": 10, |
| 1031 | "total_output_tokens": 5, |
| 1032 | "total_cache_read_tokens": 0, |
| 1033 | "total_cost_usd": 0.1, |
| 1034 | "total_cost_cny": 0.0, |
| 1035 | "cache_hit_ratio": 0.0 |
| 1036 | })) |
| 1037 | .expect("legacy baseline stays readable"); |
| 1038 | assert_eq!(metrics.money_metered_turns, 0); |
| 1039 | assert_eq!(metrics.total_cache_write_tokens, 0); |
| 1040 | } |
| 1041 | |
| 1042 | #[test] |
| 1043 | fn aggregates_tokens_and_cache_hit_ratio_independent_of_pricing() { |
| 1044 | // input_tokens includes cache hits; token_usage_for_pricing splits them: |
| 1045 | // non-cached input = 1000-200 = 800, cache_read = 200. |
| 1046 | let u1 = usage(1000, 500, 200); |
| 1047 | let u2 = usage(2000, 100, 800); // non-cached = 1200, cache_read = 800 |
| 1048 | let turns = [ |
| 1049 | TurnInput { |
| 1050 | turn_id: "t1".into(), |
| 1051 | created_at: None, |
| 1052 | provider: None, |
| 1053 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1054 | model: "unpriced-x".into(), |
| 1055 | usage: &u1, |
| 1056 | }, |
| 1057 | TurnInput { |
| 1058 | turn_id: "t2".into(), |
| 1059 | created_at: None, |
| 1060 | provider: None, |
| 1061 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1062 | model: "unpriced-x".into(), |
| 1063 | usage: &u2, |
| 1064 | }, |
| 1065 | ]; |
| 1066 | let card = Scorecard::from_turns(&turns); |
| 1067 | |
| 1068 | assert_eq!(card.metrics.turns, 2); |
| 1069 | assert_eq!(card.metrics.total_input_tokens, 800 + 1200); |
| 1070 | assert_eq!(card.metrics.total_output_tokens, 600); // 500 + 100 |
| 1071 | assert_eq!(card.metrics.total_cache_read_tokens, 1000); // 200 + 800 |
| 1072 | assert_eq!(card.metrics.unpriced_turns, 2); |
| 1073 | // cache_read / (input + cache_read) = 1000 / (2000 + 1000) |
| 1074 | let expected = 1000.0 / 3000.0; |
| 1075 | assert!((card.metrics.cache_hit_ratio - expected).abs() < 1e-9); |
| 1076 | } |
| 1077 | |
| 1078 | /// The canonical cache-efficiency denominator is |
| 1079 | /// `hit / (non-cached input + hit + write)`. Cache-write tokens are prompt |
| 1080 | /// tokens that were not served from cache, so omitting them reported a |
| 1081 | /// flattering ratio on exactly the turns that paid to populate the cache. |
| 1082 | #[test] |
| 1083 | fn cache_hit_ratio_counts_cache_write_in_the_denominator() { |
| 1084 | fn card_for(usage: &Usage) -> Scorecard { |
| 1085 | Scorecard::from_turns(&[TurnInput { |
| 1086 | turn_id: "t1".into(), |
| 1087 | created_at: None, |
| 1088 | provider: None, |
| 1089 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1090 | model: "unpriced-x".into(), |
| 1091 | usage, |
| 1092 | }]) |
| 1093 | } |
| 1094 | |
| 1095 | // Zero everything: a ratio is undefined, reported as 0.0 rather than NaN. |
| 1096 | let empty = Usage::default(); |
| 1097 | let card = card_for(&empty); |
| 1098 | assert_eq!(card.metrics.cache_hit_ratio, 0.0); |
| 1099 | assert_eq!(card.metrics.total_cache_write_tokens, 0); |
| 1100 | |
| 1101 | // Write-only: a turn that populated the cache and read nothing from it |
| 1102 | // has a 0% hit ratio, not an undefined-and-therefore-zero one that a |
| 1103 | // write-blind denominator would produce by accident. |
| 1104 | let write_only = Usage { |
| 1105 | input_tokens: 1_000, |
| 1106 | output_tokens: 10, |
| 1107 | prompt_cache_hit_tokens: Some(0), |
| 1108 | prompt_cache_write_tokens: Some(1_000), |
| 1109 | ..Default::default() |
| 1110 | }; |
| 1111 | let card = card_for(&write_only); |
| 1112 | assert_eq!(card.metrics.total_cache_write_tokens, 1_000); |
| 1113 | assert_eq!(card.metrics.total_cache_read_tokens, 0); |
| 1114 | assert_eq!(card.metrics.cache_hit_ratio, 0.0); |
| 1115 | |
| 1116 | // Mixed: 1000 prompt tokens = 200 read + 300 write + 500 non-cached. |
| 1117 | let mixed = Usage { |
| 1118 | input_tokens: 1_000, |
| 1119 | output_tokens: 10, |
| 1120 | prompt_cache_hit_tokens: Some(200), |
| 1121 | prompt_cache_write_tokens: Some(300), |
| 1122 | ..Default::default() |
| 1123 | }; |
| 1124 | let card = card_for(&mixed); |
| 1125 | assert_eq!(card.metrics.total_input_tokens, 500); |
| 1126 | assert_eq!(card.metrics.total_cache_read_tokens, 200); |
| 1127 | assert_eq!(card.metrics.total_cache_write_tokens, 300); |
| 1128 | let expected = 200.0 / (500.0 + 200.0 + 300.0); |
| 1129 | assert!( |
| 1130 | (card.metrics.cache_hit_ratio - expected).abs() < 1e-9, |
| 1131 | "got {}, want {expected}", |
| 1132 | card.metrics.cache_hit_ratio |
| 1133 | ); |
| 1134 | // The write-blind denominator would have said 200/700 — assert the two |
| 1135 | // are distinguishable so a regression is unambiguous. |
| 1136 | let write_blind = 200.0 / 700.0; |
| 1137 | assert!((card.metrics.cache_hit_ratio - write_blind).abs() > 1e-6); |
| 1138 | } |
| 1139 | |
| 1140 | #[test] |
| 1141 | fn unknown_model_is_marked_unpriced_with_zero_cost() { |
| 1142 | let u = usage(1000, 500, 0); |
| 1143 | let turns = [TurnInput { |
| 1144 | turn_id: "t1".into(), |
| 1145 | created_at: None, |
| 1146 | provider: Some("openai"), |
| 1147 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1148 | model: "definitely-not-a-real-model".into(), |
| 1149 | usage: &u, |
| 1150 | }]; |
| 1151 | let card = Scorecard::from_turns(&turns); |
| 1152 | assert!(card.per_turn[0].cost_unpriced); |
| 1153 | assert_eq!(card.per_turn[0].cost_usd, 0.0); |
| 1154 | assert_eq!(card.metrics.total_cost_usd, 0.0); |
| 1155 | assert!(card.to_summary().contains("cost_usd: unavailable")); |
| 1156 | } |
| 1157 | |
| 1158 | #[test] |
| 1159 | fn same_model_is_priced_only_for_its_authoritative_provider_route() { |
| 1160 | let u = usage(1000, 500, 0); |
| 1161 | let turns = [ |
| 1162 | TurnInput { |
| 1163 | turn_id: "api".into(), |
| 1164 | created_at: None, |
| 1165 | provider: Some("openai"), |
| 1166 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1167 | model: "gpt-5.5".into(), |
| 1168 | usage: &u, |
| 1169 | }, |
| 1170 | TurnInput { |
| 1171 | turn_id: "oauth".into(), |
| 1172 | created_at: None, |
| 1173 | provider: Some("openai-codex"), |
| 1174 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1175 | model: "gpt-5.5".into(), |
| 1176 | usage: &u, |
| 1177 | }, |
| 1178 | TurnInput { |
| 1179 | turn_id: "local".into(), |
| 1180 | created_at: None, |
| 1181 | provider: Some("ollama"), |
| 1182 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1183 | model: "gpt-5.5".into(), |
| 1184 | usage: &u, |
| 1185 | }, |
| 1186 | ]; |
| 1187 | |
| 1188 | let card = Scorecard::from_turns(&turns); |
| 1189 | |
| 1190 | assert!(!card.per_turn[0].cost_unpriced); |
| 1191 | assert!(card.per_turn[0].cost_usd > 0.0); |
| 1192 | assert!(card.per_turn[1].cost_unpriced); |
| 1193 | assert_eq!(card.per_turn[1].cost_usd, 0.0); |
| 1194 | assert!(card.per_turn[2].cost_unpriced); |
| 1195 | assert_eq!(card.per_turn[2].cost_usd, 0.0); |
| 1196 | // Codex OAuth and Ollama are *exactly* non-metered, so they leave the |
| 1197 | // money-coverage denominator entirely rather than counting as unpriced |
| 1198 | // dollars: only the OpenAI turn owes money, and it is priced. The USD |
| 1199 | // total is therefore genuinely complete for the spend it covers (#4318). |
| 1200 | assert_eq!(card.metrics.money_metered_turns, 1); |
| 1201 | assert_eq!(card.metrics.unpriced_turns, 0); |
| 1202 | assert!(card.metrics.cost_complete); |
| 1203 | for (index, expected) in [(1_usize, false), (2_usize, false)] { |
| 1204 | assert_eq!( |
| 1205 | card.per_turn[index].money_metered, expected, |
| 1206 | "turn {index} money-metered" |
| 1207 | ); |
| 1208 | assert_eq!( |
| 1209 | card.per_turn[index].cost_unpriced_reason.as_deref(), |
| 1210 | Some("not_money_metered"), |
| 1211 | "turn {index} reason" |
| 1212 | ); |
| 1213 | } |
| 1214 | assert!(card.per_turn[0].money_metered); |
| 1215 | // CNY is only published by direct DeepSeek, so the single metered turn |
| 1216 | // still has no authoritative CNY figure. |
| 1217 | assert_eq!(card.metrics.cny_unpriced_turns, 1); |
| 1218 | assert!(!card.metrics.cny_cost_complete); |
| 1219 | assert!(card.to_summary().contains("money_metered_turns: 1")); |
| 1220 | assert!(card.to_summary().contains("cost_cny: unavailable")); |
| 1221 | |
| 1222 | let json = serde_json::to_value(&card).expect("serialize scorecard"); |
| 1223 | assert_eq!(json["per_turn"][0]["provider"], "openai"); |
| 1224 | assert_eq!(json["per_turn"][1]["provider"], "openai-codex"); |
| 1225 | assert_eq!(json["per_turn"][2]["provider"], "ollama"); |
| 1226 | assert_eq!(json["metrics"]["money_metered_turns"], 1); |
| 1227 | assert_eq!(json["metrics"]["unpriced_turns"], 0); |
| 1228 | assert_eq!(json["metrics"]["cost_complete"], true); |
| 1229 | assert_eq!(json["metrics"]["cny_cost_complete"], false); |
| 1230 | } |
| 1231 | |
| 1232 | #[test] |
| 1233 | fn first_party_hand_price_survives_a_missing_catalog_offering() { |
| 1234 | let u = usage(1_000_000, 0, 0); |
| 1235 | let turns = [ |
| 1236 | TurnInput { |
| 1237 | turn_id: "openai-api".into(), |
| 1238 | created_at: None, |
| 1239 | provider: Some("openai"), |
| 1240 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1241 | model: "gpt-5-codex".into(), |
| 1242 | usage: &u, |
| 1243 | }, |
| 1244 | TurnInput { |
| 1245 | turn_id: "foreign-route".into(), |
| 1246 | created_at: None, |
| 1247 | provider: Some("ollama"), |
| 1248 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1249 | model: "gpt-5-codex".into(), |
| 1250 | usage: &u, |
| 1251 | }, |
| 1252 | ]; |
| 1253 | |
| 1254 | let card = Scorecard::from_turns(&turns); |
| 1255 | |
| 1256 | assert!(!card.per_turn[0].cost_unpriced); |
| 1257 | assert!((card.per_turn[0].cost_usd - 1.25).abs() < f64::EPSILON); |
| 1258 | assert!(card.per_turn[1].cost_unpriced); |
| 1259 | } |
| 1260 | |
| 1261 | #[test] |
| 1262 | fn documented_no_cache_discount_uses_input_without_generalizing_missing_rates() { |
| 1263 | let u = Usage { |
| 1264 | input_tokens: 1_000_000, |
| 1265 | output_tokens: 0, |
| 1266 | prompt_cache_hit_tokens: Some(250_000), |
| 1267 | prompt_cache_write_tokens: Some(100_000), |
| 1268 | ..Default::default() |
| 1269 | }; |
| 1270 | let turns = [ |
| 1271 | TurnInput { |
| 1272 | turn_id: "documented-no-discount".into(), |
| 1273 | created_at: None, |
| 1274 | provider: Some("openai"), |
| 1275 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1276 | model: "gpt-5.5-pro".into(), |
| 1277 | usage: &u, |
| 1278 | }, |
| 1279 | TurnInput { |
| 1280 | turn_id: "missing-cache-rate".into(), |
| 1281 | created_at: None, |
| 1282 | provider: Some("meta"), |
| 1283 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1284 | model: "muse-spark-1.1".into(), |
| 1285 | usage: &u, |
| 1286 | }, |
| 1287 | ]; |
| 1288 | |
| 1289 | let card = Scorecard::from_turns(&turns); |
| 1290 | |
| 1291 | assert!(!card.per_turn[0].cost_unpriced); |
| 1292 | assert!((card.per_turn[0].cost_usd - 30.0).abs() < f64::EPSILON); |
| 1293 | assert!(card.per_turn[1].cost_unpriced); |
| 1294 | assert!(!card.metrics.cost_complete); |
| 1295 | } |
| 1296 | |
| 1297 | #[test] |
| 1298 | fn anthropic_sonnet_5_uses_the_recorded_turn_time() { |
| 1299 | let u = Usage { |
| 1300 | input_tokens: 1_000_000, |
| 1301 | output_tokens: 500_000, |
| 1302 | prompt_cache_hit_tokens: Some(250_000), |
| 1303 | prompt_cache_write_tokens: Some(100_000), |
| 1304 | ..Default::default() |
| 1305 | }; |
| 1306 | let intro_at: DateTime<Utc> = "2026-08-31T23:59:59Z".parse().expect("intro time"); |
| 1307 | let standard_at: DateTime<Utc> = "2026-09-01T00:00:00Z".parse().expect("standard time"); |
| 1308 | let turns = [ |
| 1309 | TurnInput { |
| 1310 | turn_id: "sonnet-intro".into(), |
| 1311 | created_at: Some(&intro_at), |
| 1312 | provider: Some("anthropic"), |
| 1313 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1314 | model: " claude-sonnet-5 ".into(), |
| 1315 | usage: &u, |
| 1316 | }, |
| 1317 | TurnInput { |
| 1318 | turn_id: "sonnet-standard".into(), |
| 1319 | created_at: Some(&standard_at), |
| 1320 | provider: Some("anthropic"), |
| 1321 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1322 | model: "claude-sonnet-5".into(), |
| 1323 | usage: &u, |
| 1324 | }, |
| 1325 | TurnInput { |
| 1326 | turn_id: "sonnet-missing-time".into(), |
| 1327 | created_at: None, |
| 1328 | provider: Some("anthropic"), |
| 1329 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1330 | model: "claude-sonnet-5".into(), |
| 1331 | usage: &u, |
| 1332 | }, |
| 1333 | ]; |
| 1334 | |
| 1335 | let card = Scorecard::from_turns(&turns); |
| 1336 | |
| 1337 | assert!(!card.per_turn[0].cost_unpriced); |
| 1338 | assert!((card.per_turn[0].cost_usd - 6.60).abs() < 1e-12); |
| 1339 | assert_eq!(card.per_turn[0].created_at.as_ref(), Some(&intro_at)); |
| 1340 | assert!(card.per_turn[0].cost_cny_unpriced); |
| 1341 | assert!(!card.per_turn[1].cost_unpriced); |
| 1342 | assert!((card.per_turn[1].cost_usd - 9.90).abs() < 1e-12); |
| 1343 | assert!(card.per_turn[1].cost_cny_unpriced); |
| 1344 | assert!(card.per_turn[2].cost_unpriced); |
| 1345 | } |
| 1346 | |
| 1347 | #[test] |
| 1348 | fn known_zero_usage_is_zero_cost_not_unavailable() { |
| 1349 | let u = usage(0, 0, 0); |
| 1350 | let turns = [TurnInput { |
| 1351 | turn_id: "zero".into(), |
| 1352 | created_at: None, |
| 1353 | provider: Some("openai"), |
| 1354 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1355 | model: "gpt-5.5".into(), |
| 1356 | usage: &u, |
| 1357 | }]; |
| 1358 | |
| 1359 | let card = Scorecard::from_turns(&turns); |
| 1360 | |
| 1361 | assert!(!card.per_turn[0].cost_unpriced); |
| 1362 | assert_eq!(card.per_turn[0].cost_usd, 0.0); |
| 1363 | assert!(card.per_turn[0].cost_cny_unpriced); |
| 1364 | assert_eq!(card.metrics.unpriced_turns, 0); |
| 1365 | assert_eq!(card.metrics.cny_unpriced_turns, 1); |
| 1366 | assert!(card.metrics.cost_complete); |
| 1367 | assert!(!card.metrics.cny_cost_complete); |
| 1368 | assert!(card.to_summary().contains("cost_usd: $0.0000")); |
| 1369 | assert!(card.to_summary().contains("cost_cny: unavailable")); |
| 1370 | } |
| 1371 | |
| 1372 | #[test] |
| 1373 | fn direct_deepseek_route_keeps_authoritative_dual_currency_pricing() { |
| 1374 | let u = usage(1000, 500, 0); |
| 1375 | let turns = [TurnInput { |
| 1376 | turn_id: "deepseek".into(), |
| 1377 | created_at: None, |
| 1378 | provider: Some("deepseek"), |
| 1379 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1380 | model: "deepseek-v4-pro".into(), |
| 1381 | usage: &u, |
| 1382 | }]; |
| 1383 | |
| 1384 | let card = Scorecard::from_turns(&turns); |
| 1385 | |
| 1386 | assert!(!card.per_turn[0].cost_unpriced); |
| 1387 | assert!(!card.per_turn[0].cost_cny_unpriced); |
| 1388 | assert!(card.per_turn[0].cost_usd > 0.0); |
| 1389 | assert!(card.per_turn[0].cost_cny > 0.0); |
| 1390 | assert!(card.metrics.cost_complete); |
| 1391 | assert!(card.metrics.cny_cost_complete); |
| 1392 | } |
| 1393 | |
| 1394 | #[test] |
| 1395 | fn direct_deepseek_compact_aliases_use_canonical_pricing() { |
| 1396 | let u = usage(1000, 500, 100); |
| 1397 | let models = [ |
| 1398 | "deepseek-v4-pro", |
| 1399 | "pro", |
| 1400 | " DeepSeek-V4Pro ", |
| 1401 | "deepseek-v4-flash", |
| 1402 | "flash", |
| 1403 | "DEEPSEEK-V4FLASH", |
| 1404 | ]; |
| 1405 | let turns: Vec<_> = models |
| 1406 | .iter() |
| 1407 | .map(|model| TurnInput { |
| 1408 | turn_id: (*model).into(), |
| 1409 | created_at: None, |
| 1410 | provider: Some("deepseek"), |
| 1411 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1412 | model: (*model).into(), |
| 1413 | usage: &u, |
| 1414 | }) |
| 1415 | .collect(); |
| 1416 | |
| 1417 | let card = Scorecard::from_turns(&turns); |
| 1418 | |
| 1419 | for alias in [1, 2] { |
| 1420 | assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[0].cost_usd); |
| 1421 | assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[0].cost_cny); |
| 1422 | } |
| 1423 | for alias in [4, 5] { |
| 1424 | assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[3].cost_usd); |
| 1425 | assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[3].cost_cny); |
| 1426 | } |
| 1427 | assert!(card.per_turn.iter().all(|turn| !turn.cost_unpriced)); |
| 1428 | assert!(card.per_turn.iter().all(|turn| !turn.cost_cny_unpriced)); |
| 1429 | } |
| 1430 | |
| 1431 | #[test] |
| 1432 | fn direct_deepseek_compatibility_aliases_use_the_flash_route() { |
| 1433 | let u = usage(1000, 500, 100); |
| 1434 | let before_retirement: DateTime<Utc> = |
| 1435 | "2026-07-24T15:58:59Z".parse().expect("pre-retirement time"); |
| 1436 | let at_retirement: DateTime<Utc> = DEEPSEEK_ALIAS_RETIREMENT_UTC |
| 1437 | .parse() |
| 1438 | .expect("retirement time"); |
| 1439 | let turns = [ |
| 1440 | TurnInput { |
| 1441 | turn_id: "chat-alias".into(), |
| 1442 | created_at: Some(&before_retirement), |
| 1443 | provider: Some("deepseek"), |
| 1444 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1445 | model: "deepseek-chat".into(), |
| 1446 | usage: &u, |
| 1447 | }, |
| 1448 | TurnInput { |
| 1449 | turn_id: "reasoner-alias".into(), |
| 1450 | created_at: Some(&before_retirement), |
| 1451 | provider: Some("deepseek"), |
| 1452 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1453 | model: "deepseek-reasoner".into(), |
| 1454 | usage: &u, |
| 1455 | }, |
| 1456 | TurnInput { |
| 1457 | turn_id: "canonical".into(), |
| 1458 | created_at: None, |
| 1459 | provider: Some("deepseek"), |
| 1460 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1461 | model: DEEPSEEK_ALIAS_REPLACEMENT.into(), |
| 1462 | usage: &u, |
| 1463 | }, |
| 1464 | TurnInput { |
| 1465 | turn_id: "retired-alias".into(), |
| 1466 | created_at: Some(&at_retirement), |
| 1467 | provider: Some("deepseek"), |
| 1468 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1469 | model: "deepseek-chat".into(), |
| 1470 | usage: &u, |
| 1471 | }, |
| 1472 | TurnInput { |
| 1473 | turn_id: "undated-alias".into(), |
| 1474 | created_at: None, |
| 1475 | provider: Some("deepseek"), |
| 1476 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1477 | model: "deepseek-reasoner".into(), |
| 1478 | usage: &u, |
| 1479 | }, |
| 1480 | ]; |
| 1481 | |
| 1482 | let card = Scorecard::from_turns(&turns); |
| 1483 | |
| 1484 | assert_eq!(card.per_turn[0].cost_usd, card.per_turn[2].cost_usd); |
| 1485 | assert_eq!(card.per_turn[1].cost_usd, card.per_turn[2].cost_usd); |
| 1486 | assert_eq!(card.per_turn[0].cost_cny, card.per_turn[2].cost_cny); |
| 1487 | assert_eq!(card.per_turn[1].cost_cny, card.per_turn[2].cost_cny); |
| 1488 | assert!(card.per_turn[..3].iter().all(|turn| !turn.cost_unpriced)); |
| 1489 | assert!( |
| 1490 | card.per_turn[..3] |
| 1491 | .iter() |
| 1492 | .all(|turn| !turn.cost_cny_unpriced) |
| 1493 | ); |
| 1494 | assert!(card.per_turn[3].cost_unpriced); |
| 1495 | assert!(card.per_turn[4].cost_unpriced); |
| 1496 | } |
| 1497 | |
| 1498 | #[test] |
| 1499 | fn direct_arcee_aliases_do_not_cross_the_openrouter_namespace() { |
| 1500 | let u = Usage { |
| 1501 | input_tokens: 1_000_000, |
| 1502 | output_tokens: 500_000, |
| 1503 | prompt_cache_hit_tokens: Some(250_000), |
| 1504 | prompt_cache_write_tokens: Some(100_000), |
| 1505 | ..Default::default() |
| 1506 | }; |
| 1507 | let turns = [ |
| 1508 | TurnInput { |
| 1509 | turn_id: "canonical-direct".into(), |
| 1510 | created_at: None, |
| 1511 | provider: Some("arcee"), |
| 1512 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1513 | model: "trinity-large-thinking".into(), |
| 1514 | usage: &u, |
| 1515 | }, |
| 1516 | TurnInput { |
| 1517 | turn_id: "direct-alias".into(), |
| 1518 | created_at: None, |
| 1519 | provider: Some("arcee"), |
| 1520 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1521 | model: "arcee-trinity-large-thinking".into(), |
| 1522 | usage: &u, |
| 1523 | }, |
| 1524 | TurnInput { |
| 1525 | turn_id: "openrouter-namespace".into(), |
| 1526 | created_at: None, |
| 1527 | provider: Some("arcee"), |
| 1528 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1529 | model: "arcee-ai/trinity-large-thinking".into(), |
| 1530 | usage: &u, |
| 1531 | }, |
| 1532 | ]; |
| 1533 | |
| 1534 | let card = Scorecard::from_turns(&turns); |
| 1535 | |
| 1536 | assert!(!card.per_turn[0].cost_unpriced); |
| 1537 | assert!((card.per_turn[0].cost_usd - 0.65).abs() < f64::EPSILON); |
| 1538 | assert_eq!(card.per_turn[1].cost_usd, card.per_turn[0].cost_usd); |
| 1539 | assert!(!card.per_turn[1].cost_unpriced); |
| 1540 | assert!(card.per_turn[2].cost_unpriced); |
| 1541 | } |
| 1542 | |
| 1543 | #[test] |
| 1544 | fn costless_catalog_rows_fall_back_only_to_verified_provider_prices() { |
| 1545 | let u = Usage { |
| 1546 | input_tokens: 1_000_000, |
| 1547 | output_tokens: 500_000, |
| 1548 | prompt_cache_hit_tokens: Some(250_000), |
| 1549 | prompt_cache_write_tokens: Some(100_000), |
| 1550 | ..Default::default() |
| 1551 | }; |
| 1552 | let turns = [ |
| 1553 | TurnInput { |
| 1554 | turn_id: "arcee-mini".into(), |
| 1555 | created_at: None, |
| 1556 | provider: Some("arcee"), |
| 1557 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1558 | model: "trinity-mini".into(), |
| 1559 | usage: &u, |
| 1560 | }, |
| 1561 | TurnInput { |
| 1562 | turn_id: "minimax-m2.7".into(), |
| 1563 | created_at: None, |
| 1564 | provider: Some("minimax"), |
| 1565 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1566 | model: "minimax-m2.7".into(), |
| 1567 | usage: &u, |
| 1568 | }, |
| 1569 | TurnInput { |
| 1570 | turn_id: "foreign-route".into(), |
| 1571 | created_at: None, |
| 1572 | provider: Some("ollama"), |
| 1573 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1574 | model: "trinity-mini".into(), |
| 1575 | usage: &u, |
| 1576 | }, |
| 1577 | TurnInput { |
| 1578 | turn_id: "openai-hosted-deepseek".into(), |
| 1579 | created_at: None, |
| 1580 | provider: Some("openai"), |
| 1581 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1582 | model: "deepseek-v4-pro".into(), |
| 1583 | usage: &u, |
| 1584 | }, |
| 1585 | TurnInput { |
| 1586 | turn_id: "openrouter-hosted-zai".into(), |
| 1587 | created_at: None, |
| 1588 | provider: Some("openrouter"), |
| 1589 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1590 | model: "z-ai/glm-5.2".into(), |
| 1591 | usage: &u, |
| 1592 | }, |
| 1593 | ]; |
| 1594 | |
| 1595 | let card = Scorecard::from_turns(&turns); |
| 1596 | |
| 1597 | // Trinity Mini has no verified provider rate in the release metadata; |
| 1598 | // a removed hand-written estimate must stay unknown, not become zero |
| 1599 | // or leak through from a similarly named route. |
| 1600 | assert_eq!(card.per_turn[0].cost_usd, 0.0); |
| 1601 | assert!(card.per_turn[0].cost_unpriced); |
| 1602 | // MiniMax-M2.7 publishes a distinct cache-write rate (0.375/M), |
| 1603 | // retained by the provider-owned fallback even without a priced |
| 1604 | // catalog offering. |
| 1605 | assert!((card.per_turn[1].cost_usd - 0.8475).abs() < f64::EPSILON); |
| 1606 | assert!(!card.per_turn[1].cost_unpriced); |
| 1607 | assert!(card.per_turn[..2].iter().all(|turn| turn.cost_cny_unpriced)); |
| 1608 | assert!(card.per_turn[2..].iter().all(|turn| turn.cost_unpriced)); |
| 1609 | } |
| 1610 | |
| 1611 | #[test] |
| 1612 | fn stepfun_legacy_route_keeps_pricing_without_a_catalog_row() { |
| 1613 | let u = usage(1000, 500, 250); |
| 1614 | let recorded = |turn_id: &str, |
| 1615 | provider: &str, |
| 1616 | model: &str, |
| 1617 | billing_surface: Option<&str>| RecordedTurn { |
| 1618 | turn_id: turn_id.to_string(), |
| 1619 | created_at: None, |
| 1620 | model_backed: Some(true), |
| 1621 | provider: Some(provider.to_string()), |
| 1622 | billing_surface: billing_surface.map(str::to_string), |
| 1623 | model: model.to_string(), |
| 1624 | usage: Some(u.clone()), |
| 1625 | }; |
| 1626 | let turns = [ |
| 1627 | recorded( |
| 1628 | "stepfun-default", |
| 1629 | "stepfun", |
| 1630 | " STEP-3.7-FLASH ", |
| 1631 | Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE), |
| 1632 | ), |
| 1633 | recorded( |
| 1634 | "stepfun-plan", |
| 1635 | "stepfun", |
| 1636 | "step-3.7-flash", |
| 1637 | Some(crate::pricing::STEPFUN_PLAN_BILLING_SURFACE), |
| 1638 | ), |
| 1639 | recorded("stepfun-missing-surface", "stepfun", "step-3.7-flash", None), |
| 1640 | recorded("stepfun-unknown-model", "stepfun", "step-3.5-flash", None), |
| 1641 | recorded( |
| 1642 | "openrouter-stepfun-name", |
| 1643 | "openrouter", |
| 1644 | "step-3.7-flash", |
| 1645 | None, |
| 1646 | ), |
| 1647 | recorded("local-stepfun-name", "ollama", "step-3.7-flash", None), |
| 1648 | recorded( |
| 1649 | "sakana-incomplete-tier-price", |
| 1650 | "sakana", |
| 1651 | "fugu-ultra-20260615", |
| 1652 | None, |
| 1653 | ), |
| 1654 | recorded( |
| 1655 | "foreign-deepseek-name", |
| 1656 | "openmodel", |
| 1657 | "deepseek-v4-flash", |
| 1658 | None, |
| 1659 | ), |
| 1660 | ]; |
| 1661 | |
| 1662 | let card = Scorecard::from_recorded_turns(&turns); |
| 1663 | |
| 1664 | assert!((card.per_turn[0].cost_usd - 0.000_735).abs() < 1e-12); |
| 1665 | assert!(!card.per_turn[0].cost_unpriced); |
| 1666 | assert!(card.per_turn[0].cost_cny_unpriced); |
| 1667 | assert_eq!( |
| 1668 | card.per_turn[0].billing_surface.as_deref(), |
| 1669 | Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE) |
| 1670 | ); |
| 1671 | assert!(card.per_turn[1..].iter().all(|turn| turn.cost_unpriced)); |
| 1672 | } |
| 1673 | |
| 1674 | #[test] |
| 1675 | fn legacy_model_only_record_is_readable_but_unpriced() { |
| 1676 | let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({ |
| 1677 | "turn_id": "legacy", |
| 1678 | "model": "gpt-5.5", |
| 1679 | "usage": { |
| 1680 | "input_tokens": 0, |
| 1681 | "output_tokens": 0 |
| 1682 | } |
| 1683 | })) |
| 1684 | .expect("parse legacy scorecard turn"); |
| 1685 | assert_eq!(recorded.provider, None); |
| 1686 | assert_eq!(recorded.billing_surface, None); |
| 1687 | |
| 1688 | let card = Scorecard::from_recorded_turns(&[recorded]); |
| 1689 | |
| 1690 | assert!(card.per_turn[0].cost_unpriced); |
| 1691 | assert_eq!(card.per_turn[0].cost_usd, 0.0); |
| 1692 | assert_eq!(card.metrics.unpriced_turns, 1); |
| 1693 | assert!(card.to_summary().contains("cost_usd: unavailable")); |
| 1694 | } |
| 1695 | |
| 1696 | #[test] |
| 1697 | fn recorded_turn_accepts_runtime_route_aliases() { |
| 1698 | let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({ |
| 1699 | "schema_version": 1, |
| 1700 | "id": "runtime-turn", |
| 1701 | "thread_id": "thread-1", |
| 1702 | "status": "completed", |
| 1703 | "input_summary": "score this turn", |
| 1704 | "created_at": "2026-07-12T10:30:00Z", |
| 1705 | "effective_provider": "openai-codex", |
| 1706 | "effective_billing_surface": "account-subscription", |
| 1707 | "effective_model": "gpt-5.5", |
| 1708 | "usage": { |
| 1709 | "input_tokens": 1, |
| 1710 | "output_tokens": 1 |
| 1711 | } |
| 1712 | })) |
| 1713 | .expect("parse runtime scorecard turn"); |
| 1714 | |
| 1715 | assert_eq!(recorded.turn_id, "runtime-turn"); |
| 1716 | assert_eq!( |
| 1717 | recorded.created_at.as_ref().map(DateTime::to_rfc3339), |
| 1718 | Some("2026-07-12T10:30:00+00:00".to_string()) |
| 1719 | ); |
| 1720 | assert_eq!(recorded.provider.as_deref(), Some("openai-codex")); |
| 1721 | assert_eq!( |
| 1722 | recorded.billing_surface.as_deref(), |
| 1723 | Some("account-subscription") |
| 1724 | ); |
| 1725 | assert_eq!(recorded.model, "gpt-5.5"); |
| 1726 | assert!(recorded.contributes_to_scorecard()); |
| 1727 | } |
| 1728 | |
| 1729 | #[test] |
| 1730 | fn runtime_turn_without_usage_is_readable_and_filtered() { |
| 1731 | let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({ |
| 1732 | "schema_version": 1, |
| 1733 | "id": "queued-runtime-turn", |
| 1734 | "thread_id": "thread-1", |
| 1735 | "status": "queued", |
| 1736 | "input_summary": "waiting to run", |
| 1737 | "created_at": "2026-07-12T10:30:00Z", |
| 1738 | "effective_provider": "openai", |
| 1739 | "effective_model": "gpt-5.5" |
| 1740 | })) |
| 1741 | .expect("parse runtime row before usage is recorded"); |
| 1742 | |
| 1743 | assert!(recorded.usage.is_none()); |
| 1744 | assert!(!recorded.contributes_to_scorecard()); |
| 1745 | let card = Scorecard::from_recorded_turns(&[recorded]); |
| 1746 | assert_eq!(card.metrics.turns, 0); |
| 1747 | assert!(card.per_turn.is_empty()); |
| 1748 | } |
| 1749 | |
| 1750 | #[test] |
| 1751 | fn recorded_non_model_hook_turn_is_excluded_from_model_scorecard() { |
| 1752 | let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({ |
| 1753 | "turn_id": "shell-turn", |
| 1754 | "created_at": "2026-07-12T10:30:00Z", |
| 1755 | "model_backed": false, |
| 1756 | "provider": null, |
| 1757 | "model": "gpt-5.5", |
| 1758 | "usage": { |
| 1759 | "input_tokens": 0, |
| 1760 | "output_tokens": 0 |
| 1761 | } |
| 1762 | })) |
| 1763 | .expect("parse non-model turn_end record"); |
| 1764 | |
| 1765 | assert!(!recorded.contributes_to_scorecard()); |
| 1766 | } |
| 1767 | |
| 1768 | #[test] |
| 1769 | fn blank_unknown_and_custom_providers_fail_closed_as_unpriced() { |
| 1770 | let u = usage(1000, 500, 0); |
| 1771 | let turns = [ |
| 1772 | TurnInput { |
| 1773 | turn_id: "blank".into(), |
| 1774 | created_at: None, |
| 1775 | provider: Some(" "), |
| 1776 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1777 | model: "gpt-5.5".into(), |
| 1778 | usage: &u, |
| 1779 | }, |
| 1780 | TurnInput { |
| 1781 | turn_id: "named-custom".into(), |
| 1782 | created_at: None, |
| 1783 | provider: Some("my-openai-proxy"), |
| 1784 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1785 | model: "gpt-5.5".into(), |
| 1786 | usage: &u, |
| 1787 | }, |
| 1788 | TurnInput { |
| 1789 | turn_id: "generic-custom".into(), |
| 1790 | created_at: None, |
| 1791 | provider: Some("custom"), |
| 1792 | billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE), |
| 1793 | model: "gpt-5.5".into(), |
| 1794 | usage: &u, |
| 1795 | }, |
| 1796 | ]; |
| 1797 | |
| 1798 | let card = Scorecard::from_turns(&turns); |
| 1799 | |
| 1800 | assert_eq!(card.per_turn[0].provider, None); |
| 1801 | assert_eq!( |
| 1802 | card.per_turn[1].provider.as_deref(), |
| 1803 | Some("my-openai-proxy") |
| 1804 | ); |
| 1805 | assert_eq!(card.per_turn[2].provider.as_deref(), Some("custom")); |
| 1806 | assert!(card.per_turn.iter().all(|turn| turn.cost_unpriced)); |
| 1807 | assert_eq!(card.metrics.unpriced_turns, 3); |
| 1808 | assert!(!card.metrics.cost_complete); |
| 1809 | assert!(card.to_summary().contains("cost_usd: unavailable")); |
| 1810 | } |
| 1811 | |
| 1812 | #[test] |
| 1813 | fn regression_flags_cost_and_token_increases_over_threshold() { |
| 1814 | let baseline = ScorecardMetrics { |
| 1815 | turns: 1, |
| 1816 | money_metered_turns: 1, |
| 1817 | unpriced_turns: 0, |
| 1818 | cny_unpriced_turns: 0, |
| 1819 | cost_complete: true, |
| 1820 | cny_cost_complete: true, |
| 1821 | unpriced_classes: Vec::new(), |
| 1822 | total_input_tokens: 1000, |
| 1823 | total_output_tokens: 1000, |
| 1824 | total_cache_read_tokens: 0, |
| 1825 | total_cache_write_tokens: 0, |
| 1826 | total_reasoning_tokens: 0, |
| 1827 | total_cost_usd: 0.10, |
| 1828 | total_cost_cny: 0.7, |
| 1829 | cache_hit_ratio: 0.5, |
| 1830 | }; |
| 1831 | let current = ScorecardMetrics { |
| 1832 | total_cost_usd: 0.20, // +100% → regression |
| 1833 | total_input_tokens: 1010, // +1% → under 5% threshold, no regression |
| 1834 | total_output_tokens: 2000, // +100% → regression |
| 1835 | cache_hit_ratio: 0.5, // unchanged |
| 1836 | ..baseline.clone() |
| 1837 | }; |
| 1838 | let regs = current.regressions_against(&baseline, 5.0); |
| 1839 | let names: Vec<&str> = regs.iter().map(|r| r.metric.as_str()).collect(); |
| 1840 | assert!(names.contains(&"total_cost_usd")); |
| 1841 | assert!(names.contains(&"total_output_tokens")); |
| 1842 | assert!(!names.contains(&"total_input_tokens")); // under threshold |
| 1843 | } |
| 1844 | |
| 1845 | #[test] |
| 1846 | fn regression_flags_loss_of_cost_completeness_without_comparing_subtotals() { |
| 1847 | let baseline = ScorecardMetrics { |
| 1848 | cost_complete: true, |
| 1849 | total_cost_usd: 0.10, |
| 1850 | ..Default::default() |
| 1851 | }; |
| 1852 | let current = ScorecardMetrics { |
| 1853 | turns: 1, |
| 1854 | unpriced_turns: 1, |
| 1855 | total_cost_usd: 0.20, |
| 1856 | ..Default::default() |
| 1857 | }; |
| 1858 | |
| 1859 | let regs = current.regressions_against(&baseline, 5.0); |
| 1860 | assert!(!regs.iter().any(|r| r.metric == "total_cost_usd")); |
| 1861 | assert!(regs.iter().any(|r| r.metric == "cost_completeness_drop")); |
| 1862 | } |
| 1863 | |
| 1864 | #[test] |
| 1865 | fn regression_flags_loss_of_cny_cost_completeness() { |
| 1866 | let baseline = ScorecardMetrics { |
| 1867 | cny_cost_complete: true, |
| 1868 | total_cost_cny: 0.70, |
| 1869 | ..Default::default() |
| 1870 | }; |
| 1871 | let current = ScorecardMetrics { |
| 1872 | turns: 1, |
| 1873 | cny_unpriced_turns: 1, |
| 1874 | total_cost_cny: 0.0, |
| 1875 | ..Default::default() |
| 1876 | }; |
| 1877 | |
| 1878 | let regs = current.regressions_against(&baseline, 5.0); |
| 1879 | assert!( |
| 1880 | regs.iter() |
| 1881 | .any(|r| r.metric == "cny_cost_completeness_drop") |
| 1882 | ); |
| 1883 | } |
| 1884 | |
| 1885 | #[test] |
| 1886 | fn regression_flags_complete_cny_cost_increase() { |
| 1887 | let baseline = ScorecardMetrics { |
| 1888 | cny_cost_complete: true, |
| 1889 | total_cost_cny: 0.70, |
| 1890 | ..Default::default() |
| 1891 | }; |
| 1892 | let current = ScorecardMetrics { |
| 1893 | total_cost_cny: 1.40, |
| 1894 | ..baseline.clone() |
| 1895 | }; |
| 1896 | |
| 1897 | let regs = current.regressions_against(&baseline, 5.0); |
| 1898 | assert!(regs.iter().any(|r| r.metric == "total_cost_cny")); |
| 1899 | } |
| 1900 | |
| 1901 | #[test] |
| 1902 | fn legacy_baseline_is_readable_but_cost_is_not_comparable() { |
| 1903 | let baseline: ScorecardMetrics = serde_json::from_value(serde_json::json!({ |
| 1904 | "turns": 1, |
| 1905 | "total_input_tokens": 10, |
| 1906 | "total_output_tokens": 5, |
| 1907 | "total_cache_read_tokens": 0, |
| 1908 | "total_cost_usd": 0.10, |
| 1909 | "total_cost_cny": 0.0, |
| 1910 | "cache_hit_ratio": 0.0 |
| 1911 | })) |
| 1912 | .expect("parse legacy scorecard baseline"); |
| 1913 | assert!(!baseline.cost_complete); |
| 1914 | |
| 1915 | let current = ScorecardMetrics { |
| 1916 | cost_complete: true, |
| 1917 | total_cost_usd: 0.20, |
| 1918 | total_input_tokens: 10, |
| 1919 | total_output_tokens: 5, |
| 1920 | ..Default::default() |
| 1921 | }; |
| 1922 | let regs = current.regressions_against(&baseline, 5.0); |
| 1923 | assert!(!regs.iter().any(|r| r.metric == "total_cost_usd")); |
| 1924 | } |
| 1925 | |
| 1926 | #[test] |
| 1927 | fn regression_flags_cache_hit_ratio_drop() { |
| 1928 | let baseline = ScorecardMetrics { |
| 1929 | cache_hit_ratio: 0.80, |
| 1930 | ..Default::default() |
| 1931 | }; |
| 1932 | let current = ScorecardMetrics { |
| 1933 | cache_hit_ratio: 0.40, |
| 1934 | ..Default::default() |
| 1935 | }; |
| 1936 | let regs = current.regressions_against(&baseline, 10.0); |
| 1937 | assert!(regs.iter().any(|r| r.metric == "cache_hit_ratio_drop")); |
| 1938 | } |
| 1939 | |
| 1940 | #[test] |
| 1941 | fn no_regressions_when_within_threshold() { |
| 1942 | let baseline = ScorecardMetrics { |
| 1943 | total_cost_usd: 1.0, |
| 1944 | total_input_tokens: 1000, |
| 1945 | total_output_tokens: 1000, |
| 1946 | cache_hit_ratio: 0.5, |
| 1947 | ..Default::default() |
| 1948 | }; |
| 1949 | let current = baseline.clone(); |
| 1950 | assert!(current.regressions_against(&baseline, 5.0).is_empty()); |
| 1951 | } |
| 1952 | |
| 1953 | #[test] |
| 1954 | fn cache_hit_denominator_saturates_instead_of_wrapping() { |
| 1955 | assert_eq!(cacheable_token_total(u64::MAX, 1, 1), u64::MAX); |
| 1956 | assert_eq!(cacheable_token_total(1, u64::MAX, 1), u64::MAX); |
| 1957 | } |
| 1958 | } |
| 1959 |