scorecard.rs - CodeWhale

scorecard.rs

1	//! Token / cache / cost scorecard (#3388).
2	//!
3	//! A release-gate view of an agent run's token economics: per-turn input /
4	//! output / cache-read tokens and cost, aggregate totals + cache-hit ratio, and
5	//! regression detection against a committed baseline. This is the measurement
6	//! layer the "token, cache, and context discipline" EPIC asks for — it makes a
7	//! cost/token regression visible instead of silently shipping.
8	//!
9	//! The core here is pure and offline: it turns already-recorded per-turn
10	//! [`Usage`] (captured on every turn, persisted in `TurnRecord`) into a
11	//! scorecard, reusing the existing pricing layer rather than reinventing cost
12	//! math. The `scorecard` subcommand is a thin I/O wrapper over this module.
13
14	use chrono::{DateTime, Utc};
15	use serde::{Deserialize, Serialize};
16
17	use crate::config::ProviderKind;
18	#[cfg(test)]
19	use crate::config::{DEEPSEEK_ALIAS_REPLACEMENT, DEEPSEEK_ALIAS_RETIREMENT_UTC};
20	use crate::pricing::{
21	CostEstimate, TurnCostAudit, audit_turn_cost_for_route_at, token_usage_for_pricing,
22	};
23	use codewhale_models::Usage;
24
25	/// One turn's normalized token economics.
26	#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
27	pub struct TurnScore {
28	pub turn_id: String,
29	/// Timestamp used for historical/time-window pricing. `None` means the
30	/// recorder did not preserve when the turn occurred.
31	#[serde(default, skip_serializing_if = "Option::is_none")]
32	pub created_at: Option<DateTime<Utc>>,
33	/// Effective provider recorded for this turn. `None` means legacy or
34	/// otherwise unknown provenance, so cost must remain unpriced.
35	#[serde(default)]
36	pub provider: Option<String>,
37	/// Non-secret discriminator when one provider/model pair spans multiple
38	/// billing systems. Missing provenance keeps ambiguous routes unpriced.
39	#[serde(default, skip_serializing_if = "Option::is_none")]
40	pub billing_surface: Option<String>,
41	pub model: String,
42	/// Non-cached (billable) input tokens.
43	pub input_tokens: u64,
44	/// Output tokens, including reasoning output.
45	pub output_tokens: u64,
46	/// Cache-read (cache-hit) input tokens.
47	pub cache_read_tokens: u64,
48	/// Cache-write (cache-creation) input tokens. Billed at a premium on the
49	/// providers that publish one, so it is audited as its own class rather
50	/// than folded into input. Defaults to 0 for legacy records.
51	#[serde(default)]
52	pub cache_write_tokens: u64,
53	/// Reasoning tokens reported for the turn. Informational only — every
54	/// provider counts these inside `output_tokens`, so adding them here would
55	/// double-bill. Kept so a reasoning-heavy run can still be inspected.
56	#[serde(default)]
57	pub reasoning_tokens: u64,
58	pub cost_usd: f64,
59	pub cost_cny: f64,
60	/// True when provider provenance is missing/unknown or no authoritative USD
61	/// pricing row exists: numeric cost stays 0 for compatibility, while this
62	/// flag prevents it from being represented as a real zero-dollar charge.
63	pub cost_unpriced: bool,
64	/// Same availability marker for CNY. Most catalog offerings publish only
65	/// USD, so their CNY value is unavailable rather than a real zero.
66	#[serde(default)]
67	pub cost_cny_unpriced: bool,
68	/// Why USD cost is unavailable, when it is (`no_pricing_row`,
69	/// `missing_class_price`, `not_money_metered`, …). `None` for priced turns.
70	#[serde(default, skip_serializing_if = "Option::is_none")]
71	pub cost_unpriced_reason: Option<String>,
72	/// Token classes this turn used that carry no published price. Non-empty
73	/// means the estimate failed closed on purpose.
74	#[serde(default, skip_serializing_if = "Vec::is_empty")]
75	pub unpriced_classes: Vec<String>,
76	/// Provenance of the pricing row that was applied or attempted
77	/// (`models_dev_bundled`, `provider_live`, `provider_docs`,
78	/// `user_override`). `None` when no row was found at all.
79	#[serde(default, skip_serializing_if = "Option::is_none")]
80	pub pricing_provenance: Option<String>,
81	/// Live-pricing downgrade receipt: the live catalog row for this route could
82	/// not be verified (stale, or fetched from a different endpoint), so the
83	/// bundled published rates were used. Present even on priced turns, because
84	/// it explains which row the number came from.
85	#[serde(default, skip_serializing_if = "Option::is_none")]
86	pub live_pricing_defect: Option<String>,
87	/// Whether this turn is inside the money-metered coverage denominator.
88	///
89	/// False only for routes exactly identified as non-metered. Serialized so a
90	/// re-read scorecard can reproduce the coverage split without re-deriving it
91	/// from `provider` + `billing_surface`, which are also preserved above.
92	#[serde(default)]
93	pub money_metered: bool,
94	}
95
96	/// Aggregate metrics for a run. Serializes/deserializes as the baseline file.
97	#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
98	pub struct ScorecardMetrics {
99	pub turns: usize,
100	/// Turns whose route meters money, or whose billing basis could not be
101	/// established. This — not `turns` — is the denominator the USD total is
102	/// meant to cover: a local or subscription turn owes no dollars, so counting
103	/// it would understate coverage, while an unknown-basis turn must stay in
104	/// (#4318). Defaults to zero so existing baseline JSON stays readable.
105	#[serde(default)]
106	pub money_metered_turns: usize,
107	/// Money-metered turns that could not be priced authoritatively in USD.
108	/// Defaults to zero so existing baseline JSON remains readable.
109	#[serde(default)]
110	pub unpriced_turns: usize,
111	/// Turns without authoritative CNY pricing.
112	#[serde(default)]
113	pub cny_unpriced_turns: usize,
114	/// Whether every turn contributed authoritative USD pricing. Legacy
115	/// baselines lack this field and therefore default to `false`, preventing
116	/// comparisons against totals that may have been inferred from model ids
117	/// alone.
118	#[serde(default)]
119	pub cost_complete: bool,
120	/// Whether every turn contributed authoritative CNY pricing.
121	#[serde(default)]
122	pub cny_cost_complete: bool,
123	/// Token classes used somewhere in the run that had no published price, in
124	/// stable order. Non-empty means `cost_complete` is false because of a
125	/// class-level pricing gap, not merely an unknown route.
126	#[serde(default)]
127	pub unpriced_classes: Vec<String>,
128	pub total_input_tokens: u64,
129	pub total_output_tokens: u64,
130	pub total_cache_read_tokens: u64,
131	/// Cache-write (cache-creation) tokens across the run. Defaults to zero so
132	/// existing baseline JSON stays readable.
133	#[serde(default)]
134	pub total_cache_write_tokens: u64,
135	/// Reasoning tokens across the run. Informational: already inside
136	/// `total_output_tokens`, never added to it.
137	#[serde(default)]
138	pub total_reasoning_tokens: u64,
139	pub total_cost_usd: f64,
140	pub total_cost_cny: f64,
141	/// `cache_read / (input + cache_read)`; `0.0` when there are no input
142	/// tokens. Higher is better (more of the prompt was served from cache).
143	pub cache_hit_ratio: f64,
144	}
145
146	/// A metric that grew beyond the allowed threshold versus the baseline.
147	#[derive(Debug, Clone, Serialize, PartialEq)]
148	pub struct Regression {
149	pub metric: String,
150	pub baseline: f64,
151	pub current: f64,
152	/// Percent increase over baseline. `f64::INFINITY` when baseline was 0.
153	pub pct_increase: f64,
154	}
155
156	fn cacheable_token_total(input: u64, cache_read: u64, cache_write: u64) -> u64 {
157	input.saturating_add(cache_read).saturating_add(cache_write)
158	}
159
160	/// Full scorecard: per-turn breakdown plus aggregates.
161	#[derive(Debug, Clone, Serialize)]
162	pub struct Scorecard {
163	pub per_turn: Vec<TurnScore>,
164	pub metrics: ScorecardMetrics,
165	}
166
167	/// One row of input to the scorecard: a turn id, the model that served it, and
168	/// the turn's recorded usage.
169	///
170	/// `billing_surface` is explicit and has no default. The scorecard has two
171	/// entry modes, and they must agree: if this fixture mode could silently supply
172	/// an official first-party surface, every scorecard test would be asserting
173	/// against a route classification that `from_recorded_turns` never invents, and
174	/// the fail-closed path would go unexercised in the mode the tests use.
175	#[cfg(test)]
176	pub struct TurnInput<'a> {
177	pub turn_id: String,
178	pub created_at: Option<&'a DateTime<Utc>>,
179	pub provider: Option<&'a str>,
180	/// The route's recorded billing surface, or `None` when the recording did
181	/// not establish one. `None` must price exactly as it does for a recorded
182	/// turn: unknown, never official.
183	pub billing_surface: Option<&'a str>,
184	pub model: String,
185	pub usage: &'a Usage,
186	}
187
188	#[derive(Debug, Clone, Copy)]
189	struct ScorecardTurnRef<'a> {
190	turn_id: &'a str,
191	created_at: Option<&'a DateTime<Utc>>,
192	provider: Option<&'a str>,
193	billing_surface: Option<&'a str>,
194	model: &'a str,
195	usage: &'a Usage,
196	}
197
198	/// A recorded turn as read from a scorecard input file (a JSON array of these).
199	/// The base shape matches the per-turn data a `TurnEnd` hook emits. Recorders
200	/// and persisted runtime exports can add `provider` / `effective_provider` plus
201	/// non-secret billing-surface provenance. Legacy model-only recordings remain
202	/// readable but deliberately unpriced.
203	#[derive(Debug, Clone, Deserialize)]
204	pub struct RecordedTurn {
205	#[serde(default, alias = "id")]
206	pub turn_id: String,
207	#[serde(default)]
208	pub created_at: Option<DateTime<Utc>>,
209	/// New `turn_end` hooks mark shell-only lifecycle records false so the
210	/// model-cost scorecard can ignore them. Missing stays compatible with
211	/// legacy hook rows and persisted runtime turns, which are model-backed.
212	#[serde(default)]
213	pub model_backed: Option<bool>,
214	#[serde(default, alias = "effective_provider")]
215	pub provider: Option<String>,
216	#[serde(default, alias = "effective_billing_surface")]
217	pub billing_surface: Option<String>,
218	#[serde(default, alias = "effective_model")]
219	pub model: String,
220	#[serde(default)]
221	pub usage: Option<Usage>,
222	}
223
224	impl RecordedTurn {
225	#[must_use]
226	pub fn contributes_to_scorecard(&self) -> bool {
227	self.model_backed.unwrap_or(true) && self.usage.is_some() && !self.model.trim().is_empty()
228	}
229	}
230
231	#[derive(Debug, Clone, Default)]
232	struct AvailableCost {
233	usd: Option<f64>,
234	cny: Option<f64>,
235	unpriced_reason: Option<String>,
236	unpriced_classes: Vec<String>,
237	provenance: Option<String>,
238	/// Live-pricing downgrade receipt, when the row used was a bundled fallback
239	/// for an unverifiable live row.
240	live_pricing_defect: Option<String>,
241	/// Whether this turn belongs in the money-metered coverage denominator.
242	/// False only for routes exactly identified as non-metered.
243	counts_toward_money_coverage: bool,
244	}
245
246	impl AvailableCost {
247	/// Legacy/unknown provenance: no route to price against at all.
248	///
249	/// This still counts toward money coverage. A recording whose provider text
250	/// CodeWhale cannot parse is a turn whose spend is unknown, not a turn that
251	/// cost nothing — excusing it would let a legacy input file report a complete
252	/// total (#4318).
253	fn unknown_route() -> Self {
254	Self {
255	unpriced_reason: Some("unknown_route".to_string()),
256	counts_toward_money_coverage: true,
257	..Self::default()
258	}
259	}
260
261	/// Fails closed with an explicit reason, still inside money coverage.
262	fn failed_closed(reason: &str) -> Self {
263	Self {
264	unpriced_reason: Some(reason.to_string()),
265	counts_toward_money_coverage: true,
266	..Self::default()
267	}
268	}
269
270	fn from_audit(audit: &TurnCostAudit) -> Self {
271	Self {
272	usd: audit
273	.estimate
274	.and_then(\|cost\| audit.usd_priced.then_some(cost.usd)),
275	cny: audit
276	.estimate
277	.and_then(\|cost\| audit.cny_priced.then_some(cost.cny)),
278	unpriced_reason: audit
279	.unpriced_reason
280	.map(\|reason\| reason.label().to_string()),
281	unpriced_classes: audit
282	.unpriced_classes
283	.iter()
284	.map(\|class\| class.label().to_string())
285	.collect(),
286	provenance: audit
287	.provenance
288	.as_ref()
289	.map(\|provenance\| provenance.label().to_string()),
290	live_pricing_defect: audit
291	.live_pricing_defect
292	.as_ref()
293	.map(\|defect\| defect.label().to_string()),
294	counts_toward_money_coverage: audit.counts_toward_money_coverage(),
295	}
296	}
297	}
298
299	fn provider_scoped_cost(
300	provider: ProviderKind,
301	model: &str,
302	usage: &Usage,
303	created_at: Option<&DateTime<Utc>>,
304	billing_surface: Option<&str>,
305	) -> AvailableCost {
306	// These provider identities are themselves exact billing provenance and
307	// override stale/junk recorded surfaces: they cannot become PAYG merely
308	// because an older recorder wrote a bogus endpoint classification.
309	let intrinsic_surface = match provider {
310	ProviderKind::Ollama \| ProviderKind::Sglang \| ProviderKind::Vllm => {
311	Some(crate::pricing::LOCAL_BILLING_SURFACE)
312	}
313	ProviderKind::OpenaiCodex \| ProviderKind::OpencodeGo => {
314	Some(crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE)
315	}
316	_ => None,
317	};
318	let billing_surface = intrinsic_surface.or(billing_surface);
319	// Every provider that supports both PAYG and plan/OAuth routes needs the
320	// recorded surface to choose between them. A model id or provider name is
321	// not sufficient evidence in an offline scorecard.
322	if billing_surface.is_none()
323	&& matches!(
324	provider,
325	ProviderKind::Zai
326	\| ProviderKind::Moonshot
327	\| ProviderKind::Anthropic
328	\| ProviderKind::XiaomiMimo
329	\| ProviderKind::Xai
330	\| ProviderKind::Minimax
331	\| ProviderKind::MinimaxAnthropic
332	\| ProviderKind::Stepfun
333	\| ProviderKind::Custom
334	)
335	{
336	return AvailableCost::failed_closed("missing_billing_surface");
337	}
338	let direct_deepseek = matches!(
339	provider,
340	ProviderKind::Deepseek \| ProviderKind::DeepseekAnthropic
341	);
342	let normalized_model = model.trim();
343	let model_lower = normalized_model.to_ascii_lowercase();
344	// Every direct DeepSeek first-party rate is time-windowed now — the
345	// V4 flash/pro rows carry peak/off-peak tiers (01:00–04:00 and
346	// 06:00–10:00 UTC on weekdays, with the whole of a Beijing-time Saturday
347	// and Sunday billing off-peak from 2026-08-23), and the retired
348	// `deepseek-chat` / `deepseek-reasoner` aliases price through them — so an
349	// undated DeepSeek turn cannot be resolved to one price. The weekend is
350	// bounded in Beijing time, which is why the window it covers is not the
351	// one a UTC weekday would give. `claude-sonnet-5` keeps the same recorded-time
352	// contract it had during its introductory window (Anthropic later made
353	// that $2/$10 rate permanent; the row still prices at the turn's own time
354	// rather than the wall clock, and undated turns still fail closed).
355	let needs_recorded_time = direct_deepseek
356	\|\| (provider == ProviderKind::Anthropic && model_lower == "claude-sonnet-5");
357	let recorded_at = match (created_at, needs_recorded_time) {
358	(Some(recorded_at), _) => recorded_at.to_owned(),
359	// A time-windowed rate without a recorded time cannot be resolved to a
360	// single price; fail closed rather than guess a window.
361	(None, true) => return AvailableCost::failed_closed("missing_recorded_time"),
362	(None, false) => Utc::now(),
363	};
364
365	// The billing surface recorded with the turn is authoritative over any
366	// provider-level assumption, and it now covers every classification a route
367	// can carry — Z.ai Coding Plan, Kimi Code, MiniMax Token Plan, MiMo token
368	// plan, OAuth brokers, local runtimes, aggregators, first-party PAYG, and
369	// "unclassified" — not just StepFun's two surfaces (#4318).
370	match crate::pricing::endpoint_metering_for_billing_surface(billing_surface) {
371	// Exactly identified as non-metered: no dollar figure is owed, and the
372	// turn leaves the money-coverage denominator.
373	crate::pricing::EndpointMetering::ExactSubscription
374	\| crate::pricing::EndpointMetering::LocalNoBill => {
375	return AvailableCost {
376	unpriced_reason: Some(
377	crate::pricing::UnpricedReason::NotMoneyMetered
378	.label()
379	.to_string(),
380	),
381	counts_toward_money_coverage: false,
382	..AvailableCost::default()
383	};
384	}
385	// A recorded surface CodeWhale cannot place must not inherit the
386	// provider's default rates.
387	crate::pricing::EndpointMetering::Unknown if billing_surface.is_some() => {
388	return AvailableCost::failed_closed(
389	crate::pricing::UnpricedReason::UnknownBillingBasis.label(),
390	);
391	}
392	crate::pricing::EndpointMetering::Unknown \| crate::pricing::EndpointMetering::Money => {}
393	}
394
395	// The pricing layer owns the exact provider/model catalog gate, explicit
396	// first-party hand-price allowlist, cache-class completeness checks, and
397	// endpoint-derived billing surfaces. Keeping one route-aware path prevents
398	// the scorecard from drifting back to model-only pricing.
399	let audit = audit_turn_cost_for_route_at(
400	provider,
401	normalized_model,
402	billing_surface,
403	usage,
404	recorded_at,
405	);
406	AvailableCost::from_audit(&audit)
407	}
408
409	impl Scorecard {
410	/// Build a scorecard from recorded per-turn usage. Pure + offline; cost is
411	/// computed via the shared pricing layer (`None` pricing → unpriced, 0 cost).
412	#[must_use]
413	#[cfg(test)]
414	pub fn from_turns(turns: &[TurnInput<'_>]) -> Self {
415	Self::from_turn_refs(turns.iter().map(\|turn\| ScorecardTurnRef {
416	turn_id: &turn.turn_id,
417	created_at: turn.created_at,
418	provider: turn.provider,
419	billing_surface: turn.billing_surface,
420	model: &turn.model,
421	usage: turn.usage,
422	}))
423	}
424
425	/// Build directly from hook/runtime records, retaining billing provenance
426	/// while excluding explicitly non-model lifecycle rows.
427	#[must_use]
428	pub fn from_recorded_turns(turns: &[RecordedTurn]) -> Self {
429	Self::from_turn_refs(turns.iter().filter_map(\|turn\| {
430	if !turn.contributes_to_scorecard() {
431	return None;
432	}
433	let usage = turn.usage.as_ref()?;
434	Some(ScorecardTurnRef {
435	turn_id: &turn.turn_id,
436	created_at: turn.created_at.as_ref(),
437	provider: turn.provider.as_deref(),
438	billing_surface: turn.billing_surface.as_deref(),
439	model: &turn.model,
440	usage,
441	})
442	}))
443	}
444
445	fn from_turn_refs<'a>(turns: impl IntoIterator<Item = ScorecardTurnRef<'a>>) -> Self {
446	let turns = turns.into_iter();
447	let mut per_turn = Vec::with_capacity(turns.size_hint().0);
448	let mut metrics = ScorecardMetrics::default();
449	let mut unpriced_classes = std::collections::BTreeSet::new();
450
451	for turn in turns {
452	// Normalize provider usage into canonical billable classes once.
453	let classes = token_usage_for_pricing(turn.usage);
454	let provider = turn
455	.provider
456	.map(str::trim)
457	.filter(\|value\| !value.is_empty());
458	let cost = provider.and_then(ProviderKind::parse).map_or_else(
459	AvailableCost::unknown_route,
460	\|provider\| {
461	provider_scoped_cost(
462	provider,
463	turn.model,
464	turn.usage,
465	turn.created_at,
466	turn.billing_surface,
467	)
468	},
469	);
470	let cost_unpriced = cost.usd.is_none();
471	let cost_cny_unpriced = cost.cny.is_none();
472	let cost_usd = cost.usd.unwrap_or(0.0);
473	let cost_cny = cost.cny.unwrap_or(0.0);
474	let reasoning_tokens = u64::from(turn.usage.reasoning_tokens.unwrap_or(0));
475	unpriced_classes.extend(cost.unpriced_classes.iter().cloned());
476
477	metrics.turns = metrics.turns.saturating_add(1);
478	// Only money-metered turns can make a dollar total incomplete. A
479	// local or plan turn is not an unpriced dollar; an unknown one is.
480	if cost.counts_toward_money_coverage {
481	metrics.money_metered_turns = metrics.money_metered_turns.saturating_add(1);
482	metrics.unpriced_turns = metrics
483	.unpriced_turns
484	.saturating_add(usize::from(cost_unpriced));
485	metrics.cny_unpriced_turns = metrics
486	.cny_unpriced_turns
487	.saturating_add(usize::from(cost_cny_unpriced));
488	}
489	metrics.total_input_tokens = metrics.total_input_tokens.saturating_add(classes.input);
490	metrics.total_output_tokens =
491	metrics.total_output_tokens.saturating_add(classes.output);
492	metrics.total_cache_read_tokens = metrics
493	.total_cache_read_tokens
494	.saturating_add(classes.cache_read);
495	metrics.total_cache_write_tokens = metrics
496	.total_cache_write_tokens
497	.saturating_add(classes.cache_write);
498	metrics.total_reasoning_tokens = metrics
499	.total_reasoning_tokens
500	.saturating_add(reasoning_tokens);
501	metrics.total_cost_usd = CostEstimate::usd_only(metrics.total_cost_usd)
502	.saturating_add(CostEstimate::usd_only(cost_usd))
503	.usd;
504	metrics.total_cost_cny = CostEstimate {
505	usd: 0.0,
506	cny: metrics.total_cost_cny,
507	}
508	.saturating_add(CostEstimate {
509	usd: 0.0,
510	cny: cost_cny,
511	})
512	.cny;
513
514	per_turn.push(TurnScore {
515	turn_id: turn.turn_id.to_string(),
516	created_at: turn.created_at.cloned(),
517	provider: provider.map(str::to_string),
518	billing_surface: turn.billing_surface.map(str::to_string),
519	model: turn.model.to_string(),
520	input_tokens: classes.input,
521	output_tokens: classes.output,
522	cache_read_tokens: classes.cache_read,
523	cache_write_tokens: classes.cache_write,
524	reasoning_tokens,
525	cost_usd,
526	cost_cny,
527	cost_unpriced,
528	cost_cny_unpriced,
529	cost_unpriced_reason: cost.unpriced_reason,
530	unpriced_classes: cost.unpriced_classes,
531	pricing_provenance: cost.provenance,
532	live_pricing_defect: cost.live_pricing_defect,
533	money_metered: cost.counts_toward_money_coverage,
534	});
535	}
536	metrics.unpriced_classes = unpriced_classes.into_iter().collect();
537
538	// Canonical denominator: hit / (non-cached input + hit + write).
539	// `total_input_tokens` here is already the non-cached input, because
540	// `token_usage_for_pricing` splits hits and writes out of the reported
541	// prompt total. Cache-write tokens were previously missing from the
542	// denominator, which reported a better hit ratio on precisely the turns
543	// that paid to populate the cache (#4318). Write stays a separate
544	// reported total so the premium is not hidden inside the ratio.
545	let cacheable = cacheable_token_total(
546	metrics.total_input_tokens,
547	metrics.total_cache_read_tokens,
548	metrics.total_cache_write_tokens,
549	);
550	metrics.cache_hit_ratio = if cacheable > 0 {
551	metrics.total_cache_read_tokens as f64 / cacheable as f64
552	} else {
553	0.0
554	};
555	metrics.cost_complete = metrics.unpriced_turns == 0;
556	metrics.cny_cost_complete = metrics.cny_unpriced_turns == 0;
557
558	Self { per_turn, metrics }
559	}
560
561	/// Render a compact human-readable summary (used for non-JSON output).
562	#[must_use]
563	pub fn to_summary(&self) -> String {
564	let m = &self.metrics;
565	let mut out = String::new();
566	out.push_str("Token / cache / cost scorecard\n");
567	out.push_str(&format!(
568	"turns: {} money_metered_turns: {}\n",
569	m.turns, m.money_metered_turns
570	));
571	out.push_str(&format!(
572	"input_tokens: {} output_tokens: {} cache_read_tokens: {} cache_write_tokens: {}\n",
573	m.total_input_tokens,
574	m.total_output_tokens,
575	m.total_cache_read_tokens,
576	m.total_cache_write_tokens
577	));
578	out.push_str(&format!(
579	"reasoning_tokens: {} (informational; already inside output_tokens)\n",
580	m.total_reasoning_tokens
581	));
582	out.push_str(&format!(
583	"cache_hit_ratio: {:.1}%\n",
584	m.cache_hit_ratio * 100.0
585	));
586	append_currency_summary(
587	&mut out,
588	"cost_usd",
589	"priced_cost_subtotal_usd",
590	"$",
591	m.total_cost_usd,
592	m.unpriced_turns,
593	// Coverage is reported against the money-metered turns, not every
594	// turn: a local or plan turn owes no dollars, so including it in the
595	// denominator would understate how complete the figure is.
596	m.money_metered_turns,
597	);
598	append_currency_summary(
599	&mut out,
600	"cost_cny",
601	"priced_cost_subtotal_cny",
602	"¥",
603	m.total_cost_cny,
604	m.cny_unpriced_turns,
605	m.money_metered_turns,
606	);
607	if m.unpriced_turns > 0 {
608	out.push_str(&format!(
609	"note: {} turn(s) had missing/unknown provider provenance or no authoritative USD pricing row; their USD cost is unavailable and excluded.\n",
610	m.unpriced_turns
611	));
612	}
613	if m.cny_unpriced_turns > 0 {
614	out.push_str(&format!(
615	"note: {} turn(s) had no authoritative CNY pricing row; their CNY cost is unavailable and excluded.\n",
616	m.cny_unpriced_turns
617	));
618	}
619	if !m.unpriced_classes.is_empty() {
620	out.push_str(&format!(
621	"note: token class(es) with no published price on a used route: {}. Those turns fail closed rather than under-report.\n",
622	m.unpriced_classes.join(", ")
623	));
624	}
625	out
626	}
627	}
628
629	fn append_currency_summary(
630	out: &mut String,
631	complete_label: &str,
632	subtotal_label: &str,
633	symbol: &str,
634	total: f64,
635	unpriced_turns: usize,
636	turns: usize,
637	) {
638	if unpriced_turns == 0 {
639	out.push_str(&format!("{complete_label}: {symbol}{total:.4}\n"));
640	} else if unpriced_turns == turns {
641	out.push_str(&format!("{complete_label}: unavailable\n"));
642	} else {
643	out.push_str(&format!("{subtotal_label}: {symbol}{total:.4}\n"));
644	}
645	}
646
647	impl ScorecardMetrics {
648	/// Flag metrics that grew more than `threshold_pct` over `baseline`. Cost
649	/// and token counts are "lower is better", so only increases are
650	/// regressions. (Cache-hit ratio is the opposite, reported separately.)
651	///
652	/// Fails closed on values that cannot be compared: a non-finite threshold
653	/// admits no growth at all (it is treated as 0%), and a metric whose
654	/// change is `NaN` counts as regressed. Every `NaN` comparison is false,
655	/// so without this a `NaN` threshold would pass any run.
656	#[must_use]
657	pub fn regressions_against(
658	&self,
659	baseline: &ScorecardMetrics,
660	threshold_pct: f64,
661	) -> Vec<Regression> {
662	let threshold_pct = if threshold_pct.is_finite() {
663	threshold_pct
664	} else {
665	0.0
666	};
667	let mut out = Vec::new();
668	// A partial/unknown subtotal is not comparable to a complete baseline,
669	// but losing completeness is itself a regression. Otherwise removing
670	// provider provenance could turn real spend into a smaller subtotal
671	// and silently bypass the release gate.
672	if baseline.cost_complete && !self.cost_complete {
673	out.push(Regression {
674	metric: "cost_completeness_drop".to_string(),
675	baseline: 1.0,
676	current: 0.0,
677	pct_increase: 100.0,
678	});
679	} else if self.cost_complete && baseline.cost_complete {
680	push_regression(
681	&mut out,
682	"total_cost_usd",
683	baseline.total_cost_usd,
684	self.total_cost_usd,
685	threshold_pct,
686	);
687	}
688	if baseline.cny_cost_complete && !self.cny_cost_complete {
689	out.push(Regression {
690	metric: "cny_cost_completeness_drop".to_string(),
691	baseline: 1.0,
692	current: 0.0,
693	pct_increase: 100.0,
694	});
695	} else if self.cny_cost_complete && baseline.cny_cost_complete {
696	push_regression(
697	&mut out,
698	"total_cost_cny",
699	baseline.total_cost_cny,
700	self.total_cost_cny,
701	threshold_pct,
702	);
703	}
704	push_regression(
705	&mut out,
706	"total_input_tokens",
707	baseline.total_input_tokens as f64,
708	self.total_input_tokens as f64,
709	threshold_pct,
710	);
711	push_regression(
712	&mut out,
713	"total_output_tokens",
714	baseline.total_output_tokens as f64,
715	self.total_output_tokens as f64,
716	threshold_pct,
717	);
718	// Cache-hit ratio regresses when it drops; express the drop as a
719	// positive percentage so it reads like the others.
720	if baseline.cache_hit_ratio > 0.0 {
721	let drop_pct = (baseline.cache_hit_ratio - self.cache_hit_ratio)
722	/ baseline.cache_hit_ratio
723	* 100.0;
724	if drop_pct.is_nan() \|\| drop_pct > threshold_pct {
725	out.push(Regression {
726	metric: "cache_hit_ratio_drop".to_string(),
727	baseline: baseline.cache_hit_ratio,
728	current: self.cache_hit_ratio,
729	pct_increase: drop_pct,
730	});
731	}
732	}
733	out
734	}
735	}
736
737	fn push_regression(
738	out: &mut Vec<Regression>,
739	metric: &str,
740	base: f64,
741	cur: f64,
742	threshold_pct: f64,
743	) {
744	if base > 0.0 {
745	let pct = (cur - base) / base * 100.0;
746	if pct.is_nan() \|\| pct > threshold_pct {
747	out.push(Regression {
748	metric: metric.to_string(),
749	baseline: base,
750	current: cur,
751	pct_increase: pct,
752	});
753	}
754	} else if cur > 0.0 {
755	out.push(Regression {
756	metric: metric.to_string(),
757	baseline: base,
758	current: cur,
759	pct_increase: f64::INFINITY,
760	});
761	}
762	}
763
764	#[cfg(test)]
765	mod tests {
766	use super::*;
767
768	fn usage(input: u32, output: u32, cache_hit: u32) -> Usage {
769	Usage {
770	input_tokens: input,
771	output_tokens: output,
772	prompt_cache_hit_tokens: Some(cache_hit),
773	..Default::default()
774	}
775	}
776
777	/// The scorecard has two entry modes and they must classify identically.
778	///
779	/// `from_turns` is the fixture mode used by this test module;
780	/// `from_recorded_turns` is the mode that reads real recordings. The
781	/// fixture mode used to inject `first-party-payg` for every row, which
782	/// meant the whole suite was asserting against a route classification the
783	/// real mode never produces — the fail-closed path was untested precisely
784	/// where it mattered. Neither mode may invent an official surface.
785	#[test]
786	fn both_scorecard_entry_modes_agree_on_an_unestablished_billing_surface() {
787	let sample = usage(10_000, 1_000, 0);
788
789	let fixture = Scorecard::from_turns(&[TurnInput {
790	turn_id: "t1".into(),
791	created_at: None,
792	provider: Some("anthropic"),
793	billing_surface: None,
794	model: "claude-haiku-4-5".into(),
795	usage: &sample,
796	}]);
797	let recorded = Scorecard::from_recorded_turns(&[RecordedTurn {
798	turn_id: "t1".to_string(),
799	created_at: None,
800	model_backed: None,
801	provider: Some("anthropic".to_string()),
802	billing_surface: None,
803	model: "claude-haiku-4-5".to_string(),
804	usage: Some(sample.clone()),
805	}]);
806
807	assert_eq!(fixture.per_turn, recorded.per_turn);
808	assert_eq!(fixture.metrics, recorded.metrics);
809	assert!(
810	fixture.per_turn[0].cost_unpriced,
811	"a route with no established surface must not be priced"
812	);
813	assert_eq!(fixture.per_turn[0].cost_usd, 0.0);
814	assert!(!fixture.metrics.cost_complete);
815	assert_eq!(fixture.metrics.money_metered_turns, 1);
816	assert_eq!(fixture.metrics.unpriced_turns, 1);
817	let summary = fixture.to_summary();
818	assert!(summary.contains("cost_usd: unavailable"), "{summary}");
819	assert!(summary.contains("cost_cny: unavailable"), "{summary}");
820	assert!(
821	!summary.contains('$') && !summary.contains('¥'),
822	"an unpriced-only run must name no amount at all: {summary}"
823	);
824
825	// With the surface actually established, both modes price it — and
826	// still agree.
827	let priced_fixture = Scorecard::from_turns(&[TurnInput {
828	turn_id: "t1".into(),
829	created_at: None,
830	provider: Some("anthropic"),
831	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
832	model: "claude-haiku-4-5".into(),
833	usage: &sample,
834	}]);
835	let priced_recorded = Scorecard::from_recorded_turns(&[RecordedTurn {
836	turn_id: "t1".to_string(),
837	created_at: None,
838	model_backed: None,
839	provider: Some("anthropic".to_string()),
840	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE.to_string()),
841	model: "claude-haiku-4-5".to_string(),
842	usage: Some(sample),
843	}]);
844	assert_eq!(priced_fixture.per_turn, priced_recorded.per_turn);
845	assert!(!priced_fixture.per_turn[0].cost_unpriced);
846	assert!(priced_fixture.metrics.cost_complete);
847	}
848
849	#[test]
850	fn dual_mode_routes_require_surface_but_intrinsic_routes_override_junk() {
851	let usage = usage(10_000, 1_000, 0);
852	for (provider, model) in [
853	(ProviderKind::Anthropic, "claude-haiku-4-5"),
854	(ProviderKind::Moonshot, "kimi-k2.7-code"),
855	(ProviderKind::Zai, "glm-5.2"),
856	(ProviderKind::Minimax, "minimax-m3"),
857	] {
858	let cost = provider_scoped_cost(provider, model, &usage, None, None);
859	assert_eq!(
860	cost.unpriced_reason.as_deref(),
861	Some("missing_billing_surface"),
862	"{provider:?}"
863	);
864	assert!(cost.usd.is_none(), "{provider:?}");
865	}
866
867	for provider in [ProviderKind::OpenaiCodex, ProviderKind::OpencodeGo] {
868	let cost = provider_scoped_cost(
869	provider,
870	"gpt-5.5",
871	&usage,
872	None,
873	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
874	);
875	assert_eq!(cost.unpriced_reason.as_deref(), Some("not_money_metered"));
876	assert!(!cost.counts_toward_money_coverage);
877	}
878	let local = provider_scoped_cost(
879	ProviderKind::Ollama,
880	"llama3.2",
881	&usage,
882	None,
883	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
884	);
885	assert_eq!(local.unpriced_reason.as_deref(), Some("not_money_metered"));
886	assert!(!local.counts_toward_money_coverage);
887
888	let cloud = provider_scoped_cost(
889	ProviderKind::OllamaCloud,
890	crate::config::DEFAULT_OLLAMA_CLOUD_MODEL,
891	&usage,
892	None,
893	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
894	);
895	assert_eq!(
896	cloud.unpriced_reason.as_deref(),
897	Some("unknown_billing_basis")
898	);
899	assert!(
900	cloud.counts_toward_money_coverage,
901	"hosted Cloud usage must never disappear as local/free"
902	);
903	}
904
905	fn cache_write_usage(input: u32, output: u32, cache_hit: u32, cache_write: u32) -> Usage {
906	Usage {
907	input_tokens: input,
908	output_tokens: output,
909	prompt_cache_hit_tokens: Some(cache_hit),
910	prompt_cache_write_tokens: Some(cache_write),
911	reasoning_tokens: Some(output / 2),
912	..Default::default()
913	}
914	}
915
916	/// A mixed-route run: one fully-priced cache-write turn, one turn whose
917	/// route publishes no cache-write rate, and one non-metered OAuth turn.
918	/// The priced subtotal stays honest, `cost_complete` fails closed, and the
919	/// audit names the class and provenance behind each gap.
920	#[test]
921	fn mixed_route_run_audits_cache_write_classes_and_fails_closed() {
922	// 1M input of which 200k is a cache read, 100k is a cache write.
923	let priced = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
924	let unpriced_write = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
925	let oauth = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
926	let turns = [
927	TurnInput {
928	turn_id: "anthropic".into(),
929	created_at: None,
930	provider: Some("anthropic"),
931	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
932	model: "claude-haiku-4-5".into(),
933	usage: &priced,
934	},
935	TurnInput {
936	turn_id: "moonshot".into(),
937	created_at: None,
938	provider: Some("moonshot"),
939	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
940	model: "kimi-k2.7-code".into(),
941	usage: &unpriced_write,
942	},
943	TurnInput {
944	turn_id: "oauth".into(),
945	created_at: None,
946	provider: Some("openai-codex"),
947	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
948	model: "gpt-5.5".into(),
949	usage: &oauth,
950	},
951	];
952
953	let card = Scorecard::from_turns(&turns);
954
955	// Cache-write tokens are their own audited class on every turn.
956	for turn in &card.per_turn {
957	assert_eq!(turn.cache_write_tokens, 100_000, "{}", turn.turn_id);
958	assert_eq!(turn.input_tokens, 700_000, "{}", turn.turn_id);
959	assert_eq!(turn.cache_read_tokens, 200_000, "{}", turn.turn_id);
960	// Reasoning stays informational: never added to billable output.
961	assert_eq!(turn.output_tokens, 100_000, "{}", turn.turn_id);
962	assert_eq!(turn.reasoning_tokens, 50_000, "{}", turn.turn_id);
963	}
964	assert_eq!(card.metrics.total_cache_write_tokens, 300_000);
965	assert_eq!(card.metrics.total_output_tokens, 300_000);
966	assert_eq!(card.metrics.total_reasoning_tokens, 150_000);
967
968	// Anthropic publishes a 1.25/M cache-write rate, so the write premium
969	// is billed rather than silently charged at the input rate.
970	let anthropic = &card.per_turn[0];
971	assert!(!anthropic.cost_unpriced);
972	assert_eq!(anthropic.cost_unpriced_reason, None);
973	assert!(anthropic.unpriced_classes.is_empty());
974	// Provenance is recorded (bundled snapshot offline, live after a
975	// catalog refresh); the point is that it is never absent for a
976	// priced turn.
977	assert!(anthropic.pricing_provenance.is_some());
978	let expected = 0.7 * 1.0 + 0.1 * 5.0 + 0.2 * 0.1 + 0.1 * 1.25;
979	assert!((anthropic.cost_usd - expected).abs() < 1e-9);
980
981	// Moonshot's row has no published cache-write rate: the whole turn
982	// fails closed instead of under-reporting the write tokens.
983	let moonshot = &card.per_turn[1];
984	assert!(moonshot.cost_unpriced);
985	assert_eq!(moonshot.cost_usd, 0.0);
986	assert_eq!(
987	moonshot.cost_unpriced_reason.as_deref(),
988	Some("missing_class_price")
989	);
990	assert_eq!(moonshot.unpriced_classes, vec!["cache_write".to_string()]);
991	assert!(moonshot.pricing_provenance.is_some());
992
993	// A subscription route is not "free"; it is not money-metered.
994	let oauth = &card.per_turn[2];
995	assert!(oauth.cost_unpriced);
996	assert_eq!(
997	oauth.cost_unpriced_reason.as_deref(),
998	Some("not_money_metered")
999	);
1000	assert!(oauth.unpriced_classes.is_empty());
1001	assert_eq!(oauth.pricing_provenance, None);
1002
1003	// Aggregates stay honest about what the number covers. Two of the three
1004	// turns owe money (Anthropic and Moonshot); the OAuth turn does not, so
1005	// it is outside the denominator rather than counted as an unpriced dollar.
1006	assert_eq!(card.metrics.turns, 3);
1007	assert_eq!(card.metrics.money_metered_turns, 2);
1008	assert_eq!(card.metrics.unpriced_turns, 1);
1009	assert!(!card.metrics.cost_complete);
1010	assert_eq!(
1011	card.metrics.unpriced_classes,
1012	vec!["cache_write".to_string()]
1013	);
1014	assert!((card.metrics.total_cost_usd - expected).abs() < 1e-9);
1015	assert!(card.per_turn[0].money_metered);
1016	assert!(card.per_turn[1].money_metered);
1017	assert!(!card.per_turn[2].money_metered);
1018
1019	let summary = card.to_summary();
1020	assert!(summary.contains("priced_cost_subtotal_usd"));
1021	assert!(summary.contains("cache_write_tokens: 300000"));
1022	assert!(summary.contains("no published price"));
1023	// Coverage reads against the money-metered turns, not all three.
1024	assert!(summary.contains("money_metered_turns: 2"), "{summary}");
1025
1026	let json = serde_json::to_value(&card).expect("serialize scorecard");
1027	assert_eq!(json["per_turn"][1]["unpriced_classes"][0], "cache_write");
1028	assert_eq!(json["metrics"]["total_cache_write_tokens"], 300_000);
1029	assert_eq!(json["metrics"]["cost_complete"], false);
1030	assert_eq!(json["metrics"]["money_metered_turns"], 2);
1031	// Route identity survives serialization, so a re-read scorecard can be
1032	// re-explained without the original input file.
1033	assert_eq!(json["per_turn"][1]["provider"], "moonshot");
1034	assert_eq!(json["per_turn"][2]["money_metered"], false);
1035	}
1036
1037	/// Legacy baselines and per-turn records that predate the class audit must
1038	/// still deserialize; the new fields default rather than fail.
1039	#[test]
1040	fn legacy_turn_score_json_defaults_the_new_audit_fields() {
1041	let score: TurnScore = serde_json::from_value(serde_json::json!({
1042	"turn_id": "t1",
1043	"model": "gpt-5.5",
1044	"input_tokens": 10,
1045	"output_tokens": 5,
1046	"cache_read_tokens": 0,
1047	"cost_usd": 0.1,
1048	"cost_cny": 0.0,
1049	"cost_unpriced": false
1050	}))
1051	.expect("legacy per-turn record stays readable");
1052	assert_eq!(score.cache_write_tokens, 0);
1053	assert_eq!(score.reasoning_tokens, 0);
1054	assert!(score.unpriced_classes.is_empty());
1055	assert_eq!(score.pricing_provenance, None);
1056	assert_eq!(score.cost_unpriced_reason, None);
1057	assert_eq!(score.live_pricing_defect, None);
1058	// A legacy row carries no coverage evidence, so `money_metered` defaults
1059	// to false rather than asserting the row was inside a complete total.
1060	assert!(!score.money_metered);
1061
1062	// Legacy aggregate baselines stay readable too, defaulting the new
1063	// coverage denominator rather than failing the parse.
1064	let metrics: ScorecardMetrics = serde_json::from_value(serde_json::json!({
1065	"turns": 3,
1066	"total_input_tokens": 10,
1067	"total_output_tokens": 5,
1068	"total_cache_read_tokens": 0,
1069	"total_cost_usd": 0.1,
1070	"total_cost_cny": 0.0,
1071	"cache_hit_ratio": 0.0
1072	}))
1073	.expect("legacy baseline stays readable");
1074	assert_eq!(metrics.money_metered_turns, 0);
1075	assert_eq!(metrics.total_cache_write_tokens, 0);
1076	}
1077
1078	#[test]
1079	fn aggregates_tokens_and_cache_hit_ratio_independent_of_pricing() {
1080	// input_tokens includes cache hits; token_usage_for_pricing splits them:
1081	// non-cached input = 1000-200 = 800, cache_read = 200.
1082	let u1 = usage(1000, 500, 200);
1083	let u2 = usage(2000, 100, 800); // non-cached = 1200, cache_read = 800
1084	let turns = [
1085	TurnInput {
1086	turn_id: "t1".into(),
1087	created_at: None,
1088	provider: None,
1089	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1090	model: "unpriced-x".into(),
1091	usage: &u1,
1092	},
1093	TurnInput {
1094	turn_id: "t2".into(),
1095	created_at: None,
1096	provider: None,
1097	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1098	model: "unpriced-x".into(),
1099	usage: &u2,
1100	},
1101	];
1102	let card = Scorecard::from_turns(&turns);
1103
1104	assert_eq!(card.metrics.turns, 2);
1105	assert_eq!(card.metrics.total_input_tokens, 800 + 1200);
1106	assert_eq!(card.metrics.total_output_tokens, 600); // 500 + 100
1107	assert_eq!(card.metrics.total_cache_read_tokens, 1000); // 200 + 800
1108	assert_eq!(card.metrics.unpriced_turns, 2);
1109	// cache_read / (input + cache_read) = 1000 / (2000 + 1000)
1110	let expected = 1000.0 / 3000.0;
1111	assert!((card.metrics.cache_hit_ratio - expected).abs() < 1e-9);
1112	}
1113
1114	/// The canonical cache-efficiency denominator is
1115	/// `hit / (non-cached input + hit + write)`. Cache-write tokens are prompt
1116	/// tokens that were not served from cache, so omitting them reported a
1117	/// flattering ratio on exactly the turns that paid to populate the cache.
1118	#[test]
1119	fn cache_hit_ratio_counts_cache_write_in_the_denominator() {
1120	fn card_for(usage: &Usage) -> Scorecard {
1121	Scorecard::from_turns(&[TurnInput {
1122	turn_id: "t1".into(),
1123	created_at: None,
1124	provider: None,
1125	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1126	model: "unpriced-x".into(),
1127	usage,
1128	}])
1129	}
1130
1131	// Zero everything: a ratio is undefined, reported as 0.0 rather than NaN.
1132	let empty = Usage::default();
1133	let card = card_for(&empty);
1134	assert_eq!(card.metrics.cache_hit_ratio, 0.0);
1135	assert_eq!(card.metrics.total_cache_write_tokens, 0);
1136
1137	// Write-only: a turn that populated the cache and read nothing from it
1138	// has a 0% hit ratio, not an undefined-and-therefore-zero one that a
1139	// write-blind denominator would produce by accident.
1140	let write_only = Usage {
1141	input_tokens: 1_000,
1142	output_tokens: 10,
1143	prompt_cache_hit_tokens: Some(0),
1144	prompt_cache_write_tokens: Some(1_000),
1145	..Default::default()
1146	};
1147	let card = card_for(&write_only);
1148	assert_eq!(card.metrics.total_cache_write_tokens, 1_000);
1149	assert_eq!(card.metrics.total_cache_read_tokens, 0);
1150	assert_eq!(card.metrics.cache_hit_ratio, 0.0);
1151
1152	// Mixed: 1000 prompt tokens = 200 read + 300 write + 500 non-cached.
1153	let mixed = Usage {
1154	input_tokens: 1_000,
1155	output_tokens: 10,
1156	prompt_cache_hit_tokens: Some(200),
1157	prompt_cache_write_tokens: Some(300),
1158	..Default::default()
1159	};
1160	let card = card_for(&mixed);
1161	assert_eq!(card.metrics.total_input_tokens, 500);
1162	assert_eq!(card.metrics.total_cache_read_tokens, 200);
1163	assert_eq!(card.metrics.total_cache_write_tokens, 300);
1164	let expected = 200.0 / (500.0 + 200.0 + 300.0);
1165	assert!(
1166	(card.metrics.cache_hit_ratio - expected).abs() < 1e-9,
1167	"got {}, want {expected}",
1168	card.metrics.cache_hit_ratio
1169	);
1170	// The write-blind denominator would have said 200/700 — assert the two
1171	// are distinguishable so a regression is unambiguous.
1172	let write_blind = 200.0 / 700.0;
1173	assert!((card.metrics.cache_hit_ratio - write_blind).abs() > 1e-6);
1174	}
1175
1176	#[test]
1177	fn unknown_model_is_marked_unpriced_with_zero_cost() {
1178	let u = usage(1000, 500, 0);
1179	let turns = [TurnInput {
1180	turn_id: "t1".into(),
1181	created_at: None,
1182	provider: Some("openai"),
1183	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1184	model: "definitely-not-a-real-model".into(),
1185	usage: &u,
1186	}];
1187	let card = Scorecard::from_turns(&turns);
1188	assert!(card.per_turn[0].cost_unpriced);
1189	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1190	assert_eq!(card.metrics.total_cost_usd, 0.0);
1191	assert!(card.to_summary().contains("cost_usd: unavailable"));
1192	}
1193
1194	#[test]
1195	fn same_model_is_priced_only_for_its_authoritative_provider_route() {
1196	let u = usage(1000, 500, 0);
1197	let turns = [
1198	TurnInput {
1199	turn_id: "api".into(),
1200	created_at: None,
1201	provider: Some("openai"),
1202	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1203	model: "gpt-5.5".into(),
1204	usage: &u,
1205	},
1206	TurnInput {
1207	turn_id: "oauth".into(),
1208	created_at: None,
1209	provider: Some("openai-codex"),
1210	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1211	model: "gpt-5.5".into(),
1212	usage: &u,
1213	},
1214	TurnInput {
1215	turn_id: "local".into(),
1216	created_at: None,
1217	provider: Some("ollama"),
1218	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1219	model: "gpt-5.5".into(),
1220	usage: &u,
1221	},
1222	];
1223
1224	let card = Scorecard::from_turns(&turns);
1225
1226	assert!(!card.per_turn[0].cost_unpriced);
1227	assert!(card.per_turn[0].cost_usd > 0.0);
1228	assert!(card.per_turn[1].cost_unpriced);
1229	assert_eq!(card.per_turn[1].cost_usd, 0.0);
1230	assert!(card.per_turn[2].cost_unpriced);
1231	assert_eq!(card.per_turn[2].cost_usd, 0.0);
1232	// Codex OAuth and Ollama are exactly non-metered, so they leave the
1233	// money-coverage denominator entirely rather than counting as unpriced
1234	// dollars: only the OpenAI turn owes money, and it is priced. The USD
1235	// total is therefore genuinely complete for the spend it covers (#4318).
1236	assert_eq!(card.metrics.money_metered_turns, 1);
1237	assert_eq!(card.metrics.unpriced_turns, 0);
1238	assert!(card.metrics.cost_complete);
1239	for (index, expected) in [(1_usize, false), (2_usize, false)] {
1240	assert_eq!(
1241	card.per_turn[index].money_metered, expected,
1242	"turn {index} money-metered"
1243	);
1244	assert_eq!(
1245	card.per_turn[index].cost_unpriced_reason.as_deref(),
1246	Some("not_money_metered"),
1247	"turn {index} reason"
1248	);
1249	}
1250	assert!(card.per_turn[0].money_metered);
1251	// CNY is only published by direct DeepSeek, so the single metered turn
1252	// still has no authoritative CNY figure.
1253	assert_eq!(card.metrics.cny_unpriced_turns, 1);
1254	assert!(!card.metrics.cny_cost_complete);
1255	assert!(card.to_summary().contains("money_metered_turns: 1"));
1256	assert!(card.to_summary().contains("cost_cny: unavailable"));
1257
1258	let json = serde_json::to_value(&card).expect("serialize scorecard");
1259	assert_eq!(json["per_turn"][0]["provider"], "openai");
1260	assert_eq!(json["per_turn"][1]["provider"], "openai-codex");
1261	assert_eq!(json["per_turn"][2]["provider"], "ollama");
1262	assert_eq!(json["metrics"]["money_metered_turns"], 1);
1263	assert_eq!(json["metrics"]["unpriced_turns"], 0);
1264	assert_eq!(json["metrics"]["cost_complete"], true);
1265	assert_eq!(json["metrics"]["cny_cost_complete"], false);
1266	}
1267
1268	#[test]
1269	fn first_party_hand_price_survives_a_missing_catalog_offering() {
1270	let u = usage(1_000_000, 0, 0);
1271	let turns = [
1272	TurnInput {
1273	turn_id: "openai-api".into(),
1274	created_at: None,
1275	provider: Some("openai"),
1276	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1277	model: "gpt-5-codex".into(),
1278	usage: &u,
1279	},
1280	TurnInput {
1281	turn_id: "foreign-route".into(),
1282	created_at: None,
1283	provider: Some("ollama"),
1284	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1285	model: "gpt-5-codex".into(),
1286	usage: &u,
1287	},
1288	];
1289
1290	let card = Scorecard::from_turns(&turns);
1291
1292	assert!(!card.per_turn[0].cost_unpriced);
1293	assert!((card.per_turn[0].cost_usd - 1.25).abs() < f64::EPSILON);
1294	assert!(card.per_turn[1].cost_unpriced);
1295	}
1296
1297	#[test]
1298	fn documented_no_cache_discount_uses_input_without_generalizing_missing_rates() {
1299	let u = Usage {
1300	input_tokens: 1_000_000,
1301	output_tokens: 0,
1302	prompt_cache_hit_tokens: Some(250_000),
1303	prompt_cache_write_tokens: Some(100_000),
1304	..Default::default()
1305	};
1306	let turns = [
1307	TurnInput {
1308	turn_id: "documented-no-discount".into(),
1309	created_at: None,
1310	provider: Some("openai"),
1311	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1312	model: "gpt-5.5-pro".into(),
1313	usage: &u,
1314	},
1315	TurnInput {
1316	turn_id: "missing-cache-rate".into(),
1317	created_at: None,
1318	provider: Some("meta"),
1319	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1320	model: "muse-spark-1.1".into(),
1321	usage: &u,
1322	},
1323	];
1324
1325	let card = Scorecard::from_turns(&turns);
1326
1327	assert!(!card.per_turn[0].cost_unpriced);
1328	assert!((card.per_turn[0].cost_usd - 30.0).abs() < f64::EPSILON);
1329	assert!(card.per_turn[1].cost_unpriced);
1330	assert!(!card.metrics.cost_complete);
1331	}
1332
1333	#[test]
1334	fn anthropic_sonnet_5_uses_the_recorded_turn_time() {
1335	let u = Usage {
1336	input_tokens: 1_000_000,
1337	output_tokens: 500_000,
1338	prompt_cache_hit_tokens: Some(250_000),
1339	prompt_cache_write_tokens: Some(100_000),
1340	..Default::default()
1341	};
1342	// Sonnet 5's $2/$10 launch rate became the standard price (Anthropic
1343	// pricing page, 2026-08-17: the 2026-09-01 increase "will not
1344	// occur"), so both sides of the former boundary price identically:
1345	// 650K miss * 2.00 + 250K hit * 0.20 + 100K write * 2.50 + 500K out
1346	// * 10.00 = 1.30 + 0.05 + 0.25 + 5.00 = 6.60. A turn with no recorded
1347	// time still fails closed rather than guessing a window.
1348	let before_boundary: DateTime<Utc> = "2026-08-31T23:59:59Z".parse().expect("time");
1349	let after_boundary: DateTime<Utc> = "2026-09-01T00:00:00Z".parse().expect("time");
1350	let turns = [
1351	TurnInput {
1352	turn_id: "sonnet-before".into(),
1353	created_at: Some(&before_boundary),
1354	provider: Some("anthropic"),
1355	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1356	model: " claude-sonnet-5 ".into(),
1357	usage: &u,
1358	},
1359	TurnInput {
1360	turn_id: "sonnet-after".into(),
1361	created_at: Some(&after_boundary),
1362	provider: Some("anthropic"),
1363	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1364	model: "claude-sonnet-5".into(),
1365	usage: &u,
1366	},
1367	TurnInput {
1368	turn_id: "sonnet-missing-time".into(),
1369	created_at: None,
1370	provider: Some("anthropic"),
1371	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1372	model: "claude-sonnet-5".into(),
1373	usage: &u,
1374	},
1375	];
1376
1377	let card = Scorecard::from_turns(&turns);
1378
1379	assert!(!card.per_turn[0].cost_unpriced);
1380	assert!((card.per_turn[0].cost_usd - 6.60).abs() < 1e-12);
1381	assert_eq!(card.per_turn[0].created_at.as_ref(), Some(&before_boundary));
1382	assert!(card.per_turn[0].cost_cny_unpriced);
1383	assert!(!card.per_turn[1].cost_unpriced);
1384	assert!((card.per_turn[1].cost_usd - 6.60).abs() < 1e-12);
1385	assert!(card.per_turn[1].cost_cny_unpriced);
1386	assert!(card.per_turn[2].cost_unpriced);
1387	}
1388
1389	#[test]
1390	fn known_zero_usage_is_zero_cost_not_unavailable() {
1391	let u = usage(0, 0, 0);
1392	let turns = [TurnInput {
1393	turn_id: "zero".into(),
1394	created_at: None,
1395	provider: Some("openai"),
1396	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1397	model: "gpt-5.5".into(),
1398	usage: &u,
1399	}];
1400
1401	let card = Scorecard::from_turns(&turns);
1402
1403	assert!(!card.per_turn[0].cost_unpriced);
1404	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1405	assert!(card.per_turn[0].cost_cny_unpriced);
1406	assert_eq!(card.metrics.unpriced_turns, 0);
1407	assert_eq!(card.metrics.cny_unpriced_turns, 1);
1408	assert!(card.metrics.cost_complete);
1409	assert!(!card.metrics.cny_cost_complete);
1410	assert!(card.to_summary().contains("cost_usd: $0.0000"));
1411	assert!(card.to_summary().contains("cost_cny: unavailable"));
1412	}
1413
1414	#[test]
1415	fn direct_deepseek_route_keeps_authoritative_dual_currency_pricing() {
1416	let u = usage(1000, 500, 0);
1417	let recorded_at: DateTime<Utc> = "2026-08-17T15:00:00Z".parse().expect("recorded time");
1418	let turns = [TurnInput {
1419	turn_id: "deepseek".into(),
1420	created_at: Some(&recorded_at),
1421	provider: Some("deepseek"),
1422	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1423	model: "deepseek-v4-pro".into(),
1424	usage: &u,
1425	}];
1426
1427	let card = Scorecard::from_turns(&turns);
1428
1429	assert!(!card.per_turn[0].cost_unpriced);
1430	assert!(!card.per_turn[0].cost_cny_unpriced);
1431	assert!(card.per_turn[0].cost_usd > 0.0);
1432	assert!(card.per_turn[0].cost_cny > 0.0);
1433	assert!(card.metrics.cost_complete);
1434	assert!(card.metrics.cny_cost_complete);
1435	}
1436
1437	#[test]
1438	fn undated_direct_deepseek_v4_turns_fail_closed_on_the_time_window() {
1439	// V4 flash/pro are peak/off-peak tiered by the turn's recorded time; a
1440	// turn the recorder did not date cannot be resolved to one price and
1441	// must not be silently priced at whatever tier `now` happens to be.
1442	let u = usage(1000, 500, 0);
1443	let turns = [
1444	TurnInput {
1445	turn_id: "undated-pro".into(),
1446	created_at: None,
1447	provider: Some("deepseek"),
1448	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1449	model: "deepseek-v4-pro".into(),
1450	usage: &u,
1451	},
1452	TurnInput {
1453	turn_id: "undated-flash".into(),
1454	created_at: None,
1455	provider: Some("deepseek"),
1456	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1457	model: "deepseek-v4-flash".into(),
1458	usage: &u,
1459	},
1460	];
1461
1462	let card = Scorecard::from_turns(&turns);
1463
1464	assert!(card.per_turn.iter().all(\|turn\| turn.cost_unpriced));
1465	assert!(card.per_turn.iter().all(\|turn\| turn.cost_cny_unpriced));
1466	assert!(!card.metrics.cost_complete);
1467	}
1468
1469	#[test]
1470	fn direct_deepseek_compact_aliases_use_canonical_pricing() {
1471	let u = usage(1000, 500, 100);
1472	let recorded_at: DateTime<Utc> = "2026-08-17T15:00:00Z".parse().expect("recorded time");
1473	let models = [
1474	"deepseek-v4-pro",
1475	"pro",
1476	" DeepSeek-V4Pro ",
1477	"deepseek-v4-flash",
1478	"flash",
1479	"DEEPSEEK-V4FLASH",
1480	];
1481	let turns: Vec<_> = models
1482	.iter()
1483	.map(\|model\| TurnInput {
1484	turn_id: (*model).into(),
1485	created_at: Some(&recorded_at),
1486	provider: Some("deepseek"),
1487	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1488	model: (*model).into(),
1489	usage: &u,
1490	})
1491	.collect();
1492
1493	let card = Scorecard::from_turns(&turns);
1494
1495	for alias in [1, 2] {
1496	assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[0].cost_usd);
1497	assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[0].cost_cny);
1498	}
1499	for alias in [4, 5] {
1500	assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[3].cost_usd);
1501	assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[3].cost_cny);
1502	}
1503	assert!(card.per_turn.iter().all(\|turn\| !turn.cost_unpriced));
1504	assert!(card.per_turn.iter().all(\|turn\| !turn.cost_cny_unpriced));
1505	}
1506
1507	#[test]
1508	fn direct_deepseek_compatibility_aliases_use_the_flash_route() {
1509	let u = usage(1000, 500, 100);
1510	let before_retirement: DateTime<Utc> =
1511	"2026-07-24T15:58:59Z".parse().expect("pre-retirement time");
1512	let at_retirement: DateTime<Utc> = DEEPSEEK_ALIAS_RETIREMENT_UTC
1513	.parse()
1514	.expect("retirement time");
1515	let turns = [
1516	TurnInput {
1517	turn_id: "chat-alias".into(),
1518	created_at: Some(&before_retirement),
1519	provider: Some("deepseek"),
1520	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1521	model: "deepseek-chat".into(),
1522	usage: &u,
1523	},
1524	TurnInput {
1525	turn_id: "reasoner-alias".into(),
1526	created_at: Some(&before_retirement),
1527	provider: Some("deepseek"),
1528	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1529	model: "deepseek-reasoner".into(),
1530	usage: &u,
1531	},
1532	TurnInput {
1533	turn_id: "canonical".into(),
1534	created_at: Some(&before_retirement),
1535	provider: Some("deepseek"),
1536	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1537	model: DEEPSEEK_ALIAS_REPLACEMENT.into(),
1538	usage: &u,
1539	},
1540	TurnInput {
1541	turn_id: "retired-alias".into(),
1542	created_at: Some(&at_retirement),
1543	provider: Some("deepseek"),
1544	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1545	model: "deepseek-chat".into(),
1546	usage: &u,
1547	},
1548	TurnInput {
1549	turn_id: "undated-alias".into(),
1550	created_at: None,
1551	provider: Some("deepseek"),
1552	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1553	model: "deepseek-reasoner".into(),
1554	usage: &u,
1555	},
1556	];
1557
1558	let card = Scorecard::from_turns(&turns);
1559
1560	assert_eq!(card.per_turn[0].cost_usd, card.per_turn[2].cost_usd);
1561	assert_eq!(card.per_turn[1].cost_usd, card.per_turn[2].cost_usd);
1562	assert_eq!(card.per_turn[0].cost_cny, card.per_turn[2].cost_cny);
1563	assert_eq!(card.per_turn[1].cost_cny, card.per_turn[2].cost_cny);
1564	assert!(card.per_turn[..3].iter().all(\|turn\| !turn.cost_unpriced));
1565	assert!(
1566	card.per_turn[..3]
1567	.iter()
1568	.all(\|turn\| !turn.cost_cny_unpriced)
1569	);
1570	assert!(card.per_turn[3].cost_unpriced);
1571	assert!(card.per_turn[4].cost_unpriced);
1572	}
1573
1574	#[test]
1575	fn direct_arcee_aliases_do_not_cross_the_openrouter_namespace() {
1576	let u = Usage {
1577	input_tokens: 1_000_000,
1578	output_tokens: 500_000,
1579	prompt_cache_hit_tokens: Some(250_000),
1580	prompt_cache_write_tokens: Some(100_000),
1581	..Default::default()
1582	};
1583	let turns = [
1584	TurnInput {
1585	turn_id: "canonical-direct".into(),
1586	created_at: None,
1587	provider: Some("arcee"),
1588	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1589	model: "trinity-large-thinking".into(),
1590	usage: &u,
1591	},
1592	TurnInput {
1593	turn_id: "direct-alias".into(),
1594	created_at: None,
1595	provider: Some("arcee"),
1596	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1597	model: "arcee-trinity-large-thinking".into(),
1598	usage: &u,
1599	},
1600	TurnInput {
1601	turn_id: "openrouter-namespace".into(),
1602	created_at: None,
1603	provider: Some("arcee"),
1604	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1605	model: "arcee-ai/trinity-large-thinking".into(),
1606	usage: &u,
1607	},
1608	];
1609
1610	let card = Scorecard::from_turns(&turns);
1611
1612	assert!(!card.per_turn[0].cost_unpriced);
1613	assert!((card.per_turn[0].cost_usd - 0.65).abs() < f64::EPSILON);
1614	assert_eq!(card.per_turn[1].cost_usd, card.per_turn[0].cost_usd);
1615	assert!(!card.per_turn[1].cost_unpriced);
1616	assert!(card.per_turn[2].cost_unpriced);
1617	}
1618
1619	#[test]
1620	fn costless_catalog_rows_fall_back_only_to_verified_provider_prices() {
1621	let u = Usage {
1622	input_tokens: 1_000_000,
1623	output_tokens: 500_000,
1624	prompt_cache_hit_tokens: Some(250_000),
1625	prompt_cache_write_tokens: Some(100_000),
1626	..Default::default()
1627	};
1628	let turns = [
1629	TurnInput {
1630	turn_id: "arcee-mini".into(),
1631	created_at: None,
1632	provider: Some("arcee"),
1633	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1634	model: "trinity-mini".into(),
1635	usage: &u,
1636	},
1637	TurnInput {
1638	turn_id: "minimax-m2.7".into(),
1639	created_at: None,
1640	provider: Some("minimax"),
1641	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1642	model: "minimax-m2.7".into(),
1643	usage: &u,
1644	},
1645	TurnInput {
1646	turn_id: "foreign-route".into(),
1647	created_at: None,
1648	provider: Some("ollama"),
1649	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1650	model: "trinity-mini".into(),
1651	usage: &u,
1652	},
1653	TurnInput {
1654	turn_id: "openai-hosted-deepseek".into(),
1655	created_at: None,
1656	provider: Some("openai"),
1657	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1658	model: "deepseek-v4-pro".into(),
1659	usage: &u,
1660	},
1661	TurnInput {
1662	turn_id: "openrouter-hosted-zai".into(),
1663	created_at: None,
1664	provider: Some("openrouter"),
1665	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1666	model: "z-ai/glm-5.2".into(),
1667	usage: &u,
1668	},
1669	];
1670
1671	let card = Scorecard::from_turns(&turns);
1672
1673	// Trinity Mini has no verified provider rate in the release metadata;
1674	// a removed hand-written estimate must stay unknown, not become zero
1675	// or leak through from a similarly named route.
1676	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1677	assert!(card.per_turn[0].cost_unpriced);
1678	// MiniMax-M2.7 publishes a distinct cache-write rate (0.375/M),
1679	// retained by the provider-owned fallback even without a priced
1680	// catalog offering.
1681	assert!((card.per_turn[1].cost_usd - 0.8475).abs() < f64::EPSILON);
1682	assert!(!card.per_turn[1].cost_unpriced);
1683	assert!(card.per_turn[..2].iter().all(\|turn\| turn.cost_cny_unpriced));
1684	assert!(card.per_turn[2..].iter().all(\|turn\| turn.cost_unpriced));
1685	}
1686
1687	#[test]
1688	fn stepfun_legacy_route_keeps_pricing_without_a_catalog_row() {
1689	let u = usage(1000, 500, 250);
1690	let recorded = \|turn_id: &str,
1691	provider: &str,
1692	model: &str,
1693	billing_surface: Option<&str>\| RecordedTurn {
1694	turn_id: turn_id.to_string(),
1695	created_at: None,
1696	model_backed: Some(true),
1697	provider: Some(provider.to_string()),
1698	billing_surface: billing_surface.map(str::to_string),
1699	model: model.to_string(),
1700	usage: Some(u.clone()),
1701	};
1702	let turns = [
1703	recorded(
1704	"stepfun-default",
1705	"stepfun",
1706	" STEP-3.7-FLASH ",
1707	Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE),
1708	),
1709	recorded(
1710	"stepfun-plan",
1711	"stepfun",
1712	"step-3.7-flash",
1713	Some(crate::pricing::STEPFUN_PLAN_BILLING_SURFACE),
1714	),
1715	recorded("stepfun-missing-surface", "stepfun", "step-3.7-flash", None),
1716	recorded("stepfun-unknown-model", "stepfun", "step-3.5-flash", None),
1717	recorded(
1718	"openrouter-stepfun-name",
1719	"openrouter",
1720	"step-3.7-flash",
1721	None,
1722	),
1723	recorded("local-stepfun-name", "ollama", "step-3.7-flash", None),
1724	recorded(
1725	"sakana-incomplete-tier-price",
1726	"sakana",
1727	"fugu-ultra-20260615",
1728	None,
1729	),
1730	recorded(
1731	"foreign-deepseek-name",
1732	"openmodel",
1733	"deepseek-v4-flash",
1734	None,
1735	),
1736	];
1737
1738	let card = Scorecard::from_recorded_turns(&turns);
1739
1740	assert!((card.per_turn[0].cost_usd - 0.000_735).abs() < 1e-12);
1741	assert!(!card.per_turn[0].cost_unpriced);
1742	assert!(card.per_turn[0].cost_cny_unpriced);
1743	assert_eq!(
1744	card.per_turn[0].billing_surface.as_deref(),
1745	Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE)
1746	);
1747	assert!(card.per_turn[1..].iter().all(\|turn\| turn.cost_unpriced));
1748	}
1749
1750	#[test]
1751	fn legacy_model_only_record_is_readable_but_unpriced() {
1752	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1753	"turn_id": "legacy",
1754	"model": "gpt-5.5",
1755	"usage": {
1756	"input_tokens": 0,
1757	"output_tokens": 0
1758	}
1759	}))
1760	.expect("parse legacy scorecard turn");
1761	assert_eq!(recorded.provider, None);
1762	assert_eq!(recorded.billing_surface, None);
1763
1764	let card = Scorecard::from_recorded_turns(&[recorded]);
1765
1766	assert!(card.per_turn[0].cost_unpriced);
1767	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1768	assert_eq!(card.metrics.unpriced_turns, 1);
1769	assert!(card.to_summary().contains("cost_usd: unavailable"));
1770	}
1771
1772	#[test]
1773	fn recorded_turn_accepts_runtime_route_aliases() {
1774	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1775	"schema_version": 1,
1776	"id": "runtime-turn",
1777	"thread_id": "thread-1",
1778	"status": "completed",
1779	"input_summary": "score this turn",
1780	"created_at": "2026-07-12T10:30:00Z",
1781	"effective_provider": "openai-codex",
1782	"effective_billing_surface": "account-subscription",
1783	"effective_model": "gpt-5.5",
1784	"usage": {
1785	"input_tokens": 1,
1786	"output_tokens": 1
1787	}
1788	}))
1789	.expect("parse runtime scorecard turn");
1790
1791	assert_eq!(recorded.turn_id, "runtime-turn");
1792	assert_eq!(
1793	recorded.created_at.as_ref().map(DateTime::to_rfc3339),
1794	Some("2026-07-12T10:30:00+00:00".to_string())
1795	);
1796	assert_eq!(recorded.provider.as_deref(), Some("openai-codex"));
1797	assert_eq!(
1798	recorded.billing_surface.as_deref(),
1799	Some("account-subscription")
1800	);
1801	assert_eq!(recorded.model, "gpt-5.5");
1802	assert!(recorded.contributes_to_scorecard());
1803	}
1804
1805	#[test]
1806	fn runtime_turn_without_usage_is_readable_and_filtered() {
1807	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1808	"schema_version": 1,
1809	"id": "queued-runtime-turn",
1810	"thread_id": "thread-1",
1811	"status": "queued",
1812	"input_summary": "waiting to run",
1813	"created_at": "2026-07-12T10:30:00Z",
1814	"effective_provider": "openai",
1815	"effective_model": "gpt-5.5"
1816	}))
1817	.expect("parse runtime row before usage is recorded");
1818
1819	assert!(recorded.usage.is_none());
1820	assert!(!recorded.contributes_to_scorecard());
1821	let card = Scorecard::from_recorded_turns(&[recorded]);
1822	assert_eq!(card.metrics.turns, 0);
1823	assert!(card.per_turn.is_empty());
1824	}
1825
1826	#[test]
1827	fn recorded_non_model_hook_turn_is_excluded_from_model_scorecard() {
1828	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1829	"turn_id": "shell-turn",
1830	"created_at": "2026-07-12T10:30:00Z",
1831	"model_backed": false,
1832	"provider": null,
1833	"model": "gpt-5.5",
1834	"usage": {
1835	"input_tokens": 0,
1836	"output_tokens": 0
1837	}
1838	}))
1839	.expect("parse non-model turn_end record");
1840
1841	assert!(!recorded.contributes_to_scorecard());
1842	}
1843
1844	#[test]
1845	fn blank_unknown_and_custom_providers_fail_closed_as_unpriced() {
1846	let u = usage(1000, 500, 0);
1847	let turns = [
1848	TurnInput {
1849	turn_id: "blank".into(),
1850	created_at: None,
1851	provider: Some(" "),
1852	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1853	model: "gpt-5.5".into(),
1854	usage: &u,
1855	},
1856	TurnInput {
1857	turn_id: "named-custom".into(),
1858	created_at: None,
1859	provider: Some("my-openai-proxy"),
1860	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1861	model: "gpt-5.5".into(),
1862	usage: &u,
1863	},
1864	TurnInput {
1865	turn_id: "generic-custom".into(),
1866	created_at: None,
1867	provider: Some("custom"),
1868	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1869	model: "gpt-5.5".into(),
1870	usage: &u,
1871	},
1872	];
1873
1874	let card = Scorecard::from_turns(&turns);
1875
1876	assert_eq!(card.per_turn[0].provider, None);
1877	assert_eq!(
1878	card.per_turn[1].provider.as_deref(),
1879	Some("my-openai-proxy")
1880	);
1881	assert_eq!(card.per_turn[2].provider.as_deref(), Some("custom"));
1882	assert!(card.per_turn.iter().all(\|turn\| turn.cost_unpriced));
1883	assert_eq!(card.metrics.unpriced_turns, 3);
1884	assert!(!card.metrics.cost_complete);
1885	assert!(card.to_summary().contains("cost_usd: unavailable"));
1886	}
1887
1888	#[test]
1889	fn regression_flags_cost_and_token_increases_over_threshold() {
1890	let baseline = ScorecardMetrics {
1891	turns: 1,
1892	money_metered_turns: 1,
1893	unpriced_turns: 0,
1894	cny_unpriced_turns: 0,
1895	cost_complete: true,
1896	cny_cost_complete: true,
1897	unpriced_classes: Vec::new(),
1898	total_input_tokens: 1000,
1899	total_output_tokens: 1000,
1900	total_cache_read_tokens: 0,
1901	total_cache_write_tokens: 0,
1902	total_reasoning_tokens: 0,
1903	total_cost_usd: 0.10,
1904	total_cost_cny: 0.7,
1905	cache_hit_ratio: 0.5,
1906	};
1907	let current = ScorecardMetrics {
1908	total_cost_usd: 0.20, // +100% → regression
1909	total_input_tokens: 1010, // +1% → under 5% threshold, no regression
1910	total_output_tokens: 2000, // +100% → regression
1911	cache_hit_ratio: 0.5, // unchanged
1912	..baseline.clone()
1913	};
1914	let regs = current.regressions_against(&baseline, 5.0);
1915	let names: Vec<&str> = regs.iter().map(\|r\| r.metric.as_str()).collect();
1916	assert!(names.contains(&"total_cost_usd"));
1917	assert!(names.contains(&"total_output_tokens"));
1918	assert!(!names.contains(&"total_input_tokens")); // under threshold
1919	}
1920
1921	#[test]
1922	fn a_non_finite_threshold_cannot_pass_a_regressed_run() {
1923	let baseline = ScorecardMetrics {
1924	turns: 1,
1925	money_metered_turns: 1,
1926	unpriced_turns: 0,
1927	cny_unpriced_turns: 0,
1928	cost_complete: true,
1929	cny_cost_complete: true,
1930	unpriced_classes: Vec::new(),
1931	total_input_tokens: 1000,
1932	total_output_tokens: 1000,
1933	total_cache_read_tokens: 0,
1934	total_cache_write_tokens: 0,
1935	total_reasoning_tokens: 0,
1936	total_cost_usd: 0.10,
1937	total_cost_cny: 0.7,
1938	cache_hit_ratio: 0.5,
1939	};
1940	let current = ScorecardMetrics {
1941	total_output_tokens: 2000,
1942	..baseline.clone()
1943	};
1944	for threshold in [f64::NAN, f64::INFINITY] {
1945	let names: Vec<String> = current
1946	.regressions_against(&baseline, threshold)
1947	.into_iter()
1948	.map(\|r\| r.metric)
1949	.collect();
1950	assert!(
1951	names.iter().any(\|name\| name == "total_output_tokens"),
1952	"threshold {threshold} passed a +100% regression: {names:?}"
1953	);
1954	}
1955	// A metric whose change cannot be computed is not a pass either.
1956	let unknown = ScorecardMetrics {
1957	cache_hit_ratio: f64::NAN,
1958	..baseline.clone()
1959	};
1960	assert!(
1961	unknown
1962	.regressions_against(&baseline, 5.0)
1963	.iter()
1964	.any(\|r\| r.metric == "cache_hit_ratio_drop")
1965	);
1966	}
1967
1968	#[test]
1969	fn regression_flags_loss_of_cost_completeness_without_comparing_subtotals() {
1970	let baseline = ScorecardMetrics {
1971	cost_complete: true,
1972	total_cost_usd: 0.10,
1973	..Default::default()
1974	};
1975	let current = ScorecardMetrics {
1976	turns: 1,
1977	unpriced_turns: 1,
1978	total_cost_usd: 0.20,
1979	..Default::default()
1980	};
1981
1982	let regs = current.regressions_against(&baseline, 5.0);
1983	assert!(!regs.iter().any(\|r\| r.metric == "total_cost_usd"));
1984	assert!(regs.iter().any(\|r\| r.metric == "cost_completeness_drop"));
1985	}
1986
1987	#[test]
1988	fn regression_flags_loss_of_cny_cost_completeness() {
1989	let baseline = ScorecardMetrics {
1990	cny_cost_complete: true,
1991	total_cost_cny: 0.70,
1992	..Default::default()
1993	};
1994	let current = ScorecardMetrics {
1995	turns: 1,
1996	cny_unpriced_turns: 1,
1997	total_cost_cny: 0.0,
1998	..Default::default()
1999	};
2000
2001	let regs = current.regressions_against(&baseline, 5.0);
2002	assert!(
2003	regs.iter()
2004	.any(\|r\| r.metric == "cny_cost_completeness_drop")
2005	);
2006	}
2007
2008	#[test]
2009	fn regression_flags_complete_cny_cost_increase() {
2010	let baseline = ScorecardMetrics {
2011	cny_cost_complete: true,
2012	total_cost_cny: 0.70,
2013	..Default::default()
2014	};
2015	let current = ScorecardMetrics {
2016	total_cost_cny: 1.40,
2017	..baseline.clone()
2018	};
2019
2020	let regs = current.regressions_against(&baseline, 5.0);
2021	assert!(regs.iter().any(\|r\| r.metric == "total_cost_cny"));
2022	}
2023
2024	#[test]
2025	fn legacy_baseline_is_readable_but_cost_is_not_comparable() {
2026	let baseline: ScorecardMetrics = serde_json::from_value(serde_json::json!({
2027	"turns": 1,
2028	"total_input_tokens": 10,
2029	"total_output_tokens": 5,
2030	"total_cache_read_tokens": 0,
2031	"total_cost_usd": 0.10,
2032	"total_cost_cny": 0.0,
2033	"cache_hit_ratio": 0.0
2034	}))
2035	.expect("parse legacy scorecard baseline");
2036	assert!(!baseline.cost_complete);
2037
2038	let current = ScorecardMetrics {
2039	cost_complete: true,
2040	total_cost_usd: 0.20,
2041	total_input_tokens: 10,
2042	total_output_tokens: 5,
2043	..Default::default()
2044	};
2045	let regs = current.regressions_against(&baseline, 5.0);
2046	assert!(!regs.iter().any(\|r\| r.metric == "total_cost_usd"));
2047	}
2048
2049	#[test]
2050	fn regression_flags_cache_hit_ratio_drop() {
2051	let baseline = ScorecardMetrics {
2052	cache_hit_ratio: 0.80,
2053	..Default::default()
2054	};
2055	let current = ScorecardMetrics {
2056	cache_hit_ratio: 0.40,
2057	..Default::default()
2058	};
2059	let regs = current.regressions_against(&baseline, 10.0);
2060	assert!(regs.iter().any(\|r\| r.metric == "cache_hit_ratio_drop"));
2061	}
2062
2063	#[test]
2064	fn no_regressions_when_within_threshold() {
2065	let baseline = ScorecardMetrics {
2066	total_cost_usd: 1.0,
2067	total_input_tokens: 1000,
2068	total_output_tokens: 1000,
2069	cache_hit_ratio: 0.5,
2070	..Default::default()
2071	};
2072	let current = baseline.clone();
2073	assert!(current.regressions_against(&baseline, 5.0).is_empty());
2074	}
2075
2076	#[test]
2077	fn cache_hit_denominator_saturates_instead_of_wrapping() {
2078	assert_eq!(cacheable_token_total(u64::MAX, 1, 1), u64::MAX);
2079	assert_eq!(cacheable_token_total(1, u64::MAX, 1), u64::MAX);
2080	}
2081	}
2082

2082 lines RUST