| 1 | //! Offline coverage of shipped defaults, not arbitrary live/account model rosters. |
| 2 | //! Rate authority stays in the production audit; the fixture stores only exact, |
| 3 | //! source-reviewed omissions. Never regenerate it from observed audit outcomes. |
| 4 | |
| 5 | use std::collections::{BTreeMap, BTreeSet}; |
| 6 | |
| 7 | use codewhale_config::ProviderKind; |
| 8 | use codewhale_config::descriptors::{DescriptorWire, bundled_provider_descriptors}; |
| 9 | use codewhale_config::provider::all_providers; |
| 10 | use codewhale_config::route::{LogicalModelRef, RouteRequest, RouteResolver}; |
| 11 | |
| 12 | use super::*; |
| 13 | use crate::config::{Config, ProviderConfig}; |
| 14 | use crate::cost_status::EffectiveRouteEnvelope; |
| 15 | use crate::test_support::EnvVarGuard; |
| 16 | |
| 17 | // Identity, exact wire ID, exact endpoint, protocol, captured billing mode. |
| 18 | // Keep literal model/endpoint values in reviewed exemptions: referring to an |
| 19 | // expanding default/roster constant would automatically bless a new omission. |
| 20 | type RouteKey = [String; 5]; |
| 21 | |
| 22 | #[derive(serde::Deserialize)] |
| 23 | #[serde(deny_unknown_fields)] |
| 24 | struct ReviewedOmissions { |
| 25 | reason: String, |
| 26 | review: String, |
| 27 | routes: Vec<RouteKey>, |
| 28 | } |
| 29 | |
| 30 | fn reviewed_omissions() -> BTreeMap<RouteKey, UnpricedReason> { |
| 31 | let groups: Vec<ReviewedOmissions> = |
| 32 | serde_json::from_str(include_str!("default_coverage_unpriced.json")) |
| 33 | .expect("reviewed default coverage fixture"); |
| 34 | let mut reviewed = BTreeMap::new(); |
| 35 | for group in groups { |
| 36 | let reason = UnpricedReason::from_label(&group.reason); |
| 37 | assert_eq!(reason.label(), group.reason, "unknown review reason"); |
| 38 | assert!(!group.review.trim().is_empty(), "missing source review"); |
| 39 | assert!(!group.routes.is_empty(), "empty review group"); |
| 40 | for key in group.routes { |
| 41 | assert!( |
| 42 | reviewed.insert(key.clone(), reason).is_none(), |
| 43 | "duplicate default coverage exemption: {key:?}" |
| 44 | ); |
| 45 | } |
| 46 | } |
| 47 | reviewed |
| 48 | } |
| 49 | |
| 50 | #[test] |
| 51 | fn shipped_default_routes_have_reviewed_pricing_coverage() { |
| 52 | let _live = crate::provider_lake::lock_live_snapshot(); |
| 53 | // Explicit table endpoints below bypass endpoint overrides. These remaining |
| 54 | // variables can still change billing products or legacy Ollama migration. |
| 55 | let _env: Vec<_> = [ |
| 56 | "MINIMAX_API_KEY", |
| 57 | "XIAOMI_MIMO_MODE", |
| 58 | "XIAOMI_MIMO_BASE_URL", |
| 59 | "XIAOMI_MIMO_TOKEN_PLAN_API_KEY", |
| 60 | "MIMO_TOKEN_PLAN_API_KEY", |
| 61 | "XIAOMI_MIMO_API_KEY", |
| 62 | "XIAOMI_API_KEY", |
| 63 | "MIMO_API_KEY", |
| 64 | "OLLAMA_BASE_URL", |
| 65 | ] |
| 66 | .into_iter() |
| 67 | .map(EnvVarGuard::remove) |
| 68 | .collect(); |
| 69 | let _cloud = EnvVarGuard::set("CODEWHALE_DISABLE_CLOUD_FACTS", "1"); |
| 70 | crate::provider_lake::clear_live_snapshot(); |
| 71 | crate::provider_catalog_live::reset_cache_for_test(); |
| 72 | |
| 73 | let resolver = RouteResolver::new(); |
| 74 | let mut defaults = Vec::new(); |
| 75 | for entry in all_providers() { |
| 76 | if entry.kind() == ProviderKind::Antigravity { |
| 77 | continue; // Retired tombstone, not a shipped runnable choice. |
| 78 | } |
| 79 | let candidate = resolver |
| 80 | .resolve(&RouteRequest { |
| 81 | explicit_provider: Some(entry.kind()), |
| 82 | ..Default::default() |
| 83 | }) |
| 84 | .unwrap_or_else(|error| panic!("{} default cannot resolve: {error}", entry.id())); |
| 85 | let provider = entry.kind(); |
| 86 | let mut config = Config { |
| 87 | provider: Some(entry.id().to_string()), |
| 88 | ..Default::default() |
| 89 | }; |
| 90 | if provider == ProviderKind::Custom { |
| 91 | config |
| 92 | .providers |
| 93 | .get_or_insert_with(Default::default) |
| 94 | .custom |
| 95 | .insert( |
| 96 | "custom".into(), |
| 97 | ProviderConfig { |
| 98 | kind: Some("openai-compatible".into()), |
| 99 | base_url: Some(candidate.endpoint().base_url.clone()), |
| 100 | model: Some(candidate.wire_model_id().as_str().into()), |
| 101 | ..Default::default() |
| 102 | }, |
| 103 | ); |
| 104 | } |
| 105 | config |
| 106 | .provider_config_for_mut(&config.test_identity_for_kind(provider)) |
| 107 | .unwrap() |
| 108 | .base_url = Some(candidate.endpoint().base_url.clone()); |
| 109 | defaults.push((config, provider, candidate)); |
| 110 | } |
| 111 | let built_in_count = defaults.len(); |
| 112 | for descriptor in bundled_provider_descriptors() { |
| 113 | assert_eq!(descriptor.wire, DescriptorWire::OpenaiCompatible); |
| 114 | let mut config = Config { |
| 115 | provider: Some(descriptor.id.clone()), |
| 116 | ..Default::default() |
| 117 | }; |
| 118 | config |
| 119 | .providers |
| 120 | .get_or_insert_with(Default::default) |
| 121 | .custom |
| 122 | .insert( |
| 123 | descriptor.id.clone(), |
| 124 | ProviderConfig { |
| 125 | kind: Some("openai-compatible".to_string()), |
| 126 | base_url: Some(descriptor.base_url.clone()), |
| 127 | model: Some(descriptor.default_model.clone()), |
| 128 | ..Default::default() |
| 129 | }, |
| 130 | ); |
| 131 | config |
| 132 | .resolve_provider_identity(&descriptor.id) |
| 133 | .expect("named compatible default must retain a valid identity"); |
| 134 | let candidate = resolver |
| 135 | .resolve(&RouteRequest { |
| 136 | explicit_provider: Some(ProviderKind::Custom), |
| 137 | model_selector: Some(LogicalModelRef::from(descriptor.default_model.clone())), |
| 138 | base_url_override: Some( |
| 139 | config.base_url_for_route(&config.test_identity_for_kind(ProviderKind::Custom)), |
| 140 | ), |
| 141 | ..Default::default() |
| 142 | }) |
| 143 | .expect("named compatible bootstrap route must resolve"); |
| 144 | defaults.push((config, ProviderKind::Custom, candidate)); |
| 145 | } |
| 146 | |
| 147 | // A deliberate breadth receipt, including the generic Custom placeholder. |
| 148 | // Inventory still comes from production owners, not these expected counts. |
| 149 | assert_eq!(built_in_count, 51, "review changed shipped-default breadth"); |
| 150 | assert_eq!( |
| 151 | defaults.len() - built_in_count, |
| 152 | 10, |
| 153 | "review compatible-default breadth" |
| 154 | ); |
| 155 | let recorded_at = Utc.with_ymd_and_hms(2026, 9, 8, 12, 0, 0).unwrap(); |
| 156 | // Both ordinary token classes are actually used. Cache and long-context |
| 157 | // tiers have separate existing tests; this guard makes no all-tier claim. |
| 158 | let usage = Usage { |
| 159 | input_tokens: 1_000, |
| 160 | output_tokens: 100, |
| 161 | ..Default::default() |
| 162 | }; |
| 163 | let mut seen = BTreeSet::new(); |
| 164 | let mut unpriced = BTreeMap::new(); |
| 165 | for (config, provider, candidate) in defaults { |
| 166 | let base_url = config.base_url_for_route(&config.test_identity_for_kind(provider)); |
| 167 | // Config normalizes the optional trailing separator before dispatch. |
| 168 | // Retain the exact host/path comparison and audit that runtime form. |
| 169 | assert_eq!( |
| 170 | base_url, |
| 171 | candidate.endpoint().base_url.trim_end_matches('/') |
| 172 | ); |
| 173 | let route = EffectiveRouteEnvelope::capture( |
| 174 | Some(&config), |
| 175 | provider, |
| 176 | config.active_provider_identity().unwrap().key.as_str(), |
| 177 | candidate.wire_model_id().as_str(), |
| 178 | Some(&base_url), |
| 179 | recorded_at, |
| 180 | ); |
| 181 | assert!( |
| 182 | route.provider_live_pricing.is_none(), |
| 183 | "offline default captured live pricing" |
| 184 | ); |
| 185 | let key = [ |
| 186 | route.provider_identity.clone(), |
| 187 | route.model.clone(), |
| 188 | base_url, |
| 189 | serde_json::to_value(candidate.protocol()) |
| 190 | .unwrap() |
| 191 | .as_str() |
| 192 | .unwrap() |
| 193 | .to_string(), |
| 194 | serde_json::to_value(route.billing_mode) |
| 195 | .unwrap() |
| 196 | .as_str() |
| 197 | .unwrap() |
| 198 | .to_string(), |
| 199 | ]; |
| 200 | assert!(seen.insert(key.clone()), "duplicate default route: {key:?}"); |
| 201 | let audit = route.audit(&usage); |
| 202 | if let Some(reason) = audit.unpriced_reason { |
| 203 | assert!( |
| 204 | audit.estimate.is_none(), |
| 205 | "unpriced route has an estimate: {key:?}" |
| 206 | ); |
| 207 | unpriced.insert(key, reason); |
| 208 | } else { |
| 209 | assert!( |
| 210 | audit.usd_priced || audit.cny_priced, |
| 211 | "no authoritative currency: {key:?}" |
| 212 | ); |
| 213 | assert!( |
| 214 | audit |
| 215 | .provenance |
| 216 | .is_some_and(|source| source != PricingProvenance::Unknown) |
| 217 | ); |
| 218 | assert!( |
| 219 | audit.unpriced_classes.is_empty(), |
| 220 | "used token class missing: {key:?}" |
| 221 | ); |
| 222 | assert!( |
| 223 | audit |
| 224 | .estimate |
| 225 | .is_some_and(CostEstimate::is_finite_nonnegative), |
| 226 | "invalid price: {key:?}" |
| 227 | ); |
| 228 | } |
| 229 | } |
| 230 | assert_eq!( |
| 231 | unpriced, |
| 232 | reviewed_omissions(), |
| 233 | "New or changed omissions need exact source review; newly priced/removed routes must retire stale exemptions" |
| 234 | ); |
| 235 | } |
| 236 |