| 1 | //! System One decision API on the existing client (#6525). |
| 2 | //! |
| 3 | //! `[auto.router] kind = "decision"` asks a non-generative decision model |
| 4 | //! (TypeSafe's Jev) one typed Choice per turn. The wire is a plain JSON |
| 5 | //! `POST to the route’s decision endpoint` — not chat completions — served on two routes: |
| 6 | //! |
| 7 | //! * **OpenRouter** — `https://openrouter.ai/api/alpha/decisions` with the user's OpenRouter |
| 8 | //! key, built exactly like every other OpenRouter client. |
| 9 | //! * **TypeSafe direct** — `https://api.typesafe.ai/v1/systemone` with a |
| 10 | //! TypeSafe key. |
| 11 | //! |
| 12 | //! This is a child of `client` so it reuses the client's auth headers, TLS, |
| 13 | //! redaction and bounded retry plumbing; there is no second HTTP client. |
| 14 | //! |
| 15 | //! Known limits (written down so nobody assumes them): |
| 16 | //! * TypeSafe is **not** an [`ProviderKind`]: it serves no chat route, so it is |
| 17 | //! the decision router's own endpoint + key (`TYPESAFE_API_KEY`, the |
| 18 | //! `typesafe` secret-store slot, or `[providers.typesafe] api_key` / |
| 19 | //! `api_key_env`). Its tokens enter the shared usage ledger under a frozen |
| 20 | //! `custom` / `typesafe` route with unknown billing; reported cost is retained |
| 21 | //! on decision receipts. Unknown pricing is never interpreted as free. |
| 22 | //! * The routing call makes one attempt (no retry): it is bounded by the |
| 23 | //! router timeout, and a retried decision would arrive after the turn has |
| 24 | //! already fallen back. |
| 25 | //! * The router parses the `choice` answer shape; the shadow Decision Gate |
| 26 | //! (`crate::superfast`) also reads `noul`. The transport strictly validates |
| 27 | //! Choice, Noul and fractional Score responses before either policy uses them. |
| 28 | |
| 29 | use std::collections::BTreeMap; |
| 30 | |
| 31 | use serde::Deserialize; |
| 32 | use serde_json::value::RawValue; |
| 33 | |
| 34 | use super::*; |
| 35 | use crate::model_routing::AutoRouterFailure; |
| 36 | |
| 37 | /// TypeSafe's direct API base (the `/systemone` path is appended). |
| 38 | pub(crate) const TYPESAFE_DEFAULT_BASE_URL: &str = "https://api.typesafe.ai/v1"; |
| 39 | /// Environment variable holding a TypeSafe API key. |
| 40 | pub(crate) const TYPESAFE_API_KEY_ENV: &str = "TYPESAFE_API_KEY"; |
| 41 | /// Secret-store slot and `[providers.<name>]` table name for the TypeSafe key. |
| 42 | pub(crate) const TYPESAFE_KEY_NAME: &str = "typesafe"; |
| 43 | /// Bound a non-generative decision response before allocating/decoding it. |
| 44 | const DECISION_RESPONSE_MAX_BYTES: usize = 256 * 1024; |
| 45 | const DECISION_REQUEST_MAX_BYTES: usize = 1024 * 1024; |
| 46 | |
| 47 | /// Which endpoint serves a decision router. |
| 48 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 49 | #[serde(rename_all = "snake_case")] |
| 50 | pub(crate) enum DecisionRouterRoute { |
| 51 | Openrouter, |
| 52 | Typesafe, |
| 53 | } |
| 54 | |
| 55 | impl DecisionRouterRoute { |
| 56 | /// Parse an `[auto.router] provider` value for a decision router. |
| 57 | #[must_use] |
| 58 | pub(crate) fn parse(provider: &str) -> Option<Self> { |
| 59 | let provider = provider.trim(); |
| 60 | if ProviderKind::parse(provider) == Some(ProviderKind::Openrouter) { |
| 61 | Some(Self::Openrouter) |
| 62 | } else if provider.eq_ignore_ascii_case(TYPESAFE_KEY_NAME) |
| 63 | || provider.eq_ignore_ascii_case("typesafe-ai") |
| 64 | { |
| 65 | Some(Self::Typesafe) |
| 66 | } else { |
| 67 | None |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | #[must_use] |
| 72 | pub(crate) fn as_str(self) -> &'static str { |
| 73 | match self { |
| 74 | Self::Openrouter => "openrouter", |
| 75 | Self::Typesafe => TYPESAFE_KEY_NAME, |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | #[must_use] |
| 80 | pub(crate) fn display_name(self) -> &'static str { |
| 81 | match self { |
| 82 | Self::Openrouter => "OpenRouter", |
| 83 | Self::Typesafe => "TypeSafe", |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | /// Whether a credential for this route is present (never returns it). |
| 88 | #[must_use] |
| 89 | pub(crate) fn has_key(self, config: &Config) -> bool { |
| 90 | match self { |
| 91 | Self::Openrouter => config |
| 92 | .builtin_provider_identity(ProviderKind::Openrouter) |
| 93 | .is_ok_and(|identity| crate::config::has_api_key_for(config, &identity)), |
| 94 | Self::Typesafe => typesafe_api_key(config).is_some(), |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | /// Resolve the TypeSafe key: environment, then the durable secret store, then |
| 100 | /// `[providers.typesafe] api_key` / `api_key_env`. |
| 101 | pub(crate) fn typesafe_api_key(config: &Config) -> Option<String> { |
| 102 | let non_empty = |value: String| (!value.trim().is_empty()).then(|| value.trim().to_string()); |
| 103 | if let Some(key) = std::env::var(TYPESAFE_API_KEY_ENV).ok().and_then(non_empty) { |
| 104 | return Some(key); |
| 105 | } |
| 106 | if let Some(key) = crate::config::credential_secret_store() |
| 107 | .and_then(|store| store.get(TYPESAFE_KEY_NAME).ok().flatten()) |
| 108 | .and_then(non_empty) |
| 109 | { |
| 110 | return Some(key); |
| 111 | } |
| 112 | let entry = config |
| 113 | .providers |
| 114 | .as_ref() |
| 115 | .and_then(|providers| providers.custom_provider_config(TYPESAFE_KEY_NAME))?; |
| 116 | entry.api_key.clone().and_then(non_empty).or_else(|| { |
| 117 | entry |
| 118 | .api_key_env |
| 119 | .as_deref() |
| 120 | .map(str::trim) |
| 121 | .filter(|name| !name.is_empty()) |
| 122 | .and_then(|name| std::env::var(name).ok()) |
| 123 | .and_then(non_empty) |
| 124 | }) |
| 125 | } |
| 126 | |
| 127 | /// A decoded System One response. Unknown fields are ignored. |
| 128 | #[derive(Debug, Clone)] |
| 129 | pub(crate) struct SystemOneResponse { |
| 130 | pub(crate) id: Option<String>, |
| 131 | pub(crate) model: Option<String>, |
| 132 | pub(crate) answers: BTreeMap<String, SystemOneAnswer>, |
| 133 | pub(crate) usage: Option<SystemOneUsage>, |
| 134 | /// Transport validation is separate from decoding so a rejected policy |
| 135 | /// answer still preserves the provider's usage/cost evidence. |
| 136 | pub(crate) answers_validated: Option<bool>, |
| 137 | } |
| 138 | |
| 139 | impl<'de> Deserialize<'de> for SystemOneResponse { |
| 140 | fn deserialize<D: serde::Deserializer<'de>>( |
| 141 | deserializer: D, |
| 142 | ) -> std::result::Result<Self, D::Error> { |
| 143 | #[derive(Deserialize)] |
| 144 | struct Envelope { |
| 145 | #[serde(default)] |
| 146 | id: Option<Box<RawValue>>, |
| 147 | #[serde(default)] |
| 148 | model: Option<Box<RawValue>>, |
| 149 | #[serde(default)] |
| 150 | answers: Option<Box<RawValue>>, |
| 151 | #[serde(default)] |
| 152 | usage: Option<Box<RawValue>>, |
| 153 | } |
| 154 | let wire = Envelope::deserialize(deserializer)?; |
| 155 | let text = |raw: Option<&RawValue>| { |
| 156 | raw.and_then(|raw| serde_json::from_str::<String>(raw.get()).ok()) |
| 157 | }; |
| 158 | let id = text(wire.id.as_deref()); |
| 159 | let model = text(wire.model.as_deref()); |
| 160 | let malformed_identity = |
| 161 | (wire.id.is_some() && id.is_none()) || (wire.model.is_some() && model.is_none()); |
| 162 | // Invalid policy fields must fail validation without discarding the |
| 163 | // separately reported billing evidence. Empty/default answers never |
| 164 | // satisfy the required question types or probabilities. |
| 165 | let answers: BTreeMap<String, Box<RawValue>> = wire |
| 166 | .answers |
| 167 | .and_then(|raw| serde_json::from_str(raw.get()).ok()) |
| 168 | .unwrap_or_default(); |
| 169 | Ok(Self { |
| 170 | id, |
| 171 | model, |
| 172 | answers: answers |
| 173 | .into_iter() |
| 174 | .map(|(key, raw)| (key, serde_json::from_str(raw.get()).unwrap_or_default())) |
| 175 | .collect(), |
| 176 | usage: wire |
| 177 | .usage |
| 178 | .and_then(|raw| serde_json::from_str(raw.get()).ok()), |
| 179 | answers_validated: malformed_identity.then_some(false), |
| 180 | }) |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | /// One answer. The `choice` and `noul` subsets are interpreted. |
| 185 | #[derive(Debug, Clone, Default, Deserialize)] |
| 186 | pub(crate) struct SystemOneAnswer { |
| 187 | #[serde(rename = "type", default)] |
| 188 | pub(crate) kind: String, |
| 189 | #[serde(default)] |
| 190 | pub(crate) choice: Option<String>, |
| 191 | /// A `noul` answer's probability; validated by its reader. |
| 192 | #[serde(default)] |
| 193 | pub(crate) noul: Option<f64>, |
| 194 | #[serde(default)] |
| 195 | pub(crate) score: Option<f64>, |
| 196 | #[serde(default)] |
| 197 | pub(crate) legend: BTreeMap<String, Value>, |
| 198 | #[serde(default)] |
| 199 | pub(crate) probabilities: BTreeMap<String, Option<f64>>, |
| 200 | #[serde(default)] |
| 201 | pub(crate) confidence: Option<f64>, |
| 202 | } |
| 203 | |
| 204 | /// Provider usage. OpenRouter adds `cost`; the token-count casing differs |
| 205 | /// between surfaces, so both spellings are accepted. |
| 206 | #[derive(Debug, Clone)] |
| 207 | pub(crate) struct SystemOneUsage { |
| 208 | /// Kept verbatim so the receipt never re-renders a float. |
| 209 | pub(crate) cost: Option<Box<RawValue>>, |
| 210 | pub(crate) input_tokens: u32, |
| 211 | pub(crate) output_tokens: u32, |
| 212 | pub(crate) complete: bool, |
| 213 | } |
| 214 | |
| 215 | impl<'de> Deserialize<'de> for SystemOneUsage { |
| 216 | fn deserialize<D: serde::Deserializer<'de>>( |
| 217 | deserializer: D, |
| 218 | ) -> std::result::Result<Self, D::Error> { |
| 219 | let mut wire = BTreeMap::<String, Box<RawValue>>::deserialize(deserializer)?; |
| 220 | let count = |snake: &str, camel: &str| match (wire.get(snake), wire.get(camel)) { |
| 221 | (Some(raw), None) | (None, Some(raw)) => serde_json::from_str::<u32>(raw.get()).ok(), |
| 222 | // Missing, malformed or ambiguous counts cannot authorize policy |
| 223 | // or a priced subtotal; keep an independently valid raw cost. |
| 224 | _ => None, |
| 225 | }; |
| 226 | let input_tokens = count("input_tokens", "inputTokens"); |
| 227 | let output_tokens = count("output_tokens", "outputTokens"); |
| 228 | Ok(Self { |
| 229 | complete: input_tokens.is_some() && output_tokens.is_some(), |
| 230 | input_tokens: input_tokens.unwrap_or(0), |
| 231 | output_tokens: output_tokens.unwrap_or(0), |
| 232 | cost: wire.remove("cost"), |
| 233 | }) |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | impl SystemOneUsage { |
| 238 | /// The provider-reported cost as its verbatim JSON decimal, when it is a |
| 239 | /// finite, non-negative number. |
| 240 | #[must_use] |
| 241 | pub(crate) fn reported_cost(&self) -> Option<String> { |
| 242 | let raw = self.cost.as_ref()?.get().trim(); |
| 243 | let value: f64 = raw.parse().ok()?; |
| 244 | (raw.len() <= 128 && value.is_finite() && value >= 0.0).then(|| raw.to_string()) |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | impl CodewhaleClient { |
| 249 | /// Build the client that serves `route`. |
| 250 | /// |
| 251 | /// OpenRouter is the ordinary OpenRouter client (its key, base URL, |
| 252 | /// attribution headers). TypeSafe re-points a clone of the active route's |
| 253 | /// client — keeping its retry, TLS and redaction policy — at the TypeSafe |
| 254 | /// endpoint with the TypeSafe key only. Its captured request budget and |
| 255 | /// remote-control ownership continue to apply to the one decision attempt. |
| 256 | pub(crate) fn for_decision_route( |
| 257 | config: &Config, |
| 258 | route: DecisionRouterRoute, |
| 259 | base_url_override: Option<&str>, |
| 260 | ) -> Result<Self> { |
| 261 | match route { |
| 262 | DecisionRouterRoute::Openrouter => { |
| 263 | let mut scoped = config.clone(); |
| 264 | scoped.provider = Some(ProviderKind::Openrouter.as_str().to_string()); |
| 265 | // The decision model id is not a chat route; it travels in the |
| 266 | // JSON body only. |
| 267 | scoped.default_text_model = None; |
| 268 | Self::new(&scoped) |
| 269 | } |
| 270 | DecisionRouterRoute::Typesafe => { |
| 271 | let key = typesafe_api_key(config).with_context(|| { |
| 272 | format!("TypeSafe API key not configured ({TYPESAFE_API_KEY_ENV})") |
| 273 | })?; |
| 274 | let base_url = base_url_override |
| 275 | .map(str::trim) |
| 276 | .filter(|url| !url.is_empty()) |
| 277 | .unwrap_or(TYPESAFE_DEFAULT_BASE_URL) |
| 278 | .trim_end_matches('/') |
| 279 | .to_string(); |
| 280 | validate_base_url_security(&base_url, false)?; |
| 281 | let mut client = Self::new(config)?; |
| 282 | client.http_client = Self::http_client_builder_with_auth_mode( |
| 283 | &key, |
| 284 | &HashMap::new(), |
| 285 | ProviderKind::Custom, |
| 286 | &base_url, |
| 287 | WireFormat::ChatCompletions, |
| 288 | false, |
| 289 | client.force_http1, |
| 290 | config, |
| 291 | )? |
| 292 | .build()?; |
| 293 | client.base_url = base_url; |
| 294 | client.http1_client = client.http_client.clone(); |
| 295 | // Repointing auth/URL must also replace the inherited chat |
| 296 | // route identity. No TypeSafe price/product is guessed. |
| 297 | client.api_provider = ProviderKind::Custom; |
| 298 | // The fixed decision authority owns this validated key/URL; |
| 299 | // ordinary route admission supplies only its exact identity. |
| 300 | let mut route_config = config.clone(); |
| 301 | route_config.provider = Some(TYPESAFE_KEY_NAME.into()); |
| 302 | route_config |
| 303 | .providers |
| 304 | .get_or_insert_with(crate::config::ProvidersConfig::default) |
| 305 | .custom |
| 306 | .insert( |
| 307 | TYPESAFE_KEY_NAME.into(), |
| 308 | crate::config::ProviderConfig { |
| 309 | kind: Some("openai-compatible".into()), |
| 310 | base_url: Some(client.base_url.clone()), |
| 311 | api_key: Some(key.clone()), |
| 312 | ..crate::config::ProviderConfig::default() |
| 313 | }, |
| 314 | ); |
| 315 | client.admitted_identity = route_config |
| 316 | .active_provider_identity() |
| 317 | .map_err(anyhow::Error::msg)?; |
| 318 | client.openrouter_vendor = None; |
| 319 | client.billing_surface = None; |
| 320 | client.billing_mode = crate::cost_status::RouteBillingMode::Unknown; |
| 321 | client.route_limits = None; |
| 322 | // `Self::new` froze the redaction set from the chat provider's |
| 323 | // secrets; the TypeSafe key is none of them, so add it before |
| 324 | // any decision body is built from untrusted context. |
| 325 | let mut secrets = client.model_bound_secret_values.as_ref().clone(); |
| 326 | push_model_bound_secret(&mut secrets, Some(&key)); |
| 327 | client.model_bound_secret_values = Arc::new(secrets); |
| 328 | client.api_key = key; |
| 329 | Ok(client) |
| 330 | } |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | /// `POST to the route’s decision endpoint` once, isolated like the Auto chat classifier: |
| 335 | /// no global retry banners or response cache; shared admission still applies. |
| 336 | /// |
| 337 | /// Only a failure class leaves this function — provider error bodies can |
| 338 | /// echo the prompt and must never reach receipts. |
| 339 | /// |
| 340 | /// `dispatched` is set once both permits are held and the request is |
| 341 | /// handed to the transport, so a caller whose deadline cancels this |
| 342 | /// future can tell a possibly-billed request from one never sent. |
| 343 | pub(crate) async fn system_one_decide( |
| 344 | &self, |
| 345 | body: &Value, |
| 346 | dispatched: &std::sync::atomic::AtomicBool, |
| 347 | ) -> std::result::Result<SystemOneResponse, AutoRouterFailure> { |
| 348 | if serde_json::to_vec(body).map_or(true, |bytes| bytes.len() > DECISION_REQUEST_MAX_BYTES) |
| 349 | || !valid_decision_request(body) |
| 350 | { |
| 351 | return Err(AutoRouterFailure::NotRunnable); |
| 352 | } |
| 353 | let mut isolated = self.clone(); |
| 354 | isolated.isolated_request_state = true; |
| 355 | isolated.retry.max_retries = 0; |
| 356 | let _inference = isolated.acquire_remote_control_inference_permit().await; |
| 357 | let _permit = isolated.acquire_provider_request_permit().await; |
| 358 | let url = if isolated.api_provider == ProviderKind::Openrouter { |
| 359 | // The OpenRouter Decisions API is a sibling of /api/v1, so keep |
| 360 | // the configured origin/proxy prefix and replace only /v1. |
| 361 | let mut url = reqwest::Url::parse(&isolated.base_url) |
| 362 | .map_err(|_| AutoRouterFailure::NotRunnable)?; |
| 363 | let prefix = url.path().trim_end_matches('/').trim_end_matches("/v1"); |
| 364 | url.set_path(&format!("{prefix}/alpha/decisions")); |
| 365 | url.to_string() |
| 366 | } else { |
| 367 | api_url(&isolated.base_url, "systemone") |
| 368 | }; |
| 369 | dispatched.store(true, std::sync::atomic::Ordering::Release); |
| 370 | let response = isolated |
| 371 | .send_json_with_retry(&url, body) |
| 372 | .await |
| 373 | .map_err(|error| router_failure_from_error(&error))?; |
| 374 | let text = bounded_provider_catalog_text(response, DECISION_RESPONSE_MAX_BYTES) |
| 375 | .await |
| 376 | .map_err(|error| match error { |
| 377 | CatalogRefreshError::Network => AutoRouterFailure::Transport, |
| 378 | _ => AutoRouterFailure::InvalidAnswer, |
| 379 | })?; |
| 380 | let mut response: SystemOneResponse = |
| 381 | serde_json::from_str(&text).map_err(|_| AutoRouterFailure::InvalidAnswer)?; |
| 382 | response.answers_validated = Some(valid_decision_response(body, &response)); |
| 383 | response.model = response.model.map(|model| { |
| 384 | self.redact_model_bound_text(&model) |
| 385 | .chars() |
| 386 | .take(128) |
| 387 | .collect() |
| 388 | }); |
| 389 | Ok(response) |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | fn valid_decision_request(body: &Value) -> bool { |
| 394 | let Some(questions) = body.get("questions").and_then(Value::as_object) else { |
| 395 | return false; |
| 396 | }; |
| 397 | let supported_value = |
| 398 | |value: &Value| matches!(value, Value::String(_) | Value::Object(_) | Value::Array(_)); |
| 399 | let model = body.get("model").and_then(Value::as_str); |
| 400 | model.is_some_and(|m| !m.trim().is_empty() && m.len() <= 256) |
| 401 | && body.get("state").is_some_and(|state| { |
| 402 | matches!(state, Value::String(_) | Value::Object(_) | Value::Array(_)) |
| 403 | }) |
| 404 | && !questions.is_empty() |
| 405 | && questions.len() <= 64 |
| 406 | && questions |
| 407 | .values() |
| 408 | .all(|q| match q.get("type").and_then(Value::as_str) { |
| 409 | Some("noul") => true, |
| 410 | Some("choice") => q |
| 411 | .get("criteria") |
| 412 | .and_then(Value::as_object) |
| 413 | .is_some_and(|c| { |
| 414 | !c.is_empty() |
| 415 | && c.len() <= 64 |
| 416 | && c.iter().all(|(name, v)| { |
| 417 | !name.trim().is_empty() |
| 418 | && name.len() <= 128 |
| 419 | && (v.is_null() || supported_value(v)) |
| 420 | }) |
| 421 | }), |
| 422 | Some("score") => q |
| 423 | .get("criteria") |
| 424 | .and_then(Value::as_array) |
| 425 | .is_some_and(|c| { |
| 426 | !c.is_empty() && c.len() <= 10 && c.iter().all(supported_value) |
| 427 | }), |
| 428 | _ => false, |
| 429 | }) |
| 430 | } |
| 431 | |
| 432 | fn valid_decision_response(body: &Value, response: &SystemOneResponse) -> bool { |
| 433 | let Some(questions) = body.get("questions").and_then(Value::as_object) else { |
| 434 | return false; |
| 435 | }; |
| 436 | response.answers_validated != Some(false) |
| 437 | && response |
| 438 | .model |
| 439 | .as_deref() |
| 440 | .is_some_and(|m| !m.trim().is_empty() && m.len() <= 256) |
| 441 | && response.usage.as_ref().is_some_and(|usage| usage.complete) |
| 442 | && response.answers.len() == questions.len() |
| 443 | && questions.iter().all(|(name, question)| { |
| 444 | let Some(answer) = response.answers.get(name) else { |
| 445 | return false; |
| 446 | }; |
| 447 | if Some(answer.kind.as_str()) != question.get("type").and_then(Value::as_str) { |
| 448 | return false; |
| 449 | } |
| 450 | match answer.kind.as_str() { |
| 451 | "noul" => answer |
| 452 | .noul |
| 453 | .is_some_and(|v| v.is_finite() && (0.0..=1.0).contains(&v)), |
| 454 | "choice" => { |
| 455 | let Some(criteria) = question.get("criteria").and_then(Value::as_object) else { |
| 456 | return false; |
| 457 | }; |
| 458 | let options = criteria.keys().map(String::as_str).collect::<Vec<_>>(); |
| 459 | crate::model_routing::validated_choice(Some(answer), &options).is_some() |
| 460 | } |
| 461 | "score" => { |
| 462 | let Some(criteria) = question.get("criteria").and_then(Value::as_array) else { |
| 463 | return false; |
| 464 | }; |
| 465 | if criteria.is_empty() || criteria.len() > 10 { |
| 466 | return false; |
| 467 | } |
| 468 | let Some(score) = answer.score else { |
| 469 | return false; |
| 470 | }; |
| 471 | if !score.is_finite() |
| 472 | || !(0.0..=(criteria.len() - 1) as f64).contains(&score) |
| 473 | || !answer |
| 474 | .confidence |
| 475 | .is_some_and(|v| v.is_finite() && (0.0..=1.0).contains(&v)) |
| 476 | || answer.legend.len() != criteria.len() |
| 477 | || answer.probabilities.len() != criteria.len() |
| 478 | { |
| 479 | return false; |
| 480 | } |
| 481 | let mut sum = 0.0; |
| 482 | let mut expected_score = 0.0; |
| 483 | for (level, criterion) in criteria.iter().enumerate() { |
| 484 | let key = level.to_string(); |
| 485 | let Some(Some(probability)) = answer.probabilities.get(&key) else { |
| 486 | return false; |
| 487 | }; |
| 488 | if answer.legend.get(&key) != Some(criterion) |
| 489 | || !probability.is_finite() |
| 490 | || !(0.0..=1.0).contains(probability) |
| 491 | { |
| 492 | return false; |
| 493 | } |
| 494 | sum += probability; |
| 495 | expected_score += level as f64 * probability; |
| 496 | } |
| 497 | (sum - 1.0).abs() <= 0.02 && (score - expected_score).abs() <= 0.02 |
| 498 | } |
| 499 | _ => false, |
| 500 | } |
| 501 | }) |
| 502 | } |
| 503 | |
| 504 | /// Collapse a client error into a non-secret failure class. The HTTP status |
| 505 | /// is kept where the error carries it; the body never is. |
| 506 | pub(crate) fn router_failure_from_error(error: &anyhow::Error) -> AutoRouterFailure { |
| 507 | let Some(error) = error.downcast_ref::<LlmError>() else { |
| 508 | return AutoRouterFailure::Transport; |
| 509 | }; |
| 510 | match error { |
| 511 | LlmError::RateLimited { .. } => AutoRouterFailure::Http { status: 429 }, |
| 512 | LlmError::ServerError { status, .. } | LlmError::InvalidRequest { status, .. } => { |
| 513 | AutoRouterFailure::Http { status: *status } |
| 514 | } |
| 515 | LlmError::AuthenticationError(_) => AutoRouterFailure::Http { status: 401 }, |
| 516 | LlmError::AuthorizationError(_) => AutoRouterFailure::Http { status: 403 }, |
| 517 | LlmError::QuotaExhausted(_) => AutoRouterFailure::QuotaExhausted, |
| 518 | LlmError::ModelError(_) |
| 519 | | LlmError::ContextLengthError(_) |
| 520 | | LlmError::ContentPolicyError(_) => AutoRouterFailure::Rejected, |
| 521 | LlmError::Other(message) => message |
| 522 | .strip_prefix("HTTP ") |
| 523 | .and_then(|rest| rest.split(':').next()) |
| 524 | .and_then(|status| status.trim().parse::<u16>().ok()) |
| 525 | .map_or(AutoRouterFailure::Transport, |status| { |
| 526 | AutoRouterFailure::Http { status } |
| 527 | }), |
| 528 | LlmError::NetworkError(_) | LlmError::Timeout(_) | LlmError::ParseError(_) => { |
| 529 | AutoRouterFailure::Transport |
| 530 | } |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | #[cfg(test)] |
| 535 | mod decisions_compatibility_tests { |
| 536 | use super::*; |
| 537 | |
| 538 | fn request() -> Value { |
| 539 | json!({"model":"typesafe/jev-1.13", "state":{"ticket":"charged twice"}, "questions": { |
| 540 | "intent": {"type":"choice", "criteria":{"billing":"money", "other":"anything else"}}, |
| 541 | "refund": {"type":"noul", "instructions":"Does it ask for a refund?"}, |
| 542 | "urgency": {"type":"score", "criteria":["Can wait", "Needs attention this week", "Needs attention today"]} |
| 543 | }}) |
| 544 | } |
| 545 | |
| 546 | fn response() -> Value { |
| 547 | json!({"id":"fixture-decision", "model":"typesafe/jev-1.13-20260917", "usage":{"input_tokens":287,"output_tokens":20,"cost":0.000012054}, "answers": { |
| 548 | "intent":{"type":"choice","choice":"billing","confidence":0.9,"probabilities":{"billing":0.9,"other":0.1}}, |
| 549 | "refund":{"type":"noul","noul":0.99}, |
| 550 | "urgency":{"type":"score","score":1.7,"confidence":0.9,"legend":{"0":"Can wait","1":"Needs attention this week","2":"Needs attention today"},"probabilities":{"0":0.1,"1":0.1,"2":0.8}} |
| 551 | }}) |
| 552 | } |
| 553 | |
| 554 | #[test] |
| 555 | fn all_documented_primitives_validate_without_rescaling_score() { |
| 556 | let decoded: SystemOneResponse = |
| 557 | serde_json::from_value(response()).expect("documented fixture"); |
| 558 | assert!(valid_decision_request(&request())); |
| 559 | assert!(valid_decision_response(&request(), &decoded)); |
| 560 | assert_eq!(decoded.answers["urgency"].score, Some(1.7)); |
| 561 | assert_eq!( |
| 562 | decoded.usage.expect("usage").reported_cost().as_deref(), |
| 563 | Some("0.000012054") |
| 564 | ); |
| 565 | } |
| 566 | |
| 567 | #[test] |
| 568 | fn wrong_types_unoffered_choices_scores_and_partial_shapes_are_rejected() { |
| 569 | let changes = [ |
| 570 | ("/answers/refund/noul", json!(1.01)), |
| 571 | ("/answers/refund/type", json!("score")), |
| 572 | ("/answers/intent/choice", json!("not-offered")), |
| 573 | ("/answers/intent/choice", json!("other")), |
| 574 | ("/answers/intent/confidence", json!(-0.1)), |
| 575 | ("/answers/intent/probabilities", json!({"billing":0.9})), |
| 576 | ( |
| 577 | "/answers/intent/probabilities", |
| 578 | json!({"billing":0.6,"other":0.6}), |
| 579 | ), |
| 580 | ("/answers/urgency/score", json!(2.01)), |
| 581 | ("/answers/urgency/score", json!(0.5)), |
| 582 | ("/answers/urgency/legend/1", json!("different rubric")), |
| 583 | ("/answers/urgency/probabilities/1", Value::Null), |
| 584 | ("/answers/urgency/confidence", json!(1.1)), |
| 585 | ("/model", Value::Null), |
| 586 | ("/usage", Value::Null), |
| 587 | ("/usage", json!({"input_tokens": 287, "cost":0.000012054})), |
| 588 | ("/answers", json!({})), |
| 589 | ]; |
| 590 | for (pointer, value) in changes { |
| 591 | let mut body = response(); |
| 592 | *body.pointer_mut(pointer).expect("fixture field") = value; |
| 593 | let decoded: SystemOneResponse = |
| 594 | serde_json::from_value(body).expect("structural decode"); |
| 595 | assert!( |
| 596 | !valid_decision_response(&request(), &decoded), |
| 597 | "accepted {pointer}" |
| 598 | ); |
| 599 | } |
| 600 | } |
| 601 | |
| 602 | #[test] |
| 603 | fn non_finite_in_memory_answers_and_unbounded_request_shapes_are_rejected() { |
| 604 | let mut decoded: SystemOneResponse = serde_json::from_value(response()).expect("fixture"); |
| 605 | decoded.answers.get_mut("refund").expect("noul").noul = Some(f64::NAN); |
| 606 | assert!(!valid_decision_response(&request(), &decoded)); |
| 607 | let mut decoded: SystemOneResponse = serde_json::from_value(response()).expect("fixture"); |
| 608 | decoded.answers.get_mut("urgency").expect("score").score = Some(f64::INFINITY); |
| 609 | assert!(!valid_decision_response(&request(), &decoded)); |
| 610 | for shape in [ |
| 611 | json!({}), |
| 612 | json!({"model":"jev", "state":"hello", "questions":{}}), |
| 613 | json!({"model":"jev", "state":"hello", "questions":{"q":{"type":"score","criteria":vec!["level";11]}}}), |
| 614 | ] { |
| 615 | assert!(!valid_decision_request(&shape)); |
| 616 | } |
| 617 | } |
| 618 | #[tokio::test] |
| 619 | async fn decision_transport_preserves_typesafe_auth_and_shared_admission_before_dispatch() { |
| 620 | use wiremock::matchers::{header, method, path}; |
| 621 | use wiremock::{Mock, MockServer, ResponseTemplate}; |
| 622 | let server = MockServer::start().await; |
| 623 | Mock::given(method("POST")) |
| 624 | .and(path("/v1/systemone")) |
| 625 | .and(header("authorization", "Bearer decision-fixture-key")) |
| 626 | .respond_with(ResponseTemplate::new(200).set_body_json(response())) |
| 627 | .expect(1) |
| 628 | .mount(&server) |
| 629 | .await; |
| 630 | let _lock = crate::test_support::lock_test_env(); |
| 631 | let home = tempfile::tempdir().expect("home"); |
| 632 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", home.path()); |
| 633 | let _key = |
| 634 | crate::test_support::EnvVarGuard::set(TYPESAFE_API_KEY_ENV, "decision-fixture-key"); |
| 635 | let config = Config { |
| 636 | provider: Some("deepseek".to_string()), |
| 637 | providers: Some(crate::config::ProvidersConfig { |
| 638 | deepseek: crate::config::ProviderConfig { |
| 639 | api_key: Some("chat-fixture-key".into()), |
| 640 | max_concurrency: Some(1), |
| 641 | ..Default::default() |
| 642 | }, |
| 643 | ..Default::default() |
| 644 | }), |
| 645 | ..Default::default() |
| 646 | }; |
| 647 | let base = format!("{}/v1", server.uri()); |
| 648 | let ticket = crate::test_support::env_scope_ticket(); |
| 649 | let client = tokio::task::spawn_blocking(move || { |
| 650 | let _membership = crate::test_support::join_env_scope(ticket); |
| 651 | CodewhaleClient::for_decision_route(&config, DecisionRouterRoute::Typesafe, Some(&base)) |
| 652 | }) |
| 653 | .await |
| 654 | .expect("worker") |
| 655 | .expect("client"); |
| 656 | assert_eq!(client.provider_request_concurrency_limit(), Some(1)); |
| 657 | assert!( |
| 658 | client.remote_control_inference_participant, |
| 659 | "the cloned budget must remain an attached inference participant" |
| 660 | ); |
| 661 | let clone = client.clone(); |
| 662 | let held = client |
| 663 | .acquire_provider_request_permit() |
| 664 | .await |
| 665 | .expect("shared permit"); |
| 666 | let dispatched = std::sync::atomic::AtomicBool::new(false); |
| 667 | assert!( |
| 668 | tokio::time::timeout( |
| 669 | Duration::from_millis(30), |
| 670 | clone.system_one_decide(&request(), &dispatched) |
| 671 | ) |
| 672 | .await |
| 673 | .is_err() |
| 674 | ); |
| 675 | assert!( |
| 676 | !dispatched.load(std::sync::atomic::Ordering::Acquire), |
| 677 | "a queued request has no dispatched-spend marker" |
| 678 | ); |
| 679 | assert!( |
| 680 | server |
| 681 | .received_requests() |
| 682 | .await |
| 683 | .expect("requests") |
| 684 | .is_empty() |
| 685 | ); |
| 686 | drop(held); |
| 687 | let answer = client |
| 688 | .system_one_decide(&request(), &dispatched) |
| 689 | .await |
| 690 | .expect("one admitted request"); |
| 691 | assert!(dispatched.load(std::sync::atomic::Ordering::Acquire)); |
| 692 | assert_eq!(answer.answers_validated, Some(true)); |
| 693 | assert_eq!( |
| 694 | client |
| 695 | .effective_route_envelope("jev-latest", chrono::Utc::now()) |
| 696 | .provider_identity, |
| 697 | "typesafe" |
| 698 | ); |
| 699 | } |
| 700 | #[test] |
| 701 | fn decision_partial_usage_retains_raw_cost_but_cannot_authorize_policy() { |
| 702 | let mut body = response(); |
| 703 | body["usage"] = json!({"inputTokens": 287, "cost": 0.000012054}); |
| 704 | let decoded: SystemOneResponse = serde_json::from_value(body).expect("partial receipt"); |
| 705 | assert!(!valid_decision_response(&request(), &decoded)); |
| 706 | let usage = decoded.usage.expect("reported usage"); |
| 707 | assert!(!usage.complete); |
| 708 | assert_eq!(usage.input_tokens, 287); |
| 709 | assert_eq!(usage.reported_cost().as_deref(), Some("0.000012054")); |
| 710 | let route = crate::cost_status::decision_receipt_fixture("partial").route; |
| 711 | let mut body = response(); |
| 712 | body["usage"] = json!({"inputTokens": 287, "cost": 0.000012054}); |
| 713 | let response: SystemOneResponse = serde_json::from_value(body).expect("partial response"); |
| 714 | let batch = crate::model_routing::decision_usage_batch(&route, &response); |
| 715 | assert!( |
| 716 | batch.records.is_empty(), |
| 717 | "a partial token count cannot produce a complete priced subtotal" |
| 718 | ); |
| 719 | assert_eq!(batch.dropped_records, 1); |
| 720 | assert_eq!(batch.drop_records.len(), 1); |
| 721 | let mut invalid = request(); |
| 722 | invalid["questions"]["urgency"]["criteria"] = json!([null]); |
| 723 | assert!(!valid_decision_request(&invalid)); |
| 724 | } |
| 725 | |
| 726 | #[test] |
| 727 | fn malformed_policy_fields_retain_independent_raw_cost() { |
| 728 | for (pointer, value) in [ |
| 729 | ("/answers/intent/choice", json!(17)), |
| 730 | ("/answers/intent/confidence", json!("high")), |
| 731 | ("/answers/refund", Value::Null), |
| 732 | ("/answers", json!([])), |
| 733 | ("/model", json!({"invalid":"model"})), |
| 734 | ("/id", json!([])), |
| 735 | ] { |
| 736 | let mut body = response(); |
| 737 | *body.pointer_mut(pointer).expect("fixture field") = value; |
| 738 | let decoded: SystemOneResponse = |
| 739 | serde_json::from_value(body).expect("billing envelope"); |
| 740 | assert!( |
| 741 | !valid_decision_response(&request(), &decoded), |
| 742 | "accepted {pointer}" |
| 743 | ); |
| 744 | let usage = decoded |
| 745 | .usage |
| 746 | .expect("independent usage survives rejected policy"); |
| 747 | assert!(usage.complete); |
| 748 | assert_eq!(usage.input_tokens, 287); |
| 749 | assert_eq!(usage.output_tokens, 20); |
| 750 | assert_eq!(usage.reported_cost().as_deref(), Some("0.000012054")); |
| 751 | } |
| 752 | } |
| 753 | |
| 754 | #[test] |
| 755 | fn malformed_or_overflowed_counters_retain_cost_without_priced_usage() { |
| 756 | for counters in [ |
| 757 | json!({"input_tokens": u64::MAX, "output_tokens":20}), |
| 758 | json!({"input_tokens": -1, "output_tokens":20}), |
| 759 | json!({"input_tokens": "287", "output_tokens":20}), |
| 760 | json!({"input_tokens": 287, "inputTokens":287, "output_tokens":20}), |
| 761 | json!({"input_tokens": 287, "output_tokens":1.5}), |
| 762 | ] { |
| 763 | let mut body = response(); |
| 764 | body["usage"] = counters; |
| 765 | body["usage"]["cost"] = json!(0.000012054); |
| 766 | let decoded: SystemOneResponse = |
| 767 | serde_json::from_value(body).expect("billing envelope"); |
| 768 | assert!(!valid_decision_response(&request(), &decoded)); |
| 769 | let usage = decoded.usage.as_ref().expect("independent cost"); |
| 770 | assert!(!usage.complete); |
| 771 | assert_eq!(usage.reported_cost().as_deref(), Some("0.000012054")); |
| 772 | let route = crate::cost_status::decision_receipt_fixture("malformed-count").route; |
| 773 | let batch = crate::model_routing::decision_usage_batch(&route, &decoded); |
| 774 | assert!( |
| 775 | batch.records.is_empty(), |
| 776 | "invalid counters cannot fabricate priced usage" |
| 777 | ); |
| 778 | assert_eq!(batch.dropped_records, 1); |
| 779 | assert_eq!(batch.drop_records.len(), 1); |
| 780 | } |
| 781 | } |
| 782 | } |
| 783 |