| 1 | //! Codewhale's corrections to Models.dev rows (#6396). |
| 2 | //! |
| 3 | //! Some Models.dev facts are true but misleading for a Codewhale route: a flat |
| 4 | //! price for a plan that bills quota, a base rate that doubles past a prompt |
| 5 | //! size, an output limit the provider publishes differently. These used to be |
| 6 | //! hand edits to the offline seed, so they held only until the first live |
| 7 | //! refresh replaced the row. Here they are field patches applied to every |
| 8 | //! Models.dev row as it is hydrated, bundled seed and live refresh alike, so a |
| 9 | //! correction means the same thing on every install. |
| 10 | //! |
| 11 | //! Corrections rank above bundled and live Models.dev and below signed cloud |
| 12 | //! facts (which may correct a correction) and provider rosters. A corrected |
| 13 | //! row keeps its own source, so layer code that sorts rows by source still |
| 14 | //! places it correctly; a price the correction owns (set or withheld) is |
| 15 | //! attributed to [`CatalogSource::CodewhaleBundled`] through `cost_source`. |
| 16 | //! It reuses the signed layer's [`ModelFact`] shape and patch code, so there |
| 17 | //! is one way to correct a catalog row. Corrections never add a row and never |
| 18 | //! hide one. |
| 19 | |
| 20 | use std::collections::BTreeMap; |
| 21 | use std::sync::OnceLock; |
| 22 | |
| 23 | use serde::Deserialize; |
| 24 | |
| 25 | use super::{CatalogOffering, CatalogSource}; |
| 26 | use crate::cloud_facts::catalog_patch::apply_patches; |
| 27 | use crate::cloud_facts::{ModelFact, ModelOp}; |
| 28 | |
| 29 | /// The committed corrections asset. |
| 30 | pub const CATALOG_CORRECTIONS_JSON: &str = include_str!("../../assets/catalog_corrections.json"); |
| 31 | |
| 32 | /// Parsed corrections. |
| 33 | #[derive(Debug, Clone, PartialEq, Deserialize)] |
| 34 | #[serde(deny_unknown_fields)] |
| 35 | pub struct CatalogCorrections { |
| 36 | #[serde(default, rename = "_about")] |
| 37 | pub about: String, |
| 38 | /// Stamped as [`CatalogSource::CodewhaleBundled`] on prices it owns. |
| 39 | pub revision: String, |
| 40 | /// Pure reviewed model/transport data copied into the deterministic bundled seed. |
| 41 | #[serde(default)] |
| 42 | pub reviewed: super::reviewed::ReviewedCatalog, |
| 43 | /// Rules for every row a provider serves. |
| 44 | #[serde(default)] |
| 45 | pub providers: Vec<ProviderCorrection>, |
| 46 | /// Rules for one `(provider, wire id)` row. |
| 47 | #[serde(default)] |
| 48 | pub models: Vec<ModelCorrection>, |
| 49 | } |
| 50 | |
| 51 | /// A rule for every row of one provider. |
| 52 | #[derive(Debug, Clone, PartialEq, Deserialize)] |
| 53 | #[serde(deny_unknown_fields)] |
| 54 | pub struct ProviderCorrection { |
| 55 | pub provider: String, |
| 56 | /// Why no price is shown for any of this provider's rows. |
| 57 | pub pricing_withheld: String, |
| 58 | } |
| 59 | |
| 60 | /// A field patch for one row, with the reason it exists. |
| 61 | #[derive(Debug, Clone, PartialEq, Deserialize)] |
| 62 | pub struct ModelCorrection { |
| 63 | /// Why the patch exists. Required unless the patch only withholds pricing |
| 64 | /// (whose text is its own reason). |
| 65 | #[serde(default)] |
| 66 | pub reason: Option<String>, |
| 67 | #[serde(flatten)] |
| 68 | pub fact: ModelFact, |
| 69 | } |
| 70 | |
| 71 | impl CatalogCorrections { |
| 72 | /// Parse and validate a corrections document. |
| 73 | /// |
| 74 | /// # Errors |
| 75 | /// Returns a message naming the first entry that is malformed or does more |
| 76 | /// than a correction may (hide, deprecate, annotate, or add a row). |
| 77 | pub fn parse(json: &str) -> Result<Self, String> { |
| 78 | let corrections: Self = serde_json::from_str(json).map_err(|err| err.to_string())?; |
| 79 | corrections.validate()?; |
| 80 | Ok(corrections) |
| 81 | } |
| 82 | |
| 83 | fn validate(&self) -> Result<(), String> { |
| 84 | if self.revision.trim().is_empty() { |
| 85 | return Err("revision must be set".into()); |
| 86 | } |
| 87 | self.reviewed.validate()?; |
| 88 | for rule in &self.providers { |
| 89 | if rule.provider.trim().is_empty() || rule.pricing_withheld.trim().is_empty() { |
| 90 | return Err("provider rules need a provider and a reason".into()); |
| 91 | } |
| 92 | } |
| 93 | for correction in &self.models { |
| 94 | let fact = &correction.fact; |
| 95 | let name = format!("{}/{}", fact.provider, fact.id); |
| 96 | if fact.provider.trim().is_empty() || fact.id.trim().is_empty() { |
| 97 | return Err(format!("{name}: provider and id are required")); |
| 98 | } |
| 99 | if fact.op != ModelOp::Upsert || fact.allow_unlisted { |
| 100 | return Err(format!("{name}: a correction may only patch a listed row")); |
| 101 | } |
| 102 | if fact.display_name.is_some() |
| 103 | || fact.note.is_some() |
| 104 | || fact.deprecated_at.is_some() |
| 105 | || fact.replacement.is_some() |
| 106 | || fact.applies_to.is_some() |
| 107 | { |
| 108 | return Err(format!("{name}: a correction changes facts, not labels")); |
| 109 | } |
| 110 | if fact.pricing.is_some() && fact.pricing_withheld.is_some() { |
| 111 | return Err(format!("{name}: sets a price and withholds it")); |
| 112 | } |
| 113 | let patches_more_than_price = fact.context_window.is_some() |
| 114 | || fact.max_output.is_some() |
| 115 | || fact.pricing.is_some() |
| 116 | || fact.reasoning.is_some() |
| 117 | || fact.reasoning_options.is_some(); |
| 118 | if !patches_more_than_price && fact.pricing_withheld.is_none() { |
| 119 | return Err(format!("{name}: changes nothing")); |
| 120 | } |
| 121 | if patches_more_than_price |
| 122 | && correction |
| 123 | .reason |
| 124 | .as_deref() |
| 125 | .is_none_or(|reason| reason.trim().is_empty()) |
| 126 | { |
| 127 | return Err(format!("{name}: needs a reason")); |
| 128 | } |
| 129 | } |
| 130 | Ok(()) |
| 131 | } |
| 132 | |
| 133 | /// Validate committed referents against the seed, not a partial live refresh. |
| 134 | fn validate_targets(&self, seed: &[CatalogOffering]) -> Result<(), String> { |
| 135 | for rule in &self.providers { |
| 136 | if !seed.iter().any(|row| row.provider == rule.provider) { |
| 137 | return Err(format!( |
| 138 | "{}: correction provider missing from bundled seed", |
| 139 | rule.provider |
| 140 | )); |
| 141 | } |
| 142 | } |
| 143 | for correction in &self.models { |
| 144 | let fact = &correction.fact; |
| 145 | if !seed |
| 146 | .iter() |
| 147 | .any(|row| row.provider == fact.provider && row.wire_model_id == fact.id) |
| 148 | { |
| 149 | return Err(format!( |
| 150 | "{}/{}: correction model missing from bundled seed", |
| 151 | fact.provider, fact.id |
| 152 | )); |
| 153 | } |
| 154 | } |
| 155 | Ok(()) |
| 156 | } |
| 157 | |
| 158 | /// Apply every correction to the rows it names, in place. |
| 159 | /// |
| 160 | /// Provider rules run first, then per-row patches, so a row patch can add |
| 161 | /// to a provider rule (a DeepSeek output limit on top of its withheld |
| 162 | /// price). Rows no rule names are untouched. Live refreshes may omit |
| 163 | /// correction targets; only the committed seed must contain every target. |
| 164 | pub fn apply_to(&self, rows: &mut [CatalogOffering]) { |
| 165 | let mut patches: BTreeMap<(String, String), Vec<ModelFact>> = BTreeMap::new(); |
| 166 | for rule in &self.providers { |
| 167 | for row in rows.iter().filter(|row| row.provider == rule.provider) { |
| 168 | patches |
| 169 | .entry((row.provider.clone(), row.wire_model_id.clone())) |
| 170 | .or_default() |
| 171 | .push(ModelFact { |
| 172 | provider: row.provider.clone(), |
| 173 | id: row.wire_model_id.clone(), |
| 174 | pricing_withheld: Some(rule.pricing_withheld.clone()), |
| 175 | ..ModelFact::default() |
| 176 | }); |
| 177 | } |
| 178 | } |
| 179 | for correction in &self.models { |
| 180 | patches |
| 181 | .entry((correction.fact.provider.clone(), correction.fact.id.clone())) |
| 182 | .or_default() |
| 183 | .push(correction.fact.clone()); |
| 184 | } |
| 185 | if patches.is_empty() { |
| 186 | return; |
| 187 | } |
| 188 | let source = CatalogSource::CodewhaleBundled { |
| 189 | revision: self.revision.clone(), |
| 190 | }; |
| 191 | for row in rows.iter_mut() { |
| 192 | let key = (row.provider.clone(), row.wire_model_id.clone()); |
| 193 | let Some(row_patches) = patches.get(&key) else { |
| 194 | continue; |
| 195 | }; |
| 196 | // The row stays on its own layer: a hydrated row is classified by |
| 197 | // its source (a live Models.dev row must stay below signed facts), |
| 198 | // so only the price a correction owns is attributed to it. |
| 199 | let origin = row.source.clone(); |
| 200 | let modalities_source = row.modalities_source.clone(); |
| 201 | let cost_source = row.cost_source.clone(); |
| 202 | let mut single = BTreeMap::from([(key.clone(), std::mem::take(row))]); |
| 203 | apply_patches(&mut single, row_patches, &source, false); |
| 204 | if let Some(mut patched) = single.remove(&key) { |
| 205 | patched.source = origin; |
| 206 | patched.modalities_source = modalities_source; |
| 207 | if !row_patches.iter().any(owns_price) { |
| 208 | patched.cost_source = cost_source; |
| 209 | } |
| 210 | *row = patched; |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | |
| 216 | fn owns_price(fact: &ModelFact) -> bool { |
| 217 | fact.pricing.is_some() || fact.pricing_withheld.is_some() |
| 218 | } |
| 219 | |
| 220 | /// The committed corrections, parsed once. |
| 221 | /// |
| 222 | /// # Panics |
| 223 | /// Panics if the committed asset is invalid or names a missing seed target. |
| 224 | #[must_use] |
| 225 | pub fn bundled_corrections() -> &'static CatalogCorrections { |
| 226 | static CORRECTIONS: OnceLock<CatalogCorrections> = OnceLock::new(); |
| 227 | CORRECTIONS.get_or_init(|| { |
| 228 | let corrections = CatalogCorrections::parse(CATALOG_CORRECTIONS_JSON) |
| 229 | .expect("committed catalog corrections must be valid"); |
| 230 | let seed = super::bundled_offerings_from_models_dev(super::bundled_models_dev_catalog()); |
| 231 | corrections |
| 232 | .validate_targets(&seed) |
| 233 | .expect("committed catalog correction targets must exist"); |
| 234 | corrections |
| 235 | }) |
| 236 | } |
| 237 | |
| 238 | #[cfg(test)] |
| 239 | mod tests { |
| 240 | use super::*; |
| 241 | |
| 242 | #[test] |
| 243 | fn missing_seed_targets_fail_validation_but_partial_live_refreshes_are_allowed() { |
| 244 | let corrections = bundled_corrections(); |
| 245 | let seed = super::super::bundled_offerings_from_models_dev( |
| 246 | super::super::bundled_models_dev_catalog(), |
| 247 | ); |
| 248 | let provider = &corrections.providers[0].provider; |
| 249 | let without_provider: Vec<_> = seed |
| 250 | .iter() |
| 251 | .filter(|row| &row.provider != provider) |
| 252 | .cloned() |
| 253 | .collect(); |
| 254 | let error = corrections.validate_targets(&without_provider).unwrap_err(); |
| 255 | assert!(error.contains(provider), "{error}"); |
| 256 | |
| 257 | let target = &corrections.models[0].fact; |
| 258 | let mut without_model = seed; |
| 259 | without_model |
| 260 | .retain(|row| row.provider != target.provider || row.wire_model_id != target.id); |
| 261 | let error = corrections.validate_targets(&without_model).unwrap_err(); |
| 262 | assert!( |
| 263 | error.contains(&format!("{}/{}", target.provider, target.id)), |
| 264 | "{error}" |
| 265 | ); |
| 266 | |
| 267 | // Absence from a live refresh does not invalidate committed corrections |
| 268 | // or invent a row to patch. |
| 269 | corrections.apply_to(&mut without_model); |
| 270 | assert!( |
| 271 | !without_model |
| 272 | .iter() |
| 273 | .any(|row| row.provider == target.provider && row.wire_model_id == target.id) |
| 274 | ); |
| 275 | corrections.apply_to(&mut []); |
| 276 | } |
| 277 | } |
| 278 |