| 1 | //! Provider-free contract tests for the bundled starter pack (#4698). |
| 2 | //! |
| 3 | //! Scope, stated precisely so these assertions are not over-read: |
| 4 | //! |
| 5 | //! * They validate **deterministic Codewhale behavior** — which skills install, |
| 6 | //! which parse, which become ambient catalogue entries, which stay explicitly |
| 7 | //! loadable, how aliases resolve, and how much prompt budget the catalogue |
| 8 | //! costs. |
| 9 | //! * They validate **nothing about semantic routing**. Whether a model picks |
| 10 | //! `debug` for a stack trace is a live-provider question and is deliberately |
| 11 | //! out of scope here (see `docs/LIVE_SMOKE.md` for the opt-in harness). |
| 12 | //! |
| 13 | //! The expectation table lives in `assets/skills-catalog-matrix.json` and is |
| 14 | //! authored, not generated. The bijection assertion means a shipped-catalog |
| 15 | //! change cannot land without an explicit fixture update. |
| 16 | |
| 17 | use std::collections::{BTreeSet, HashMap}; |
| 18 | use std::path::Path; |
| 19 | |
| 20 | use tempfile::TempDir; |
| 21 | |
| 22 | use super::system::{bundled_skill_generation, bundled_skill_names, install_system_skills}; |
| 23 | use super::{ |
| 24 | BundledSkillTier, MAX_AVAILABLE_SKILLS_CHARS, MAX_SKILL_DESCRIPTION_CHARS, Skill, |
| 25 | SkillInvocation, SkillRegistry, bundled_skill_tier, render_skills_block, |
| 26 | }; |
| 27 | |
| 28 | const MATRIX_JSON: &str = include_str!("../../assets/skills-catalog-matrix.json"); |
| 29 | |
| 30 | #[derive(Debug, serde::Deserialize)] |
| 31 | struct MatrixFixture { |
| 32 | generation: String, |
| 33 | skills: Vec<MatrixEntry>, |
| 34 | } |
| 35 | |
| 36 | #[derive(Debug, serde::Deserialize)] |
| 37 | struct MatrixEntry { |
| 38 | name: String, |
| 39 | tier: String, |
| 40 | invocation: String, |
| 41 | #[serde(default)] |
| 42 | aliases: Vec<String>, |
| 43 | in_model_catalogue: bool, |
| 44 | #[serde(default)] |
| 45 | shadowed_aliases: Vec<String>, |
| 46 | } |
| 47 | |
| 48 | fn fixture() -> MatrixFixture { |
| 49 | serde_json::from_str(MATRIX_JSON).expect("skills-catalog-matrix.json must be valid JSON") |
| 50 | } |
| 51 | |
| 52 | /// Install the shipped pack into a temp dir and discover it, exactly as a |
| 53 | /// fresh user install would. No network, no provider, no ambient home. |
| 54 | fn installed_registry() -> (TempDir, SkillRegistry) { |
| 55 | let tmp = TempDir::new().expect("temp dir"); |
| 56 | install_system_skills(tmp.path()).expect("bundled install"); |
| 57 | let registry = SkillRegistry::discover(tmp.path()); |
| 58 | assert!( |
| 59 | registry.warnings().is_empty(), |
| 60 | "bundled pack must parse without warnings: {:?}", |
| 61 | registry.warnings() |
| 62 | ); |
| 63 | (tmp, registry) |
| 64 | } |
| 65 | |
| 66 | fn rendered_catalogue(registry: &SkillRegistry, locale: &str, workspace: &Path) -> String { |
| 67 | render_skills_block(registry, locale, workspace).expect("non-empty registry renders a block") |
| 68 | } |
| 69 | |
| 70 | /// Canonical names that appear as `- <name>: …` entries under |
| 71 | /// `### Available skills` (and only there — the warnings section also uses |
| 72 | /// `- ` bullets). |
| 73 | fn catalogue_entry_names(block: &str) -> Vec<String> { |
| 74 | let mut names = Vec::new(); |
| 75 | let mut inside = false; |
| 76 | for line in block.lines() { |
| 77 | if line.starts_with("###") { |
| 78 | inside = line.trim() == "### Available skills"; |
| 79 | continue; |
| 80 | } |
| 81 | if !inside { |
| 82 | continue; |
| 83 | } |
| 84 | if let Some(rest) = line.strip_prefix("- ") |
| 85 | && let Some((name, _)) = rest.split_once(':') |
| 86 | { |
| 87 | names.push(name.trim().to_string()); |
| 88 | } |
| 89 | } |
| 90 | names |
| 91 | } |
| 92 | |
| 93 | fn by_name(registry: &SkillRegistry) -> HashMap<&str, &Skill> { |
| 94 | registry |
| 95 | .list() |
| 96 | .iter() |
| 97 | .map(|skill| (skill.name.as_str(), skill)) |
| 98 | .collect() |
| 99 | } |
| 100 | |
| 101 | // ── fixture ↔ bundle bijection ────────────────────────────────────────────── |
| 102 | |
| 103 | #[test] |
| 104 | fn fixture_matrix_covers_exactly_the_shipped_bundle() { |
| 105 | let fixture = fixture(); |
| 106 | let fixture_names: BTreeSet<&str> = fixture.skills.iter().map(|e| e.name.as_str()).collect(); |
| 107 | let shipped_names: BTreeSet<&str> = bundled_skill_names().into_iter().collect(); |
| 108 | |
| 109 | assert_eq!( |
| 110 | fixture_names, shipped_names, |
| 111 | "assets/skills-catalog-matrix.json must be updated whenever the bundled \ |
| 112 | starter pack changes; left = fixture, right = BUNDLED_SKILLS" |
| 113 | ); |
| 114 | assert_eq!( |
| 115 | fixture.skills.len(), |
| 116 | fixture_names.len(), |
| 117 | "fixture must not list the same skill twice" |
| 118 | ); |
| 119 | assert_eq!( |
| 120 | fixture.generation, |
| 121 | bundled_skill_generation(), |
| 122 | "fixture generation must track BUNDLED_SKILL_VERSION" |
| 123 | ); |
| 124 | } |
| 125 | |
| 126 | #[test] |
| 127 | fn fixture_matrix_matches_parsed_frontmatter_for_every_bundled_skill() { |
| 128 | let (_tmp, registry) = installed_registry(); |
| 129 | let skills = by_name(®istry); |
| 130 | |
| 131 | for entry in fixture().skills { |
| 132 | let skill = skills |
| 133 | .get(entry.name.as_str()) |
| 134 | .unwrap_or_else(|| panic!("{} must install and parse", entry.name)); |
| 135 | |
| 136 | let expected_invocation = match entry.invocation.as_str() { |
| 137 | "explicit-only" => SkillInvocation::ExplicitOnly, |
| 138 | "model+user" => SkillInvocation::ModelAndUser, |
| 139 | other => panic!("{}: unknown fixture invocation {other}", entry.name), |
| 140 | }; |
| 141 | assert_eq!( |
| 142 | skill.invocation, expected_invocation, |
| 143 | "{} invocation drifted from the fixture", |
| 144 | entry.name |
| 145 | ); |
| 146 | |
| 147 | let actual_aliases: BTreeSet<&str> = skill.aliases.iter().map(String::as_str).collect(); |
| 148 | let expected_aliases: BTreeSet<&str> = entry.aliases.iter().map(String::as_str).collect(); |
| 149 | assert_eq!( |
| 150 | actual_aliases, expected_aliases, |
| 151 | "{} aliases drifted from the fixture", |
| 152 | entry.name |
| 153 | ); |
| 154 | |
| 155 | let expected_tier = match entry.tier.as_str() { |
| 156 | "core" => BundledSkillTier::CoreAgentic, |
| 157 | "tools" => BundledSkillTier::FormatTooling, |
| 158 | other => panic!("{}: unknown fixture tier {other}", entry.name), |
| 159 | }; |
| 160 | assert_eq!( |
| 161 | bundled_skill_tier(&entry.name), |
| 162 | Some(expected_tier), |
| 163 | "{} tier drifted from the fixture", |
| 164 | entry.name |
| 165 | ); |
| 166 | |
| 167 | assert!( |
| 168 | !skill.description.trim().is_empty(), |
| 169 | "{} must ship a routing description", |
| 170 | entry.name |
| 171 | ); |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | // ── positive: eligibility and explicit load ───────────────────────────────── |
| 176 | |
| 177 | #[test] |
| 178 | fn every_bundled_skill_is_explicitly_loadable_including_explicit_only() { |
| 179 | let (_tmp, registry) = installed_registry(); |
| 180 | for name in bundled_skill_names() { |
| 181 | let resolved = registry |
| 182 | .get(name) |
| 183 | .unwrap_or_else(|| panic!("{name} must resolve by canonical name")); |
| 184 | assert_eq!(resolved.name, name); |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | #[test] |
| 189 | fn ambient_catalogue_is_a_progressive_subset_of_eligible_bundled_skills() { |
| 190 | let (tmp, registry) = installed_registry(); |
| 191 | let block = rendered_catalogue(®istry, "en", tmp.path()); |
| 192 | let rendered: BTreeSet<String> = catalogue_entry_names(&block).into_iter().collect(); |
| 193 | |
| 194 | let expected: BTreeSet<String> = fixture() |
| 195 | .skills |
| 196 | .into_iter() |
| 197 | .filter(|entry| entry.in_model_catalogue) |
| 198 | .map(|entry| entry.name) |
| 199 | .collect(); |
| 200 | |
| 201 | assert!( |
| 202 | !rendered.is_empty(), |
| 203 | "the ambient routing page must not be empty" |
| 204 | ); |
| 205 | assert!( |
| 206 | rendered.is_subset(&expected), |
| 207 | "the ambient page may contain only model+user bundled skills" |
| 208 | ); |
| 209 | if rendered.len() < expected.len() { |
| 210 | assert!( |
| 211 | block.contains("`load_skill` with `name=\"list\""), |
| 212 | "a truncated ambient page must point to complete discovery" |
| 213 | ); |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | // ── negative: non-activation and explicit-only exclusion ──────────────────── |
| 218 | |
| 219 | #[test] |
| 220 | fn explicit_only_skills_are_absent_from_the_ambient_catalogue() { |
| 221 | let (tmp, registry) = installed_registry(); |
| 222 | let block = rendered_catalogue(®istry, "en", tmp.path()); |
| 223 | |
| 224 | for entry in fixture() |
| 225 | .skills |
| 226 | .into_iter() |
| 227 | .filter(|e| !e.in_model_catalogue) |
| 228 | { |
| 229 | assert!( |
| 230 | !block.contains(&format!("- {}: ", entry.name)), |
| 231 | "{} is explicit-only and must not become ambient context", |
| 232 | entry.name |
| 233 | ); |
| 234 | assert!( |
| 235 | registry.get(&entry.name).is_some(), |
| 236 | "{} must still be loadable by explicit name", |
| 237 | entry.name |
| 238 | ); |
| 239 | } |
| 240 | } |
| 241 | |
| 242 | #[test] |
| 243 | fn unrelated_and_unshipped_names_do_not_activate_a_bundled_skill() { |
| 244 | let (_tmp, registry) = installed_registry(); |
| 245 | for name in [ |
| 246 | "", |
| 247 | " ", |
| 248 | "imagine", |
| 249 | "image-generation", |
| 250 | "codereview", |
| 251 | "checkwork", |
| 252 | "gh-file-issue", |
| 253 | "codew-release-qa-sweep", |
| 254 | "totally-unknown-skill", |
| 255 | ] { |
| 256 | assert!( |
| 257 | registry.get(name).is_none(), |
| 258 | "{name:?} must not resolve to a bundled skill" |
| 259 | ); |
| 260 | } |
| 261 | |
| 262 | // Lookup normalization is punctuation-insensitive by design, so an alias |
| 263 | // typed with a space still lands on its canonical skill. Pin that so the |
| 264 | // negative cases above stay meaningful rather than accidental. |
| 265 | assert_eq!( |
| 266 | registry.get("check work").map(|skill| skill.name.as_str()), |
| 267 | Some("verify") |
| 268 | ); |
| 269 | } |
| 270 | |
| 271 | // ── alias behavior ────────────────────────────────────────────────────────── |
| 272 | |
| 273 | #[test] |
| 274 | fn aliases_resolve_to_their_canonical_skill_and_add_no_catalogue_entries() { |
| 275 | let (tmp, registry) = installed_registry(); |
| 276 | let block = rendered_catalogue(®istry, "en", tmp.path()); |
| 277 | let rendered = catalogue_entry_names(&block); |
| 278 | |
| 279 | for entry in fixture().skills { |
| 280 | for alias in &entry.aliases { |
| 281 | let resolved = registry |
| 282 | .get(alias) |
| 283 | .unwrap_or_else(|| panic!("alias {alias} must resolve")); |
| 284 | let expected = if entry.shadowed_aliases.contains(alias) { |
| 285 | // A canonical bundled name always wins over another skill's |
| 286 | // alias, so `docx` resolves to `docx`, never to `documents`. |
| 287 | alias.as_str() |
| 288 | } else { |
| 289 | entry.name.as_str() |
| 290 | }; |
| 291 | assert_eq!( |
| 292 | resolved.name, expected, |
| 293 | "alias {alias} resolved to the wrong canonical skill" |
| 294 | ); |
| 295 | |
| 296 | if !entry.shadowed_aliases.contains(alias) { |
| 297 | assert!( |
| 298 | !rendered.iter().any(|name| name == alias), |
| 299 | "alias {alias} must not become a second catalogue entry" |
| 300 | ); |
| 301 | } |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | #[test] |
| 307 | fn grok_compatibility_aliases_map_to_shipped_codewhale_workflows() { |
| 308 | let (_tmp, registry) = installed_registry(); |
| 309 | for (alias, canonical) in [ |
| 310 | ("check-work", "verify"), |
| 311 | ("code-review", "review"), |
| 312 | ("create-skill", "skill-creator"), |
| 313 | ] { |
| 314 | let resolved = registry |
| 315 | .get(alias) |
| 316 | .unwrap_or_else(|| panic!("{alias} must resolve")); |
| 317 | assert_eq!(resolved.name, canonical, "{alias} must map to {canonical}"); |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | #[test] |
| 322 | fn no_two_bundled_skills_claim_the_same_alias() { |
| 323 | let mut owners: HashMap<String, String> = HashMap::new(); |
| 324 | for entry in fixture().skills { |
| 325 | for alias in entry.aliases { |
| 326 | if let Some(previous) = owners.insert(alias.clone(), entry.name.clone()) { |
| 327 | panic!( |
| 328 | "alias {alias} is claimed by both {previous} and {}", |
| 329 | entry.name |
| 330 | ); |
| 331 | } |
| 332 | } |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | // ── prompt-budget invariants ──────────────────────────────────────────────── |
| 337 | |
| 338 | #[test] |
| 339 | fn catalogue_has_unique_entries_and_the_complete_block_fits_the_prompt_budget() { |
| 340 | let (tmp, registry) = installed_registry(); |
| 341 | let block = rendered_catalogue(®istry, "en", tmp.path()); |
| 342 | let rendered = catalogue_entry_names(&block); |
| 343 | |
| 344 | let unique: BTreeSet<&String> = rendered.iter().collect(); |
| 345 | assert_eq!( |
| 346 | rendered.len(), |
| 347 | unique.len(), |
| 348 | "metadata must never duplicate a prompt entry: {rendered:?}" |
| 349 | ); |
| 350 | |
| 351 | assert!( |
| 352 | block.chars().count() <= MAX_AVAILABLE_SKILLS_CHARS, |
| 353 | "complete ambient block is {} chars, over the {MAX_AVAILABLE_SKILLS_CHARS} budget", |
| 354 | block.chars().count() |
| 355 | ); |
| 356 | if block.contains("additional skills omitted") { |
| 357 | assert!( |
| 358 | block.contains("`load_skill` with `name=\"list\""), |
| 359 | "budget overflow must advertise complete on-demand discovery" |
| 360 | ); |
| 361 | } |
| 362 | |
| 363 | // No entry may smuggle newlines or an oversized description into the |
| 364 | // prompt prefix — that is how a catalogue line would poison context. |
| 365 | for line in block.lines() { |
| 366 | assert!( |
| 367 | line.chars().count() <= MAX_SKILL_DESCRIPTION_CHARS + 200, |
| 368 | "catalogue line is too long for a routing hint: {line}" |
| 369 | ); |
| 370 | } |
| 371 | for skill in registry.list() { |
| 372 | assert!( |
| 373 | !skill.description.contains('\n'), |
| 374 | "{} description must stay single-line", |
| 375 | skill.name |
| 376 | ); |
| 377 | } |
| 378 | } |
| 379 | |
| 380 | // ── locale-aware routing metadata ─────────────────────────────────────────── |
| 381 | |
| 382 | #[test] |
| 383 | fn every_shipped_locale_falls_back_to_canonical_english_routing_descriptions() { |
| 384 | // The bundled pack ships no `description_<tag>` frontmatter. Rather than |
| 385 | // fabricate translations, the contract is an explicit, tested fallback: |
| 386 | // every shipped locale sees the canonical English routing description. |
| 387 | let (_tmp, registry) = installed_registry(); |
| 388 | for skill in registry.list() { |
| 389 | assert!( |
| 390 | skill.localized_descriptions.is_empty(), |
| 391 | "{} ships localized routing metadata; add source-backed coverage \ |
| 392 | to this test instead of relying on the English-fallback contract", |
| 393 | skill.name |
| 394 | ); |
| 395 | for locale in crate::localization::Locale::shipped() { |
| 396 | assert_eq!( |
| 397 | skill.description_for_locale(locale.tag()), |
| 398 | skill.description, |
| 399 | "{} must fall back to canonical English for {}", |
| 400 | skill.name, |
| 401 | locale.tag() |
| 402 | ); |
| 403 | } |
| 404 | } |
| 405 | } |
| 406 | |
| 407 | #[test] |
| 408 | fn rendered_catalogue_is_identical_across_every_shipped_locale() { |
| 409 | let (tmp, registry) = installed_registry(); |
| 410 | let english = rendered_catalogue(®istry, "en", tmp.path()); |
| 411 | for locale in crate::localization::Locale::shipped() { |
| 412 | let localized = rendered_catalogue(®istry, locale.tag(), tmp.path()); |
| 413 | assert_eq!( |
| 414 | localized, |
| 415 | english, |
| 416 | "{} catalogue must match English parity while no localized \ |
| 417 | routing descriptions are shipped", |
| 418 | locale.tag() |
| 419 | ); |
| 420 | } |
| 421 | } |
| 422 | |
| 423 | #[test] |
| 424 | fn locale_resolution_covers_exact_primary_tag_and_english_fallback() { |
| 425 | // Synthetic fixture: exercises the three resolution paths the bundled pack |
| 426 | // does not currently reach, for every shipped locale tag. |
| 427 | let content = "---\n\ |
| 428 | name: locale-probe\n\ |
| 429 | description: English routing description\n\ |
| 430 | description_ja: 日本語のルーティング説明\n\ |
| 431 | description_pt: Descrição de roteamento\n\ |
| 432 | description_zh-hant: 繁體路由說明\n\ |
| 433 | ---\n\n# body\n"; |
| 434 | let skill = SkillRegistry::parse_skill(Path::new("SKILL.md"), content).expect("parses"); |
| 435 | |
| 436 | // Exact tag match (lowercased key lookup). |
| 437 | assert_eq!( |
| 438 | skill.description_for_locale("ja"), |
| 439 | "日本語のルーティング説明" |
| 440 | ); |
| 441 | assert_eq!(skill.description_for_locale("zh-Hant"), "繁體路由說明"); |
| 442 | // Primary-subtag fallback: pt-BR → description_pt. |
| 443 | assert_eq!( |
| 444 | skill.description_for_locale("pt-BR"), |
| 445 | "Descrição de roteamento" |
| 446 | ); |
| 447 | // English fallback for shipped locales with no authored variant. |
| 448 | for tag in ["en", "ko", "vi", "es-419", "zh-Hans"] { |
| 449 | assert_eq!( |
| 450 | skill.description_for_locale(tag), |
| 451 | "English routing description", |
| 452 | "{tag} must fall back to the canonical description" |
| 453 | ); |
| 454 | } |
| 455 | |
| 456 | // Every shipped locale tag must resolve to *some* non-empty description. |
| 457 | for locale in crate::localization::Locale::shipped() { |
| 458 | assert!( |
| 459 | !skill.description_for_locale(locale.tag()).is_empty(), |
| 460 | "{} must resolve to a non-empty routing description", |
| 461 | locale.tag() |
| 462 | ); |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | // ── starter-pack boundaries ───────────────────────────────────────────────── |
| 467 | |
| 468 | #[test] |
| 469 | fn repository_maintenance_helpers_stay_out_of_the_end_user_starter_pack() { |
| 470 | // These live in `docs/skills/` for maintainers and must never be installed |
| 471 | // for end users (#4698 item 5). Plugin delivery is #4836, out of scope. |
| 472 | for name in [ |
| 473 | "gh-file-issue", |
| 474 | "gh-compile-issues", |
| 475 | "gh-assign-issues", |
| 476 | "gh-find-prs", |
| 477 | "gh-treasure-hunt", |
| 478 | "gh-close-issues", |
| 479 | "gh-credit-harvest", |
| 480 | "codew-release-qa-sweep", |
| 481 | ] { |
| 482 | assert!( |
| 483 | !bundled_skill_names().contains(&name), |
| 484 | "{name} is a maintainer helper and must not ship in the starter pack" |
| 485 | ); |
| 486 | } |
| 487 | } |
| 488 | |
| 489 | #[test] |
| 490 | fn starter_pack_does_not_advertise_capabilities_the_runtime_lacks() { |
| 491 | // `imagine` stays out of scope while no image-generation/edit tool exists. |
| 492 | let names = bundled_skill_names(); |
| 493 | for name in ["imagine", "image", "image-gen"] { |
| 494 | assert!( |
| 495 | !names.contains(&name), |
| 496 | "{name} must not ship without a real image-generation tool" |
| 497 | ); |
| 498 | } |
| 499 | } |
| 500 | |
| 501 | #[test] |
| 502 | fn help_is_a_bounded_explicit_only_router() { |
| 503 | let (_tmp, registry) = installed_registry(); |
| 504 | let help = registry.get("help").expect("help must be installed"); |
| 505 | assert_eq!(help.invocation, SkillInvocation::ExplicitOnly); |
| 506 | // Bounded: a routing card, not an embedded manual. |
| 507 | assert!( |
| 508 | help.body.lines().count() < 80, |
| 509 | "help must stay a router, not a manual: {} lines", |
| 510 | help.body.lines().count() |
| 511 | ); |
| 512 | for surface in ["/help", "/skills", "/config", "doctor"] { |
| 513 | assert!( |
| 514 | help.body.contains(surface), |
| 515 | "help must route to the installed {surface} surface" |
| 516 | ); |
| 517 | } |
| 518 | } |
| 519 |