| 1 | //! Provider-neutral experimental search authoring for Workflow. |
| 2 | //! |
| 3 | //! This module is an authoring and freeze boundary, not a new runtime or |
| 4 | //! scheduler. A validated search still has to be lowered by the Workflow host |
| 5 | //! into Fleet workers plus a runtime-owned evaluator. In particular, worker |
| 6 | //! self-reports are never promoted to hard-gate evidence here. |
| 7 | |
| 8 | use serde::{Deserialize, Serialize}; |
| 9 | use sha2::{Digest, Sha256}; |
| 10 | use std::path::{Component, Path}; |
| 11 | use thiserror::Error; |
| 12 | |
| 13 | use crate::{DEFAULT_FLEET_WORKFLOW_MAX_AGENTS, experimental_search::SearchSpecError::*}; |
| 14 | |
| 15 | pub const WORKFLOW_SEARCH_SCHEMA_VERSION: u32 = 1; |
| 16 | /// Live-worker ceiling for one search admission batch. |
| 17 | /// |
| 18 | /// 16 matches the Workflow host's live-child ceiling today |
| 19 | /// (`codewhale_workflow_js::WORKFLOW_MAX_CONCURRENT`, from which the tui |
| 20 | /// driver sizes its per-run admission semaphore). This crate cannot import |
| 21 | /// that constant directly because `codewhale-workflow-js` depends on |
| 22 | /// `codewhale-workflow`, so 16 is documented here as today's default — not a |
| 23 | /// new configuration knob. Keep it in sync with the host constant. |
| 24 | pub const WORKFLOW_SEARCH_MAX_CONCURRENT: u16 = 16; |
| 25 | |
| 26 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 27 | pub struct WorkflowSearchSpec { |
| 28 | #[serde(default = "default_schema_version")] |
| 29 | pub schema_version: u32, |
| 30 | pub name: String, |
| 31 | pub objective: String, |
| 32 | pub population: u16, |
| 33 | pub rounds: Vec<u16>, |
| 34 | pub concurrency: u16, |
| 35 | pub worker: SearchWorkerSpec, |
| 36 | #[serde(default)] |
| 37 | pub budget: SearchBudgetSpec, |
| 38 | pub hard_gates: SearchHardGateSpec, |
| 39 | pub score: SearchScoreSpec, |
| 40 | #[serde(default)] |
| 41 | pub selection: SearchSelectionSpec, |
| 42 | #[serde(default)] |
| 43 | pub integration_policy: SearchIntegrationPolicy, |
| 44 | } |
| 45 | |
| 46 | impl WorkflowSearchSpec { |
| 47 | pub fn from_toml(source: &str) -> Result<Self, SearchSpecError> { |
| 48 | let spec: Self = toml::from_str(source).map_err(|error| Parse(error.to_string()))?; |
| 49 | spec.validate()?; |
| 50 | Ok(spec) |
| 51 | } |
| 52 | |
| 53 | pub fn validate(&self) -> Result<(), SearchSpecError> { |
| 54 | if self.schema_version != WORKFLOW_SEARCH_SCHEMA_VERSION { |
| 55 | return Err(UnsupportedSchemaVersion(self.schema_version)); |
| 56 | } |
| 57 | validate_name(&self.name)?; |
| 58 | validate_text("objective", &self.objective, 32_768)?; |
| 59 | if !(2..=DEFAULT_FLEET_WORKFLOW_MAX_AGENTS as u16).contains(&self.population) { |
| 60 | return Err(InvalidPopulation(self.population)); |
| 61 | } |
| 62 | if self.concurrency == 0 |
| 63 | || self.concurrency > WORKFLOW_SEARCH_MAX_CONCURRENT |
| 64 | || self.concurrency > self.population |
| 65 | { |
| 66 | return Err(InvalidConcurrency { |
| 67 | concurrency: self.concurrency, |
| 68 | population: self.population, |
| 69 | }); |
| 70 | } |
| 71 | validate_rounds(self.population, &self.rounds)?; |
| 72 | validate_text("worker.model", &self.worker.model, 256)?; |
| 73 | if self.worker.write_roots.is_empty() && self.worker.exact_files.is_empty() { |
| 74 | return Err(UnboundedWriteScope); |
| 75 | } |
| 76 | validate_repo_relative_paths("worker.write_roots", &self.worker.write_roots, 128)?; |
| 77 | validate_repo_relative_paths("worker.exact_files", &self.worker.exact_files, 256)?; |
| 78 | if self.budget.max_cost_microusd == Some(0) { |
| 79 | return Err(ZeroBudget("max_cost_microusd")); |
| 80 | } |
| 81 | if self.budget.max_tokens == Some(0) { |
| 82 | return Err(ZeroBudget("max_tokens")); |
| 83 | } |
| 84 | if !self.hard_gates.forbid_test_changes { |
| 85 | return Err(TestWeakeningAllowed); |
| 86 | } |
| 87 | if self.hard_gates.commands.is_empty() { |
| 88 | return Err(MissingHardGates); |
| 89 | } |
| 90 | validate_string_list("hard_gates.commands", &self.hard_gates.commands, 32, 4_096)?; |
| 91 | validate_string_list( |
| 92 | "hard_gates.protected_paths", |
| 93 | &self.hard_gates.protected_paths, |
| 94 | 256, |
| 95 | 1_024, |
| 96 | )?; |
| 97 | validate_text("score.command", &self.score.command, 4_096)?; |
| 98 | validate_text("score.metric", &self.score.metric, 256)?; |
| 99 | if !(1..=25).contains(&self.score.trials) { |
| 100 | return Err(InvalidTrials(self.score.trials)); |
| 101 | } |
| 102 | if self.score.tie_breakers.is_empty() { |
| 103 | return Err(MissingTieBreakers); |
| 104 | } |
| 105 | Ok(()) |
| 106 | } |
| 107 | |
| 108 | /// Freeze the exact public inputs and evaluator identity before admission. |
| 109 | /// The evaluator bytes are hashed, not exposed to generation workers. |
| 110 | pub fn freeze( |
| 111 | &self, |
| 112 | baseline_commit: &str, |
| 113 | resolved_model: &str, |
| 114 | public_evidence: &[u8], |
| 115 | evaluator: &[u8], |
| 116 | ) -> Result<FrozenWorkflowSearch, SearchSpecError> { |
| 117 | self.validate()?; |
| 118 | validate_commit(baseline_commit)?; |
| 119 | validate_text("resolved_model", resolved_model, 256)?; |
| 120 | if evaluator.is_empty() { |
| 121 | return Err(EmptyEvaluator); |
| 122 | } |
| 123 | |
| 124 | let public_evidence_hash = sha256_label(public_evidence); |
| 125 | let evaluator_hash = sha256_label(evaluator); |
| 126 | let freeze_input = FreezeInput { |
| 127 | spec: self, |
| 128 | baseline_commit, |
| 129 | requested_model: &self.worker.model, |
| 130 | resolved_model, |
| 131 | public_evidence_hash: &public_evidence_hash, |
| 132 | evaluator_hash: &evaluator_hash, |
| 133 | }; |
| 134 | let encoded = |
| 135 | serde_json::to_vec(&freeze_input).map_err(|error| FreezeEncoding(error.to_string()))?; |
| 136 | let preregistration_hash = sha256_label(&encoded); |
| 137 | let search_id = format!("search-{}", &preregistration_hash[7..23]); |
| 138 | |
| 139 | Ok(FrozenWorkflowSearch { |
| 140 | schema_version: self.schema_version, |
| 141 | search_id, |
| 142 | baseline_commit: baseline_commit.to_string(), |
| 143 | preregistration_hash, |
| 144 | public_evidence_hash, |
| 145 | evaluator_hash, |
| 146 | requested_model: self.worker.model.clone(), |
| 147 | resolved_model: resolved_model.to_string(), |
| 148 | candidate_ids: self.candidate_ids(), |
| 149 | }) |
| 150 | } |
| 151 | |
| 152 | #[must_use] |
| 153 | pub fn candidate_ids(&self) -> Vec<String> { |
| 154 | let width = self.population.to_string().len().max(3); |
| 155 | (1..=self.population) |
| 156 | .map(|index| format!("cand_{index:0width$}")) |
| 157 | .collect() |
| 158 | } |
| 159 | |
| 160 | /// Deterministic admission batches. Fleet owns actual scheduling and may |
| 161 | /// run fewer workers when its configured pool or provider quota is lower. |
| 162 | pub fn admission_batches(&self) -> Result<Vec<Vec<String>>, SearchSpecError> { |
| 163 | self.validate()?; |
| 164 | Ok(self |
| 165 | .candidate_ids() |
| 166 | .chunks(usize::from(self.concurrency)) |
| 167 | .map(<[String]>::to_vec) |
| 168 | .collect()) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 173 | pub struct SearchWorkerSpec { |
| 174 | #[serde(default)] |
| 175 | pub provider: Option<String>, |
| 176 | pub model: String, |
| 177 | #[serde(default)] |
| 178 | pub reasoning_effort: SearchReasoningEffort, |
| 179 | #[serde(default)] |
| 180 | pub write_authority: SearchWriteAuthority, |
| 181 | #[serde(default)] |
| 182 | pub write_roots: Vec<String>, |
| 183 | #[serde(default)] |
| 184 | pub exact_files: Vec<String>, |
| 185 | } |
| 186 | |
| 187 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 188 | #[serde(rename_all = "snake_case")] |
| 189 | pub enum SearchReasoningEffort { |
| 190 | Off, |
| 191 | Low, |
| 192 | Medium, |
| 193 | #[default] |
| 194 | High, |
| 195 | Max, |
| 196 | } |
| 197 | |
| 198 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 199 | #[serde(rename_all = "snake_case")] |
| 200 | pub enum SearchWriteAuthority { |
| 201 | #[default] |
| 202 | WorktreeWrite, |
| 203 | } |
| 204 | |
| 205 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 206 | pub struct SearchBudgetSpec { |
| 207 | #[serde(default)] |
| 208 | pub max_cost_microusd: Option<u64>, |
| 209 | #[serde(default)] |
| 210 | pub max_tokens: Option<u64>, |
| 211 | } |
| 212 | |
| 213 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 214 | pub struct SearchHardGateSpec { |
| 215 | pub commands: Vec<String>, |
| 216 | #[serde(default = "default_true")] |
| 217 | pub forbid_test_changes: bool, |
| 218 | #[serde(default)] |
| 219 | pub protected_paths: Vec<String>, |
| 220 | } |
| 221 | |
| 222 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 223 | pub struct SearchScoreSpec { |
| 224 | pub command: String, |
| 225 | pub metric: String, |
| 226 | #[serde(default)] |
| 227 | pub direction: SearchDirection, |
| 228 | #[serde(default = "default_trials")] |
| 229 | pub trials: u16, |
| 230 | #[serde(default)] |
| 231 | pub tie_breakers: Vec<SearchTieBreaker>, |
| 232 | } |
| 233 | |
| 234 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 235 | #[serde(rename_all = "snake_case")] |
| 236 | pub enum SearchDirection { |
| 237 | #[default] |
| 238 | Minimize, |
| 239 | Maximize, |
| 240 | } |
| 241 | |
| 242 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 243 | #[serde(rename_all = "snake_case")] |
| 244 | pub enum SearchTieBreaker { |
| 245 | DiffLines, |
| 246 | CostMicrousd, |
| 247 | Score, |
| 248 | } |
| 249 | |
| 250 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 251 | pub struct SearchSelectionSpec { |
| 252 | #[serde(default)] |
| 253 | pub policy: SearchSelectionPolicy, |
| 254 | #[serde(default = "default_true")] |
| 255 | pub retain_diversity: bool, |
| 256 | #[serde(default)] |
| 257 | pub ordering: Vec<SearchSelectionMetric>, |
| 258 | } |
| 259 | |
| 260 | impl Default for SearchSelectionSpec { |
| 261 | fn default() -> Self { |
| 262 | Self { |
| 263 | policy: SearchSelectionPolicy::Pareto, |
| 264 | retain_diversity: true, |
| 265 | ordering: Vec::new(), |
| 266 | } |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 271 | #[serde(rename_all = "snake_case")] |
| 272 | pub enum SearchSelectionPolicy { |
| 273 | #[default] |
| 274 | Pareto, |
| 275 | Ordered, |
| 276 | } |
| 277 | |
| 278 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 279 | #[serde(rename_all = "snake_case")] |
| 280 | pub enum SearchSelectionMetric { |
| 281 | Score, |
| 282 | Runtime, |
| 283 | DiffLines, |
| 284 | CostMicrousd, |
| 285 | } |
| 286 | |
| 287 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 288 | #[serde(rename_all = "snake_case")] |
| 289 | pub enum SearchIntegrationPolicy { |
| 290 | /// Produce a verified, reviewable winner or NONE. Never apply or merge it. |
| 291 | #[default] |
| 292 | ReviewOnly, |
| 293 | } |
| 294 | |
| 295 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 296 | pub struct FrozenWorkflowSearch { |
| 297 | pub schema_version: u32, |
| 298 | pub search_id: String, |
| 299 | pub baseline_commit: String, |
| 300 | pub preregistration_hash: String, |
| 301 | pub public_evidence_hash: String, |
| 302 | pub evaluator_hash: String, |
| 303 | pub requested_model: String, |
| 304 | pub resolved_model: String, |
| 305 | pub candidate_ids: Vec<String>, |
| 306 | } |
| 307 | |
| 308 | #[derive(Serialize)] |
| 309 | struct FreezeInput<'a> { |
| 310 | spec: &'a WorkflowSearchSpec, |
| 311 | baseline_commit: &'a str, |
| 312 | requested_model: &'a str, |
| 313 | resolved_model: &'a str, |
| 314 | public_evidence_hash: &'a str, |
| 315 | evaluator_hash: &'a str, |
| 316 | } |
| 317 | |
| 318 | #[derive(Debug, Clone, PartialEq, Eq, Error)] |
| 319 | pub enum SearchSpecError { |
| 320 | #[error("failed to parse Workflow search TOML: {0}")] |
| 321 | Parse(String), |
| 322 | #[error("unsupported Workflow search schema version {0}")] |
| 323 | UnsupportedSchemaVersion(u32), |
| 324 | #[error("search name must be a 1-96 character lowercase token")] |
| 325 | InvalidName, |
| 326 | #[error("{field} must be non-empty and no longer than {max} characters")] |
| 327 | InvalidText { field: &'static str, max: usize }, |
| 328 | #[error("population {0} must be between 2 and 1000")] |
| 329 | InvalidPopulation(u16), |
| 330 | #[error( |
| 331 | "concurrency {concurrency} must be between 1 and 16 and no greater than population {population}" |
| 332 | )] |
| 333 | InvalidConcurrency { concurrency: u16, population: u16 }, |
| 334 | #[error("rounds must start at population, decrease strictly, and end at 1")] |
| 335 | InvalidRounds, |
| 336 | #[error("write-capable search workers require write_roots or exact_files")] |
| 337 | UnboundedWriteScope, |
| 338 | #[error("{field} contains an empty, oversized, or duplicate entry")] |
| 339 | InvalidStringList { field: &'static str }, |
| 340 | #[error("{field} entries must be bounded repo-relative paths without parent traversal")] |
| 341 | InvalidWriteScope { field: &'static str }, |
| 342 | #[error("{0} must be greater than zero when set")] |
| 343 | ZeroBudget(&'static str), |
| 344 | #[error("experimental search must forbid test changes")] |
| 345 | TestWeakeningAllowed, |
| 346 | #[error("experimental search requires at least one runtime-owned hard-gate command")] |
| 347 | MissingHardGates, |
| 348 | #[error("score.trials must be between 1 and 25, got {0}")] |
| 349 | InvalidTrials(u16), |
| 350 | #[error("experimental search requires at least one deterministic tie-breaker")] |
| 351 | MissingTieBreakers, |
| 352 | #[error("baseline_commit must be a 7-64 character hexadecimal commit id")] |
| 353 | InvalidBaselineCommit, |
| 354 | #[error("evaluator bytes must be non-empty")] |
| 355 | EmptyEvaluator, |
| 356 | #[error("failed to encode frozen Workflow search: {0}")] |
| 357 | FreezeEncoding(String), |
| 358 | } |
| 359 | |
| 360 | fn default_schema_version() -> u32 { |
| 361 | WORKFLOW_SEARCH_SCHEMA_VERSION |
| 362 | } |
| 363 | |
| 364 | fn default_trials() -> u16 { |
| 365 | 5 |
| 366 | } |
| 367 | |
| 368 | fn default_true() -> bool { |
| 369 | true |
| 370 | } |
| 371 | |
| 372 | fn validate_name(value: &str) -> Result<(), SearchSpecError> { |
| 373 | if value.is_empty() |
| 374 | || value.len() > 96 |
| 375 | || !value |
| 376 | .bytes() |
| 377 | .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_".contains(&byte)) |
| 378 | { |
| 379 | return Err(InvalidName); |
| 380 | } |
| 381 | Ok(()) |
| 382 | } |
| 383 | |
| 384 | fn validate_text(field: &'static str, value: &str, max: usize) -> Result<(), SearchSpecError> { |
| 385 | if value.trim().is_empty() || value.chars().count() > max { |
| 386 | return Err(InvalidText { field, max }); |
| 387 | } |
| 388 | Ok(()) |
| 389 | } |
| 390 | |
| 391 | fn validate_rounds(population: u16, rounds: &[u16]) -> Result<(), SearchSpecError> { |
| 392 | if rounds.len() < 2 |
| 393 | || rounds.first() != Some(&population) |
| 394 | || rounds.last() != Some(&1) |
| 395 | || rounds.windows(2).any(|pair| pair[0] <= pair[1]) |
| 396 | { |
| 397 | return Err(InvalidRounds); |
| 398 | } |
| 399 | Ok(()) |
| 400 | } |
| 401 | |
| 402 | fn validate_string_list( |
| 403 | field: &'static str, |
| 404 | values: &[String], |
| 405 | max_items: usize, |
| 406 | max_chars: usize, |
| 407 | ) -> Result<(), SearchSpecError> { |
| 408 | if values.len() > max_items |
| 409 | || values |
| 410 | .iter() |
| 411 | .any(|value| value.trim().is_empty() || value.chars().count() > max_chars) |
| 412 | || values |
| 413 | .iter() |
| 414 | .enumerate() |
| 415 | .any(|(index, value)| values[..index].contains(value)) |
| 416 | { |
| 417 | return Err(InvalidStringList { field }); |
| 418 | } |
| 419 | Ok(()) |
| 420 | } |
| 421 | |
| 422 | fn validate_repo_relative_paths( |
| 423 | field: &'static str, |
| 424 | values: &[String], |
| 425 | max_items: usize, |
| 426 | ) -> Result<(), SearchSpecError> { |
| 427 | validate_string_list(field, values, max_items, 1_024)?; |
| 428 | if values.iter().any(|value| { |
| 429 | let trimmed = value.trim(); |
| 430 | let normalized = trimmed.replace('\\', "/"); |
| 431 | let windows_drive = normalized.as_bytes().get(1) == Some(&b':') |
| 432 | && normalized |
| 433 | .as_bytes() |
| 434 | .first() |
| 435 | .is_some_and(u8::is_ascii_alphabetic); |
| 436 | trimmed != value |
| 437 | || trimmed.chars().any(|ch| matches!(ch, '\0' | '\r' | '\n')) |
| 438 | || windows_drive |
| 439 | || Path::new(&normalized).is_absolute() |
| 440 | || Path::new(&normalized).components().any(|component| { |
| 441 | matches!( |
| 442 | component, |
| 443 | Component::ParentDir | Component::RootDir | Component::Prefix(_) |
| 444 | ) |
| 445 | }) |
| 446 | }) { |
| 447 | return Err(InvalidWriteScope { field }); |
| 448 | } |
| 449 | Ok(()) |
| 450 | } |
| 451 | |
| 452 | fn validate_commit(value: &str) -> Result<(), SearchSpecError> { |
| 453 | if !(7..=64).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { |
| 454 | return Err(InvalidBaselineCommit); |
| 455 | } |
| 456 | Ok(()) |
| 457 | } |
| 458 | |
| 459 | fn sha256_label(bytes: &[u8]) -> String { |
| 460 | let digest = Sha256::digest(bytes); |
| 461 | let mut output = String::with_capacity(71); |
| 462 | output.push_str("sha256:"); |
| 463 | for byte in digest { |
| 464 | use std::fmt::Write as _; |
| 465 | let _ = write!(output, "{byte:02x}"); |
| 466 | } |
| 467 | output |
| 468 | } |
| 469 | |
| 470 | #[cfg(test)] |
| 471 | mod tests { |
| 472 | use super::*; |
| 473 | |
| 474 | const SPEC: &str = r#" |
| 475 | name = "speed-up-certificate" |
| 476 | objective = "Reduce runtime without changing exact results" |
| 477 | population = 32 |
| 478 | rounds = [32, 8, 3, 1] |
| 479 | concurrency = 16 |
| 480 | integration_policy = "review_only" |
| 481 | |
| 482 | [worker] |
| 483 | provider = "deepseek" |
| 484 | model = "deepseek-v4-flash" |
| 485 | reasoning_effort = "max" |
| 486 | write_authority = "worktree_write" |
| 487 | write_roots = ["code"] |
| 488 | |
| 489 | [budget] |
| 490 | max_cost_microusd = 5000000 |
| 491 | max_tokens = 10000000 |
| 492 | |
| 493 | [hard_gates] |
| 494 | commands = ["cargo test --locked", "git diff --exit-code -- expected.json"] |
| 495 | forbid_test_changes = true |
| 496 | protected_paths = ["tests", "expected.json"] |
| 497 | |
| 498 | [score] |
| 499 | command = "./scripts/benchmark_candidate.sh" |
| 500 | direction = "minimize" |
| 501 | metric = "median_runtime_ms" |
| 502 | trials = 5 |
| 503 | tie_breakers = ["diff_lines", "cost_microusd"] |
| 504 | |
| 505 | [selection] |
| 506 | policy = "pareto" |
| 507 | retain_diversity = true |
| 508 | "#; |
| 509 | |
| 510 | #[test] |
| 511 | fn parses_valid_search_and_queues_through_live_cap() { |
| 512 | let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 513 | |
| 514 | assert_eq!(spec.population, 32); |
| 515 | let batches = spec.admission_batches().expect("validated admission"); |
| 516 | assert_eq!(batches.len(), 2); |
| 517 | assert_eq!(batches[0].len(), 16); |
| 518 | assert_eq!(spec.candidate_ids()[0], "cand_001"); |
| 519 | assert_eq!(spec.candidate_ids()[31], "cand_032"); |
| 520 | } |
| 521 | |
| 522 | #[test] |
| 523 | fn freeze_is_deterministic_and_model_version_sensitive() { |
| 524 | let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 525 | let first = spec |
| 526 | .freeze( |
| 527 | "33bc6a98", |
| 528 | "DeepSeek-V4-Flash-0731", |
| 529 | b"public evidence", |
| 530 | b"private evaluator", |
| 531 | ) |
| 532 | .expect("freeze succeeds"); |
| 533 | let replay = spec |
| 534 | .freeze( |
| 535 | "33bc6a98", |
| 536 | "DeepSeek-V4-Flash-0731", |
| 537 | b"public evidence", |
| 538 | b"private evaluator", |
| 539 | ) |
| 540 | .expect("freeze succeeds"); |
| 541 | let drifted = spec |
| 542 | .freeze( |
| 543 | "33bc6a98", |
| 544 | "DeepSeek-V4-Flash-next", |
| 545 | b"public evidence", |
| 546 | b"private evaluator", |
| 547 | ) |
| 548 | .expect("freeze succeeds"); |
| 549 | |
| 550 | assert_eq!(first, replay); |
| 551 | assert_ne!(first.search_id, drifted.search_id); |
| 552 | assert_ne!(first.preregistration_hash, drifted.preregistration_hash); |
| 553 | assert_eq!(first.requested_model, "deepseek-v4-flash"); |
| 554 | assert_eq!(first.resolved_model, "DeepSeek-V4-Flash-0731"); |
| 555 | } |
| 556 | |
| 557 | #[test] |
| 558 | fn rejects_unsafe_or_unbounded_searches() { |
| 559 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 560 | spec.hard_gates.forbid_test_changes = false; |
| 561 | assert_eq!(spec.validate(), Err(SearchSpecError::TestWeakeningAllowed)); |
| 562 | |
| 563 | spec.hard_gates.forbid_test_changes = true; |
| 564 | spec.worker.write_roots.clear(); |
| 565 | assert_eq!(spec.validate(), Err(SearchSpecError::UnboundedWriteScope)); |
| 566 | } |
| 567 | |
| 568 | #[test] |
| 569 | fn rejects_invalid_rounds_and_excess_live_concurrency() { |
| 570 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 571 | spec.rounds = vec![32, 8, 8, 1]; |
| 572 | assert_eq!(spec.validate(), Err(SearchSpecError::InvalidRounds)); |
| 573 | |
| 574 | spec.rounds = vec![32, 1]; |
| 575 | spec.concurrency = 17; |
| 576 | assert_eq!( |
| 577 | spec.validate(), |
| 578 | Err(SearchSpecError::InvalidConcurrency { |
| 579 | concurrency: 17, |
| 580 | population: 32, |
| 581 | }) |
| 582 | ); |
| 583 | } |
| 584 | |
| 585 | #[test] |
| 586 | fn admission_refuses_unvalidated_zero_concurrency_without_panicking() { |
| 587 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 588 | spec.concurrency = 0; |
| 589 | |
| 590 | assert_eq!( |
| 591 | spec.admission_batches(), |
| 592 | Err(SearchSpecError::InvalidConcurrency { |
| 593 | concurrency: 0, |
| 594 | population: 32, |
| 595 | }) |
| 596 | ); |
| 597 | } |
| 598 | |
| 599 | #[test] |
| 600 | fn rejects_write_scopes_that_escape_or_obscure_the_repo_boundary() { |
| 601 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 602 | for unsafe_path in ["../outside", "/tmp/outside", r"C:\outside"] { |
| 603 | spec.worker.write_roots = vec![unsafe_path.to_string()]; |
| 604 | assert_eq!( |
| 605 | spec.validate(), |
| 606 | Err(SearchSpecError::InvalidWriteScope { |
| 607 | field: "worker.write_roots", |
| 608 | }), |
| 609 | "path should be rejected: {unsafe_path}" |
| 610 | ); |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | #[test] |
| 615 | fn deserialization_refuses_auto_merge_policy() { |
| 616 | let source = SPEC.replace("review_only", "auto_merge"); |
| 617 | let error = WorkflowSearchSpec::from_toml(&source).expect_err("must reject auto merge"); |
| 618 | |
| 619 | assert!(matches!(error, SearchSpecError::Parse(_))); |
| 620 | } |
| 621 | } |
| 622 |