runtime_threads.rs - CodeWhale

runtime_threads.rs

1	//! Durable thread/turn/item runtime for the HTTP API and background tasks.
2	//!
3	//! Execution follows the configured provider route while exposing Codex-like
4	//! lifecycle semantics (threads, turns, items, interrupt/steer, and replayable
5	//! events).
6
7	// Background-task runtime — runs alongside the TUI. Raw stdio prints
8	// here would still land in the alt-screen on whichever terminal the
9	// foreground TUI happens to own. Route everything through `tracing::*`
10	// instead — see `runtime_log` for the rationale.
11	#![deny(clippy::print_stdout)]
12	#![deny(clippy::print_stderr)]
13
14	use std::collections::{HashMap, HashSet, VecDeque};
15	use std::fs::{self, File, OpenOptions};
16	use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write};
17	use std::path::{Component, Path, PathBuf};
18	use std::sync::Arc;
19	use std::time::{Duration, Instant};
20
21	use anyhow::{Context, Result, anyhow, bail};
22	use chrono::{DateTime, Utc};
23	use serde::{Deserialize, Serialize};
24	use serde_json::{Value, json};
25	use tokio::sync::{Mutex, broadcast, mpsc, oneshot, watch};
26	use tokio_util::sync::CancellationToken;
27	use uuid::Uuid;
28
29	use crate::compaction::CompactionConfig;
30	#[cfg(test)]
31	use crate::config::DEFAULT_TEXT_MODEL;
32	use crate::config::{ApiProvider, Config, MAX_SUBAGENTS, ProviderIdentity};
33	use crate::core::engine::{
34	EngineConfig, EngineHandle, spawn_engine_with_authoritative_route_config,
35	};
36	use crate::core::events::{Event as EngineEvent, TurnOutcomeStatus};
37	use crate::core::ops::Op;
38	use crate::cost_status::{
39	EffectiveRouteEnvelope, EffectiveRouteUsage, RouteBillingMode, RuntimeUsageRecord,
40	};
41	use crate::models::{ContentBlock, Message, SystemPrompt, Usage};
42	use crate::route_budget::{
43	auto_compact_default_for_route, compaction_threshold_for_route_at_percent, known_route_limits,
44	route_context_window_tokens,
45	};
46	use crate::route_runtime::{
47	ResolvedRuntimeRoute, resolve_runtime_route, resolve_runtime_route_for_identity,
48	};
49	use crate::runtime_policy::RuntimePolicyProjection;
50	use crate::tools::plan::new_shared_plan_state;
51	use crate::tools::subagent::SubAgentStatus;
52	use crate::tools::todo::new_shared_todo_list;
53	#[cfg(test)]
54	use crate::tui::app::AppMode;
55	use codewhale_protocol::runtime::{
56	DynamicToolCallContent, DynamicToolCallParams, DynamicToolCallResult, DynamicToolSpec,
57	TurnEnvironmentParams,
58	};
59
60	const EVENT_CHANNEL_CAPACITY: usize = 1024;
61	pub(crate) const RUNTIME_EVENT_REPLAY_BATCH_SIZE: usize = 256;
62	pub(crate) const MAX_RUNTIME_EVENT_REPLAY_TAIL: usize = 4096;
63	const MAX_ACTIVE_THREADS_DEFAULT: usize = 8;
64	const MAX_PENDING_DYNAMIC_TOOL_CALLS: usize = 128;
65	const SUMMARY_LIMIT: usize = 280;
66	const STREAM_DELTA_BATCH_MAX_LATENCY: Duration = Duration::from_millis(32);
67	const STREAM_DELTA_BATCH_MAX_BYTES: usize = 16 * 1024;
68	const EVENT_TRANSACTION_LOCK_TIMEOUT: Duration = Duration::from_secs(5);
69	const EVENT_TRANSACTION_LOCK_POLL: Duration = Duration::from_millis(5);
70	const EVENT_TRANSACTION_LOCK_FILE: &str = "events.lock";
71	const REQUEST_USER_INPUT_TOOL_NAME: &str = "request_user_input";
72	const REDACTED_USER_INPUT_RECEIPT: &str = "User input submitted";
73	pub(crate) const MAX_ROUTED_USAGE_RECORDS_PER_TURN: usize = 64;
74
75	#[cfg(test)]
76	#[derive(Clone, Copy, PartialEq, Eq)]
77	pub(crate) enum EventAppendTestFault {
78	AfterFlush,
79	AfterSync,
80	}
81
82	#[cfg(test)]
83	static TEST_EVENT_APPEND_FAULTS: std::sync::Mutex<Vec<(String, EventAppendTestFault, usize)>> =
84	std::sync::Mutex::new(Vec::new());
85
86	#[cfg(test)]
87	pub(crate) type EventAppendTestFaultRestore = (String, Option<(EventAppendTestFault, usize)>);
88
89	#[cfg(test)]
90	pub(crate) fn set_test_event_append_fault(
91	thread_id: &str,
92	fault: EventAppendTestFault,
93	remaining: usize,
94	) -> EventAppendTestFaultRestore {
95	assert!(remaining > 0, "event append fault count must be positive");
96	let mut pending = TEST_EVENT_APPEND_FAULTS
97	.lock()
98	.unwrap_or_else(std::sync::PoisonError::into_inner);
99	let previous = pending
100	.iter()
101	.position(\|(target, _, _)\| target == thread_id)
102	.map(\|index\| {
103	let (_, previous_fault, previous_remaining) = pending.remove(index);
104	(previous_fault, previous_remaining)
105	});
106	pending.push((thread_id.to_string(), fault, remaining));
107	(thread_id.to_string(), previous)
108	}
109
110	#[cfg(test)]
111	pub(crate) fn restore_test_event_append_fault(restore: EventAppendTestFaultRestore) {
112	let (thread_id, previous) = restore;
113	let mut pending = TEST_EVENT_APPEND_FAULTS
114	.lock()
115	.unwrap_or_else(std::sync::PoisonError::into_inner);
116	if let Some(index) = pending
117	.iter()
118	.position(\|(target, _, _)\| target == &thread_id)
119	{
120	pending.remove(index);
121	}
122	if let Some((fault, remaining)) = previous {
123	pending.push((thread_id, fault, remaining));
124	}
125	}
126
127	#[cfg(test)]
128	fn take_test_event_append_fault(thread_id: &str, expected: EventAppendTestFault) -> bool {
129	let mut pending = TEST_EVENT_APPEND_FAULTS
130	.lock()
131	.unwrap_or_else(std::sync::PoisonError::into_inner);
132	let Some(index) = pending
133	.iter()
134	.position(\|(target, fault, _)\| target == thread_id && *fault == expected)
135	else {
136	return false;
137	};
138	if pending[index].2 > 1 {
139	pending[index].2 -= 1;
140	} else {
141	pending.remove(index);
142	}
143	true
144	}
145
146	#[derive(Clone, Copy, PartialEq, Eq)]
147	enum StreamDeltaKind {
148	Message,
149	Reasoning,
150	}
151
152	struct StreamDeltaBatch {
153	content: String,
154	pending_event: Option<EngineEvent>,
155	channel_closed: bool,
156	}
157
158	async fn coalesce_stream_delta(
159	engine: &EngineHandle,
160	kind: StreamDeltaKind,
161	mut content: String,
162	) -> StreamDeltaBatch {
163	let deadline = tokio::time::Instant::now() + STREAM_DELTA_BATCH_MAX_LATENCY;
164	let mut pending_event = None;
165	let mut channel_closed = false;
166	let mut rx = engine.rx_event.write().await;
167
168	while content.len() < STREAM_DELTA_BATCH_MAX_BYTES {
169	let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
170	if remaining.is_zero() {
171	break;
172	}
173	let next = match tokio::time::timeout(remaining, rx.recv()).await {
174	Ok(Some(event)) => event,
175	Ok(None) => {
176	channel_closed = true;
177	break;
178	}
179	Err(_) => break,
180	};
181	match next {
182	EngineEvent::MessageDelta { content: next, .. } if kind == StreamDeltaKind::Message => {
183	content.push_str(&next);
184	}
185	EngineEvent::ThinkingDelta { content: next, .. }
186	if kind == StreamDeltaKind::Reasoning =>
187	{
188	content.push_str(&next);
189	}
190	event => {
191	pending_event = Some(event);
192	break;
193	}
194	}
195	}
196
197	StreamDeltaBatch {
198	content,
199	pending_event,
200	channel_closed,
201	}
202	}
203
204	/// Sentinel delimiters wrapping the compaction summary section persisted in a
205	/// thread record's `system_prompt`. The section carries the engine-rendered
206	/// summary (which contains the `Conversation Summary (Auto-Generated)` marker,
207	/// so `SyncSession` → `extract_compaction_summary_prompt` restores it on
208	/// engine reload). Delimiters make replacement idempotent: each completed
209	/// compaction swaps the section in place instead of stacking duplicates.
210	/// External `PATCH /v1/threads/{id}` callers that rewrite `system_prompt`
211	/// should preserve this section verbatim or the summary is lost on reload.
212	const COMPACTION_SUMMARY_BEGIN: &str = "<!-- compaction-summary:begin -->";
213	const COMPACTION_SUMMARY_END: &str = "<!-- compaction-summary:end -->";
214
215	/// Merge a rendered compaction summary into a thread record's system prompt,
216	/// replacing any previously persisted summary section.
217	fn merge_summary_into_prompt(base: Option<&str>, summary_text: &str) -> String {
218	let stripped = base.map(strip_summary_section).unwrap_or_default();
219	let mut out = stripped.trim_end().to_string();
220	if !out.is_empty() {
221	out.push_str("\n\n");
222	}
223	out.push_str(COMPACTION_SUMMARY_BEGIN);
224	out.push('\n');
225	out.push_str(summary_text.trim());
226	out.push('\n');
227	out.push_str(COMPACTION_SUMMARY_END);
228	out
229	}
230
231	/// Remove a previously persisted compaction summary section, if present.
232	fn strip_summary_section(base: &str) -> String {
233	let Some(start) = base.find(COMPACTION_SUMMARY_BEGIN) else {
234	return base.to_string();
235	};
236	let end = base[start..]
237	.find(COMPACTION_SUMMARY_END)
238	.map(\|rel\| start + rel + COMPACTION_SUMMARY_END.len());
239	let mut out = base[..start].trim_end().to_string();
240	if let Some(end) = end {
241	let tail = base[end..].trim_start();
242	if !tail.is_empty() {
243	if !out.is_empty() {
244	out.push_str("\n\n");
245	}
246	out.push_str(tail);
247	}
248	}
249	out
250	}
251
252	fn validated_record_id<'a>(id: &'a str, label: &str) -> Result<&'a str> {
253	let trimmed = id.trim();
254	if trimmed.is_empty() {
255	bail!("{label} cannot be empty");
256	}
257	if trimmed != id {
258	bail!("{label} cannot contain leading or trailing whitespace");
259	}
260	if !trimmed
261	.chars()
262	.all(\|c\| c.is_ascii_alphanumeric() \|\| c == '-' \|\| c == '_')
263	{
264	bail!("{label} contains unsupported characters");
265	}
266	Ok(trimmed)
267	}
268
269	fn sort_turn_items_by_start(items: &mut [TurnItemRecord]) {
270	let fallback = Utc::now();
271	items.sort_by(\|a, b\| {
272	let left = a.started_at.unwrap_or(fallback);
273	let right = b.started_at.unwrap_or(fallback);
274	left.cmp(&right)
275	});
276	}
277
278	/// Bumped to 2 for v0.6.6 after live engine semantics changed. The persisted
279	/// thread/turn/item records did not change shape, but a v1 reader on a v2
280	/// session should still fail closed rather than silently mis-replay.
281	const CURRENT_RUNTIME_SCHEMA_VERSION: u32 = 2;
282
283	fn is_zero_u64(value: &u64) -> bool {
284	*value == 0
285	}
286
287	fn serialize_route_label_option<S>(
288	value: &Option<String>,
289	serializer: S,
290	) -> std::result::Result<S::Ok, S::Error>
291	where
292	S: serde::Serializer,
293	{
294	value
295	.as_deref()
296	.map(crate::cost_status::sanitize_persisted_route_label)
297	.serialize(serializer)
298	}
299
300	fn serialize_endpoint_fingerprint_option<S>(
301	value: &Option<String>,
302	serializer: S,
303	) -> std::result::Result<S::Ok, S::Error>
304	where
305	S: serde::Serializer,
306	{
307	value
308	.as_deref()
309	.filter(\|fingerprint\| {
310	fingerprint.len() == 64 && fingerprint.bytes().all(\|byte\| byte.is_ascii_hexdigit())
311	})
312	.map(str::to_ascii_lowercase)
313	.serialize(serializer)
314	}
315
316	fn serialize_routed_usage_source_ids<S>(
317	values: &[String],
318	serializer: S,
319	) -> std::result::Result<S::Ok, S::Error>
320	where
321	S: serde::Serializer,
322	{
323	values
324	.iter()
325	.map(\|value\| {
326	if value.len() == 64 && value.bytes().all(\|byte\| byte.is_ascii_hexdigit()) {
327	value.to_ascii_lowercase()
328	} else {
329	codewhale_config::catalog::base_url_fingerprint(value)
330	}
331	})
332	.collect::<Vec<_>>()
333	.serialize(serializer)
334	}
335	const RUNTIME_RESTART_REASON: &str = "Interrupted by process restart";
336	const EMPTY_TURN_REASON: &str = "Turn completed without engine output";
337	const APPROVAL_DECISION_TIMEOUT: Duration = Duration::from_secs(300);
338	const DYNAMIC_TOOL_RESULT_TIMEOUT: Duration = Duration::from_secs(300);
339
340	#[cfg(test)]
341	static TEST_APPROVAL_DECISION_TIMEOUT_MS: std::sync::atomic::AtomicU64 =
342	std::sync::atomic::AtomicU64::new(0);
343
344	#[cfg(test)]
345	static TEST_DYNAMIC_TOOL_RESULT_TIMEOUT_MS: std::sync::atomic::AtomicU64 =
346	std::sync::atomic::AtomicU64::new(0);
347
348	fn approval_decision_timeout() -> Duration {
349	#[cfg(test)]
350	{
351	let ms = TEST_APPROVAL_DECISION_TIMEOUT_MS.load(std::sync::atomic::Ordering::SeqCst);
352	if ms > 0 {
353	return Duration::from_millis(ms);
354	}
355	}
356	APPROVAL_DECISION_TIMEOUT
357	}
358
359	fn dynamic_tool_result_timeout() -> Duration {
360	#[cfg(test)]
361	{
362	let ms = TEST_DYNAMIC_TOOL_RESULT_TIMEOUT_MS.load(std::sync::atomic::Ordering::SeqCst);
363	if ms > 0 {
364	return Duration::from_millis(ms);
365	}
366	}
367	DYNAMIC_TOOL_RESULT_TIMEOUT
368	}
369
370	#[cfg(test)]
371	pub(crate) fn set_test_approval_decision_timeout_ms(ms: u64) -> u64 {
372	TEST_APPROVAL_DECISION_TIMEOUT_MS.swap(ms, std::sync::atomic::Ordering::SeqCst)
373	}
374
375	#[cfg(test)]
376	pub(crate) fn set_test_dynamic_tool_result_timeout_ms(ms: u64) -> u64 {
377	TEST_DYNAMIC_TOOL_RESULT_TIMEOUT_MS.swap(ms, std::sync::atomic::Ordering::SeqCst)
378	}
379
380	const fn default_runtime_schema_version() -> u32 {
381	CURRENT_RUNTIME_SCHEMA_VERSION
382	}
383
384	#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
385	#[serde(rename_all = "snake_case")]
386	pub enum RuntimeTurnStatus {
387	Queued,
388	InProgress,
389	Completed,
390	Failed,
391	Interrupted,
392	Canceled,
393	}
394
395	#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
396	#[serde(rename_all = "snake_case")]
397	pub enum TurnItemKind {
398	UserMessage,
399	AgentMessage,
400	AgentReasoning,
401	ToolCall,
402	FileChange,
403	CommandExecution,
404	ContextCompaction,
405	Status,
406	Error,
407	}
408
409	#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
410	#[serde(rename_all = "snake_case")]
411	pub enum TurnItemLifecycleStatus {
412	Queued,
413	InProgress,
414	Completed,
415	Failed,
416	Interrupted,
417	Canceled,
418	}
419
420	#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
421	pub struct ThreadRecord {
422	#[serde(default = "default_runtime_schema_version")]
423	pub schema_version: u32,
424	pub id: String,
425	pub created_at: DateTime<Utc>,
426	pub updated_at: DateTime<Utc>,
427	pub model: String,
428	/// Generic provider kind for this thread's model route. Named custom
429	/// routes remain `custom` for compatibility with enum-only consumers.
430	#[serde(default, skip_serializing_if = "Option::is_none")]
431	pub model_provider: Option<String>,
432	/// Exact non-secret configured provider key.
433	#[serde(default, skip_serializing_if = "Option::is_none")]
434	pub model_provider_id: Option<String>,
435	pub workspace: PathBuf,
436	pub mode: String,
437	/// Named default permission posture for new turns. Absent on legacy
438	/// records, whose effective posture is derived from the old fields.
439	#[serde(default, skip_serializing_if = "Option::is_none")]
440	pub permission_posture: Option<String>,
441	pub allow_shell: bool,
442	pub trust_mode: bool,
443	pub auto_approve: bool,
444	#[serde(skip_serializing_if = "Option::is_none")]
445	pub latest_turn_id: Option<String>,
446	#[serde(skip_serializing_if = "Option::is_none")]
447	pub latest_response_bookmark: Option<String>,
448	#[serde(default)]
449	pub archived: bool,
450	#[serde(default, skip_serializing_if = "Option::is_none")]
451	pub system_prompt: Option<String>,
452	#[serde(default, skip_serializing_if = "Option::is_none")]
453	pub task_id: Option<String>,
454	/// User-set title for the thread. When `None`, consumers fall back to a
455	/// derived title (typically the latest turn's input summary). Added in
456	/// v0.8.10 (#562); old runtime records simply have no `title` and behave
457	/// as before. Schema version is not bumped because this field is purely
458	/// additive metadata — older readers ignore it without misinterpretation.
459	#[serde(default, skip_serializing_if = "Option::is_none")]
460	pub title: Option<String>,
461	/// The session ID associated with this thread. When set, `ensure_engine_loaded`
462	/// loads the full message history (including thinking/tool blocks) from the
463	/// session file instead of reconstructing from turns (which loses process info).
464	#[serde(default, skip_serializing_if = "Option::is_none")]
465	pub session_id: Option<String>,
466	}
467
468	fn thread_execution_state_matches(left: &ThreadRecord, right: &ThreadRecord) -> bool {
469	left.schema_version == right.schema_version
470	&& left.id == right.id
471	&& left.model == right.model
472	&& left.model_provider == right.model_provider
473	&& left.model_provider_id == right.model_provider_id
474	&& left.workspace == right.workspace
475	&& left.mode == right.mode
476	&& left.permission_posture == right.permission_posture
477	&& left.allow_shell == right.allow_shell
478	&& left.trust_mode == right.trust_mode
479	&& left.auto_approve == right.auto_approve
480	&& left.latest_turn_id == right.latest_turn_id
481	&& left.latest_response_bookmark == right.latest_response_bookmark
482	&& left.archived == right.archived
483	&& left.system_prompt == right.system_prompt
484	&& left.task_id == right.task_id
485	&& left.session_id == right.session_id
486	}
487
488	#[derive(Debug, Clone, Serialize, Deserialize)]
489	pub struct TurnRecord {
490	#[serde(default = "default_runtime_schema_version")]
491	pub schema_version: u32,
492	pub id: String,
493	pub thread_id: String,
494	pub status: RuntimeTurnStatus,
495	pub input_summary: String,
496	pub created_at: DateTime<Utc>,
497	#[serde(skip_serializing_if = "Option::is_none")]
498	pub started_at: Option<DateTime<Utc>>,
499	#[serde(skip_serializing_if = "Option::is_none")]
500	pub ended_at: Option<DateTime<Utc>>,
501	#[serde(skip_serializing_if = "Option::is_none")]
502	pub duration_ms: Option<u64>,
503	#[serde(skip_serializing_if = "Option::is_none")]
504	pub usage: Option<Usage>,
505	/// Canonical posture that governed this turn. New records always carry
506	/// this receipt; old records deserialize with no fabricated value.
507	#[serde(default, skip_serializing_if = "Option::is_none")]
508	pub permission_posture: Option<String>,
509	/// Concrete generic provider kind selected for this turn.
510	#[serde(
511	default,
512	skip_serializing_if = "Option::is_none",
513	serialize_with = "serialize_route_label_option"
514	)]
515	pub effective_provider: Option<String>,
516	/// Exact non-secret configured provider key selected for this turn.
517	#[serde(
518	default,
519	skip_serializing_if = "Option::is_none",
520	serialize_with = "serialize_route_label_option"
521	)]
522	pub effective_provider_id: Option<String>,
523	/// Non-secret discriminator for routes whose provider/model pair spans
524	/// different billing systems (for example StepFun PAYG vs Step Plan).
525	#[serde(
526	default,
527	skip_serializing_if = "Option::is_none",
528	serialize_with = "serialize_route_label_option"
529	)]
530	pub effective_billing_surface: Option<String>,
531	/// SHA-256 fingerprint of the concrete dispatch endpoint. Raw URLs are
532	/// intentionally never persisted.
533	#[serde(
534	default,
535	skip_serializing_if = "Option::is_none",
536	serialize_with = "serialize_endpoint_fingerprint_option"
537	)]
538	pub effective_endpoint_fingerprint: Option<String>,
539	/// Immutable billing classification captured before dispatch.
540	#[serde(default, skip_serializing_if = "Option::is_none")]
541	pub effective_billing_mode: Option<RouteBillingMode>,
542	/// Dispatch timestamp used for historical/live pricing lookup.
543	#[serde(default, skip_serializing_if = "Option::is_none")]
544	pub effective_dispatched_at: Option<DateTime<Utc>>,
545	/// Concrete wire model selected for this turn (especially important when
546	/// the thread is configured as `auto`).
547	#[serde(
548	default,
549	skip_serializing_if = "Option::is_none",
550	serialize_with = "serialize_route_label_option"
551	)]
552	pub effective_model: Option<String>,
553	/// Model calls made beneath this parent turn, each paired with its own
554	/// immutable route. These are exclusive of `usage`, which is only the
555	/// parent engine turn.
556	#[serde(default, skip_serializing_if = "Vec::is_empty")]
557	pub routed_usage: Vec<EffectiveRouteUsage>,
558	/// Fingerprints of provider-call identities already appended to this turn.
559	/// This durable ledger makes mailbox delivery, direct sinks, fallback
560	/// recovery, and process restart idempotent without persisting raw ids.
561	#[serde(
562	default,
563	skip_serializing_if = "Vec::is_empty",
564	serialize_with = "serialize_routed_usage_source_ids"
565	)]
566	pub routed_usage_source_ids: Vec<String>,
567	/// Background provider calls discarded from the bounded fallback journal.
568	/// Non-zero means token/cost aggregation is necessarily incomplete.
569	#[serde(default, skip_serializing_if = "is_zero_u64")]
570	pub routed_usage_dropped_records: u64,
571	#[serde(skip_serializing_if = "Option::is_none")]
572	pub error: Option<String>,
573	#[serde(default)]
574	pub item_ids: Vec<String>,
575	#[serde(default)]
576	pub steer_count: usize,
577	}
578
579	impl TurnRecord {
580	pub(crate) fn effective_provider_label(&self) -> Option<&str> {
581	self.effective_provider_id
582	.as_deref()
583	.filter(\|identity\| !identity.trim().is_empty())
584	.or_else(\|\| {
585	self.effective_provider
586	.as_deref()
587	.filter(\|provider\| !provider.trim().is_empty())
588	})
589	}
590
591	fn persist_effective_route(&mut self, route: &EffectiveRouteEnvelope) {
592	let route = route.sanitized_for_persistence();
593	self.effective_provider = Some(route.provider.as_str().to_string());
594	self.effective_provider_id = Some(route.provider_identity);
595	self.effective_billing_surface = route.billing_surface;
596	self.effective_endpoint_fingerprint = route.endpoint_fingerprint;
597	self.effective_billing_mode = Some(route.billing_mode);
598	self.effective_dispatched_at = Some(route.dispatched_at);
599	self.effective_model = Some(route.model);
600	}
601
602	/// Rehydrate only a complete persisted dispatch record. Legacy rows must
603	/// not borrow a provider identity or timestamp from the current thread.
604	fn effective_route_envelope(&self) -> Option<EffectiveRouteEnvelope> {
605	let provider = self
606	.effective_provider
607	.as_deref()
608	.and_then(ApiProvider::parse)?;
609	let provider_identity = self
610	.effective_provider_id
611	.as_deref()
612	.filter(\|identity\| !identity.trim().is_empty())?
613	.to_string();
614	let model = self
615	.effective_model
616	.as_deref()
617	.filter(\|model\| !model.trim().is_empty())?
618	.to_string();
619	let dispatched_at = self.effective_dispatched_at?;
620	Some(
621	EffectiveRouteEnvelope {
622	provider,
623	provider_identity,
624	model,
625	billing_surface: self.effective_billing_surface.clone(),
626	endpoint_fingerprint: self.effective_endpoint_fingerprint.clone(),
627	billing_mode: self
628	.effective_billing_mode
629	.unwrap_or(RouteBillingMode::Unknown),
630	dispatched_at,
631	}
632	.sanitized_for_persistence(),
633	)
634	}
635	}
636
637	fn routed_usage_source_fingerprint(source_id: &str) -> String {
638	codewhale_config::catalog::base_url_fingerprint(source_id.trim())
639	}
640
641	/// The only mutation path for routed provider usage. Every source is recorded
642	/// once, route labels are sanitized at the boundary, and retained records are
643	/// bounded regardless of whether they arrived synchronously, by mailbox, or
644	/// from the fallback journal.
645	fn append_routed_usage_record(
646	turn: &mut TurnRecord,
647	source_id: &str,
648	usage: EffectiveRouteUsage,
649	) -> bool {
650	let source_fingerprint = routed_usage_source_fingerprint(source_id);
651	if turn
652	.routed_usage_source_ids
653	.iter()
654	.any(\|persisted\| persisted == &source_fingerprint)
655	{
656	return false;
657	}
658	turn.routed_usage_source_ids.push(source_fingerprint);
659	if turn.routed_usage.len() == MAX_ROUTED_USAGE_RECORDS_PER_TURN {
660	turn.routed_usage.remove(0);
661	turn.routed_usage_dropped_records = turn.routed_usage_dropped_records.saturating_add(1);
662	}
663	turn.routed_usage.push(EffectiveRouteUsage {
664	route: usage.route.sanitized_for_persistence(),
665	usage: usage.usage,
666	});
667	true
668	}
669
670	#[derive(Debug, Clone, Serialize, Deserialize)]
671	pub struct TurnItemRecord {
672	#[serde(default = "default_runtime_schema_version")]
673	pub schema_version: u32,
674	pub id: String,
675	pub turn_id: String,
676	pub kind: TurnItemKind,
677	pub status: TurnItemLifecycleStatus,
678	pub summary: String,
679	#[serde(skip_serializing_if = "Option::is_none")]
680	pub detail: Option<String>,
681	#[serde(default, skip_serializing_if = "Option::is_none")]
682	pub metadata: Option<Value>,
683	#[serde(default)]
684	pub artifact_refs: Vec<PathBuf>,
685	#[serde(skip_serializing_if = "Option::is_none")]
686	pub started_at: Option<DateTime<Utc>>,
687	#[serde(skip_serializing_if = "Option::is_none")]
688	pub ended_at: Option<DateTime<Utc>>,
689	}
690
691	#[derive(Debug, Clone, Serialize, Deserialize)]
692	pub struct RuntimeEventRecord {
693	#[serde(default = "default_runtime_schema_version")]
694	pub schema_version: u32,
695	pub seq: u64,
696	pub timestamp: DateTime<Utc>,
697	pub thread_id: String,
698	#[serde(skip_serializing_if = "Option::is_none")]
699	pub turn_id: Option<String>,
700	#[serde(skip_serializing_if = "Option::is_none")]
701	pub item_id: Option<String>,
702	pub event: String,
703	pub payload: Value,
704	}
705
706	pub(crate) struct RuntimeEventReplay {
707	/// Cursor immediately before the first replayed event. For a tail-limited
708	/// replay this advances past omitted history so continuity remains exact.
709	pub(crate) base_seq: u64,
710	/// Filesystem parsing happens on the blocking pool and publishes bounded
711	/// chunks through this small channel, applying backpressure instead of
712	/// allocating an unbounded backlog on a Tokio worker.
713	pub(crate) batches: mpsc::Receiver<std::result::Result<Vec<RuntimeEventRecord>, String>>,
714	}
715
716	type RuntimeEventReader = BufReader<std::io::Take<File>>;
717
718	enum RuntimeEventMatch {
719	TurnCompleted { turn_id: String },
720	DynamicTerminal { turn_id: String, call_id: String },
721	}
722
723	#[derive(Debug, Clone, Serialize, Deserialize)]
724	pub struct RuntimeStoreState {
725	#[serde(default = "default_runtime_schema_version")]
726	schema_version: u32,
727	next_seq: u64,
728	}
729
730	impl Default for RuntimeStoreState {
731	fn default() -> Self {
732	Self {
733	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
734	next_seq: 1,
735	}
736	}
737	}
738
739	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
740	enum EventAppendFailureDisposition {
741	RolledBack,
742	Indeterminate,
743	}
744
745	#[derive(Debug)]
746	struct RuntimeEventAppendError {
747	disposition: EventAppendFailureDisposition,
748	append_error: String,
749	rollback_error: Option<String>,
750	}
751
752	#[derive(Debug, thiserror::Error)]
753	#[error("Runtime event lock timed out after {0:?}")]
754	struct RuntimeEventLockTimeout(Duration);
755
756	impl RuntimeEventAppendError {
757	const fn retry_safe(&self) -> bool {
758	matches!(self.disposition, EventAppendFailureDisposition::RolledBack)
759	}
760	}
761
762	impl std::fmt::Display for RuntimeEventAppendError {
763	fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
764	match &self.rollback_error {
765	Some(rollback_error) => write!(
766	formatter,
767	"Runtime event append is indeterminate after append error ({}) and rollback error ({})",
768	self.append_error, rollback_error
769	),
770	None => write!(
771	formatter,
772	"Runtime event append failed and was rolled back: {}",
773	self.append_error
774	),
775	}
776	}
777	}
778
779	impl std::error::Error for RuntimeEventAppendError {}
780
781	fn event_append_is_indeterminate(error: &anyhow::Error) -> bool {
782	error.chain().any(\|source\| {
783	source
784	.downcast_ref::<RuntimeEventAppendError>()
785	.is_some_and(\|append\| !append.retry_safe())
786	})
787	}
788
789	#[derive(Debug, Clone)]
790	pub struct RuntimeThreadStore {
791	threads_dir: PathBuf,
792	turns_dir: PathBuf,
793	items_dir: PathBuf,
794	events_dir: PathBuf,
795	state_path: PathBuf,
796	event_lock_path: PathBuf,
797	/// Serializes load-modify-save operations on thread records. The guard is
798	/// synchronous and must never cross an `.await`; JSON records are small,
799	/// and one global guard avoids per-thread lock lifecycle races.
800	thread_mutation: Arc<parking_lot::Mutex<()>>,
801	/// Serializes load-modify-save operations on turn records. Like the
802	/// thread guard, it is synchronous and never crosses an `.await`.
803	turn_mutation: Arc<parking_lot::Mutex<()>>,
804	}
805
806	impl RuntimeThreadStore {
807	pub fn open(root: PathBuf) -> Result<Self> {
808	let root = checked_runtime_store_root(root)?;
809	ensure_runtime_store_dir(&root)?;
810	let threads_dir = root.join("threads");
811	let turns_dir = root.join("turns");
812	let items_dir = root.join("items");
813	let events_dir = root.join("events");
814	ensure_runtime_store_dir(&threads_dir)?;
815	ensure_runtime_store_dir(&turns_dir)?;
816	ensure_runtime_store_dir(&items_dir)?;
817	ensure_runtime_store_dir(&events_dir)?;
818	let state_path = root.join("state.json");
819	let store = Self {
820	threads_dir,
821	turns_dir,
822	items_dir,
823	events_dir,
824	state_path,
825	event_lock_path: root.join(EVENT_TRANSACTION_LOCK_FILE),
826	thread_mutation: Arc::new(parking_lot::Mutex::new(())),
827	turn_mutation: Arc::new(parking_lot::Mutex::new(())),
828	};
829	store.with_event_transaction(EVENT_TRANSACTION_LOCK_TIMEOUT, \|\| {
830	repair_torn_event_log_tails(&store.events_dir)?;
831	if store.state_path.exists() {
832	load_runtime_store_state(&store.state_path)?;
833	} else {
834	write_json_atomic(&store.state_path, &RuntimeStoreState::default())?;
835	}
836	Ok(())
837	})?;
838	Ok(store)
839	}
840
841	fn open_event_lock(&self) -> Result<File> {
842	let file =
843	open_runtime_store_file(&self.event_lock_path, "Runtime event lock", \|options\| {
844	options.create(true).truncate(false).read(true).write(true);
845	})?;
846	#[cfg(unix)]
847	{
848	use std::os::unix::fs::PermissionsExt as _;
849	file.set_permissions(fs::Permissions::from_mode(0o600))
850	.context("Failed to secure Runtime event lock")?;
851	}
852	Ok(file)
853	}
854
855	fn with_event_transaction<T>(
856	&self,
857	timeout: Duration,
858	operation: impl FnOnce() -> Result<T>,
859	) -> Result<T> {
860	let mut lock = fd_lock::RwLock::new(self.open_event_lock()?);
861	let started = Instant::now();
862	let mut operation = Some(operation);
863	loop {
864	match lock
865	.try_write()
866	.map(\|_guard\| operation.take().expect("event transaction runs once")())
867	{
868	Ok(result) => return result,
869	Err(error)
870	if matches!(
871	error.kind(),
872	std::io::ErrorKind::WouldBlock \| std::io::ErrorKind::Interrupted
873	) =>
874	{
875	wait_for_event_lock(started, timeout)?;
876	}
877	Err(error) => return Err(error).context("Failed to lock Runtime events"),
878	}
879	}
880	}
881
882	fn record_path(base: &Path, id: &str, extension: &str, label: &str) -> Result<PathBuf> {
883	let id = validated_record_id(id, label)?;
884	Ok(base.join(format!("{id}.{extension}")))
885	}
886
887	fn thread_path(&self, thread_id: &str) -> Result<PathBuf> {
888	Self::record_path(&self.threads_dir, thread_id, "json", "thread id")
889	}
890
891	fn turn_path(&self, turn_id: &str) -> Result<PathBuf> {
892	Self::record_path(&self.turns_dir, turn_id, "json", "turn id")
893	}
894
895	fn item_path(&self, item_id: &str) -> Result<PathBuf> {
896	Self::record_path(&self.items_dir, item_id, "json", "item id")
897	}
898
899	fn events_path(&self, thread_id: &str) -> Result<PathBuf> {
900	Self::record_path(&self.events_dir, thread_id, "jsonl", "thread id")
901	}
902
903	pub fn save_thread(&self, thread: &ThreadRecord) -> Result<()> {
904	write_json_atomic(&self.thread_path(&thread.id)?, thread)
905	}
906
907	pub fn save_turn(&self, turn: &TurnRecord) -> Result<()> {
908	validated_record_id(&turn.thread_id, "thread id")?;
909	write_json_atomic(&self.turn_path(&turn.id)?, turn)
910	}
911
912	pub fn save_item(&self, item: &TurnItemRecord) -> Result<()> {
913	validated_record_id(&item.turn_id, "turn id")?;
914	write_json_atomic(&self.item_path(&item.id)?, item)
915	}
916
917	fn remove_turn(&self, turn_id: &str) -> Result<()> {
918	remove_file_if_exists(&self.turn_path(turn_id)?)
919	}
920
921	fn remove_thread(&self, thread_id: &str) -> Result<()> {
922	remove_file_if_exists(&self.thread_path(thread_id)?)
923	}
924
925	fn remove_item(&self, item_id: &str) -> Result<()> {
926	remove_file_if_exists(&self.item_path(item_id)?)
927	}
928
929	pub fn load_thread(&self, thread_id: &str) -> Result<ThreadRecord> {
930	let path = self.thread_path(thread_id)?;
931	let raw = read_store_file(&path)
932	.with_context(\|\| format!("Failed to read thread {}", path.display()))?;
933	let record: ThreadRecord = serde_json::from_str(&raw)
934	.with_context(\|\| format!("Failed to parse thread {}", path.display()))?;
935	if record.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
936	bail!(
937	"Thread schema v{} is newer than supported v{}",
938	record.schema_version,
939	CURRENT_RUNTIME_SCHEMA_VERSION
940	);
941	}
942	Ok(record)
943	}
944
945	pub fn load_turn(&self, turn_id: &str) -> Result<TurnRecord> {
946	let path = self.turn_path(turn_id)?;
947	let raw = read_store_file(&path)
948	.with_context(\|\| format!("Failed to read turn {}", path.display()))?;
949	let record: TurnRecord = serde_json::from_str(&raw)
950	.with_context(\|\| format!("Failed to parse turn {}", path.display()))?;
951	if record.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
952	bail!(
953	"Turn schema v{} is newer than supported v{}",
954	record.schema_version,
955	CURRENT_RUNTIME_SCHEMA_VERSION
956	);
957	}
958	Ok(record)
959	}
960
961	pub fn load_item(&self, item_id: &str) -> Result<TurnItemRecord> {
962	let path = self.item_path(item_id)?;
963	let raw = read_store_file(&path)
964	.with_context(\|\| format!("Failed to read item {}", path.display()))?;
965	let record: TurnItemRecord = serde_json::from_str(&raw)
966	.with_context(\|\| format!("Failed to parse item {}", path.display()))?;
967	if record.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
968	bail!(
969	"Item schema v{} is newer than supported v{}",
970	record.schema_version,
971	CURRENT_RUNTIME_SCHEMA_VERSION
972	);
973	}
974	Ok(record)
975	}
976
977	pub fn list_threads(&self) -> Result<Vec<ThreadRecord>> {
978	let mut out = Vec::new();
979	let threads_dir = checked_existing_runtime_store_dir(&self.threads_dir)?;
980	for entry in fs::read_dir(&threads_dir)
981	.with_context(\|\| format!("Failed to read {}", threads_dir.display()))?
982	{
983	let entry = entry?;
984	let path = entry.path();
985	if path.extension().is_none_or(\|ext\| ext != "json") {
986	continue;
987	}
988	let raw = read_store_file(&path)
989	.with_context(\|\| format!("Failed to read {}", path.display()))?;
990	let thread: ThreadRecord = serde_json::from_str(&raw)
991	.with_context(\|\| format!("Failed to parse {}", path.display()))?;
992	if thread.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
993	bail!(
994	"Thread schema v{} is newer than supported v{}",
995	thread.schema_version,
996	CURRENT_RUNTIME_SCHEMA_VERSION
997	);
998	}
999	out.push(thread);
1000	}
1001	out.sort_by_key(\|t\| std::cmp::Reverse(t.updated_at));
1002	Ok(out)
1003	}
1004
1005	pub fn list_turns_for_thread(&self, thread_id: &str) -> Result<Vec<TurnRecord>> {
1006	validated_record_id(thread_id, "thread id")?;
1007	let mut out = self.list_all_turns()?;
1008	out.retain(\|turn\| turn.thread_id == thread_id);
1009	Ok(out)
1010	}
1011
1012	/// Every turn in the store, sorted by creation time. One directory scan;
1013	/// callers that need multiple threads' turns (boot recovery) use this
1014	/// instead of paying a full scan per thread (#3757).
1015	pub fn list_all_turns(&self) -> Result<Vec<TurnRecord>> {
1016	let mut out = Vec::new();
1017	let turns_dir = checked_existing_runtime_store_dir(&self.turns_dir)?;
1018	for entry in fs::read_dir(&turns_dir)
1019	.with_context(\|\| format!("Failed to read {}", turns_dir.display()))?
1020	{
1021	let entry = entry?;
1022	let path = entry.path();
1023	if path.extension().is_none_or(\|ext\| ext != "json") {
1024	continue;
1025	}
1026	let raw = read_store_file(&path)
1027	.with_context(\|\| format!("Failed to read {}", path.display()))?;
1028	let turn: TurnRecord = serde_json::from_str(&raw)
1029	.with_context(\|\| format!("Failed to parse {}", path.display()))?;
1030	if turn.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
1031	bail!(
1032	"Turn schema v{} is newer than supported v{}",
1033	turn.schema_version,
1034	CURRENT_RUNTIME_SCHEMA_VERSION
1035	);
1036	}
1037	out.push(turn);
1038	}
1039	out.sort_by_key(\|a\| a.created_at);
1040	Ok(out)
1041	}
1042
1043	pub fn list_items_for_turn(&self, turn_id: &str) -> Result<Vec<TurnItemRecord>> {
1044	validated_record_id(turn_id, "turn id")?;
1045	let mut out = Vec::new();
1046	let items_dir = checked_existing_runtime_store_dir(&self.items_dir)?;
1047	for entry in fs::read_dir(&items_dir)
1048	.with_context(\|\| format!("Failed to read {}", items_dir.display()))?
1049	{
1050	let entry = entry?;
1051	let path = entry.path();
1052	if path.extension().is_none_or(\|ext\| ext != "json") {
1053	continue;
1054	}
1055	let raw = read_store_file(&path)
1056	.with_context(\|\| format!("Failed to read {}", path.display()))?;
1057	let item: TurnItemRecord = serde_json::from_str(&raw)
1058	.with_context(\|\| format!("Failed to parse {}", path.display()))?;
1059	if item.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
1060	bail!(
1061	"Item schema v{} is newer than supported v{}",
1062	item.schema_version,
1063	CURRENT_RUNTIME_SCHEMA_VERSION
1064	);
1065	}
1066	if item.turn_id == turn_id {
1067	out.push(item);
1068	}
1069	}
1070	sort_turn_items_by_start(&mut out);
1071	Ok(out)
1072	}
1073
1074	pub fn list_items_for_turns_map(
1075	&self,
1076	turn_ids: &[String],
1077	) -> Result<HashMap<String, Vec<TurnItemRecord>>> {
1078	if turn_ids.is_empty() {
1079	return Ok(HashMap::new());
1080	}
1081
1082	for turn_id in turn_ids {
1083	validated_record_id(turn_id, "turn id")?;
1084	}
1085
1086	let wanted: HashSet<&str> = turn_ids.iter().map(String::as_str).collect();
1087	let mut out: HashMap<String, Vec<TurnItemRecord>> = HashMap::new();
1088	let items_dir = checked_existing_runtime_store_dir(&self.items_dir)?;
1089	for entry in fs::read_dir(&items_dir)
1090	.with_context(\|\| format!("Failed to read {}", items_dir.display()))?
1091	{
1092	let entry = entry?;
1093	let path = entry.path();
1094	if path.extension().is_none_or(\|ext\| ext != "json") {
1095	continue;
1096	}
1097	let raw = read_store_file(&path)
1098	.with_context(\|\| format!("Failed to read {}", path.display()))?;
1099	let item: TurnItemRecord = serde_json::from_str(&raw)
1100	.with_context(\|\| format!("Failed to parse {}", path.display()))?;
1101	if item.schema_version > CURRENT_RUNTIME_SCHEMA_VERSION {
1102	bail!(
1103	"Item schema v{} is newer than supported v{}",
1104	item.schema_version,
1105	CURRENT_RUNTIME_SCHEMA_VERSION
1106	);
1107	}
1108	if wanted.contains(item.turn_id.as_str()) {
1109	out.entry(item.turn_id.clone()).or_default().push(item);
1110	}
1111	}
1112
1113	for items in out.values_mut() {
1114	sort_turn_items_by_start(items);
1115	}
1116	Ok(out)
1117	}
1118
1119	pub async fn append_event(
1120	&self,
1121	thread_id: &str,
1122	turn_id: Option<&str>,
1123	item_id: Option<&str>,
1124	event: impl Into<String>,
1125	payload: Value,
1126	) -> Result<RuntimeEventRecord> {
1127	validated_record_id(thread_id, "thread id")?;
1128	if let Some(turn_id) = turn_id {
1129	validated_record_id(turn_id, "turn id")?;
1130	}
1131	if let Some(item_id) = item_id {
1132	validated_record_id(item_id, "item id")?;
1133	}
1134	let store = self.clone();
1135	let thread_id = thread_id.to_string();
1136	let turn_id = turn_id.map(ToString::to_string);
1137	let item_id = item_id.map(ToString::to_string);
1138	let event = event.into();
1139	tokio::task::spawn_blocking(move \|\| {
1140	store.append_event_transaction(
1141	thread_id,
1142	turn_id,
1143	item_id,
1144	event,
1145	payload,
1146	EVENT_TRANSACTION_LOCK_TIMEOUT,
1147	)
1148	})
1149	.await
1150	.context("Runtime event transaction worker failed")?
1151	}
1152
1153	fn append_event_transaction(
1154	&self,
1155	thread_id: String,
1156	turn_id: Option<String>,
1157	item_id: Option<String>,
1158	event: String,
1159	payload: Value,
1160	lock_timeout: Duration,
1161	) -> Result<RuntimeEventRecord> {
1162	let path = self.events_path(&thread_id)?;
1163	self.with_event_transaction(lock_timeout, \|\| {
1164	reject_symlinked_store_dir(&self.events_dir)?;
1165	repair_torn_event_log_tail(&path)?;
1166	let mut state = load_runtime_store_state(&self.state_path)?;
1167	let seq = state.next_seq;
1168	state.next_seq = seq
1169	.checked_add(1)
1170	.context("Runtime event sequence exhausted")?;
1171	write_json_atomic(&self.state_path, &state)?;
1172
1173	let record = RuntimeEventRecord {
1174	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
1175	seq,
1176	timestamp: Utc::now(),
1177	thread_id,
1178	turn_id,
1179	item_id,
1180	event,
1181	payload,
1182	};
1183
1184	let mut file = open_runtime_store_file(&path, "event append", \|options\| {
1185	options.create(true).append(true);
1186	})?;
1187	let rollback_file =
1188	open_runtime_store_file(&path, "Runtime event rollback", \|options\| {
1189	options.write(true);
1190	})?;
1191	validate_same_runtime_store_file_handles(&file, &rollback_file, &path)?;
1192	let original_len = file
1193	.metadata()
1194	.with_context(\|\| format!("Failed to inspect {}", path.display()))?
1195	.len();
1196	let mut line = serde_json::to_vec(&record)?;
1197	// A trailing newline is the commit marker. Startup removes a
1198	// parseable but unterminated tail without reusing its sequence.
1199	line.push(b'\n');
1200	let append_result = (\|\| -> std::io::Result<()> {
1201	file.write_all(&line)?;
1202	file.flush()?;
1203	#[cfg(test)]
1204	if take_test_event_append_fault(&record.thread_id, EventAppendTestFault::AfterFlush)
1205	{
1206	return Err(std::io::Error::other(
1207	"injected Runtime event failure after flush",
1208	));
1209	}
1210	file.sync_all()?;
1211	#[cfg(test)]
1212	if take_test_event_append_fault(&record.thread_id, EventAppendTestFault::AfterSync)
1213	{
1214	return Err(std::io::Error::other(
1215	"injected Runtime event failure after fsync",
1216	));
1217	}
1218	Ok(())
1219	})();
1220	if let Err(append_error) = append_result {
1221	// A failed flush/fsync can still leave the complete JSONL record
1222	// visible (or even durable). Roll back to the exact pre-append
1223	// offset and fsync that truncation before reporting a retryable
1224	// error. If rollback itself fails, classify the write as
1225	// indeterminate so callers never restore/retry and duplicate a
1226	// possibly committed terminal receipt.
1227	// The pre-opened rollback handle was identity-checked before
1228	// any bytes were written and stays live across this transaction.
1229	drop(file);
1230	let rollback_result =
1231	rollback_failed_event_append_handle(&rollback_file, original_len);
1232	let error = match rollback_result {
1233	Ok(()) => RuntimeEventAppendError {
1234	disposition: EventAppendFailureDisposition::RolledBack,
1235	append_error: append_error.to_string(),
1236	rollback_error: None,
1237	},
1238	Err(rollback_error) => RuntimeEventAppendError {
1239	disposition: EventAppendFailureDisposition::Indeterminate,
1240	append_error: append_error.to_string(),
1241	rollback_error: Some(rollback_error.to_string()),
1242	},
1243	};
1244	return Err(anyhow!(error));
1245	}
1246	Ok(record)
1247	})
1248	}
1249
1250	pub fn events_since(
1251	&self,
1252	thread_id: &str,
1253	since_seq: Option<u64>,
1254	) -> Result<Vec<RuntimeEventRecord>> {
1255	let path = self.events_path(thread_id)?;
1256	let Some(mut reader) = self.open_event_reader(thread_id)? else {
1257	return Ok(Vec::new());
1258	};
1259	let mut out = Vec::new();
1260	while let Some(event) = read_complete_event(&mut reader, &path)? {
1261	if let Some(since) = since_seq
1262	&& event.seq <= since
1263	{
1264	continue;
1265	}
1266	out.push(event);
1267	}
1268	Ok(out)
1269	}
1270
1271	fn publish_event_replay(
1272	&self,
1273	thread_id: &str,
1274	since_seq: Option<u64>,
1275	tail_limit: Option<usize>,
1276	base_tx: oneshot::Sender<std::result::Result<u64, String>>,
1277	batch_tx: mpsc::Sender<std::result::Result<Vec<RuntimeEventRecord>, String>>,
1278	) {
1279	let mut base_tx = Some(base_tx);
1280	let result = match tail_limit {
1281	Some(limit) => {
1282	self.publish_tail_event_replay(thread_id, since_seq, limit, &mut base_tx, &batch_tx)
1283	}
1284	None => self.publish_full_event_replay(thread_id, since_seq, &mut base_tx, &batch_tx),
1285	};
1286	if let Err(error) = result {
1287	let message = format!("{error:#}");
1288	if let Some(base_tx) = base_tx.take() {
1289	let _ = base_tx.send(Err(message));
1290	} else {
1291	let _ = batch_tx.blocking_send(Err(message));
1292	}
1293	}
1294	}
1295
1296	fn open_event_reader(&self, thread_id: &str) -> Result<Option<RuntimeEventReader>> {
1297	let path = self.events_path(thread_id)?;
1298	self.with_event_transaction(EVENT_TRANSACTION_LOCK_TIMEOUT, \|\| {
1299	reject_symlinked_store_dir(&self.events_dir)?;
1300	if !path.exists() {
1301	return Ok(None);
1302	}
1303	let file = open_runtime_store_file(&path, "Runtime event replay", \|options\| {
1304	options.read(true);
1305	})?;
1306	let committed_len = file
1307	.metadata()
1308	.with_context(\|\| format!("Failed to inspect {}", path.display()))?
1309	.len();
1310	Ok(Some(BufReader::new(file.take(committed_len))))
1311	})
1312	}
1313
1314	fn contains_event(&self, thread_id: &str, expected: &RuntimeEventMatch) -> Result<bool> {
1315	let Some(mut reader) = self.open_event_reader(thread_id)? else {
1316	return Ok(false);
1317	};
1318	let path = self.events_path(thread_id)?;
1319	while let Some(event) = read_complete_event(&mut reader, &path)? {
1320	let matches = match expected {
1321	RuntimeEventMatch::TurnCompleted { turn_id } => {
1322	event.event == "turn.completed"
1323	&& event.turn_id.as_deref() == Some(turn_id.as_str())
1324	}
1325	RuntimeEventMatch::DynamicTerminal { turn_id, call_id } => {
1326	matches!(
1327	event.event.as_str(),
1328	"tool_call.resolved" \| "tool_call.canceled" \| "tool_call.timeout"
1329	) && event.turn_id.as_deref() == Some(turn_id.as_str())
1330	&& event.payload.get("call_id").and_then(Value::as_str)
1331	== Some(call_id.as_str())
1332	}
1333	};
1334	if matches {
1335	return Ok(true);
1336	}
1337	}
1338	Ok(false)
1339	}
1340
1341	fn publish_full_event_replay(
1342	&self,
1343	thread_id: &str,
1344	since_seq: Option<u64>,
1345	base_tx: &mut Option<oneshot::Sender<std::result::Result<u64, String>>>,
1346	batch_tx: &mpsc::Sender<std::result::Result<Vec<RuntimeEventRecord>, String>>,
1347	) -> Result<()> {
1348	let Some(mut reader) = self.open_event_reader(thread_id)? else {
1349	if let Some(base_tx) = base_tx.take() {
1350	let _ = base_tx.send(Ok(since_seq.unwrap_or(0)));
1351	}
1352	return Ok(());
1353	};
1354	if base_tx
1355	.take()
1356	.is_some_and(\|base_tx\| base_tx.send(Ok(since_seq.unwrap_or(0))).is_err())
1357	{
1358	return Ok(());
1359	}
1360
1361	let path = self.events_path(thread_id)?;
1362	let mut batch = Vec::with_capacity(RUNTIME_EVENT_REPLAY_BATCH_SIZE);
1363	while let Some(event) = read_complete_event(&mut reader, &path)? {
1364	if since_seq.is_some_and(\|since\| event.seq <= since) {
1365	continue;
1366	}
1367	batch.push(event);
1368	if batch.len() == RUNTIME_EVENT_REPLAY_BATCH_SIZE {
1369	if batch_tx.blocking_send(Ok(batch)).is_err() {
1370	return Ok(());
1371	}
1372	batch = Vec::with_capacity(RUNTIME_EVENT_REPLAY_BATCH_SIZE);
1373	}
1374	}
1375	if !batch.is_empty() {
1376	let _ = batch_tx.blocking_send(Ok(batch));
1377	}
1378	Ok(())
1379	}
1380
1381	fn publish_tail_event_replay(
1382	&self,
1383	thread_id: &str,
1384	since_seq: Option<u64>,
1385	tail_limit: usize,
1386	base_tx: &mut Option<oneshot::Sender<std::result::Result<u64, String>>>,
1387	batch_tx: &mpsc::Sender<std::result::Result<Vec<RuntimeEventRecord>, String>>,
1388	) -> Result<()> {
1389	let Some(mut reader) = self.open_event_reader(thread_id)? else {
1390	if let Some(base_tx) = base_tx.take() {
1391	let _ = base_tx.send(Ok(since_seq.unwrap_or(0)));
1392	}
1393	return Ok(());
1394	};
1395	let path = self.events_path(thread_id)?;
1396	let mut base_seq = since_seq.unwrap_or(0);
1397	let mut tail = VecDeque::with_capacity(tail_limit.min(RUNTIME_EVENT_REPLAY_BATCH_SIZE));
1398	while let Some(event) = read_complete_event(&mut reader, &path)? {
1399	if since_seq.is_some_and(\|since\| event.seq <= since) {
1400	continue;
1401	}
1402	if tail_limit == 0 {
1403	base_seq = event.seq;
1404	continue;
1405	}
1406	tail.push_back(event);
1407	if tail.len() > tail_limit
1408	&& let Some(omitted) = tail.pop_front()
1409	{
1410	base_seq = omitted.seq;
1411	}
1412	}
1413	if base_tx
1414	.take()
1415	.is_some_and(\|base_tx\| base_tx.send(Ok(base_seq)).is_err())
1416	{
1417	return Ok(());
1418	}
1419	while !tail.is_empty() {
1420	let take = tail.len().min(RUNTIME_EVENT_REPLAY_BATCH_SIZE);
1421	let batch = tail.drain(..take).collect::<Vec<_>>();
1422	if batch_tx.blocking_send(Ok(batch)).is_err() {
1423	return Ok(());
1424	}
1425	}
1426	Ok(())
1427	}
1428
1429	pub async fn current_seq(&self) -> Result<u64> {
1430	let store = self.clone();
1431	tokio::task::spawn_blocking(move \|\| {
1432	store.with_event_transaction(EVENT_TRANSACTION_LOCK_TIMEOUT, \|\| {
1433	Ok(load_runtime_store_state(&store.state_path)?
1434	.next_seq
1435	.saturating_sub(1))
1436	})
1437	})
1438	.await
1439	.context("Runtime event cursor worker failed")?
1440	}
1441	}
1442
1443	#[derive(Debug, Clone)]
1444	pub struct RuntimeThreadManagerConfig {
1445	pub data_dir: PathBuf,
1446	pub task_data_dir: PathBuf,
1447	pub max_active_threads: usize,
1448	}
1449
1450	impl RuntimeThreadManagerConfig {
1451	#[must_use]
1452	pub fn from_task_data_dir(task_data_dir: PathBuf) -> Self {
1453	let data_dir = std::env::var("CODEWHALE_RUNTIME_DIR")
1454	.or_else(\|_\| std::env::var("DEEPSEEK_RUNTIME_DIR"))
1455	.ok()
1456	.filter(\|override_dir\| !override_dir.trim().is_empty())
1457	.map_or_else(\|\| task_data_dir.join("runtime"), PathBuf::from);
1458	Self {
1459	data_dir,
1460	task_data_dir,
1461	max_active_threads: MAX_ACTIVE_THREADS_DEFAULT,
1462	}
1463	}
1464	}
1465
1466	/// Visibility filter for `list_threads`. Default is `ActiveOnly`. The runtime
1467	/// API exposes this as the combination of `include_archived` and
1468	/// `archived_only` query params (see `runtime_api.rs`); whalescale#260 / #563.
1469	#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1470	pub enum ThreadListFilter {
1471	/// Only `archived = false` threads. The original default.
1472	#[default]
1473	ActiveOnly,
1474	/// Active and archived threads, sorted as the store returns them.
1475	IncludeArchived,
1476	/// Only `archived = true` threads.
1477	ArchivedOnly,
1478	}
1479
1480	#[derive(Debug, Clone, Serialize, Deserialize, Default)]
1481	pub struct CreateThreadRequest {
1482	pub model: Option<String>,
1483	/// Generic provider kind or, for legacy clients, an exact provider id.
1484	#[serde(default)]
1485	pub model_provider: Option<String>,
1486	/// Exact configured provider key. Takes precedence over `model_provider`.
1487	#[serde(default)]
1488	pub model_provider_id: Option<String>,
1489	pub workspace: Option<PathBuf>,
1490	pub mode: Option<String>,
1491	#[serde(default)]
1492	pub permission_posture: Option<String>,
1493	pub allow_shell: Option<bool>,
1494	pub trust_mode: Option<bool>,
1495	pub auto_approve: Option<bool>,
1496	#[serde(default)]
1497	pub archived: bool,
1498	#[serde(default)]
1499	pub system_prompt: Option<String>,
1500	#[serde(default)]
1501	pub task_id: Option<String>,
1502	#[serde(default)]
1503	pub dynamic_tools: Vec<DynamicToolSpec>,
1504	#[serde(default)]
1505	pub environments: Vec<TurnEnvironmentParams>,
1506	}
1507
1508	/// Mutable fields accepted by `PATCH /v1/threads/{id}`.
1509	///
1510	/// Each field is optional — missing means "no change". Extended in v0.8.10
1511	/// (#562, whalescale#256) so the UI can flip persistent thread state without
1512	/// having to recreate a thread or pass per-turn overrides on every send.
1513	#[derive(Debug, Clone, Serialize, Deserialize, Default)]
1514	pub struct UpdateThreadRequest {
1515	pub archived: Option<bool>,
1516	pub allow_shell: Option<bool>,
1517	pub trust_mode: Option<bool>,
1518	pub auto_approve: Option<bool>,
1519	pub model: Option<String>,
1520	pub mode: Option<String>,
1521	pub permission_posture: Option<String>,
1522	pub title: Option<String>,
1523	pub system_prompt: Option<String>,
1524	pub workspace: Option<PathBuf>,
1525	}
1526
1527	#[derive(Debug, Clone, Serialize, Deserialize, Default)]
1528	pub struct StartTurnRequest {
1529	pub prompt: String,
1530	#[serde(default)]
1531	pub input_summary: Option<String>,
1532	pub model: Option<String>,
1533	pub mode: Option<String>,
1534	#[serde(default)]
1535	pub permission_posture: Option<String>,
1536	pub allow_shell: Option<bool>,
1537	pub trust_mode: Option<bool>,
1538	pub auto_approve: Option<bool>,
1539	#[serde(default)]
1540	pub dynamic_tools: Vec<DynamicToolSpec>,
1541	#[serde(default)]
1542	pub environment_id: Option<String>,
1543	}
1544
1545	#[derive(Debug, Clone, Serialize, Deserialize)]
1546	pub struct SteerTurnRequest {
1547	pub prompt: String,
1548	}
1549
1550	#[derive(Debug, Clone, Serialize, Deserialize, Default)]
1551	pub struct CompactThreadRequest {
1552	#[serde(default)]
1553	pub reason: Option<String>,
1554	}
1555
1556	#[derive(Debug, Clone, Serialize, Deserialize)]
1557	pub struct ThreadDetail {
1558	pub thread: ThreadRecord,
1559	pub turns: Vec<TurnRecord>,
1560	pub items: Vec<TurnItemRecord>,
1561	pub latest_seq: u64,
1562	/// Approval prompts that are still waiting for a decision. These are part
1563	/// of the canonical snapshot so clients can recover attention UI after a
1564	/// tab reload without replaying events older than `latest_seq`.
1565	#[serde(default)]
1566	pub pending_approvals: Vec<PendingApprovalRequest>,
1567	/// User-input prompts that are still waiting for answers. As with
1568	/// approvals, the snapshot is authoritative across client reconnects.
1569	#[serde(default)]
1570	pub pending_user_inputs: Vec<PendingUserInputRequest>,
1571	/// Client-executed dynamic tool calls that are still waiting for a result.
1572	/// Keeping the typed request in the canonical snapshot lets an external
1573	/// Runtime client reload from `latest_seq` without stranding a call whose
1574	/// `tool_call.requested` event is already behind that cursor.
1575	#[serde(default)]
1576	pub pending_dynamic_tool_calls: Vec<DynamicToolCallParams>,
1577	}
1578
1579	#[derive(Debug, Clone, Serialize, Deserialize)]
1580	pub struct PendingApprovalRequest {
1581	pub id: String,
1582	pub turn_id: String,
1583	pub tool_name: String,
1584	pub description: String,
1585	#[serde(default, skip_serializing_if = "Option::is_none")]
1586	pub intent_summary: Option<String>,
1587	}
1588
1589	#[derive(Debug, Clone, Serialize, Deserialize)]
1590	pub struct PendingUserInputRequest {
1591	pub id: String,
1592	pub turn_id: String,
1593	pub request: crate::tools::user_input::UserInputRequest,
1594	}
1595
1596	/// Aggregation key for `aggregate_usage`. Whalescale#261 / #564.
1597	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1598	pub enum UsageGroupBy {
1599	Day,
1600	Model,
1601	Provider,
1602	Thread,
1603	}
1604
1605	#[derive(Debug, Clone, Default, Serialize)]
1606	pub struct UsageTotals {
1607	pub input_tokens: u64,
1608	pub output_tokens: u64,
1609	pub cached_tokens: u64,
1610	pub reasoning_tokens: u64,
1611	pub reasoning_replay_tokens: u64,
1612	pub cache_write_tokens: u64,
1613	pub cost_usd: f64,
1614	/// Authoritative USD coverage for this aggregate. `cost_usd` is a priced
1615	/// subtotal whenever `unpriced_turns > 0`.
1616	pub priced_turns: u64,
1617	pub unpriced_turns: u64,
1618	pub nonmetered_turns: u64,
1619	pub cost_complete: bool,
1620	pub unpriced_reasons: std::collections::BTreeSet<String>,
1621	pub unpriced_classes: std::collections::BTreeSet<String>,
1622	pub pricing_provenances: std::collections::BTreeSet<String>,
1623	pub live_pricing_defects: std::collections::BTreeSet<String>,
1624	pub live_pricing_unusable_defects: std::collections::BTreeSet<String>,
1625	pub route_receipts: std::collections::BTreeSet<String>,
1626	/// Provider-call receipts lost from a bounded fallback journal. A non-zero
1627	/// value always makes `cost_complete` false.
1628	pub dropped_usage_records: u64,
1629	/// Number of provider-call usage records (parent turns plus child and
1630	/// compaction calls), including zero-token audited calls.
1631	pub turns: u64,
1632	}
1633
1634	#[derive(Debug, Clone, Default, Serialize)]
1635	pub struct UsageBucket {
1636	pub key: String,
1637	pub input_tokens: u64,
1638	pub output_tokens: u64,
1639	pub cached_tokens: u64,
1640	pub reasoning_tokens: u64,
1641	pub reasoning_replay_tokens: u64,
1642	pub cache_write_tokens: u64,
1643	pub cost_usd: f64,
1644	pub priced_turns: u64,
1645	pub unpriced_turns: u64,
1646	pub nonmetered_turns: u64,
1647	pub cost_complete: bool,
1648	pub unpriced_reasons: std::collections::BTreeSet<String>,
1649	pub unpriced_classes: std::collections::BTreeSet<String>,
1650	pub pricing_provenances: std::collections::BTreeSet<String>,
1651	pub live_pricing_defects: std::collections::BTreeSet<String>,
1652	pub live_pricing_unusable_defects: std::collections::BTreeSet<String>,
1653	pub route_receipts: std::collections::BTreeSet<String>,
1654	pub dropped_usage_records: u64,
1655	/// Provider-call usage records contributing to this bucket.
1656	pub turns: u64,
1657	}
1658
1659	#[derive(Debug, Clone, Serialize)]
1660	pub struct UsageAggregation {
1661	pub since: Option<DateTime<Utc>>,
1662	pub until: Option<DateTime<Utc>>,
1663	pub group_by: String,
1664	pub totals: UsageTotals,
1665	pub buckets: Vec<UsageBucket>,
1666	}
1667
1668	fn accumulate_runtime_cost_coverage(
1669	audit: Option<&crate::pricing::TurnCostAudit>,
1670	priced_turns: &mut u64,
1671	unpriced_turns: &mut u64,
1672	nonmetered_turns: &mut u64,
1673	reasons: &mut std::collections::BTreeSet<String>,
1674	provenances: &mut std::collections::BTreeSet<String>,
1675	) {
1676	let Some(audit) = audit else {
1677	unpriced_turns = (unpriced_turns).saturating_add(1);
1678	reasons.insert("unknown_provider_route".to_string());
1679	return;
1680	};
1681	if let Some(provenance) = audit.provenance.as_ref() {
1682	provenances.insert(provenance.label().to_string());
1683	}
1684	if !audit.counts_toward_money_coverage() {
1685	nonmetered_turns = (nonmetered_turns).saturating_add(1);
1686	return;
1687	}
1688	if audit.usd_priced {
1689	priced_turns = (priced_turns).saturating_add(1);
1690	} else {
1691	unpriced_turns = (unpriced_turns).saturating_add(1);
1692	if let Some(reason) = audit.unpriced_reason {
1693	reasons.insert(reason.label().to_string());
1694	}
1695	}
1696	}
1697
1698	fn accumulate_runtime_cost_details(
1699	audit: Option<&crate::pricing::TurnCostAudit>,
1700	unpriced_classes: &mut std::collections::BTreeSet<String>,
1701	live_pricing_defects: &mut std::collections::BTreeSet<String>,
1702	live_pricing_unusable_defects: &mut std::collections::BTreeSet<String>,
1703	) {
1704	let Some(audit) = audit else {
1705	return;
1706	};
1707	unpriced_classes.extend(
1708	audit
1709	.unpriced_classes
1710	.iter()
1711	.map(\|class\| class.label().to_string()),
1712	);
1713	if let Some(defect) = audit.live_pricing_defect.as_ref() {
1714	if audit.estimate.is_some() {
1715	live_pricing_defects.insert(defect.label().to_string());
1716	} else {
1717	live_pricing_unusable_defects.insert(defect.label().to_string());
1718	}
1719	}
1720	}
1721
1722	fn saturating_add_usd(total: &mut f64, delta: f64) {
1723	total = crate::pricing::CostEstimate::usd_only(total)
1724	.saturating_add(crate::pricing::CostEstimate::usd_only(delta))
1725	.usd;
1726	}
1727
1728	fn runtime_usage_bucket_key(
1729	group_by: UsageGroupBy,
1730	route: Option<&EffectiveRouteEnvelope>,
1731	turn: &TurnRecord,
1732	thread: &ThreadRecord,
1733	) -> String {
1734	match group_by {
1735	UsageGroupBy::Day => route
1736	.map_or(turn.created_at, \|route\| route.dispatched_at)
1737	.format("%Y-%m-%d")
1738	.to_string(),
1739	UsageGroupBy::Model => crate::cost_status::sanitize_persisted_route_label(
1740	route
1741	.map(\|route\| route.model.as_str())
1742	.or_else(\|\| {
1743	turn.effective_model
1744	.as_deref()
1745	.filter(\|model\| !model.trim().is_empty())
1746	})
1747	.unwrap_or(&thread.model),
1748	),
1749	UsageGroupBy::Provider => crate::cost_status::sanitize_persisted_route_label(
1750	route
1751	.map(\|route\| {
1752	if route.provider_identity.trim().is_empty() {
1753	route.provider.as_str()
1754	} else {
1755	route.provider_identity.as_str()
1756	}
1757	})
1758	.or_else(\|\| turn.effective_provider_label())
1759	.unwrap_or("unknown"),
1760	),
1761	UsageGroupBy::Thread => thread.id.clone(),
1762	}
1763	}
1764
1765	fn accumulate_runtime_usage_record(
1766	totals: &mut UsageTotals,
1767	buckets: &mut std::collections::BTreeMap<String, UsageBucket>,
1768	group_by: UsageGroupBy,
1769	route: Option<&EffectiveRouteEnvelope>,
1770	usage: &Usage,
1771	turn: &TurnRecord,
1772	thread: &ThreadRecord,
1773	) {
1774	let classes = crate::pricing::token_usage_for_pricing(usage);
1775	let reasoning = u64::from(usage.reasoning_tokens.unwrap_or(0));
1776	let reasoning_replay = u64::from(usage.reasoning_replay_tokens.unwrap_or(0));
1777	let audit = route.map(\|route\| route.audit(usage));
1778	let cost = audit
1779	.as_ref()
1780	.filter(\|audit\| audit.usd_priced)
1781	.and_then(\|audit\| audit.estimate)
1782	.map_or(0.0, \|estimate\| estimate.usd);
1783	let receipt = route.zip(audit.as_ref()).map(\|(route, audit)\| {
1784	crate::cost_status::effective_route_usage_receipt(route, audit, usage)
1785	});
1786
1787	totals.input_tokens = totals.input_tokens.saturating_add(classes.input);
1788	totals.output_tokens = totals.output_tokens.saturating_add(classes.output);
1789	totals.cached_tokens = totals.cached_tokens.saturating_add(classes.cache_read);
1790	totals.reasoning_tokens = totals.reasoning_tokens.saturating_add(reasoning);
1791	totals.reasoning_replay_tokens = totals
1792	.reasoning_replay_tokens
1793	.saturating_add(reasoning_replay);
1794	totals.cache_write_tokens = totals
1795	.cache_write_tokens
1796	.saturating_add(classes.cache_write);
1797	saturating_add_usd(&mut totals.cost_usd, cost);
1798	accumulate_runtime_cost_coverage(
1799	audit.as_ref(),
1800	&mut totals.priced_turns,
1801	&mut totals.unpriced_turns,
1802	&mut totals.nonmetered_turns,
1803	&mut totals.unpriced_reasons,
1804	&mut totals.pricing_provenances,
1805	);
1806	accumulate_runtime_cost_details(
1807	audit.as_ref(),
1808	&mut totals.unpriced_classes,
1809	&mut totals.live_pricing_defects,
1810	&mut totals.live_pricing_unusable_defects,
1811	);
1812	if let Some(receipt) = receipt.as_ref() {
1813	totals.route_receipts.insert(receipt.clone());
1814	}
1815	totals.turns = totals.turns.saturating_add(1);
1816
1817	let key = runtime_usage_bucket_key(group_by, route, turn, thread);
1818	let bucket = buckets.entry(key.clone()).or_insert_with(\|\| UsageBucket {
1819	key,
1820	..UsageBucket::default()
1821	});
1822	bucket.input_tokens = bucket.input_tokens.saturating_add(classes.input);
1823	bucket.output_tokens = bucket.output_tokens.saturating_add(classes.output);
1824	bucket.cached_tokens = bucket.cached_tokens.saturating_add(classes.cache_read);
1825	bucket.reasoning_tokens = bucket.reasoning_tokens.saturating_add(reasoning);
1826	bucket.reasoning_replay_tokens = bucket
1827	.reasoning_replay_tokens
1828	.saturating_add(reasoning_replay);
1829	bucket.cache_write_tokens = bucket
1830	.cache_write_tokens
1831	.saturating_add(classes.cache_write);
1832	saturating_add_usd(&mut bucket.cost_usd, cost);
1833	accumulate_runtime_cost_coverage(
1834	audit.as_ref(),
1835	&mut bucket.priced_turns,
1836	&mut bucket.unpriced_turns,
1837	&mut bucket.nonmetered_turns,
1838	&mut bucket.unpriced_reasons,
1839	&mut bucket.pricing_provenances,
1840	);
1841	accumulate_runtime_cost_details(
1842	audit.as_ref(),
1843	&mut bucket.unpriced_classes,
1844	&mut bucket.live_pricing_defects,
1845	&mut bucket.live_pricing_unusable_defects,
1846	);
1847	if let Some(receipt) = receipt {
1848	bucket.route_receipts.insert(receipt);
1849	}
1850	bucket.turns = bucket.turns.saturating_add(1);
1851	}
1852
1853	fn usage_timestamp_in_range(
1854	timestamp: DateTime<Utc>,
1855	since: Option<DateTime<Utc>>,
1856	until: Option<DateTime<Utc>>,
1857	) -> bool {
1858	since.is_none_or(\|lower\| timestamp >= lower) && until.is_none_or(\|upper\| timestamp <= upper)
1859	}
1860
1861	fn accumulate_truncated_runtime_usage(
1862	totals: &mut UsageTotals,
1863	buckets: &mut std::collections::BTreeMap<String, UsageBucket>,
1864	group_by: UsageGroupBy,
1865	dropped: u64,
1866	turn: &TurnRecord,
1867	thread: &ThreadRecord,
1868	) {
1869	if dropped == 0 {
1870	return;
1871	}
1872	totals.dropped_usage_records = totals.dropped_usage_records.saturating_add(dropped);
1873	totals.unpriced_turns = totals.unpriced_turns.saturating_add(dropped);
1874	totals.turns = totals.turns.saturating_add(dropped);
1875	totals
1876	.unpriced_reasons
1877	.insert("runtime_usage_journal_truncated".to_string());
1878
1879	let key = match group_by {
1880	UsageGroupBy::Day => turn.created_at.format("%Y-%m-%d").to_string(),
1881	UsageGroupBy::Model \| UsageGroupBy::Provider => "unknown-truncated".to_string(),
1882	UsageGroupBy::Thread => thread.id.clone(),
1883	};
1884	let bucket = buckets.entry(key.clone()).or_insert_with(\|\| UsageBucket {
1885	key,
1886	..UsageBucket::default()
1887	});
1888	bucket.dropped_usage_records = bucket.dropped_usage_records.saturating_add(dropped);
1889	bucket.unpriced_turns = bucket.unpriced_turns.saturating_add(dropped);
1890	bucket.turns = bucket.turns.saturating_add(dropped);
1891	bucket
1892	.unpriced_reasons
1893	.insert("runtime_usage_journal_truncated".to_string());
1894	}
1895
1896	fn resolve_runtime_thread_route(
1897	config: &Config,
1898	provider: ApiProvider,
1899	model_selector: Option<&str>,
1900	) -> Result<ResolvedRuntimeRoute> {
1901	resolve_runtime_route(config, provider, model_selector)
1902	.map_err(\|reason\| anyhow!("Failed to resolve runtime thread route: {reason}"))
1903	}
1904
1905	fn resolve_runtime_thread_route_for_identity(
1906	config: &Config,
1907	identity: &ProviderIdentity,
1908	model_selector: Option<&str>,
1909	) -> Result<ResolvedRuntimeRoute> {
1910	resolve_runtime_route_for_identity(config, identity, model_selector)
1911	.map_err(\|reason\| anyhow!("Failed to resolve runtime thread route: {reason}"))
1912	}
1913
1914	fn runtime_compaction_config(
1915	provider: ApiProvider,
1916	model: &str,
1917	route_limits: Option<codewhale_config::route::RouteLimits>,
1918	auto_compact: bool,
1919	auto_compact_explicit: bool,
1920	threshold_percent: f64,
1921	) -> CompactionConfig {
1922	CompactionConfig {
1923	enabled: if auto_compact_explicit {
1924	auto_compact
1925	} else {
1926	auto_compact_default_for_route(provider, model, route_limits)
1927	},
1928	model: model.to_string(),
1929	token_threshold: compaction_threshold_for_route_at_percent(
1930	provider,
1931	model,
1932	route_limits,
1933	threshold_percent,
1934	),
1935	effective_context_window: Some(route_context_window_tokens(provider, model, route_limits)),
1936	..Default::default()
1937	}
1938	}
1939
1940	#[derive(Debug, Clone)]
1941	struct ActiveTurnState {
1942	turn_id: String,
1943	interrupt_requested: bool,
1944	}
1945
1946	#[derive(Debug, Clone, Copy)]
1947	enum ClaimedTurnKind {
1948	Message,
1949	Compaction,
1950	}
1951
1952	impl ClaimedTurnKind {
1953	const fn label(self) -> &'static str {
1954	match self {
1955	Self::Message => "turn",
1956	Self::Compaction => "compaction turn",
1957	}
1958	}
1959	}
1960
1961	#[derive(Clone)]
1962	struct ActiveThreadState {
1963	engine: EngineHandle,
1964	active_turn: Option<ActiveTurnState>,
1965	route_identity: ProviderIdentity,
1966	route_model: String,
1967	/// Real engines client-preflight before an in-progress record is written.
1968	/// Explicitly injected test engines own their client seam.
1969	client_preflight_required: bool,
1970	}
1971
1972	#[derive(Default)]
1973	struct ActiveThreads {
1974	engines: HashMap<String, ActiveThreadState>,
1975	lru: VecDeque<String>,
1976	}
1977
1978	pub type SharedRuntimeThreadManager = Arc<RuntimeThreadManager>;
1979
1980	#[derive(Clone)]
1981	struct RecoveredTurnReceipt {
1982	turn: TurnRecord,
1983	unresolved_dynamic_tools: Vec<DynamicToolCallParams>,
1984	}
1985
1986	/// Manages active engine threads, lifecycle, and event persistence.
1987	///
1988	/// # Lock ordering invariant
1989	///
1990	/// Runtime state uses eight lock classes:
1991	/// - `RuntimeThreadManager::engine_load` — serializes cache-miss engine builds.
1992	/// It may cross awaits and is always acquired before `active`.
1993	/// - `RuntimeThreadManager::event_emit` — preserves append-to-broadcast event
1994	/// order and is only acquired after all record/engine guards are released.
1995	/// - `RuntimeThreadManager::projection_locks` — one async lock per thread,
1996	/// held while a streamed item checkpoint and its event are published or
1997	/// while a terminal turn projection, receipt, and active-claim cleanup are
1998	/// published, or while a snapshot captures its cursor and reads projections.
1999	/// - `RuntimeThreadManager::recovery_flush` — serializes deferred receipt
2000	/// reconciliation before it acquires a projection lock and `event_emit`.
2001	/// - the Runtime event-file transaction lock — serializes writes across processes.
2002	/// - `RuntimeThreadStore::thread_mutation` — synchronizes short, synchronous
2003	/// thread-record load-modify-save transactions and never crosses `.await`.
2004	/// - `RuntimeThreadStore::turn_mutation` — does the same for turn records.
2005	/// - `RuntimeThreadManager::active` — protects the set of loaded engine handles.
2006	///
2007	/// `state` is never held with `active`, either record-mutation guard, or
2008	/// `engine_load`. Streaming projection publication acquires its per-thread
2009	/// projection lock before `event_emit`, which acquires `state`; snapshots
2010	/// acquire only the projection lock and then `state`. All guards are released
2011	/// before returning. All
2012	/// `emit_event` calls happen after `active`, `thread_mutation`, and
2013	/// `turn_mutation` have been released. When record and engine state must change
2014	/// atomically, acquire `active` before the applicable record-mutation guard and
2015	/// release both before awaiting.
2016	#[derive(Clone)]
2017	pub struct RuntimeThreadManager {
2018	config: Arc<parking_lot::RwLock<Config>>,
2019	workspace: PathBuf,
2020	plugin_registry: Option<Arc<crate::plugins::PluginRegistry>>,
2021	store: RuntimeThreadStore,
2022	engine_load: Arc<Mutex<()>>,
2023	active: Arc<Mutex<ActiveThreads>>,
2024	event_emit: Arc<Mutex<()>>,
2025	projection_locks: Arc<parking_lot::Mutex<HashMap<String, Arc<Mutex<()>>>>>,
2026	event_tx: broadcast::Sender<RuntimeEventRecord>,
2027	manager_cfg: RuntimeThreadManagerConfig,
2028	cancel_token: CancellationToken,
2029	task_manager: Arc<parking_lot::Mutex<Option<crate::task_manager::SharedTaskManager>>>,
2030	automations:
2031	Arc<parking_lot::Mutex<Option<crate::automation_manager::SharedAutomationManager>>>,
2032	pending_approvals: Arc<parking_lot::Mutex<HashMap<String, PendingApprovalEntry>>>,
2033	pending_user_inputs: Arc<parking_lot::Mutex<HashMap<(String, String), PendingUserInputEntry>>>,
2034	pending_dynamic_tools: Arc<parking_lot::Mutex<HashMap<String, PendingDynamicToolEntry>>>,
2035	recovery_receipts: Arc<parking_lot::Mutex<HashMap<String, Vec<RecoveredTurnReceipt>>>>,
2036	recovery_flush: Arc<Mutex<()>>,
2037	#[cfg(test)]
2038	snapshot_test_hook: Arc<parking_lot::Mutex<Option<mpsc::UnboundedSender<SnapshotTestPoint>>>>,
2039	}
2040
2041	#[cfg(test)]
2042	pub(crate) struct SnapshotTestPoint {
2043	pub thread_id: String,
2044	pub latest_seq: u64,
2045	pub resume: oneshot::Sender<()>,
2046	}
2047
2048	/// Helper types for `seed_thread_from_messages` — intermediate representation
2049	/// of a turn being built from session messages before persisting as items.
2050	///
2051	/// A single content block extracted from an assistant message.
2052	enum SeedItem {
2053	Text(String),
2054	Thinking(String),
2055	ToolUse {
2056	id: String,
2057	name: String,
2058	input: serde_json::Value,
2059	},
2060	ToolResult {
2061	tool_use_id: String,
2062	content: String,
2063	is_error: bool,
2064	content_blocks: Option<Vec<serde_json::Value>>,
2065	},
2066	}
2067
2068	/// A turn being assembled from session messages.
2069	struct TurnSeed {
2070	user_text: String,
2071	items: Vec<SeedItem>,
2072	}
2073
2074	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2075	enum RuntimeApprovalDecision {
2076	ApproveTool,
2077	DenyTool,
2078	RetryWithFullAccess,
2079	}
2080
2081	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2082	pub enum ExternalApprovalDecision {
2083	Allow { remember: bool },
2084	Deny { remember: bool },
2085	}
2086
2087	struct PendingApprovalEntry {
2088	thread_id: String,
2089	request: PendingApprovalRequest,
2090	sender: oneshot::Sender<ExternalApprovalDecision>,
2091	}
2092
2093	struct PendingUserInputEntry {
2094	request: PendingUserInputRequest,
2095	/// A request remains snapshot-visible while its winner appends the
2096	/// secret-free terminal receipt. This prevents a snapshot cursor from
2097	/// observing neither the pending prompt nor its settlement event.
2098	settling: bool,
2099	settlement_tx: watch::Sender<u64>,
2100	/// An append whose rollback failed may or may not be durable. Never send
2101	/// the answer or allow a retry in that state: either could disclose or
2102	/// duplicate a response whose receipt cannot be established safely.
2103	indeterminate: bool,
2104	}
2105
2106	enum PendingUserInputClaim {
2107	Claimed(PendingUserInputRequest),
2108	Settling,
2109	Indeterminate,
2110	Missing,
2111	}
2112
2113	enum UserInputTerminalOutcome {
2114	Answered(crate::tools::user_input::UserInputResponse),
2115	Canceled { terminal: bool },
2116	}
2117
2118	struct PendingDynamicToolEntry {
2119	params: DynamicToolCallParams,
2120	/// Present while the call can still be claimed by result delivery,
2121	/// timeout, or turn termination. The entry remains in the registry after
2122	/// the winner takes this sender so snapshots continue to advertise the
2123	/// request until its terminal receipt is durably appended.
2124	sender: Option<oneshot::Sender<DynamicToolCallResult>>,
2125	settlement_tx: watch::Sender<u64>,
2126	indeterminate: bool,
2127	}
2128
2129	struct ClaimedDynamicToolSettlement {
2130	params: DynamicToolCallParams,
2131	sender: oneshot::Sender<DynamicToolCallResult>,
2132	settlement_tx: watch::Sender<u64>,
2133	}
2134
2135	enum PendingDynamicToolClaim {
2136	Claimed(ClaimedDynamicToolSettlement),
2137	Settling(watch::Receiver<u64>),
2138	Indeterminate,
2139	Missing,
2140	}
2141
2142	enum DynamicToolTerminalOutcome {
2143	Resolved(DynamicToolCallResult),
2144	Canceled {
2145	reason: &'static str,
2146	terminal: bool,
2147	},
2148	Timeout {
2149	timeout: Duration,
2150	},
2151	}
2152
2153	struct DynamicToolSettlementAck {
2154	result_accepted: bool,
2155	}
2156
2157	impl RuntimeThreadManager {
2158	/// Helper to read the current config under RwLock.
2159	pub(crate) fn read_config(&self) -> parking_lot::RwLockReadGuard<'_, Config> {
2160	self.config.read()
2161	}
2162
2163	fn resolved_route_for_thread(
2164	&self,
2165	config: &Config,
2166	thread: &ThreadRecord,
2167	) -> Result<ResolvedRuntimeRoute> {
2168	let provider_identity = self.provider_identity_for_thread(config, thread)?;
2169	if !thread.model.trim().eq_ignore_ascii_case("auto") {
2170	return resolve_runtime_thread_route_for_identity(
2171	config,
2172	&provider_identity,
2173	Some(&thread.model),
2174	);
2175	}
2176
2177	let mut thread_config = config.clone();
2178	thread_config.scope_to_provider_identity(&provider_identity);
2179
2180	let restored = self
2181	.store
2182	.list_turns_for_thread(&thread.id)?
2183	.into_iter()
2184	.rev()
2185	.find_map(\|turn\| {
2186	let model = turn.effective_model?.trim().to_string();
2187	let provider_kind = turn
2188	.effective_provider
2189	.filter(\|provider\| !provider.trim().is_empty());
2190	// Preserve an explicitly empty additive id so malformed
2191	// imported receipts fail closed instead of becoming an
2192	// id-less legacy custom route.
2193	let provider_id = turn.effective_provider_id;
2194	((provider_kind.is_some() \|\| provider_id.is_some()) && !model.is_empty())
2195	.then_some((provider_kind, provider_id, model))
2196	});
2197	match restored {
2198	Some((restored_kind, restored_id, model)) => {
2199	let identity = thread_config
2200	.resolve_persisted_provider_identity(
2201	restored_kind.as_deref(),
2202	restored_id.as_deref(),
2203	)
2204	.map_err(\|reason\| anyhow!(reason))?;
2205	resolve_runtime_thread_route_for_identity(config, &identity, Some(&model))
2206	}
2207	None => resolve_runtime_thread_route_for_identity(config, &provider_identity, None),
2208	}
2209	}
2210
2211	fn provider_identity_for_thread(
2212	&self,
2213	config: &Config,
2214	thread: &ThreadRecord,
2215	) -> Result<ProviderIdentity> {
2216	let has_persisted_route = thread
2217	.model_provider
2218	.as_deref()
2219	.is_some_and(\|provider\| !provider.trim().is_empty())
2220	\|\| thread.model_provider_id.is_some();
2221	let identity = if has_persisted_route {
2222	config.resolve_persisted_provider_identity(
2223	thread.model_provider.as_deref(),
2224	thread.model_provider_id.as_deref(),
2225	)
2226	} else {
2227	config.active_provider_identity(config.api_provider())
2228	};
2229	identity.map_err(\|reason\| anyhow!(reason))
2230	}
2231
2232	/// Atomically replace the authoritative runtime config after preflighting
2233	/// every loaded thread's exact route. Active turns retain their immutable
2234	/// descriptor; the next `start_turn` resolves and installs the new route.
2235	pub async fn reload_config(&self, new_config: Config) -> Result<()> {
2236	let _engine_load = self.engine_load.lock().await;
2237	let entries: Vec<(
2238	String,
2239	EngineHandle,
2240	ProviderIdentity,
2241	String,
2242	Option<String>,
2243	)> = {
2244	let active = self.active.lock().await;
2245	active
2246	.engines
2247	.iter()
2248	.map(\|(id, state)\| {
2249	(
2250	id.clone(),
2251	state.engine.clone(),
2252	state.route_identity.clone(),
2253	state.route_model.clone(),
2254	state
2255	.active_turn
2256	.as_ref()
2257	.map(\|active\| active.turn_id.clone()),
2258	)
2259	})
2260	.collect()
2261	};
2262
2263	let mut validated = Vec::with_capacity(entries.len());
2264	let mut failures = Vec::new();
2265	for (thread_id, engine, provider_identity, engine_model, active_turn_id) in entries {
2266	match resolve_runtime_thread_route_for_identity(
2267	&new_config,
2268	&provider_identity,
2269	Some(&engine_model),
2270	) {
2271	Ok(route) => validated.push((thread_id, engine, route, active_turn_id)),
2272	Err(err) => failures.push(format!("{thread_id}: {err}")),
2273	}
2274	}
2275	if !failures.is_empty() {
2276	bail!(
2277	"Config reload rejected because active thread routes are invalid: {}",
2278	failures.join("; ")
2279	);
2280	}
2281
2282	{
2283	let mut guard = self.config.write();
2284	*guard = new_config;
2285	}
2286
2287	let settings = crate::settings::Settings::load().unwrap_or_default();
2288	let stream_chunk_timeout_secs = self.read_config().stream_chunk_timeout_secs();
2289	for (thread_id, engine, route, active_turn_id) in validated {
2290	let provider = route.identity.provider;
2291	let route_limits = known_route_limits(route.candidate.limits());
2292	let mut engine_compaction = runtime_compaction_config(
2293	provider,
2294	&route.model,
2295	route_limits,
2296	settings.auto_compact,
2297	crate::settings::Settings::auto_compact_explicitly_configured(),
2298	settings.auto_compact_threshold_percent,
2299	);
2300	engine_compaction.runtime_cost_owner = active_turn_id;
2301	let route_config = route.config;
2302	let _ = engine
2303	.send(Op::SetCompaction {
2304	config: engine_compaction,
2305	})
2306	.await;
2307	let _ = engine
2308	.send(Op::SetStreamChunkTimeout {
2309	timeout_secs: stream_chunk_timeout_secs,
2310	})
2311	.await;
2312	let _ = engine
2313	.send(Op::SetSubagentRuntimeConfig {
2314	enabled: route_config.subagents_enabled_for_provider(provider),
2315	max_subagents: route_config
2316	.max_subagents_for_provider(provider)
2317	.clamp(1, crate::config::MAX_SUBAGENTS),
2318	launch_concurrency: route_config.launch_concurrency_for_provider(provider),
2319	max_spawn_depth: route_config.subagent_max_spawn_depth_for_provider(provider),
2320	api_timeout_secs: route_config.subagent_api_timeout_secs_for_provider(provider),
2321	heartbeat_timeout_secs: route_config
2322	.subagent_heartbeat_timeout_secs_for_provider(provider),
2323	})
2324	.await;
2325	tracing::info!(
2326	thread_id = %thread_id,
2327	"Reloaded runtime controls; provider route will apply on the next turn"
2328	);
2329	}
2330	Ok(())
2331	}
2332
2333	#[cfg(test)]
2334	pub fn open(
2335	config: Config,
2336	workspace: PathBuf,
2337	manager_cfg: RuntimeThreadManagerConfig,
2338	) -> Result<Self> {
2339	Self::open_inner(config, workspace, manager_cfg, None)
2340	}
2341
2342	pub fn open_with_plugin_registry(
2343	config: Config,
2344	workspace: PathBuf,
2345	manager_cfg: RuntimeThreadManagerConfig,
2346	plugin_registry: Arc<crate::plugins::PluginRegistry>,
2347	) -> Result<Self> {
2348	Self::open_inner(config, workspace, manager_cfg, Some(plugin_registry))
2349	}
2350
2351	fn open_inner(
2352	config: Config,
2353	workspace: PathBuf,
2354	manager_cfg: RuntimeThreadManagerConfig,
2355	plugin_registry: Option<Arc<crate::plugins::PluginRegistry>>,
2356	) -> Result<Self> {
2357	let store = RuntimeThreadStore::open(manager_cfg.data_dir.clone())?;
2358	let (event_tx, _event_rx) = broadcast::channel(EVENT_CHANNEL_CAPACITY);
2359	let manager = Self {
2360	config: Arc::new(parking_lot::RwLock::new(config)),
2361	workspace,
2362	plugin_registry,
2363	store,
2364	engine_load: Arc::new(Mutex::new(())),
2365	active: Arc::new(Mutex::new(ActiveThreads::default())),
2366	event_emit: Arc::new(Mutex::new(())),
2367	projection_locks: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2368	event_tx,
2369	manager_cfg,
2370	cancel_token: CancellationToken::new(),
2371	task_manager: Arc::new(parking_lot::Mutex::new(None)),
2372	automations: Arc::new(parking_lot::Mutex::new(None)),
2373	pending_approvals: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2374	pending_user_inputs: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2375	pending_dynamic_tools: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2376	recovery_receipts: Arc::new(parking_lot::Mutex::new(HashMap::new())),
2377	recovery_flush: Arc::new(Mutex::new(())),
2378	#[cfg(test)]
2379	snapshot_test_hook: Arc::new(parking_lot::Mutex::new(None)),
2380	};
2381	manager.recover_interrupted_state()?;
2382	Ok(manager)
2383	}
2384
2385	/// Attach the durable task manager so model-visible task tools work inside
2386	/// runtime thread turns as well as interactive TUI turns.
2387	pub fn attach_task_manager(&self, task_manager: crate::task_manager::SharedTaskManager) {
2388	*self.task_manager.lock() = Some(task_manager);
2389	}
2390
2391	/// Attach the automation manager for model-visible scheduling tools.
2392	pub fn attach_automation_manager(
2393	&self,
2394	automations: crate::automation_manager::SharedAutomationManager,
2395	) {
2396	*self.automations.lock() = Some(automations);
2397	}
2398
2399	#[allow(dead_code)] // Public API for external callers (runtime API, task manager)
2400	pub fn shutdown(&self) {
2401	self.cancel_token.cancel();
2402	self.pending_approvals.lock().clear();
2403	self.pending_user_inputs.lock().clear();
2404	self.pending_dynamic_tools.lock().clear();
2405	}
2406
2407	#[allow(dead_code)] // Public API for external callers
2408	pub fn is_shutdown(&self) -> bool {
2409	self.cancel_token.is_cancelled()
2410	}
2411
2412	fn register_pending_approval(
2413	&self,
2414	thread_id: &str,
2415	request: PendingApprovalRequest,
2416	) -> oneshot::Receiver<ExternalApprovalDecision> {
2417	let (tx, rx) = oneshot::channel();
2418	self.pending_approvals.lock().insert(
2419	request.id.clone(),
2420	PendingApprovalEntry {
2421	thread_id: thread_id.to_string(),
2422	request,
2423	sender: tx,
2424	},
2425	);
2426	rx
2427	}
2428
2429	fn cancel_pending_approval(&self, approval_id: &str) {
2430	self.pending_approvals.lock().remove(approval_id);
2431	}
2432
2433	fn register_pending_user_input(&self, thread_id: &str, request: PendingUserInputRequest) {
2434	let (settlement_tx, _settlement_rx) = watch::channel(0);
2435	self.pending_user_inputs.lock().insert(
2436	(thread_id.to_string(), request.id.clone()),
2437	PendingUserInputEntry {
2438	request,
2439	settling: false,
2440	settlement_tx,
2441	indeterminate: false,
2442	},
2443	);
2444	}
2445
2446	fn claim_pending_user_input(&self, thread_id: &str, input_id: &str) -> PendingUserInputClaim {
2447	let mut pending = self.pending_user_inputs.lock();
2448	let Some(entry) = pending.get_mut(&(thread_id.to_string(), input_id.to_string())) else {
2449	return PendingUserInputClaim::Missing;
2450	};
2451	if entry.indeterminate {
2452	return PendingUserInputClaim::Indeterminate;
2453	}
2454	if entry.settling {
2455	return PendingUserInputClaim::Settling;
2456	}
2457	entry.settling = true;
2458	PendingUserInputClaim::Claimed(entry.request.clone())
2459	}
2460
2461	fn discard_pending_user_input_registration(&self, thread_id: &str, input_id: &str) {
2462	let key = (thread_id.to_string(), input_id.to_string());
2463	let mut pending = self.pending_user_inputs.lock();
2464	if pending.get(&key).is_some_and(\|entry\| !entry.settling) {
2465	pending.remove(&key);
2466	}
2467	}
2468
2469	fn claim_pending_user_inputs_for_turn(
2470	&self,
2471	thread_id: &str,
2472	turn_id: &str,
2473	) -> Result<(Vec<PendingUserInputRequest>, Vec<watch::Receiver<u64>>)> {
2474	let mut pending = self.pending_user_inputs.lock();
2475	if let Some((_, entry)) = pending.iter().find(\|((pending_thread_id, _), entry)\| {
2476	pending_thread_id == thread_id
2477	&& entry.request.turn_id == turn_id
2478	&& entry.indeterminate
2479	}) {
2480	bail!(
2481	"User-input request '{}' has an indeterminate terminal receipt; inspect Runtime storage before completing turn '{turn_id}'",
2482	entry.request.id
2483	);
2484	}
2485	let mut claims = Vec::new();
2486	let mut settling = Vec::new();
2487	for ((pending_thread_id, _), entry) in pending.iter_mut() {
2488	if pending_thread_id != thread_id \|\| entry.request.turn_id != turn_id {
2489	continue;
2490	}
2491	if entry.settling {
2492	settling.push(entry.settlement_tx.subscribe());
2493	continue;
2494	}
2495	entry.settling = true;
2496	claims.push(entry.request.clone());
2497	}
2498	Ok((claims, settling))
2499	}
2500
2501	fn restore_pending_user_input_claim(&self, thread_id: &str, request: &PendingUserInputRequest) {
2502	let settlement_tx = if let Some(entry) = self
2503	.pending_user_inputs
2504	.lock()
2505	.get_mut(&(thread_id.to_string(), request.id.clone()))
2506	&& entry.request.turn_id == request.turn_id
2507	{
2508	entry.settling = false;
2509	entry.indeterminate = false;
2510	Some(entry.settlement_tx.clone())
2511	} else {
2512	None
2513	};
2514	if let Some(settlement_tx) = settlement_tx {
2515	settlement_tx.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
2516	}
2517	}
2518
2519	fn mark_pending_user_input_indeterminate(
2520	&self,
2521	thread_id: &str,
2522	request: &PendingUserInputRequest,
2523	) {
2524	let settlement_tx = if let Some(entry) = self
2525	.pending_user_inputs
2526	.lock()
2527	.get_mut(&(thread_id.to_string(), request.id.clone()))
2528	&& entry.request.turn_id == request.turn_id
2529	{
2530	entry.settling = true;
2531	entry.indeterminate = true;
2532	Some(entry.settlement_tx.clone())
2533	} else {
2534	None
2535	};
2536	if let Some(settlement_tx) = settlement_tx {
2537	settlement_tx.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
2538	}
2539	}
2540
2541	fn finish_pending_user_input_settlement(
2542	&self,
2543	thread_id: &str,
2544	request: &PendingUserInputRequest,
2545	) -> Option<watch::Sender<u64>> {
2546	let mut pending = self.pending_user_inputs.lock();
2547	let key = (thread_id.to_string(), request.id.clone());
2548	let settlement_tx = if pending.get(&key).is_some_and(\|entry\| {
2549	entry.request.turn_id == request.turn_id && entry.settling && !entry.indeterminate
2550	}) {
2551	pending.remove(&key).map(\|entry\| entry.settlement_tx)
2552	} else {
2553	None
2554	};
2555	drop(pending);
2556	settlement_tx
2557	}
2558
2559	fn pending_requests_for_thread(
2560	&self,
2561	thread_id: &str,
2562	) -> (Vec<PendingApprovalRequest>, Vec<PendingUserInputRequest>) {
2563	let mut approvals = self
2564	.pending_approvals
2565	.lock()
2566	.values()
2567	.filter(\|entry\| entry.thread_id == thread_id)
2568	.map(\|entry\| entry.request.clone())
2569	.collect::<Vec<_>>();
2570	approvals.sort_by(\|left, right\| {
2571	left.turn_id
2572	.cmp(&right.turn_id)
2573	.then_with(\|\| left.id.cmp(&right.id))
2574	});
2575
2576	let mut user_inputs = self
2577	.pending_user_inputs
2578	.lock()
2579	.iter()
2580	.filter(\|((pending_thread_id, _), _)\| pending_thread_id == thread_id)
2581	.map(\|(_, entry)\| entry.request.clone())
2582	.collect::<Vec<_>>();
2583	user_inputs.sort_by(\|left, right\| {
2584	left.turn_id
2585	.cmp(&right.turn_id)
2586	.then_with(\|\| left.id.cmp(&right.id))
2587	});
2588	(approvals, user_inputs)
2589	}
2590
2591	fn register_pending_dynamic_tool(
2592	&self,
2593	params: DynamicToolCallParams,
2594	) -> Result<oneshot::Receiver<DynamicToolCallResult>> {
2595	let (tx, rx) = oneshot::channel();
2596	let (settlement_tx, _settlement_rx) = watch::channel(0);
2597	let mut pending = self.pending_dynamic_tools.lock();
2598	if pending.len() >= MAX_PENDING_DYNAMIC_TOOL_CALLS {
2599	bail!(
2600	"Runtime has reached the pending dynamic tool call limit ({MAX_PENDING_DYNAMIC_TOOL_CALLS})"
2601	);
2602	}
2603	if pending.contains_key(&params.call_id) {
2604	bail!("Dynamic tool call '{}' is already pending", params.call_id);
2605	}
2606	pending.insert(
2607	params.call_id.clone(),
2608	PendingDynamicToolEntry {
2609	params,
2610	sender: Some(tx),
2611	settlement_tx,
2612	indeterminate: false,
2613	},
2614	);
2615	Ok(rx)
2616	}
2617
2618	/// Atomically select the single terminal owner for a dynamic tool call.
2619	///
2620	/// The registry entry intentionally remains present with an empty sender
2621	/// while the winner commits its receipt. `get_thread_detail` therefore
2622	/// cannot publish a cursor that has neither the pending request nor the
2623	/// terminal event, and competing result/timeout/cancel paths cannot claim
2624	/// the same call twice.
2625	fn claim_pending_dynamic_tool(
2626	&self,
2627	thread_id: &str,
2628	turn_id: &str,
2629	call_id: &str,
2630	) -> PendingDynamicToolClaim {
2631	let mut pending = self.pending_dynamic_tools.lock();
2632	let Some(entry) = pending.get_mut(call_id) else {
2633	return PendingDynamicToolClaim::Missing;
2634	};
2635	let matches_route = entry.params.thread_id == thread_id && entry.params.turn_id == turn_id;
2636	if !matches_route {
2637	return PendingDynamicToolClaim::Missing;
2638	}
2639	if entry.indeterminate {
2640	return PendingDynamicToolClaim::Indeterminate;
2641	}
2642	match entry.sender.take() {
2643	Some(sender) => PendingDynamicToolClaim::Claimed(ClaimedDynamicToolSettlement {
2644	params: entry.params.clone(),
2645	sender,
2646	settlement_tx: entry.settlement_tx.clone(),
2647	}),
2648	None => PendingDynamicToolClaim::Settling(entry.settlement_tx.subscribe()),
2649	}
2650	}
2651
2652	fn remove_pending_dynamic_tool(
2653	&self,
2654	thread_id: &str,
2655	turn_id: &str,
2656	call_id: &str,
2657	) -> Option<PendingDynamicToolEntry> {
2658	let mut pending = self.pending_dynamic_tools.lock();
2659	let matches_route = pending.get(call_id).is_some_and(\|entry\| {
2660	entry.params.thread_id == thread_id && entry.params.turn_id == turn_id
2661	});
2662	matches_route.then(\|\| pending.remove(call_id)).flatten()
2663	}
2664
2665	fn pending_dynamic_tool_calls_for_thread(&self, thread_id: &str) -> Vec<DynamicToolCallParams> {
2666	let mut calls = self
2667	.pending_dynamic_tools
2668	.lock()
2669	.values()
2670	.filter(\|entry\| entry.params.thread_id == thread_id)
2671	.map(\|entry\| entry.params.clone())
2672	.collect::<Vec<_>>();
2673	calls.sort_by(\|left, right\| {
2674	left.turn_id
2675	.cmp(&right.turn_id)
2676	.then_with(\|\| left.call_id.cmp(&right.call_id))
2677	});
2678	calls
2679	}
2680
2681	fn claim_or_watch_pending_dynamic_tools_for_turn(
2682	&self,
2683	thread_id: &str,
2684	turn_id: &str,
2685	) -> (
2686	Vec<ClaimedDynamicToolSettlement>,
2687	Vec<watch::Receiver<u64>>,
2688	bool,
2689	) {
2690	let mut pending = self.pending_dynamic_tools.lock();
2691	let mut claims = Vec::new();
2692	let mut settling = Vec::new();
2693	let mut indeterminate = false;
2694	for entry in pending
2695	.values_mut()
2696	.filter(\|entry\| entry.params.thread_id == thread_id && entry.params.turn_id == turn_id)
2697	{
2698	if entry.indeterminate {
2699	indeterminate = true;
2700	continue;
2701	}
2702	match entry.sender.take() {
2703	Some(sender) => claims.push(ClaimedDynamicToolSettlement {
2704	params: entry.params.clone(),
2705	sender,
2706	settlement_tx: entry.settlement_tx.clone(),
2707	}),
2708	None => settling.push(entry.settlement_tx.subscribe()),
2709	}
2710	}
2711	(claims, settling, indeterminate)
2712	}
2713
2714	fn finish_dynamic_tool_settlement(&self, params: &DynamicToolCallParams) {
2715	let mut pending = self.pending_dynamic_tools.lock();
2716	let can_remove = pending.get(&params.call_id).is_some_and(\|entry\| {
2717	entry.params.thread_id == params.thread_id
2718	&& entry.params.turn_id == params.turn_id
2719	&& entry.sender.is_none()
2720	});
2721	if can_remove {
2722	pending.remove(&params.call_id);
2723	}
2724	}
2725
2726	fn restore_dynamic_tool_claim(&self, claim: ClaimedDynamicToolSettlement) {
2727	let settlement_tx = claim.settlement_tx.clone();
2728	let mut pending = self.pending_dynamic_tools.lock();
2729	if let Some(entry) = pending.get_mut(&claim.params.call_id)
2730	&& entry.params.thread_id == claim.params.thread_id
2731	&& entry.params.turn_id == claim.params.turn_id
2732	&& entry.sender.is_none()
2733	{
2734	entry.sender = Some(claim.sender);
2735	entry.indeterminate = false;
2736	}
2737	settlement_tx.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
2738	}
2739
2740	fn mark_dynamic_tool_claim_indeterminate(&self, claim: &ClaimedDynamicToolSettlement) {
2741	let mut pending = self.pending_dynamic_tools.lock();
2742	if let Some(entry) = pending.get_mut(&claim.params.call_id)
2743	&& entry.params.thread_id == claim.params.thread_id
2744	&& entry.params.turn_id == claim.params.turn_id
2745	&& entry.sender.is_none()
2746	{
2747	entry.indeterminate = true;
2748	}
2749	claim
2750	.settlement_tx
2751	.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
2752	}
2753
2754	pub fn deliver_external_approval(
2755	&self,
2756	approval_id: &str,
2757	decision: ExternalApprovalDecision,
2758	) -> bool {
2759	let entry = self.pending_approvals.lock().remove(approval_id);
2760	match entry {
2761	Some(entry) => entry.sender.send(decision).is_ok(),
2762	None => false,
2763	}
2764	}
2765
2766	pub async fn deliver_dynamic_tool_result(
2767	&self,
2768	thread_id: &str,
2769	turn_id: &str,
2770	call_id: &str,
2771	result: DynamicToolCallResult,
2772	) -> Result<bool> {
2773	let claim = match self.claim_pending_dynamic_tool(thread_id, turn_id, call_id) {
2774	PendingDynamicToolClaim::Claimed(claim) => claim,
2775	PendingDynamicToolClaim::Settling(_) \| PendingDynamicToolClaim::Missing => {
2776	return Ok(false);
2777	}
2778	PendingDynamicToolClaim::Indeterminate => {
2779	bail!(
2780	"Dynamic tool call '{call_id}' has an indeterminate terminal receipt; inspect Runtime storage before retrying"
2781	);
2782	}
2783	};
2784	let ack =
2785	self.spawn_dynamic_tool_settlement(claim, DynamicToolTerminalOutcome::Resolved(result));
2786	Ok(Self::await_dynamic_tool_settlement(ack)
2787	.await?
2788	.result_accepted)
2789	}
2790
2791	pub async fn submit_user_input(
2792	&self,
2793	thread_id: &str,
2794	input_id: &str,
2795	response: crate::tools::user_input::UserInputResponse,
2796	) -> Result<bool> {
2797	let engine = {
2798	let active = self.active.lock().await;
2799	let Some(state) = active.engines.get(thread_id) else {
2800	bail!("thread '{thread_id}' not found");
2801	};
2802	state.engine.clone()
2803	};
2804	let request = match self.claim_pending_user_input(thread_id, input_id) {
2805	PendingUserInputClaim::Claimed(request) => request,
2806	PendingUserInputClaim::Missing \| PendingUserInputClaim::Settling => {
2807	return Ok(false);
2808	}
2809	PendingUserInputClaim::Indeterminate => {
2810	bail!(
2811	"User-input request '{input_id}' has an indeterminate terminal receipt; inspect Runtime storage before retrying"
2812	);
2813	}
2814	};
2815
2816	// This child task deliberately outlives the HTTP future. Once a
2817	// request is claimed, client disconnect/cancellation cannot strand it
2818	// between durable acceptance and engine delivery.
2819	let manager = self.clone();
2820	let thread_id = thread_id.to_string();
2821	tokio::spawn(async move {
2822	manager
2823	.settle_claimed_user_input(
2824	&thread_id,
2825	Some(engine),
2826	request,
2827	UserInputTerminalOutcome::Answered(response),
2828	)
2829	.await
2830	})
2831	.await
2832	.context("User-input settlement task failed")?
2833	}
2834
2835	#[allow(dead_code)]
2836	pub async fn cancel_user_input(&self, thread_id: &str, input_id: &str) -> Result<bool> {
2837	let engine = {
2838	let active = self.active.lock().await;
2839	let Some(state) = active.engines.get(thread_id) else {
2840	bail!("thread '{thread_id}' not found");
2841	};
2842	state.engine.clone()
2843	};
2844	let request = match self.claim_pending_user_input(thread_id, input_id) {
2845	PendingUserInputClaim::Claimed(request) => request,
2846	PendingUserInputClaim::Missing \| PendingUserInputClaim::Settling => {
2847	return Ok(false);
2848	}
2849	PendingUserInputClaim::Indeterminate => {
2850	bail!(
2851	"User-input request '{input_id}' has an indeterminate terminal receipt; inspect Runtime storage before retrying"
2852	);
2853	}
2854	};
2855	let manager = self.clone();
2856	let thread_id = thread_id.to_string();
2857	tokio::spawn(async move {
2858	manager
2859	.settle_claimed_user_input(
2860	&thread_id,
2861	Some(engine),
2862	request,
2863	UserInputTerminalOutcome::Canceled { terminal: false },
2864	)
2865	.await
2866	})
2867	.await
2868	.context("User-input cancellation task failed")?
2869	}
2870
2871	async fn settle_claimed_user_input(
2872	&self,
2873	thread_id: &str,
2874	engine: Option<EngineHandle>,
2875	request: PendingUserInputRequest,
2876	outcome: UserInputTerminalOutcome,
2877	) -> Result<bool> {
2878	let projection_lock = self.projection_lock(thread_id);
2879	let _projection = projection_lock.lock().await;
2880	let (event, payload) = match &outcome {
2881	UserInputTerminalOutcome::Answered(_) => (
2882	"user_input.answered",
2883	json!({ "id": &request.id, "input_id": &request.id }),
2884	),
2885	UserInputTerminalOutcome::Canceled { terminal } => (
2886	"user_input.canceled",
2887	json!({
2888	"id": &request.id,
2889	"input_id": &request.id,
2890	"terminal": terminal,
2891	}),
2892	),
2893	};
2894	if let Err(error) = self
2895	.emit_event(thread_id, Some(&request.turn_id), None, event, payload)
2896	.await
2897	{
2898	if event_append_is_indeterminate(&error) {
2899	self.mark_pending_user_input_indeterminate(thread_id, &request);
2900	} else {
2901	self.restore_pending_user_input_claim(thread_id, &request);
2902	}
2903	return Err(error);
2904	}
2905	let settlement_tx = self.finish_pending_user_input_settlement(thread_id, &request);
2906	drop(_projection);
2907
2908	let delivery_result = match (engine, outcome) {
2909	(Some(engine), UserInputTerminalOutcome::Answered(response)) => {
2910	engine.submit_user_input(&request.id, response).await
2911	}
2912	(Some(engine), UserInputTerminalOutcome::Canceled { .. }) => {
2913	if let Err(error) = engine.cancel_user_input(&request.id).await {
2914	tracing::debug!(
2915	thread_id,
2916	input_id = %request.id,
2917	"User-input cancellation was durable after engine mailbox closed: {error}"
2918	);
2919	}
2920	Ok(())
2921	}
2922	(None, _) => Ok(()),
2923	};
2924	if let Some(settlement_tx) = settlement_tx {
2925	settlement_tx.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
2926	}
2927	delivery_result?;
2928	Ok(true)
2929	}
2930
2931	async fn settle_user_inputs_for_terminal_turn(
2932	&self,
2933	thread_id: &str,
2934	turn_id: &str,
2935	engine: Option<EngineHandle>,
2936	) -> Result<()> {
2937	loop {
2938	let (requests, settling) =
2939	self.claim_pending_user_inputs_for_turn(thread_id, turn_id)?;
2940	for request in requests {
2941	self.settle_claimed_user_input(
2942	thread_id,
2943	engine.clone(),
2944	request,
2945	UserInputTerminalOutcome::Canceled { terminal: true },
2946	)
2947	.await?;
2948	}
2949	if settling.is_empty() {
2950	return Ok(());
2951	}
2952	for mut progress in settling {
2953	let _ = progress.changed().await;
2954	}
2955	}
2956	}
2957
2958	#[allow(dead_code)]
2959	pub fn pending_approvals_count(&self) -> usize {
2960	self.pending_approvals.lock().len()
2961	}
2962
2963	#[allow(dead_code)]
2964	pub fn pending_dynamic_tools_count(&self) -> usize {
2965	self.pending_dynamic_tools.lock().len()
2966	}
2967
2968	#[cfg(test)]
2969	pub(crate) fn register_pending_approval_for_test(
2970	&self,
2971	approval_id: &str,
2972	) -> oneshot::Receiver<ExternalApprovalDecision> {
2973	self.register_pending_approval(
2974	"test-thread",
2975	PendingApprovalRequest {
2976	id: approval_id.to_string(),
2977	turn_id: "test-turn".to_string(),
2978	tool_name: "test-tool".to_string(),
2979	description: "test approval".to_string(),
2980	intent_summary: None,
2981	},
2982	)
2983	}
2984
2985	#[cfg(test)]
2986	pub(crate) fn register_pending_dynamic_tool_for_test(
2987	&self,
2988	thread_id: &str,
2989	turn_id: &str,
2990	call_id: &str,
2991	) -> Result<oneshot::Receiver<DynamicToolCallResult>> {
2992	self.register_pending_dynamic_tool(DynamicToolCallParams {
2993	thread_id: thread_id.to_string(),
2994	turn_id: turn_id.to_string(),
2995	call_id: call_id.to_string(),
2996	namespace: Some("test".to_string()),
2997	tool: "test_tool".to_string(),
2998	arguments: json!({ "input": "test" }),
2999	})
3000	}
3001
3002	async fn remember_thread_auto_approve(&self, thread_id: &str) {
3003	let thread = {
3004	let _thread_mutation = self.store.thread_mutation.lock();
3005	let Ok(mut thread) = self.store.load_thread(thread_id) else {
3006	return;
3007	};
3008	if !thread.auto_approve \|\| thread.permission_posture.as_deref() != Some("full_access") {
3009	thread.auto_approve = true;
3010	thread.permission_posture = Some("full_access".to_string());
3011	thread.updated_at = Utc::now();
3012	if let Err(err) = self.store.save_thread(&thread) {
3013	tracing::warn!(
3014	"Failed to persist full-access posture for thread {}: {}",
3015	thread_id,
3016	err
3017	);
3018	return;
3019	}
3020	}
3021	thread
3022	};
3023
3024	let engine = {
3025	let active = self.active.lock().await;
3026	active
3027	.engines
3028	.get(thread_id)
3029	.map(\|state\| state.engine.clone())
3030	};
3031	if let Some(engine) = engine {
3032	let configured_sandbox_mode = self.read_config().sandbox_mode.clone();
3033	let policy = RuntimePolicyProjection::from_persisted(
3034	&thread.mode,
3035	thread.permission_posture.as_deref(),
3036	thread.auto_approve,
3037	);
3038	let _ = engine.try_send(Op::ChangeMode {
3039	mode: policy.mode,
3040	allow_shell: thread.allow_shell,
3041	trust_mode: thread.trust_mode,
3042	auto_approve: policy.auto_approve(),
3043	approval_mode: policy.permission,
3044	configured_sandbox_mode,
3045	});
3046	}
3047	}
3048
3049	#[must_use]
3050	pub fn subscribe_events(&self) -> broadcast::Receiver<RuntimeEventRecord> {
3051	self.event_tx.subscribe()
3052	}
3053
3054	fn projection_lock(&self, thread_id: &str) -> Arc<Mutex<()>> {
3055	let mut locks = self.projection_locks.lock();
3056	Arc::clone(
3057	locks
3058	.entry(thread_id.to_string())
3059	.or_insert_with(\|\| Arc::new(Mutex::new(()))),
3060	)
3061	}
3062
3063	async fn emit_event(
3064	&self,
3065	thread_id: &str,
3066	turn_id: Option<&str>,
3067	item_id: Option<&str>,
3068	event: impl Into<String>,
3069	payload: Value,
3070	) -> Result<RuntimeEventRecord> {
3071	let _emit_order = self.event_emit.lock().await;
3072	self.append_and_broadcast_event(thread_id, turn_id, item_id, event, payload)
3073	.await
3074	}
3075
3076	/// Append and broadcast an event while the caller owns `event_emit`.
3077	/// Keeping this primitive separate lets dynamic-tool settlement hold its
3078	/// projection boundary through durable append, registry removal, and the
3079	/// non-awaiting result send.
3080	async fn append_and_broadcast_event(
3081	&self,
3082	thread_id: &str,
3083	turn_id: Option<&str>,
3084	item_id: Option<&str>,
3085	event: impl Into<String>,
3086	payload: Value,
3087	) -> Result<RuntimeEventRecord> {
3088	let record = self
3089	.store
3090	.append_event(thread_id, turn_id, item_id, event, payload)
3091	.await?;
3092	if let Err(e) = self.event_tx.send(record.clone()) {
3093	tracing::debug!(
3094	"Runtime event broadcast failed (no receivers or channel full): {}",
3095	e
3096	);
3097	}
3098	Ok(record)
3099	}
3100
3101	async fn emit_turn_completed_if_missing(
3102	&self,
3103	turn: &TurnRecord,
3104	recovered: bool,
3105	) -> Result<bool> {
3106	let _emit_order = self.event_emit.lock().await;
3107	let store = self.store.clone();
3108	let thread_id = turn.thread_id.clone();
3109	let expected = RuntimeEventMatch::TurnCompleted {
3110	turn_id: turn.id.clone(),
3111	};
3112	let already_emitted =
3113	tokio::task::spawn_blocking(move \|\| store.contains_event(&thread_id, &expected))
3114	.await
3115	.context("Runtime turn-completion dedupe scan failed")??;
3116	if already_emitted {
3117	return Ok(false);
3118	}
3119	let mut payload = json!({ "turn": turn });
3120	if recovered && let Some(object) = payload.as_object_mut() {
3121	object.insert("recovered".to_string(), json!(true));
3122	}
3123	self.append_and_broadcast_event(
3124	&turn.thread_id,
3125	Some(&turn.id),
3126	None,
3127	"turn.completed",
3128	payload,
3129	)
3130	.await?;
3131	Ok(true)
3132	}
3133
3134	async fn emit_recovered_dynamic_cancellation_if_missing(
3135	&self,
3136	params: &DynamicToolCallParams,
3137	) -> Result<bool> {
3138	let _emit_order = self.event_emit.lock().await;
3139	let store = self.store.clone();
3140	let thread_id = params.thread_id.clone();
3141	let expected = RuntimeEventMatch::DynamicTerminal {
3142	turn_id: params.turn_id.clone(),
3143	call_id: params.call_id.clone(),
3144	};
3145	let already_emitted =
3146	tokio::task::spawn_blocking(move \|\| store.contains_event(&thread_id, &expected))
3147	.await
3148	.context("Runtime dynamic-tool terminal dedupe scan failed")??;
3149	if already_emitted {
3150	return Ok(false);
3151	}
3152	let mut payload =
3153	dynamic_tool_terminal_payload(params, "canceled", None, Some("process_restart"));
3154	if let Some(object) = payload.as_object_mut() {
3155	object.insert("terminal".to_string(), json!(true));
3156	object.insert("recovered".to_string(), json!(true));
3157	}
3158	self.append_and_broadcast_event(
3159	&params.thread_id,
3160	Some(&params.turn_id),
3161	None,
3162	"tool_call.canceled",
3163	payload,
3164	)
3165	.await?;
3166	Ok(true)
3167	}
3168
3169	async fn flush_recovery_receipts_for_thread(&self, thread_id: &str) -> Result<()> {
3170	if !self.recovery_receipts.lock().contains_key(thread_id) {
3171	return Ok(());
3172	}
3173	let _recovery_flush = self.recovery_flush.lock().await;
3174	loop {
3175	let next = self
3176	.recovery_receipts
3177	.lock()
3178	.get(thread_id)
3179	.and_then(\|receipts\| receipts.first())
3180	.cloned();
3181	let Some(receipt) = next else {
3182	return Ok(());
3183	};
3184
3185	// An in-process monitor failure may leave retry-safe calls in the
3186	// live registry. Retry their supervised cancellation before the
3187	// static restart-recovery receipts below. Startup recovery has no
3188	// live registry entries, so this is a no-op in that case.
3189	self.settle_dynamic_tools_for_terminal_turn(thread_id, &receipt.turn.id)
3190	.await?;
3191	let engine = {
3192	let active = self.active.lock().await;
3193	active
3194	.engines
3195	.get(thread_id)
3196	.map(\|state\| state.engine.clone())
3197	};
3198	self.settle_user_inputs_for_terminal_turn(thread_id, &receipt.turn.id, engine)
3199	.await?;
3200
3201	let projection_lock = self.projection_lock(thread_id);
3202	let _projection = projection_lock.lock().await;
3203	for params in &receipt.unresolved_dynamic_tools {
3204	self.emit_recovered_dynamic_cancellation_if_missing(params)
3205	.await?;
3206	}
3207	self.emit_turn_completed_if_missing(&receipt.turn, true)
3208	.await?;
3209	drop(_projection);
3210
3211	let mut queued = self.recovery_receipts.lock();
3212	let remove_thread = if let Some(receipts) = queued.get_mut(thread_id) {
3213	receipts.retain(\|candidate\| candidate.turn.id != receipt.turn.id);
3214	receipts.is_empty()
3215	} else {
3216	false
3217	};
3218	if remove_thread {
3219	queued.remove(thread_id);
3220	}
3221	}
3222	}
3223
3224	fn queue_recovery_receipt(&self, receipt: RecoveredTurnReceipt) {
3225	let thread_id = receipt.turn.thread_id.clone();
3226	let turn_id = receipt.turn.id.clone();
3227	let mut queued = self.recovery_receipts.lock();
3228	let receipts = queued.entry(thread_id).or_default();
3229	if let Some(existing) = receipts
3230	.iter_mut()
3231	.find(\|candidate\| candidate.turn.id == turn_id)
3232	{
3233	let mut known_calls = existing
3234	.unresolved_dynamic_tools
3235	.iter()
3236	.map(\|params\| params.call_id.clone())
3237	.collect::<HashSet<_>>();
3238	existing.unresolved_dynamic_tools.extend(
3239	receipt
3240	.unresolved_dynamic_tools
3241	.into_iter()
3242	.filter(\|params\| known_calls.insert(params.call_id.clone())),
3243	);
3244	return;
3245	}
3246	receipts.push(receipt);
3247	receipts.sort_by_key(\|candidate\| candidate.turn.created_at);
3248	}
3249
3250	fn spawn_dynamic_tool_settlement(
3251	&self,
3252	claim: ClaimedDynamicToolSettlement,
3253	outcome: DynamicToolTerminalOutcome,
3254	) -> oneshot::Receiver<std::result::Result<DynamicToolSettlementAck, String>> {
3255	let (ack_tx, ack_rx) = oneshot::channel();
3256	let manager = self.clone();
3257	tokio::spawn(async move {
3258	use futures_util::FutureExt;
3259
3260	let mut claim = Some(claim);
3261	let mut outcome = Some(outcome);
3262	let settlement = std::panic::AssertUnwindSafe(async {
3263	let claim_ref = claim
3264	.as_ref()
3265	.ok_or_else(\|\| "Dynamic tool settlement lost its claim".to_string())?;
3266	let outcome_ref = outcome
3267	.as_ref()
3268	.ok_or_else(\|\| "Dynamic tool settlement lost its outcome".to_string())?;
3269	let projection_lock = manager.projection_lock(&claim_ref.params.thread_id);
3270	let _projection = projection_lock.lock().await;
3271	let emit_order = manager.event_emit.lock().await;
3272
3273	// `resolved` linearizes durable acceptance by the Runtime. It
3274	// deliberately does not claim that the model consumed the
3275	// result: the receiver may close at any point before the
3276	// post-receipt, non-awaiting send.
3277	let (event, payload) = match outcome_ref {
3278	DynamicToolTerminalOutcome::Resolved(result) => {
3279	let mut payload = dynamic_tool_terminal_payload(
3280	&claim_ref.params,
3281	"resolved",
3282	Some(result.success),
3283	None,
3284	);
3285	if let Some(object) = payload.as_object_mut() {
3286	object.insert("result_accepted".to_string(), json!(true));
3287	}
3288	("tool_call.resolved", payload)
3289	}
3290	DynamicToolTerminalOutcome::Canceled { reason, terminal } => {
3291	let mut payload = dynamic_tool_terminal_payload(
3292	&claim_ref.params,
3293	"canceled",
3294	None,
3295	Some(reason),
3296	);
3297	if *terminal && let Some(object) = payload.as_object_mut() {
3298	object.insert("terminal".to_string(), json!(true));
3299	}
3300	("tool_call.canceled", payload)
3301	}
3302	DynamicToolTerminalOutcome::Timeout { timeout } => {
3303	let mut payload =
3304	dynamic_tool_terminal_payload(&claim_ref.params, "timeout", None, None);
3305	if let Some(object) = payload.as_object_mut() {
3306	object.insert("timeout_secs".to_string(), json!(timeout.as_secs()));
3307	}
3308	("tool_call.timeout", payload)
3309	}
3310	};
3311
3312	if let Err(error) = manager
3313	.append_and_broadcast_event(
3314	&claim_ref.params.thread_id,
3315	Some(&claim_ref.params.turn_id),
3316	None,
3317	event,
3318	payload,
3319	)
3320	.await
3321	{
3322	drop(emit_order);
3323	if let Some(claim) = claim.take() {
3324	let retry_safe = error
3325	.downcast_ref::<RuntimeEventAppendError>()
3326	.is_none_or(RuntimeEventAppendError::retry_safe);
3327	if retry_safe {
3328	// Definite pre-write failures and transactionally
3329	// rolled-back appends return the call to Awaiting.
3330	manager.restore_dynamic_tool_claim(claim);
3331	} else {
3332	// A failed rollback means the JSONL tail may already
3333	// contain the terminal line. Keep the request
3334	// explicitly indeterminate so neither an API retry
3335	// nor turn timeout can append a duplicate.
3336	manager.mark_dynamic_tool_claim_indeterminate(&claim);
3337	drop(claim);
3338	}
3339	}
3340	return Err(error.to_string());
3341	}
3342
3343	let claim = claim
3344	.take()
3345	.ok_or_else(\|\| "Dynamic tool settlement lost its claim".to_string())?;
3346	let outcome = outcome
3347	.take()
3348	.ok_or_else(\|\| "Dynamic tool settlement lost its outcome".to_string())?;
3349
3350	// The snapshot boundary stays held until the request
3351	// disappears. The model-facing channel is only woken after the
3352	// terminal event is on disk, and send itself cannot suspend or
3353	// be caller-canceled.
3354	manager.finish_dynamic_tool_settlement(&claim.params);
3355	claim
3356	.settlement_tx
3357	.send_modify(\|epoch\| *epoch = epoch.saturating_add(1));
3358	let result_accepted = matches!(&outcome, DynamicToolTerminalOutcome::Resolved(_));
3359	match outcome {
3360	DynamicToolTerminalOutcome::Resolved(result) => {
3361	if claim.sender.send(result).is_err() {
3362	tracing::debug!(
3363	call_id = %claim.params.call_id,
3364	"Durably accepted dynamic tool result had no remaining model receiver"
3365	);
3366	}
3367	}
3368	DynamicToolTerminalOutcome::Canceled { .. }
3369	\| DynamicToolTerminalOutcome::Timeout { .. } => drop(claim.sender),
3370	}
3371	Ok(DynamicToolSettlementAck { result_accepted })
3372	})
3373	.catch_unwind()
3374	.await;
3375
3376	let result = match settlement {
3377	Ok(result) => result,
3378	Err(payload) => {
3379	// A panic before durable completion must not leave a
3380	// Settling tombstone. Reacquire the same projection
3381	// boundary before returning the sender to Awaiting.
3382	if let Some(claim) = claim.take() {
3383	let projection_lock = manager.projection_lock(&claim.params.thread_id);
3384	let _projection = projection_lock.lock().await;
3385	manager.restore_dynamic_tool_claim(claim);
3386	}
3387	Err(format!(
3388	"Dynamic tool settlement task panicked: {}",
3389	panic_payload_message(&*payload)
3390	))
3391	}
3392	};
3393	let _ = ack_tx.send(result);
3394	});
3395	ack_rx
3396	}
3397
3398	async fn await_dynamic_tool_settlement(
3399	ack: oneshot::Receiver<std::result::Result<DynamicToolSettlementAck, String>>,
3400	) -> Result<DynamicToolSettlementAck> {
3401	match ack.await {
3402	Ok(Ok(ack)) => Ok(ack),
3403	Ok(Err(error)) => bail!("{error}"),
3404	Err(_) => bail!("Dynamic tool settlement task ended before acknowledgement"),
3405	}
3406	}
3407
3408	async fn settle_dynamic_tool_timeout(
3409	&self,
3410	claim: ClaimedDynamicToolSettlement,
3411	timeout: Duration,
3412	) -> Result<()> {
3413	let ack = self
3414	.spawn_dynamic_tool_settlement(claim, DynamicToolTerminalOutcome::Timeout { timeout });
3415	Self::await_dynamic_tool_settlement(ack).await?;
3416	Ok(())
3417	}
3418
3419	async fn settle_dynamic_tools_for_terminal_turn(
3420	&self,
3421	thread_id: &str,
3422	turn_id: &str,
3423	) -> Result<()> {
3424	loop {
3425	let (claims, mut settling, indeterminate) =
3426	self.claim_or_watch_pending_dynamic_tools_for_turn(thread_id, turn_id);
3427	if indeterminate {
3428	bail!(
3429	"Turn {turn_id} has an indeterminate dynamic-tool receipt; refusing to publish turn completion"
3430	);
3431	}
3432	if claims.is_empty() && settling.is_empty() {
3433	return Ok(());
3434	}
3435
3436	let mut first_error = None;
3437	for claim in claims {
3438	let ack = self.spawn_dynamic_tool_settlement(
3439	claim,
3440	DynamicToolTerminalOutcome::Canceled {
3441	reason: "turn_terminal",
3442	terminal: true,
3443	},
3444	);
3445	if let Err(error) = Self::await_dynamic_tool_settlement(ack).await
3446	&& first_error.is_none()
3447	{
3448	first_error = Some(error);
3449	}
3450	}
3451
3452	// If result delivery or timeout already owned a call, wait for its
3453	// supervised completion/rollback before publishing turn.completed.
3454	// On rollback the next iteration claims terminal cancellation; on
3455	// success the completed entry is gone.
3456	for progress in &mut settling {
3457	let _ = progress.changed().await;
3458	}
3459
3460	// Every claim selected above has now either committed, restored
3461	// itself to Awaiting, or entered the explicit indeterminate state.
3462	// Returning only after supervising the whole batch prevents an
3463	// early failure from dropping unstarted senders into permanent
3464	// Settling tombstones.
3465	if let Some(error) = first_error {
3466	return Err(error);
3467	}
3468	}
3469	}
3470
3471	/// Persist a streaming item without blocking the Tokio worker that drives
3472	/// engine events. Each delta must reach the item projection before its
3473	/// durable event is sequenced, otherwise a snapshot at that cursor can
3474	/// expose stale text. Keeping the full record in memory avoids rereading
3475	/// and reparsing the same item for every provider chunk.
3476	async fn save_streaming_item(&self, item: &TurnItemRecord) -> Result<()> {
3477	let store = self.store.clone();
3478	let item = item.clone();
3479	tokio::task::spawn_blocking(move \|\| store.save_item(&item))
3480	.await
3481	.context("Streaming item persistence task failed")??;
3482	Ok(())
3483	}
3484
3485	#[cfg(test)]
3486	pub(crate) async fn emit_event_for_test(
3487	&self,
3488	thread_id: &str,
3489	turn_id: Option<&str>,
3490	event: &str,
3491	payload: Value,
3492	) -> Result<RuntimeEventRecord> {
3493	self.emit_event(thread_id, turn_id, None, event, payload)
3494	.await
3495	}
3496
3497	#[cfg(test)]
3498	pub(crate) fn set_snapshot_test_hook(&self, hook: mpsc::UnboundedSender<SnapshotTestPoint>) {
3499	*self.snapshot_test_hook.lock() = Some(hook);
3500	}
3501
3502	pub async fn create_thread(&self, req: CreateThreadRequest) -> Result<ThreadRecord> {
3503	let now = Utc::now();
3504	let (model_provider, model_provider_id, default_model) = {
3505	let config = self.read_config().clone();
3506	let requested_kind = req
3507	.model_provider
3508	.as_deref()
3509	.filter(\|provider\| !provider.trim().is_empty());
3510	// `Some("")` is malformed provenance, not absence. Pass it to
3511	// the resolver so an imported/API-created record cannot silently
3512	// acquire the root custom route.
3513	let requested_id = req.model_provider_id.as_deref().map(str::trim);
3514	let identity = if requested_kind.is_some() \|\| requested_id.is_some() {
3515	config.resolve_persisted_provider_identity(requested_kind, requested_id)
3516	} else {
3517	let selected = config
3518	.provider
3519	.as_deref()
3520	.unwrap_or(ApiProvider::Deepseek.as_str());
3521	config.resolve_provider_identity(selected)
3522	}
3523	.map_err(\|reason\| anyhow!(reason))?;
3524	let default_model = resolve_runtime_route_for_identity(&config, &identity, None)
3525	.map_err(\|reason\| anyhow!(reason))?
3526	.model;
3527	(
3528	identity.provider.as_str().to_string(),
3529	identity.exact_id,
3530	default_model,
3531	)
3532	};
3533	let model = req
3534	.model
3535	.filter(\|m\| !m.trim().is_empty())
3536	.unwrap_or(default_model);
3537	let workspace = req.workspace.unwrap_or_else(\|\| self.workspace.clone());
3538	let requested_mode = req
3539	.mode
3540	.filter(\|m\| !m.trim().is_empty())
3541	.unwrap_or_else(\|\| "agent".to_string());
3542	let policy = RuntimePolicyProjection::from_request(
3543	&requested_mode,
3544	req.permission_posture.as_deref(),
3545	req.auto_approve,
3546	)?;
3547	let mode = policy.mode_setting().to_string();
3548	let permission_posture = Some(policy.permission_wire().to_string());
3549	let allow_shell = req
3550	.allow_shell
3551	.unwrap_or_else(\|\| self.read_config().allow_shell());
3552	let trust_mode = req.trust_mode.unwrap_or(false);
3553	let auto_approve = policy.auto_approve();
3554
3555	let thread = ThreadRecord {
3556	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
3557	id: format!("thr_{}", &Uuid::new_v4().to_string()[..8]),
3558	created_at: now,
3559	updated_at: now,
3560	model,
3561	model_provider: Some(model_provider),
3562	model_provider_id,
3563	workspace,
3564	mode,
3565	permission_posture,
3566	allow_shell,
3567	trust_mode,
3568	auto_approve,
3569	latest_turn_id: None,
3570	latest_response_bookmark: None,
3571	archived: req.archived,
3572	system_prompt: req.system_prompt,
3573	task_id: req.task_id,
3574	title: None,
3575	session_id: None,
3576	};
3577	self.store.save_thread(&thread)?;
3578	self.emit_event(
3579	&thread.id,
3580	None,
3581	None,
3582	"thread.started",
3583	json!({ "thread": thread }),
3584	)
3585	.await?;
3586	Ok(thread)
3587	}
3588
3589	pub async fn list_threads(
3590	&self,
3591	filter: ThreadListFilter,
3592	limit: Option<usize>,
3593	) -> Result<Vec<ThreadRecord>> {
3594	let mut threads = self.store.list_threads()?;
3595	match filter {
3596	ThreadListFilter::ActiveOnly => threads.retain(\|t\| !t.archived),
3597	ThreadListFilter::ArchivedOnly => threads.retain(\|t\| t.archived),
3598	ThreadListFilter::IncludeArchived => {}
3599	}
3600	if let Some(limit) = limit {
3601	threads.truncate(limit);
3602	}
3603	Ok(threads)
3604	}
3605
3606	/// Aggregate token + cost usage across all threads/turns inside the time
3607	/// range `[since, until]`. Each parent, child, and compaction call is
3608	/// computed via provider-aware pricing using its persisted concrete route.
3609	/// Legacy turns without provider provenance and providers without an
3610	/// authoritative runtime price (including ChatGPT/Codex OAuth) accrue
3611	/// tokens but no fabricated dollar cost. Whalescale#261 / #564.
3612	///
3613	/// Buckets are sorted by ascending key for deterministic output. Empty
3614	/// ranges produce empty `buckets` (never an error).
3615	pub async fn aggregate_usage(
3616	&self,
3617	since: Option<DateTime<Utc>>,
3618	until: Option<DateTime<Utc>>,
3619	group_by: UsageGroupBy,
3620	) -> Result<UsageAggregation> {
3621	let mut buckets: std::collections::BTreeMap<String, UsageBucket> =
3622	std::collections::BTreeMap::new();
3623	let mut totals = UsageTotals::default();
3624	for thread in self.store.list_threads()? {
3625	let turns = self.store.list_turns_for_thread(&thread.id)?;
3626	for turn in turns {
3627	let parent_route = turn.effective_route_envelope();
3628	let parent_dispatched_at = parent_route
3629	.as_ref()
3630	.map_or(turn.created_at, \|route\| route.dispatched_at);
3631	if let Some(usage) = turn.usage.as_ref()
3632	&& usage_timestamp_in_range(parent_dispatched_at, since, until)
3633	{
3634	accumulate_runtime_usage_record(
3635	&mut totals,
3636	&mut buckets,
3637	group_by,
3638	parent_route.as_ref(),
3639	usage,
3640	&turn,
3641	&thread,
3642	);
3643	}
3644	for child in &turn.routed_usage {
3645	if usage_timestamp_in_range(child.route.dispatched_at, since, until) {
3646	accumulate_runtime_usage_record(
3647	&mut totals,
3648	&mut buckets,
3649	group_by,
3650	Some(&child.route),
3651	&child.usage,
3652	&turn,
3653	&thread,
3654	);
3655	}
3656	}
3657	// Dropped fallback receipts no longer carry a trustworthy
3658	// dispatch timestamp. Use the owning turn timestamp only to
3659	// decide whether the explicit incompleteness marker belongs
3660	// in this query window; never fabricate a model/provider.
3661	if usage_timestamp_in_range(turn.created_at, since, until) {
3662	accumulate_truncated_runtime_usage(
3663	&mut totals,
3664	&mut buckets,
3665	group_by,
3666	turn.routed_usage_dropped_records,
3667	&turn,
3668	&thread,
3669	);
3670	}
3671	}
3672	}
3673
3674	let group_by_str = match group_by {
3675	UsageGroupBy::Day => "day",
3676	UsageGroupBy::Model => "model",
3677	UsageGroupBy::Provider => "provider",
3678	UsageGroupBy::Thread => "thread",
3679	}
3680	.to_string();
3681
3682	totals.cost_complete = totals.unpriced_turns == 0;
3683	for bucket in buckets.values_mut() {
3684	bucket.cost_complete = bucket.unpriced_turns == 0;
3685	}
3686
3687	Ok(UsageAggregation {
3688	since,
3689	until,
3690	group_by: group_by_str,
3691	totals,
3692	buckets: buckets.into_values().collect(),
3693	})
3694	}
3695
3696	pub async fn get_thread(&self, id: &str) -> Result<ThreadRecord> {
3697	self.flush_recovery_receipts_for_thread(id).await?;
3698	self.store
3699	.load_thread(id)
3700	.with_context(\|\| format!("Thread not found: {id}"))
3701	}
3702
3703	pub async fn update_thread(&self, id: &str, req: UpdateThreadRequest) -> Result<ThreadRecord> {
3704	if req.archived.is_none()
3705	&& req.allow_shell.is_none()
3706	&& req.trust_mode.is_none()
3707	&& req.auto_approve.is_none()
3708	&& req.model.is_none()
3709	&& req.mode.is_none()
3710	&& req.permission_posture.is_none()
3711	&& req.title.is_none()
3712	&& req.system_prompt.is_none()
3713	&& req.workspace.is_none()
3714	{
3715	bail!("At least one thread field is required");
3716	}
3717
3718	if let Some(model) = req.model.as_ref()
3719	&& model.trim().is_empty()
3720	{
3721	bail!("model must not be empty");
3722	}
3723	if let Some(mode) = req.mode.as_ref()
3724	&& mode.trim().is_empty()
3725	{
3726	bail!("mode must not be empty");
3727	}
3728	if let Some(permission_posture) = req.permission_posture.as_ref()
3729	&& permission_posture.trim().is_empty()
3730	{
3731	bail!("permission_posture must not be empty");
3732	}
3733	if let Some(workspace) = req.workspace.as_ref()
3734	&& workspace.as_os_str().is_empty()
3735	{
3736	bail!("workspace must not be empty");
3737	}
3738
3739	let configured_sandbox_mode = self.read_config().sandbox_mode.clone();
3740	let (thread, changes, evicted_engine, posture_engine) = {
3741	// Take the active guard first so a workspace mutation can check
3742	// and evict the cached engine atomically with the durable update.
3743	// Using the same order as start/compact avoids lock inversion.
3744	let mut active = self.active.lock().await;
3745	let _thread_mutation = self.store.thread_mutation.lock();
3746	let mut thread = self
3747	.store
3748	.load_thread(id)
3749	.with_context(\|\| format!("Thread not found: {id}"))?;
3750	let mut changes = serde_json::Map::new();
3751	let policy_patch = if req.mode.is_some()
3752	\|\| req.permission_posture.is_some()
3753	\|\| req.auto_approve.is_some()
3754	{
3755	Some(runtime_policy_with_overrides(
3756	&thread,
3757	req.mode.as_deref(),
3758	req.permission_posture.as_deref(),
3759	req.auto_approve,
3760	)?)
3761	} else {
3762	None
3763	};
3764
3765	if let Some(archived) = req.archived
3766	&& thread.archived != archived
3767	{
3768	thread.archived = archived;
3769	changes.insert("archived".to_string(), json!(archived));
3770	}
3771	if let Some(allow_shell) = req.allow_shell
3772	&& thread.allow_shell != allow_shell
3773	{
3774	thread.allow_shell = allow_shell;
3775	changes.insert("allow_shell".to_string(), json!(allow_shell));
3776	}
3777	if let Some(trust_mode) = req.trust_mode
3778	&& thread.trust_mode != trust_mode
3779	{
3780	thread.trust_mode = trust_mode;
3781	changes.insert("trust_mode".to_string(), json!(trust_mode));
3782	}
3783	if let Some(model) = req.model
3784	&& thread.model != model
3785	{
3786	thread.model = model.clone();
3787	changes.insert("model".to_string(), json!(model));
3788	}
3789	if let Some(policy) = policy_patch {
3790	let mode = policy.mode_setting().to_string();
3791	let permission_posture = Some(policy.permission_wire().to_string());
3792	let auto_approve = policy.auto_approve();
3793	if thread.mode != mode {
3794	thread.mode = mode.clone();
3795	changes.insert("mode".to_string(), json!(mode));
3796	}
3797	if thread.permission_posture != permission_posture {
3798	thread.permission_posture = permission_posture.clone();
3799	changes.insert("permission_posture".to_string(), json!(permission_posture));
3800	}
3801	if thread.auto_approve != auto_approve {
3802	thread.auto_approve = auto_approve;
3803	changes.insert("auto_approve".to_string(), json!(auto_approve));
3804	}
3805	}
3806	if let Some(title) = req.title {
3807	// Empty string clears a previously-set title and reverts to derived.
3808	let new_title = if title.trim().is_empty() {
3809	None
3810	} else {
3811	Some(title)
3812	};
3813	if thread.title != new_title {
3814	thread.title = new_title.clone();
3815	changes.insert("title".to_string(), json!(new_title));
3816	}
3817	}
3818	if let Some(system_prompt) = req.system_prompt {
3819	let new_sys = if system_prompt.trim().is_empty() {
3820	None
3821	} else {
3822	Some(system_prompt)
3823	};
3824	if thread.system_prompt != new_sys {
3825	thread.system_prompt = new_sys.clone();
3826	changes.insert("system_prompt".to_string(), json!(new_sys));
3827	}
3828	}
3829	if let Some(workspace) = req.workspace
3830	&& thread.workspace != workspace
3831	{
3832	changes.insert("workspace".to_string(), json!(workspace));
3833	thread.workspace = workspace;
3834	}
3835
3836	let workspace_changed = changes.contains_key("workspace");
3837	if workspace_changed
3838	&& active
3839	.engines
3840	.get(id)
3841	.and_then(\|state\| state.active_turn.as_ref())
3842	.is_some()
3843	{
3844	bail!("workspace cannot be changed while the thread has an active turn");
3845	}
3846
3847	// A posture/mode edit must reach the live engine even while a
3848	// turn is running. EngineHandle publishes the authority snapshot
3849	// before queueing ChangeMode; the turn loop applies that pending
3850	// update before the next tool batch.
3851	let posture_changed = changes.contains_key("auto_approve")
3852	\|\| changes.contains_key("permission_posture")
3853	\|\| changes.contains_key("trust_mode")
3854	\|\| changes.contains_key("allow_shell")
3855	\|\| changes.contains_key("mode");
3856
3857	let evicted_engine = if changes.is_empty() {
3858	None
3859	} else {
3860	thread.updated_at = Utc::now();
3861	self.store.save_thread(&thread)?;
3862	if workspace_changed {
3863	active.lru.retain(\|thread_id\| thread_id != id);
3864	active.engines.remove(id).map(\|state\| state.engine)
3865	} else {
3866	None
3867	}
3868	};
3869	let posture_engine = if posture_changed && !workspace_changed {
3870	active.engines.get(id).map(\|state\| state.engine.clone())
3871	} else {
3872	None
3873	};
3874	(thread, changes, evicted_engine, posture_engine)
3875	};
3876
3877	if let Some(engine) = evicted_engine {
3878	let _ = engine.send(Op::Shutdown).await;
3879	}
3880
3881	// Keep the live engine session converged with the thread record.
3882	// Idle engines apply it immediately; a running turn applies it at
3883	// the next mid-turn drain (before the next tool batch).
3884	if let Some(engine) = posture_engine {
3885	let policy = RuntimePolicyProjection::from_persisted(
3886	&thread.mode,
3887	thread.permission_posture.as_deref(),
3888	thread.auto_approve,
3889	);
3890	let _ = engine.try_send(Op::ChangeMode {
3891	mode: policy.mode,
3892	allow_shell: thread.allow_shell,
3893	trust_mode: thread.trust_mode,
3894	auto_approve: policy.auto_approve(),
3895	approval_mode: policy.permission,
3896	configured_sandbox_mode: configured_sandbox_mode.clone(),
3897	});
3898	}
3899
3900	if !changes.is_empty() {
3901	self.emit_event(
3902	&thread.id,
3903	None,
3904	None,
3905	"thread.updated",
3906	json!({
3907	"thread": thread.clone(),
3908	"changes": Value::Object(changes),
3909	}),
3910	)
3911	.await?;
3912	}
3913
3914	Ok(thread)
3915	}
3916
3917	/// Link a session to a thread so that `ensure_engine_loaded` can restore
3918	/// the full message history (including thinking/tool blocks) from the
3919	/// session file instead of reconstructing from turns.
3920	pub async fn set_thread_session_id(&self, thread_id: &str, session_id: &str) -> Result<()> {
3921	let thread = {
3922	let _thread_mutation = self.store.thread_mutation.lock();
3923	let mut thread = self
3924	.store
3925	.load_thread(thread_id)
3926	.with_context(\|\| format!("Thread not found: {thread_id}"))?;
3927	if thread.session_id.as_deref() == Some(session_id) {
3928	return Ok(());
3929	}
3930	thread.session_id = Some(session_id.to_string());
3931	thread.updated_at = Utc::now();
3932	self.store.save_thread(&thread)?;
3933	thread
3934	};
3935	self.emit_event(
3936	thread_id,
3937	None,
3938	None,
3939	"thread.updated",
3940	json!({ "thread": thread, "changes": { "session_id": session_id } }),
3941	)
3942	.await?;
3943	Ok(())
3944	}
3945
3946	pub async fn get_thread_detail(&self, id: &str) -> Result<ThreadDetail> {
3947	self.flush_recovery_receipts_for_thread(id).await?;
3948	// Hold the per-thread projection boundary from cursor capture through
3949	// item reads. A streamed delta is therefore either entirely before
3950	// this snapshot (materialized item + included cursor) or entirely
3951	// after it (old item + replayable delta), never both.
3952	let projection_lock = self.projection_lock(id);
3953	let _projection = projection_lock.lock().await;
3954	let latest_seq = self.store.current_seq().await?;
3955
3956	#[cfg(test)]
3957	let snapshot_test_hook = { self.snapshot_test_hook.lock().take() };
3958	#[cfg(test)]
3959	if let Some(hook) = snapshot_test_hook {
3960	let (resume, wait_for_resume) = oneshot::channel();
3961	hook.send(SnapshotTestPoint {
3962	thread_id: id.to_string(),
3963	latest_seq,
3964	resume,
3965	})
3966	.map_err(\|_\| anyhow!("snapshot test hook closed"))?;
3967	wait_for_resume
3968	.await
3969	.map_err(\|_\| anyhow!("snapshot test hook dropped resume"))?;
3970	}
3971
3972	// Recovery was flushed before taking the non-reentrant projection
3973	// lock. Do not call `get_thread` here: a receipt queued between that
3974	// flush and this read would re-enter recovery and wait forever on the
3975	// projection lock held by this snapshot.
3976	let store = self.store.clone();
3977	let snapshot_thread_id = id.to_string();
3978	let (thread, turns, items) = tokio::task::spawn_blocking(move \|\| {
3979	let thread = store
3980	.load_thread(&snapshot_thread_id)
3981	.with_context(\|\| format!("Thread not found: {snapshot_thread_id}"))?;
3982	let turns = store.list_turns_for_thread(&snapshot_thread_id)?;
3983	let turn_ids: Vec<String> = turns.iter().map(\|turn\| turn.id.clone()).collect();
3984	let mut items_by_turn = store.list_items_for_turns_map(&turn_ids)?;
3985	let mut items = Vec::new();
3986	for turn in &turns {
3987	if let Some(mut turn_items) = items_by_turn.remove(&turn.id) {
3988	items.append(&mut turn_items);
3989	}
3990	}
3991	Ok::<_, anyhow::Error>((thread, turns, items))
3992	})
3993	.await
3994	.context("Runtime thread projection task failed")??;
3995	let (pending_approvals, pending_user_inputs) = self.pending_requests_for_thread(id);
3996	let pending_dynamic_tool_calls = self.pending_dynamic_tool_calls_for_thread(id);
3997	Ok(ThreadDetail {
3998	thread,
3999	turns,
4000	items,
4001	latest_seq,
4002	pending_approvals,
4003	pending_user_inputs,
4004	pending_dynamic_tool_calls,
4005	})
4006	}
4007
4008	pub async fn resume_thread(&self, id: &str) -> Result<ThreadRecord> {
4009	let thread = self.get_thread(id).await?;
4010	self.ensure_engine_loaded(&thread).await?;
4011	Ok(thread)
4012	}
4013
4014	/// Resume a thread and recover the sub-agent rebind hints needed to
4015	/// reconstruct in-transcript cards (issue #128). Drains the persisted
4016	/// `agent.*` event stream and collapses it into the latest known
4017	/// status per `agent_id` — the UI consumes this to seed empty
4018	/// `DelegateCard` / `FanoutCard` placeholders so subsequent live
4019	/// mailbox envelopes mutate them in place.
4020	#[allow(dead_code)] // exposed for the runtime API resume flow; consumed by #128 follow-up.
4021	pub async fn resume_thread_with_agent_rebind(
4022	&self,
4023	id: &str,
4024	) -> Result<(ThreadRecord, Vec<AgentRebindHint>)> {
4025	let thread = self.resume_thread(id).await?;
4026	let events = self.events_since_async(&thread.id, None).await?;
4027	let hints = collect_agent_rebind_hints(&events);
4028	Ok((thread, hints))
4029	}
4030
4031	pub async fn fork_thread(&self, id: &str) -> Result<ThreadRecord> {
4032	let source = self.get_thread(id).await?;
4033	let mut forked = source.clone();
4034	let now = Utc::now();
4035	forked.id = format!("thr_{}", &Uuid::new_v4().to_string()[..8]);
4036	forked.created_at = now;
4037	forked.updated_at = now;
4038	forked.latest_turn_id = None;
4039	forked.archived = false;
4040
4041	let source_turns = self.store.list_turns_for_thread(&source.id)?;
4042	let mut cloned_records = Vec::with_capacity(source_turns.len());
4043	for source_turn in source_turns {
4044	let mut cloned_turn = source_turn.clone();
4045	cloned_turn.id = format!("turn_{}", &Uuid::new_v4().to_string()[..8]);
4046	cloned_turn.thread_id = forked.id.clone();
4047	cloned_turn.item_ids.clear();
4048
4049	let items = self.store.list_items_for_turn(&source_turn.id)?;
4050	let mut cloned_items = Vec::with_capacity(items.len());
4051	for item in items {
4052	let mut cloned_item = item.clone();
4053	cloned_item.id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
4054	cloned_item.turn_id = cloned_turn.id.clone();
4055	cloned_turn.item_ids.push(cloned_item.id.clone());
4056	cloned_items.push(cloned_item);
4057	}
4058	forked.latest_turn_id = Some(cloned_turn.id.clone());
4059	forked.updated_at = now;
4060	cloned_records.push((cloned_turn, cloned_items));
4061	}
4062	self.publish_fork(&forked, &cloned_records)?;
4063
4064	self.emit_event(
4065	&forked.id,
4066	None,
4067	None,
4068	"thread.forked",
4069	json!({
4070	"thread": forked,
4071	"source_thread_id": source.id,
4072	}),
4073	)
4074	.await?;
4075	Ok(forked)
4076	}
4077
4078	/// Fork a thread, dropping every turn from the Nth-from-tail user
4079	/// message onward (issue #133 — Esc-Esc backtrack).
4080	///
4081	/// `depth_from_tail` selects which user turn to roll back to:
4082	///
4083	/// - `0` — drop the most recent turn (the freshest user message and
4084	/// everything after it)
4085	/// - `1` — drop the two most recent turns (rewind one further)
4086	/// - …and so on
4087	///
4088	/// Returns a tuple of `(forked_thread, original_user_text)` where the
4089	/// second element is the `detail` of the first `UserMessage` item in
4090	/// the first dropped turn — i.e. the input the user typed to start
4091	/// that turn — so the caller can pre-populate the composer with it.
4092	/// `None` when no detail was recorded (defensive — every persisted
4093	/// `UserMessage` since v0.6 carries a detail string).
4094	///
4095	/// Counts user turns by iterating `list_turns_for_thread` (sorted
4096	/// oldest → newest) backwards. A turn is counted as a "user turn"
4097	/// when at least one of its items has `kind ==
4098	/// TurnItemKind::UserMessage`. Steered turns (which append additional
4099	/// `UserMessage` items) still count as one turn — backtrack rewinds
4100	/// at the turn boundary, not at the steer boundary.
4101	///
4102	/// Errors:
4103	/// - `depth_from_tail` exceeds the number of user turns
4104	/// - source thread not found
4105	#[allow(dead_code)] // exposed for the runtime/HTTP fork-on-backtrack path; the in-TUI Esc-Esc flow trims `App` state directly. Issue #133.
4106	pub async fn fork_at_user_message(
4107	&self,
4108	id: &str,
4109	depth_from_tail: usize,
4110	) -> Result<(ThreadRecord, Option<String>)> {
4111	let source = self.get_thread(id).await?;
4112	let source_turns = self.store.list_turns_for_thread(&source.id)?;
4113
4114	// Walk turns from newest to oldest. For each turn, ask: does it
4115	// contain a UserMessage item? If yes, it counts toward the depth.
4116	let mut user_turn_indices: Vec<usize> = Vec::new();
4117	for (idx, turn) in source_turns.iter().enumerate().rev() {
4118	let items = self.store.list_items_for_turn(&turn.id)?;
4119	if items
4120	.iter()
4121	.any(\|item\| item.kind == TurnItemKind::UserMessage)
4122	{
4123	user_turn_indices.push(idx);
4124	}
4125	}
4126	if depth_from_tail >= user_turn_indices.len() {
4127	bail!(
4128	"fork_at_user_message: depth {} exceeds {} user turn(s)",
4129	depth_from_tail,
4130	user_turn_indices.len()
4131	);
4132	}
4133	// `user_turn_indices` is newest-first because we iterated in
4134	// reverse, so the Nth element is exactly the Nth-from-tail user
4135	// turn in the original chronological list.
4136	let target_turn_idx = user_turn_indices[depth_from_tail];
4137	let target_turn_id = source_turns[target_turn_idx].id.clone();
4138
4139	// Pull the original user-message text out of the dropped turn so
4140	// the caller can drop it back into the composer.
4141	let target_items = self.store.list_items_for_turn(&target_turn_id)?;
4142	let original_user_text = target_items
4143	.iter()
4144	.find(\|item\| item.kind == TurnItemKind::UserMessage)
4145	.and_then(\|item\| item.detail.clone());
4146
4147	// Copy turns strictly before `target_turn_idx` into a new thread.
4148	// Mirrors `fork_thread` but stops at the cutoff instead of copying
4149	// every turn. Kept structurally close so future parity reviews
4150	// can spot drift between the two paths.
4151	let mut forked = source.clone();
4152	let now = Utc::now();
4153	forked.id = format!("thr_{}", &Uuid::new_v4().to_string()[..8]);
4154	forked.created_at = now;
4155	forked.updated_at = now;
4156	forked.latest_turn_id = None;
4157	forked.archived = false;
4158
4159	let mut cloned_records = Vec::with_capacity(target_turn_idx);
4160	for source_turn in source_turns.iter().take(target_turn_idx) {
4161	let mut cloned_turn = source_turn.clone();
4162	cloned_turn.id = format!("turn_{}", &Uuid::new_v4().to_string()[..8]);
4163	cloned_turn.thread_id = forked.id.clone();
4164	cloned_turn.item_ids.clear();
4165
4166	let items = self.store.list_items_for_turn(&source_turn.id)?;
4167	let mut cloned_items = Vec::with_capacity(items.len());
4168	for item in items {
4169	let mut cloned_item = item.clone();
4170	cloned_item.id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
4171	cloned_item.turn_id = cloned_turn.id.clone();
4172	cloned_turn.item_ids.push(cloned_item.id.clone());
4173	cloned_items.push(cloned_item);
4174	}
4175	forked.latest_turn_id = Some(cloned_turn.id.clone());
4176	forked.updated_at = now;
4177	cloned_records.push((cloned_turn, cloned_items));
4178	}
4179	self.publish_fork(&forked, &cloned_records)?;
4180
4181	self.emit_event(
4182	&forked.id,
4183	None,
4184	None,
4185	"thread.forked",
4186	json!({
4187	"thread": forked,
4188	"source_thread_id": source.id,
4189	"backtrack_depth_from_tail": depth_from_tail,
4190	"dropped_turn_id": target_turn_id,
4191	}),
4192	)
4193	.await?;
4194	Ok((forked, original_user_text))
4195	}
4196
4197	/// Persist cloned records before publishing their thread. Until the final
4198	/// atomic thread write succeeds, list/get/start callers cannot observe a
4199	/// partial fork. Any failed write removes all unpublished clone artifacts.
4200	fn publish_fork(
4201	&self,
4202	thread: &ThreadRecord,
4203	records: &[(TurnRecord, Vec<TurnItemRecord>)],
4204	) -> Result<()> {
4205	let mut saved_turn_ids = Vec::new();
4206	let mut saved_item_ids = Vec::new();
4207	let persistence = (\|\| -> Result<()> {
4208	for (turn, items) in records {
4209	for item in items {
4210	self.store.save_item(item)?;
4211	saved_item_ids.push(item.id.clone());
4212	}
4213	self.store.save_turn(turn)?;
4214	saved_turn_ids.push(turn.id.clone());
4215	}
4216	self.store.save_thread(thread)
4217	})();
4218
4219	if let Err(persistence_error) = persistence {
4220	let mut cleanup_errors = Vec::new();
4221	if let Err(error) = self.store.remove_thread(&thread.id) {
4222	cleanup_errors.push(format!("remove thread: {error}"));
4223	}
4224	for turn_id in saved_turn_ids.iter().rev() {
4225	if let Err(error) = self.store.remove_turn(turn_id) {
4226	cleanup_errors.push(format!("remove turn {turn_id}: {error}"));
4227	}
4228	}
4229	for item_id in saved_item_ids.iter().rev() {
4230	if let Err(error) = self.store.remove_item(item_id) {
4231	cleanup_errors.push(format!("remove item {item_id}: {error}"));
4232	}
4233	}
4234	if cleanup_errors.is_empty() {
4235	return Err(persistence_error);
4236	}
4237	bail!(
4238	"Failed to persist fork: {persistence_error}; cleanup also failed: {}",
4239	cleanup_errors.join("; ")
4240	);
4241	}
4242	Ok(())
4243	}
4244
4245	/// Seed a thread with messages from a saved session so subsequent turns
4246	/// continue with the prior conversation context.
4247	///
4248	/// Unlike the old text-only implementation, this preserves all content
4249	/// block types (thinking, tool_use, tool_result, etc.) as separate turn
4250	/// items so that `loadHistory` in the GUI can reconstruct the full
4251	/// conversation including process information.
4252	pub async fn seed_thread_from_messages(
4253	&self,
4254	thread_id: &str,
4255	messages: &[Message],
4256	) -> Result<()> {
4257	// Session seeding writes turns/items and then advances the existing
4258	// thread pointer as one synchronous record transaction.
4259	let thread_mutation = self.store.thread_mutation.lock();
4260	let mut thread = self
4261	.store
4262	.load_thread(thread_id)
4263	.with_context(\|\| format!("Thread not found: {thread_id}"))?;
4264	let now = Utc::now();
4265
4266	// Group messages into turns. A turn starts with a user message and
4267	// includes all subsequent assistant messages (which may contain
4268	// thinking, tool_use, tool_result blocks) until the next user message.
4269	let mut turns: Vec<TurnSeed> = Vec::new();
4270	let mut current_turn: Option<TurnSeed> = None;
4271
4272	for msg in messages {
4273	match msg.role.as_str() {
4274	"user" => {
4275	let mut user_text = String::new();
4276	let mut tool_results = Vec::new();
4277
4278	for block in &msg.content {
4279	match block {
4280	ContentBlock::Text { text, .. } if !text.trim().is_empty() => {
4281	if !user_text.is_empty() {
4282	user_text.push('\n');
4283	}
4284	user_text.push_str(text);
4285	}
4286	ContentBlock::ToolResult {
4287	tool_use_id,
4288	content,
4289	is_error,
4290	content_blocks,
4291	} => {
4292	tool_results.push(SeedItem::ToolResult {
4293	tool_use_id: tool_use_id.clone(),
4294	content: content.clone(),
4295	is_error: is_error.unwrap_or(false),
4296	content_blocks: content_blocks.clone(),
4297	});
4298	}
4299	// Other block types in user messages are rare;
4300	// skip them gracefully.
4301	_ => {}
4302	}
4303	}
4304
4305	if !user_text.is_empty() {
4306	// A real user prompt begins a new turn. Tool results
4307	// without text belong to the preceding assistant turn.
4308	if let Some(t) = current_turn.take() {
4309	turns.push(t);
4310	}
4311	current_turn = Some(TurnSeed {
4312	user_text,
4313	items: tool_results,
4314	});
4315	} else if !tool_results.is_empty() {
4316	let turn = current_turn.get_or_insert_with(\|\| TurnSeed {
4317	user_text: String::new(),
4318	items: Vec::new(),
4319	});
4320	turn.items.extend(tool_results);
4321	} else {
4322	if let Some(t) = current_turn.take() {
4323	turns.push(t);
4324	}
4325	current_turn = Some(TurnSeed {
4326	user_text: String::new(),
4327	items: Vec::new(),
4328	});
4329	}
4330	}
4331	"assistant" => {
4332	// If no current turn exists (e.g. session starts with
4333	// an assistant message), create a placeholder turn.
4334	let turn = current_turn.get_or_insert_with(\|\| TurnSeed {
4335	user_text: String::new(),
4336	items: Vec::new(),
4337	});
4338	for block in &msg.content {
4339	match block {
4340	ContentBlock::Text { text, .. } if !text.trim().is_empty() => {
4341	turn.items.push(SeedItem::Text(text.clone()));
4342	}
4343	ContentBlock::Thinking { thinking, .. }
4344	if !thinking.trim().is_empty() =>
4345	{
4346	turn.items.push(SeedItem::Thinking(thinking.clone()));
4347	}
4348	ContentBlock::ToolUse {
4349	id, name, input, ..
4350	} => {
4351	turn.items.push(SeedItem::ToolUse {
4352	id: id.clone(),
4353	name: name.clone(),
4354	input: input.clone(),
4355	});
4356	}
4357	ContentBlock::ServerToolUse {
4358	id, name, input, ..
4359	} => {
4360	turn.items.push(SeedItem::ToolUse {
4361	id: id.clone(),
4362	name: name.clone(),
4363	input: input.clone(),
4364	});
4365	}
4366	// Skip other block types (image_url, etc.)
4367	_ => {}
4368	}
4369	}
4370	}
4371	// System messages and other roles are ignored for turn seeding.
4372	_ => {}
4373	}
4374	}
4375	// Flush the last turn.
4376	if let Some(t) = current_turn.take() {
4377	turns.push(t);
4378	}
4379
4380	for turn_seed in turns {
4381	let turn_id = format!("turn_{}", &Uuid::new_v4().to_string()[..8]);
4382	let summary =
4383	crate::utils::truncate_with_ellipsis(&turn_seed.user_text, SUMMARY_LIMIT, "...");
4384	let mut item_ids = Vec::new();
4385
4386	// Save user message item.
4387	if !turn_seed.user_text.is_empty() {
4388	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
4389	self.store.save_item(&TurnItemRecord {
4390	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4391	id: item_id.clone(),
4392	turn_id: turn_id.clone(),
4393	kind: TurnItemKind::UserMessage,
4394	status: TurnItemLifecycleStatus::Completed,
4395	summary: summary.clone(),
4396	detail: Some(turn_seed.user_text.clone()),
4397	metadata: None,
4398	artifact_refs: Vec::new(),
4399	started_at: Some(now),
4400	ended_at: Some(now),
4401	})?;
4402	item_ids.push(item_id);
4403	}
4404
4405	// Save assistant content items in order.
4406	for seed_item in &turn_seed.items {
4407	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
4408	match seed_item {
4409	SeedItem::Text(text) => {
4410	let asst_summary = if text.len() > SUMMARY_LIMIT {
4411	crate::utils::truncate_with_ellipsis(text, SUMMARY_LIMIT, "...")
4412	} else {
4413	text.clone()
4414	};
4415	self.store.save_item(&TurnItemRecord {
4416	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4417	id: item_id.clone(),
4418	turn_id: turn_id.clone(),
4419	kind: TurnItemKind::AgentMessage,
4420	status: TurnItemLifecycleStatus::Completed,
4421	summary: asst_summary,
4422	detail: Some(text.clone()),
4423	metadata: None,
4424	artifact_refs: Vec::new(),
4425	started_at: Some(now),
4426	ended_at: Some(now),
4427	})?;
4428	}
4429	SeedItem::Thinking(thinking) => {
4430	let thinking_summary = if thinking.len() > SUMMARY_LIMIT {
4431	crate::utils::truncate_with_ellipsis(thinking, SUMMARY_LIMIT, "...")
4432	} else {
4433	thinking.clone()
4434	};
4435	self.store.save_item(&TurnItemRecord {
4436	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4437	id: item_id.clone(),
4438	turn_id: turn_id.clone(),
4439	kind: TurnItemKind::AgentReasoning,
4440	status: TurnItemLifecycleStatus::Completed,
4441	summary: thinking_summary,
4442	detail: Some(thinking.clone()),
4443	metadata: None,
4444	artifact_refs: Vec::new(),
4445	started_at: Some(now),
4446	ended_at: Some(now),
4447	})?;
4448	}
4449	SeedItem::ToolUse {
4450	id: tool_id,
4451	name,
4452	input,
4453	} => {
4454	let input_str =
4455	serde_json::to_string(input).unwrap_or_else(\|_\| input.to_string());
4456	let tool_summary = format!("{name}({})", {
4457	let s = &input_str;
4458	if s.len() > 80 {
4459	crate::utils::truncate_with_ellipsis(s, 80, "...")
4460	} else {
4461	s.clone()
4462	}
4463	});
4464	self.store.save_item(&TurnItemRecord {
4465	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4466	id: item_id.clone(),
4467	turn_id: turn_id.clone(),
4468	kind: TurnItemKind::ToolCall,
4469	status: TurnItemLifecycleStatus::Completed,
4470	summary: tool_summary,
4471	detail: Some(input_str),
4472	metadata: Some(serde_json::Value::Object(
4473	serde_json::json!({
4474	"tool_use_id": tool_id,
4475	"tool_name": name,
4476	})
4477	.as_object()
4478	.unwrap()
4479	.clone(),
4480	)),
4481	artifact_refs: Vec::new(),
4482	started_at: Some(now),
4483	ended_at: Some(now),
4484	})?;
4485	}
4486	SeedItem::ToolResult {
4487	tool_use_id,
4488	content,
4489	is_error,
4490	content_blocks,
4491	} => {
4492	let result_summary = if content.len() > SUMMARY_LIMIT {
4493	crate::utils::truncate_with_ellipsis(content, SUMMARY_LIMIT, "...")
4494	} else {
4495	content.clone()
4496	};
4497	let mut metadata = serde_json::Map::new();
4498	metadata.insert("tool_result_for".to_string(), json!(tool_use_id));
4499	metadata.insert("is_error".to_string(), json!(is_error));
4500	if let Some(blocks) = content_blocks {
4501	metadata
4502	.insert("content_blocks".to_string(), Value::Array(blocks.clone()));
4503	}
4504	self.store.save_item(&TurnItemRecord {
4505	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4506	id: item_id.clone(),
4507	turn_id: turn_id.clone(),
4508	kind: TurnItemKind::ToolCall,
4509	status: if *is_error {
4510	TurnItemLifecycleStatus::Failed
4511	} else {
4512	TurnItemLifecycleStatus::Completed
4513	},
4514	summary: result_summary,
4515	detail: Some(content.clone()),
4516	metadata: Some(Value::Object(metadata)),
4517	artifact_refs: Vec::new(),
4518	started_at: Some(now),
4519	ended_at: Some(now),
4520	})?;
4521	}
4522	}
4523	item_ids.push(item_id);
4524	}
4525
4526	// Only create a turn if there's content.
4527	if !item_ids.is_empty() {
4528	self.store.save_turn(&TurnRecord {
4529	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
4530	id: turn_id.clone(),
4531	thread_id: thread_id.to_string(),
4532	status: RuntimeTurnStatus::Completed,
4533	input_summary: summary,
4534	created_at: now,
4535	started_at: Some(now),
4536	ended_at: Some(now),
4537	duration_ms: Some(0),
4538	usage: None,
4539	permission_posture: None,
4540	effective_provider: None,
4541	effective_provider_id: None,
4542	effective_billing_surface: None,
4543	effective_endpoint_fingerprint: None,
4544	effective_billing_mode: None,
4545	effective_dispatched_at: None,
4546	effective_model: None,
4547	routed_usage: Vec::new(),
4548	routed_usage_source_ids: Vec::new(),
4549	routed_usage_dropped_records: 0,
4550	error: None,
4551	item_ids,
4552	steer_count: 0,
4553	})?;
4554
4555	thread.latest_turn_id = Some(turn_id);
4556	thread.updated_at = now;
4557	}
4558	}
4559
4560	self.store.save_thread(&thread)?;
4561	drop(thread_mutation);
4562	self.emit_event(
4563	thread_id,
4564	None,
4565	None,
4566	"thread.updated",
4567	json!({ "thread": thread, "reason": "session_resume" }),
4568	)
4569	.await?;
4570	Ok(())
4571	}
4572
4573	fn cleanup_unaccepted_turn_records(&self, turn_id: &str, item_id: Option<&str>) -> Result<()> {
4574	let mut errors = Vec::new();
4575	if let Some(item_id) = item_id
4576	&& let Err(err) = self.store.remove_item(item_id)
4577	{
4578	errors.push(format!("remove item: {err}"));
4579	}
4580	if let Err(err) = self.store.remove_turn(turn_id) {
4581	errors.push(format!("remove turn: {err}"));
4582	}
4583	if errors.is_empty() {
4584	Ok(())
4585	} else {
4586	bail!(errors.join("; "))
4587	}
4588	}
4589
4590	async fn emit_claimed_turn_started(
4591	&self,
4592	turn: &TurnRecord,
4593	user_item: Option<&TurnItemRecord>,
4594	kind: ClaimedTurnKind,
4595	) {
4596	let start_payload = match kind {
4597	ClaimedTurnKind::Message => json!({ "turn": turn.clone() }),
4598	ClaimedTurnKind::Compaction => {
4599	json!({ "turn": turn.clone(), "manual_compaction": true })
4600	}
4601	};
4602	if let Err(err) = self
4603	.emit_event(
4604	&turn.thread_id,
4605	Some(&turn.id),
4606	None,
4607	"turn.started",
4608	start_payload,
4609	)
4610	.await
4611	{
4612	tracing::warn!(
4613	"Failed to persist {}.started after engine acceptance: {err}",
4614	kind.label()
4615	);
4616	}
4617
4618	if let Some(user_item) = user_item {
4619	if let Err(err) = self
4620	.emit_event(
4621	&turn.thread_id,
4622	Some(&turn.id),
4623	Some(&user_item.id),
4624	"item.started",
4625	json!({ "item": user_item.clone() }),
4626	)
4627	.await
4628	{
4629	tracing::warn!("Failed to persist item.started after engine acceptance: {err}");
4630	}
4631	if let Err(err) = self
4632	.emit_event(
4633	&turn.thread_id,
4634	Some(&turn.id),
4635	Some(&user_item.id),
4636	"item.completed",
4637	json!({ "item": user_item.clone() }),
4638	)
4639	.await
4640	{
4641	tracing::warn!("Failed to persist item.completed after engine acceptance: {err}");
4642	}
4643	}
4644	}
4645
4646	async fn settle_claimed_turn_failure(&self, thread_id: &str, turn_id: &str, reason: &str) {
4647	// Block steer attempts while terminal receipts are being settled; the
4648	// active claim remains present so a replacement turn cannot start.
4649	{
4650	let mut active = self.active.lock().await;
4651	if let Some(turn) = active
4652	.engines
4653	.get_mut(thread_id)
4654	.and_then(\|state\| state.active_turn.as_mut())
4655	&& turn.turn_id == turn_id
4656	{
4657	turn.interrupt_requested = true;
4658	}
4659	}
4660	let now = Utc::now();
4661	crate::cost_status::finish_runtime_usage_owner(turn_id);
4662	let background_usage = crate::cost_status::take_runtime_usage(turn_id);
4663	let mut terminal_items = Vec::new();
4664	match self.store.list_items_for_turn(turn_id) {
4665	Ok(items) => {
4666	for mut item in items {
4667	if matches!(
4668	item.status,
4669	TurnItemLifecycleStatus::Queued \| TurnItemLifecycleStatus::InProgress
4670	) {
4671	item.status = TurnItemLifecycleStatus::Failed;
4672	item.ended_at = Some(now);
4673	match self.store.save_item(&item) {
4674	Ok(()) => terminal_items.push(item),
4675	Err(err) => tracing::error!(
4676	item_id = %item.id,
4677	"Failed to terminalize item after monitor failure: {err}"
4678	),
4679	}
4680	}
4681	}
4682	}
4683	Err(err) => tracing::error!(
4684	"Failed to list turn items after monitor failure for {turn_id}: {err}"
4685	),
4686	}
4687	let terminal_turn = {
4688	let _turn_mutation = self.store.turn_mutation.lock();
4689	match self.store.load_turn(turn_id) {
4690	Ok(mut turn) => {
4691	for record in background_usage.records.iter().cloned() {
4692	append_routed_usage_record(&mut turn, &record.source_id, record.usage);
4693	}
4694	turn.routed_usage_dropped_records = turn
4695	.routed_usage_dropped_records
4696	.saturating_add(background_usage.dropped_records);
4697	if turn.status == RuntimeTurnStatus::InProgress {
4698	turn.status = RuntimeTurnStatus::Failed;
4699	turn.ended_at = Some(now);
4700	turn.duration_ms = turn.started_at.map(\|start\| duration_ms(start, now));
4701	turn.error = Some(reason.to_string());
4702	}
4703	matches!(
4704	turn.status,
4705	RuntimeTurnStatus::Completed
4706	\| RuntimeTurnStatus::Failed
4707	\| RuntimeTurnStatus::Interrupted
4708	\| RuntimeTurnStatus::Canceled
4709	)
4710	.then_some(turn)
4711	}
4712	Err(err) => {
4713	tracing::error!("Failed to load turn after monitor failure: {err}");
4714	None
4715	}
4716	}
4717	};
4718
4719	for item in terminal_items {
4720	if let Err(err) = self
4721	.emit_event(
4722	thread_id,
4723	Some(turn_id),
4724	Some(&item.id),
4725	"item.failed",
4726	json!({ "item": item, "error": reason }),
4727	)
4728	.await
4729	{
4730	tracing::error!("Failed to emit terminal item failure: {err}");
4731	}
4732	}
4733
4734	// A failed turn can no longer answer an outstanding prompt. Mirror the
4735	// happy terminal path's receipt-before-removal ordering.
4736	let engine_for_cancel = {
4737	let active = self.active.lock().await;
4738	active
4739	.engines
4740	.get(thread_id)
4741	.map(\|state\| state.engine.clone())
4742	};
4743	let user_inputs_settled = if let Err(err) = self
4744	.settle_user_inputs_for_terminal_turn(thread_id, turn_id, engine_for_cancel)
4745	.await
4746	{
4747	tracing::error!("Failed to emit user-input cancellation after monitor failure: {err}");
4748	false
4749	} else {
4750	true
4751	};
4752
4753	let dynamic_tools_settled = if let Err(err) = self
4754	.settle_dynamic_tools_for_terminal_turn(thread_id, turn_id)
4755	.await
4756	{
4757	tracing::error!(
4758	"Failed to emit dynamic-tool cancellation after monitor failure: {err}"
4759	);
4760	false
4761	} else {
4762	true
4763	};
4764
4765	// A terminal record is the externally visible lifecycle boundary.
4766	// Keep snapshots outside that boundary until its terminal receipt and
4767	// active-claim cleanup are also ordered. The dedupe scan may yield to
4768	// a blocking worker while this projection guard remains held.
4769	let projection_lock = self.projection_lock(thread_id);
4770	let _projection = projection_lock.lock().await;
4771	let terminal_turn = terminal_turn.and_then(\|turn\| {
4772	let _turn_mutation = self.store.turn_mutation.lock();
4773	match self.store.save_turn(&turn) {
4774	Ok(()) => Some(turn),
4775	Err(err) => {
4776	tracing::error!("Failed to persist terminal monitor failure: {err}");
4777	None
4778	}
4779	}
4780	});
4781	if let Some(turn) = terminal_turn.as_ref() {
4782	if user_inputs_settled && dynamic_tools_settled {
4783	if let Err(err) = self.emit_turn_completed_if_missing(turn, false).await {
4784	tracing::error!("Failed to emit terminal monitor failure: {err}");
4785	self.queue_recovery_receipt(RecoveredTurnReceipt {
4786	turn: turn.clone(),
4787	unresolved_dynamic_tools: Vec::new(),
4788	});
4789	}
4790	} else {
4791	self.queue_recovery_receipt(RecoveredTurnReceipt {
4792	turn: turn.clone(),
4793	unresolved_dynamic_tools: Vec::new(),
4794	});
4795	}
4796	}
4797
4798	// Keep the failed claim in place until its terminal receipts are
4799	// ordered. Then poison and evict this engine so the next turn gets a
4800	// distinct event receiver and cannot consume stale terminal events.
4801	let evicted_engine = {
4802	let mut active = self.active.lock().await;
4803	let owns_failed_turn = active
4804	.engines
4805	.get(thread_id)
4806	.and_then(\|state\| state.active_turn.as_ref())
4807	.is_some_and(\|turn\| turn.turn_id == turn_id);
4808	if owns_failed_turn {
4809	active.lru.retain(\|id\| id != thread_id);
4810	active.engines.remove(thread_id).map(\|state\| state.engine)
4811	} else {
4812	None
4813	}
4814	};
4815	if let Some(engine) = evicted_engine {
4816	drop(_projection);
4817	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
4818	let _ = engine.try_send(Op::Shutdown);
4819	}
4820	}
4821
4822	async fn monitor_claimed_turn(
4823	&self,
4824	thread_id: String,
4825	turn_id: String,
4826	engine: EngineHandle,
4827	kind: ClaimedTurnKind,
4828	) {
4829	if self.cancel_token.is_cancelled() {
4830	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
4831	self.settle_claimed_turn_failure(
4832	&thread_id,
4833	&turn_id,
4834	"Runtime shutdown requested before turn monitoring started",
4835	)
4836	.await;
4837	return;
4838	}
4839
4840	use futures_util::FutureExt;
4841	let result = std::panic::AssertUnwindSafe(self.monitor_turn(
4842	thread_id.clone(),
4843	turn_id.clone(),
4844	engine.clone(),
4845	))
4846	.catch_unwind()
4847	.await;
4848	let failure = match result {
4849	Ok(Ok(())) => return,
4850	Ok(Err(error)) => format!("Failed to monitor {}: {error}", kind.label()),
4851	Err(payload) => format!(
4852	"{} monitor panicked: {}",
4853	kind.label(),
4854	panic_payload_message(&*payload)
4855	),
4856	};
4857	tracing::error!("{failure}");
4858	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
4859	self.settle_claimed_turn_failure(&thread_id, &turn_id, &failure)
4860	.await;
4861	}
4862
4863	fn spawn_claimed_turn_monitor(
4864	&self,
4865	turn: TurnRecord,
4866	user_item: Option<TurnItemRecord>,
4867	engine: EngineHandle,
4868	kind: ClaimedTurnKind,
4869	) -> oneshot::Receiver<std::result::Result<TurnRecord, String>> {
4870	let (acceptance_tx, acceptance_rx) = oneshot::channel();
4871	let manager = Arc::new(self.clone());
4872	tokio::spawn(async move {
4873	use futures_util::FutureExt;
4874	let start_events = std::panic::AssertUnwindSafe(manager.emit_claimed_turn_started(
4875	&turn,
4876	user_item.as_ref(),
4877	kind,
4878	))
4879	.catch_unwind()
4880	.await;
4881	if let Err(payload) = start_events {
4882	let failure = format!(
4883	"{} start-event recording panicked after engine acceptance: {}",
4884	kind.label(),
4885	panic_payload_message(&*payload)
4886	);
4887	tracing::error!("{failure}");
4888	let _ = acceptance_tx.send(Ok(turn.clone()));
4889	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
4890	manager
4891	.settle_claimed_turn_failure(&turn.thread_id, &turn.id, &failure)
4892	.await;
4893	return;
4894	}
4895
4896	let _ = acceptance_tx.send(Ok(turn.clone()));
4897	manager
4898	.monitor_claimed_turn(turn.thread_id.clone(), turn.id.clone(), engine, kind)
4899	.await;
4900	});
4901	acceptance_rx
4902	}
4903
4904	fn spawn_steer_receipts(
4905	&self,
4906	turn: TurnRecord,
4907	item: TurnItemRecord,
4908	prompt: String,
4909	) -> oneshot::Receiver<TurnRecord> {
4910	let (receipt_tx, receipt_rx) = oneshot::channel();
4911	let manager = Arc::new(self.clone());
4912	tokio::spawn(async move {
4913	use futures_util::FutureExt;
4914	let receipts = std::panic::AssertUnwindSafe(async {
4915	if let Err(err) = manager
4916	.emit_event(
4917	&turn.thread_id,
4918	Some(&turn.id),
4919	Some(&item.id),
4920	"turn.steered",
4921	json!({
4922	"thread_id": turn.thread_id.clone(),
4923	"turn_id": turn.id.clone(),
4924	"input": prompt,
4925	}),
4926	)
4927	.await
4928	{
4929	tracing::warn!("Failed to persist turn.steered after engine acceptance: {err}");
4930	}
4931	if let Err(err) = manager
4932	.emit_event(
4933	&turn.thread_id,
4934	Some(&turn.id),
4935	Some(&item.id),
4936	"item.completed",
4937	json!({ "item": item }),
4938	)
4939	.await
4940	{
4941	tracing::warn!("Failed to persist steer item.completed: {err}");
4942	}
4943	})
4944	.catch_unwind()
4945	.await;
4946	if let Err(payload) = receipts {
4947	tracing::error!(
4948	"Steer receipt task panicked after engine acceptance: {}",
4949	panic_payload_message(&*payload)
4950	);
4951	}
4952	let _ = receipt_tx.send(turn);
4953	});
4954	receipt_rx
4955	}
4956
4957	pub async fn start_turn(&self, thread_id: &str, req: StartTurnRequest) -> Result<TurnRecord> {
4958	// Heap-allocate the turn-start state machine. Its future holds two full
4959	// Config clones plus ThreadRecord/EngineHandle/TurnRecord/TurnItemRecord
4960	// and the Op::SendMessage, and inlines the large ensure_engine_loaded
4961	// sub-future (which builds a full EngineConfig), all across ~8 sequential
4962	// .awaits. On Windows the runtime thread stack is ~1 MiB and this
4963	// monolithic frame overflowed it (test
4964	// start_turn_accepts_dynamic_tools_and_environment_id on windows-latest,
4965	// STATUS_STACK_OVERFLOW). Box::pin moves the whole frame to the heap so
4966	// no caller's stack carries it; behavior is unchanged.
4967	Box::pin(async move {
4968	let prompt = req.prompt.trim().to_string();
4969	if prompt.is_empty() {
4970	bail!("prompt is required");
4971	}
4972
4973	let thread = self.get_thread(thread_id).await?;
4974	let policy =
4975	if req.mode.is_some() \|\| req.permission_posture.is_some() \|\| req.auto_approve.is_some()
4976	{
4977	runtime_policy_with_overrides(
4978	&thread,
4979	req.mode.as_deref(),
4980	req.permission_posture.as_deref(),
4981	req.auto_approve,
4982	)?
4983	} else {
4984	RuntimePolicyProjection::from_persisted(
4985	&thread.mode,
4986	thread.permission_posture.as_deref(),
4987	thread.auto_approve,
4988	)
4989	};
4990	let mode = policy.mode;
4991	let engine = self.ensure_engine_loaded(&thread).await?;
4992
4993	let client_preflight_required = {
4994	let active = self.active.lock().await;
4995	if let Some(active_thread) = active.engines.get(thread_id)
4996	&& active_thread.active_turn.is_some()
4997	{
4998	bail!("Thread already has an active turn");
4999	}
5000	active
5001	.engines
5002	.get(thread_id)
5003	.is_none_or(\|state\| state.client_preflight_required)
5004	};
5005
5006	// Resolve the concrete provider/model before persisting a turn. Auto
5007	// routing can fail, and such a failure must not leave a zombie
5008	// in-progress record behind.
5009	let requested_model = req.model.as_deref().unwrap_or(&thread.model).to_string();
5010	let auto_model = requested_model.trim().eq_ignore_ascii_case("auto");
5011	let cfg_snapshot = self.config.read().clone();
5012	let identity = self.provider_identity_for_thread(&cfg_snapshot, &thread)?;
5013	let mut thread_config = cfg_snapshot.clone();
5014	thread_config.scope_to_provider_identity(&identity);
5015	let verbosity = thread_config.verbosity.clone();
5016	let reasoning_preference = thread_config
5017	.reasoning_effort()
5018	.filter(\|_\| thread_config.reasoning_effort_is_explicit())
5019	.map(crate::tui::app::ReasoningEffort::from_setting);
5020	let (route, reasoning_effort, auto_controls_reasoning) = if auto_model {
5021	let selection = crate::model_routing::resolve_auto_route_with_inventory(
5022	&thread_config,
5023	&prompt,
5024	"",
5025	"auto",
5026	"auto",
5027	)
5028	.await?;
5029	let route = resolve_runtime_thread_route(
5030	&thread_config,
5031	selection.provider,
5032	Some(&selection.model),
5033	)?;
5034	let (selected_reasoning, auto_controls_reasoning) =
5035	crate::model_routing::resolve_auto_model_reasoning(
5036	reasoning_preference,
5037	selection.reasoning_effort,
5038	);
5039	let reasoning_effort = selected_reasoning.map(\|effort\| {
5040	effort
5041	.normalize_for_route(
5042	route.identity.provider,
5043	&route.candidate.endpoint().base_url,
5044	&route.model,
5045	)
5046	.as_setting()
5047	.to_string()
5048	});
5049	(route, reasoning_effort, auto_controls_reasoning)
5050	} else {
5051	(
5052	resolve_runtime_thread_route_for_identity(
5053	&cfg_snapshot,
5054	&identity,
5055	Some(&requested_model),
5056	)?,
5057	None,
5058	false,
5059	)
5060	};
5061	let route = if client_preflight_required {
5062	route
5063	.preflight()
5064	.map_err(\|reason\| anyhow!("Failed to validate runtime thread route: {reason}"))?
5065	} else {
5066	route
5067	};
5068	let configured_sandbox_mode = route.config.sandbox_mode.clone();
5069	let provider = route.identity.provider;
5070	let provider_identity = route.identity.clone();
5071	let model = route.model.clone();
5072	let route_limits = known_route_limits(route.candidate.limits());
5073	let settings = crate::settings::Settings::load().unwrap_or_default();
5074	let mut compaction = runtime_compaction_config(
5075	provider,
5076	&model,
5077	route_limits,
5078	settings.auto_compact,
5079	crate::settings::Settings::auto_compact_explicitly_configured(),
5080	settings.auto_compact_threshold_percent,
5081	);
5082	let now = Utc::now();
5083	let turn_id = format!("turn_{}", &Uuid::new_v4().to_string()[..8]);
5084	compaction.runtime_cost_owner = Some(turn_id.clone());
5085	let mut turn = TurnRecord {
5086	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
5087	id: turn_id.clone(),
5088	thread_id: thread_id.to_string(),
5089	status: RuntimeTurnStatus::InProgress,
5090	input_summary: req
5091	.input_summary
5092	.unwrap_or_else(\|\| summarize_text(&prompt, SUMMARY_LIMIT)),
5093	created_at: now,
5094	started_at: Some(now),
5095	ended_at: None,
5096	duration_ms: None,
5097	usage: None,
5098	permission_posture: Some(policy.permission_wire().to_string()),
5099	effective_provider: Some(provider.as_str().to_string()),
5100	effective_provider_id: provider_identity
5101	.exact_id
5102	.as_deref()
5103	.map(crate::cost_status::sanitize_persisted_route_label),
5104	effective_billing_surface: None,
5105	effective_endpoint_fingerprint: None,
5106	effective_billing_mode: None,
5107	effective_dispatched_at: None,
5108	effective_model: Some(crate::cost_status::sanitize_persisted_route_label(&model)),
5109	routed_usage: Vec::new(),
5110	routed_usage_source_ids: Vec::new(),
5111	routed_usage_dropped_records: 0,
5112	error: None,
5113	item_ids: Vec::new(),
5114	steer_count: 0,
5115	};
5116
5117	let user_item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
5118	let user_item = TurnItemRecord {
5119	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
5120	id: user_item_id.clone(),
5121	turn_id: turn_id.clone(),
5122	kind: TurnItemKind::UserMessage,
5123	status: TurnItemLifecycleStatus::Completed,
5124	summary: summarize_text(&prompt, SUMMARY_LIMIT),
5125	detail: Some(prompt.clone()),
5126	metadata: None,
5127	artifact_refs: Vec::new(),
5128	started_at: Some(now),
5129	ended_at: Some(now),
5130	};
5131	turn.item_ids.push(user_item_id.clone());
5132
5133	let allow_shell = req.allow_shell.unwrap_or(thread.allow_shell);
5134	let trust_mode = req.trust_mode.unwrap_or(thread.trust_mode);
5135	let auto_approve = policy.auto_approve();
5136	let op = Op::SendMessage {
5137	content: prompt,
5138	mode,
5139	route: Box::new(route),
5140	compaction: Box::new(compaction),
5141	goal_objective: None,
5142	goal_token_budget: None,
5143	goal_status: crate::tools::goal::GoalStatus::Active,
5144	reasoning_effort,
5145	reasoning_effort_auto: auto_controls_reasoning,
5146	auto_model,
5147	allow_shell,
5148	trust_mode,
5149	auto_approve,
5150	translation_enabled: false,
5151	allowed_tools: None,
5152	dynamic_tools: req.dynamic_tools,
5153	hook_executor: None,
5154	approval_mode: policy.permission,
5155	verbosity,
5156	provenance: crate::core::ops::UserInputProvenance::ExternalUser,
5157	};
5158
5159	// Reserve mailbox capacity before claiming or persisting anything.
5160	// If the caller is cancelled while capacity is unavailable, no
5161	// durable or in-memory turn state has changed.
5162	let permit = engine
5163	.tx_op
5164	.clone()
5165	.reserve_owned()
5166	.await
5167	.map_err(\|_\| anyhow!("Failed to start turn: engine operation channel closed"))?;
5168
5169	let acceptance_rx = {
5170	// Lock order is active -> thread_mutation. Neither guard crosses
5171	// an await, and spawning the owned lifecycle task is synchronous.
5172	let mut active = self.active.lock().await;
5173	let Some(state) = active.engines.get_mut(thread_id) else {
5174	bail!("Thread engine not loaded");
5175	};
5176	if state.active_turn.is_some() {
5177	bail!("Thread already has an active turn");
5178	}
5179	let _thread_mutation = self.store.thread_mutation.lock();
5180	let mut current_thread = self.store.load_thread(thread_id)?;
5181	if !thread_execution_state_matches(&thread, &current_thread) {
5182	bail!("Thread execution settings changed while preparing the turn; retry");
5183	}
5184	let previous_active_route = (state.route_identity.clone(), state.route_model.clone());
5185	state.active_turn = Some(ActiveTurnState {
5186	turn_id: turn_id.clone(),
5187	interrupt_requested: false,
5188	});
5189	state.route_identity = provider_identity;
5190	state.route_model.clone_from(&model);
5191
5192	let persistence_result = (\|\| -> Result<()> {
5193	self.store.save_item(&user_item)?;
5194	self.store.save_turn(&turn)?;
5195	current_thread.latest_turn_id = Some(turn_id.clone());
5196	current_thread.updated_at = now;
5197	self.store.save_thread(&current_thread)
5198	})();
5199	if let Err(persistence_error) = persistence_result {
5200	let cleanup_error = self
5201	.cleanup_unaccepted_turn_records(&turn_id, Some(&user_item_id))
5202	.err();
5203	state.active_turn = None;
5204	state.route_identity = previous_active_route.0;
5205	state.route_model = previous_active_route.1;
5206	return match cleanup_error {
5207	None => Err(anyhow!("Failed to persist turn: {persistence_error}")),
5208	Some(cleanup_error) => Err(anyhow!(
5209	"Failed to persist turn: {persistence_error}; cleanup also failed: {cleanup_error}"
5210	)),
5211	};
5212	}
5213
5214	self.register_runtime_usage_sink(&turn_id);
5215	// Sending through an owned permit cannot await or fail. From this
5216	// point the engine owns the operation and the spawned task owns
5217	// lifecycle events, monitoring, and terminal cleanup even if the
5218	// HTTP/client future is dropped.
5219	engine.publish_turn_authority(
5220	mode,
5221	allow_shell,
5222	trust_mode,
5223	auto_approve,
5224	policy.permission,
5225	configured_sandbox_mode,
5226	);
5227	let _sender = permit.send(op);
5228	touch_lru(&mut active.lru, thread_id);
5229	self.spawn_claimed_turn_monitor(
5230	turn.clone(),
5231	Some(user_item),
5232	engine.clone(),
5233	ClaimedTurnKind::Message,
5234	)
5235	};
5236
5237	acceptance_rx
5238	.await
5239	.map_err(\|_\| anyhow!("Turn lifecycle task ended before acknowledgement"))?
5240	.map_err(anyhow::Error::msg)
5241	})
5242	.await
5243	}
5244
5245	pub async fn interrupt_turn(&self, thread_id: &str, turn_id: &str) -> Result<TurnRecord> {
5246	{
5247	let mut active = self.active.lock().await;
5248	let Some(active_thread) = active.engines.get_mut(thread_id) else {
5249	bail!("Thread is not loaded");
5250	};
5251	let Some(active_turn) = active_thread.active_turn.as_mut() else {
5252	bail!("No active turn on thread {thread_id}");
5253	};
5254	if active_turn.turn_id != turn_id {
5255	bail!("Turn {turn_id} is not active on thread {thread_id}");
5256	}
5257	active_turn.interrupt_requested = true;
5258	active_thread.engine.cancel();
5259	touch_lru(&mut active.lru, thread_id);
5260	}
5261
5262	self.emit_event(
5263	thread_id,
5264	Some(turn_id),
5265	None,
5266	"turn.interrupt_requested",
5267	json!({ "thread_id": thread_id, "turn_id": turn_id }),
5268	)
5269	.await?;
5270
5271	self.store.load_turn(turn_id)
5272	}
5273
5274	pub async fn steer_turn(
5275	&self,
5276	thread_id: &str,
5277	turn_id: &str,
5278	req: SteerTurnRequest,
5279	) -> Result<TurnRecord> {
5280	let prompt = req.prompt.trim().to_string();
5281	if prompt.is_empty() {
5282	bail!("prompt is required");
5283	}
5284
5285	let engine = {
5286	let mut active = self.active.lock().await;
5287	let engine = {
5288	let Some(active_thread) = active.engines.get_mut(thread_id) else {
5289	bail!("Thread is not loaded");
5290	};
5291	let Some(active_turn) = active_thread.active_turn.as_mut() else {
5292	bail!("No active turn on thread {thread_id}");
5293	};
5294	if active_turn.turn_id != turn_id {
5295	bail!("Turn {turn_id} is not active on thread {thread_id}");
5296	}
5297	if active_turn.interrupt_requested {
5298	bail!("Turn {turn_id} is stopping and cannot be steered");
5299	}
5300	active_thread.engine.clone()
5301	};
5302	touch_lru(&mut active.lru, thread_id);
5303	engine
5304	};
5305
5306	let permit = engine
5307	.reserve_steer()
5308	.await
5309	.map_err(\|error\| anyhow!("Failed to steer turn: {error}"))?;
5310
5311	let now = Utc::now();
5312	let item = TurnItemRecord {
5313	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
5314	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
5315	turn_id: turn_id.to_string(),
5316	kind: TurnItemKind::UserMessage,
5317	status: TurnItemLifecycleStatus::Completed,
5318	summary: summarize_text(&prompt, SUMMARY_LIMIT),
5319	detail: Some(prompt.clone()),
5320	metadata: None,
5321	artifact_refs: Vec::new(),
5322	started_at: Some(now),
5323	ended_at: Some(now),
5324	};
5325	let receipt_rx = {
5326	let mut active = self.active.lock().await;
5327	let Some(active_thread) = active.engines.get(thread_id) else {
5328	bail!("Thread is not loaded");
5329	};
5330	let Some(active_turn) = active_thread.active_turn.as_ref() else {
5331	bail!("No active turn on thread {thread_id}");
5332	};
5333	if active_turn.turn_id != turn_id {
5334	bail!("Turn {turn_id} is not active on thread {thread_id}");
5335	}
5336	if active_turn.interrupt_requested {
5337	bail!("Turn {turn_id} is stopping and cannot be steered");
5338	}
5339	if !active_thread.engine.tx_op.same_channel(&engine.tx_op) {
5340	bail!("Thread engine changed while preparing steer; retry");
5341	}
5342	let _turn_mutation = self.store.turn_mutation.lock();
5343	let persistence = (\|\| -> Result<TurnRecord> {
5344	let mut turn = self.store.load_turn(turn_id)?;
5345	if turn.status != RuntimeTurnStatus::InProgress {
5346	bail!("Turn {turn_id} is no longer in progress and cannot be steered");
5347	}
5348	self.store.save_item(&item)?;
5349	turn.steer_count = turn.steer_count.saturating_add(1);
5350	if !turn.item_ids.iter().any(\|id\| id == &item.id) {
5351	turn.item_ids.push(item.id.clone());
5352	}
5353	self.store.save_turn(&turn)?;
5354	Ok(turn)
5355	})();
5356	let turn = match persistence {
5357	Ok(turn) => turn,
5358	Err(error) => {
5359	let cleanup = self.store.remove_item(&item.id);
5360	return match cleanup {
5361	Ok(()) => Err(error),
5362	Err(cleanup_error) => Err(anyhow!(
5363	"Failed to persist steer: {error}; cleanup also failed: {cleanup_error}"
5364	)),
5365	};
5366	}
5367	};
5368	// The reserved send has no await/failure point. From here the
5369	// engine and durable record agree even if the API caller drops.
5370	let _sender = permit.send(prompt.clone());
5371	touch_lru(&mut active.lru, thread_id);
5372	self.spawn_steer_receipts(turn, item, prompt)
5373	};
5374	receipt_rx
5375	.await
5376	.map_err(\|_\| anyhow!("Steer receipt task ended before acknowledgement"))
5377	}
5378
5379	pub async fn compact_thread(
5380	&self,
5381	thread_id: &str,
5382	req: CompactThreadRequest,
5383	) -> Result<TurnRecord> {
5384	let thread = self.get_thread(thread_id).await?;
5385	let engine = self.ensure_engine_loaded(&thread).await?;
5386
5387	let client_preflight_required = {
5388	let active = self.active.lock().await;
5389	let Some(active_thread) = active.engines.get(thread_id) else {
5390	bail!("Thread engine not loaded");
5391	};
5392	if active_thread.active_turn.is_some() {
5393	bail!("Thread already has an active turn");
5394	}
5395	active_thread.client_preflight_required
5396	};
5397	let route = self.resolved_route_for_thread(&self.read_config(), &thread)?;
5398	let route = if client_preflight_required {
5399	route
5400	.preflight()
5401	.map_err(\|reason\| anyhow!("Failed to validate runtime thread route: {reason}"))?
5402	} else {
5403	route
5404	};
5405	let configured_sandbox_mode = route.config.sandbox_mode.clone();
5406	let route_provider = route.identity.provider;
5407	let route_identity = route.identity.clone();
5408	let route_model = route.model.clone();
5409	let route_limits = known_route_limits(route.candidate.limits());
5410	let settings = crate::settings::Settings::load().unwrap_or_default();
5411	let mut compaction = runtime_compaction_config(
5412	route_provider,
5413	&route_model,
5414	route_limits,
5415	settings.auto_compact,
5416	crate::settings::Settings::auto_compact_explicitly_configured(),
5417	settings.auto_compact_threshold_percent,
5418	);
5419
5420	let now = Utc::now();
5421	let turn_id = format!("turn_{}", &Uuid::new_v4().to_string()[..8]);
5422	compaction.runtime_cost_owner = Some(turn_id.clone());
5423	let turn = TurnRecord {
5424	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
5425	id: turn_id.clone(),
5426	thread_id: thread_id.to_string(),
5427	status: RuntimeTurnStatus::InProgress,
5428	input_summary: req
5429	.reason
5430	.as_deref()
5431	.map(\|s\| summarize_text(s, SUMMARY_LIMIT))
5432	.unwrap_or_else(\|\| "Manual context compaction".to_string()),
5433	created_at: now,
5434	started_at: Some(now),
5435	ended_at: None,
5436	duration_ms: None,
5437	usage: None,
5438	permission_posture: Some(
5439	RuntimePolicyProjection::from_persisted(
5440	&thread.mode,
5441	thread.permission_posture.as_deref(),
5442	thread.auto_approve,
5443	)
5444	.permission_wire()
5445	.to_string(),
5446	),
5447	effective_provider: Some(route_provider.as_str().to_string()),
5448	effective_provider_id: route_identity
5449	.exact_id
5450	.as_deref()
5451	.map(crate::cost_status::sanitize_persisted_route_label),
5452	effective_billing_surface: None,
5453	effective_endpoint_fingerprint: None,
5454	effective_billing_mode: None,
5455	effective_dispatched_at: None,
5456	effective_model: Some(crate::cost_status::sanitize_persisted_route_label(
5457	&route_model,
5458	)),
5459	routed_usage: Vec::new(),
5460	routed_usage_source_ids: Vec::new(),
5461	routed_usage_dropped_records: 0,
5462	error: None,
5463	item_ids: Vec::new(),
5464	steer_count: 0,
5465	};
5466	let op = Op::CompactContext {
5467	route: Box::new(route),
5468	compaction: Box::new(compaction),
5469	};
5470	let permit = engine.tx_op.clone().reserve_owned().await.map_err(\|_\| {
5471	anyhow!("Failed to trigger compaction: engine operation channel closed")
5472	})?;
5473
5474	let acceptance_rx = {
5475	let mut active = self.active.lock().await;
5476	let Some(state) = active.engines.get_mut(thread_id) else {
5477	bail!("Thread engine not loaded");
5478	};
5479	if state.active_turn.is_some() {
5480	bail!("Thread already has an active turn");
5481	}
5482	let _thread_mutation = self.store.thread_mutation.lock();
5483	let mut current_thread = self.store.load_thread(thread_id)?;
5484	if !thread_execution_state_matches(&thread, &current_thread) {
5485	bail!("Thread execution settings changed while preparing compaction; retry");
5486	}
5487	let previous_active_route = (state.route_identity.clone(), state.route_model.clone());
5488	state.active_turn = Some(ActiveTurnState {
5489	turn_id: turn_id.clone(),
5490	interrupt_requested: false,
5491	});
5492	state.route_identity = route_identity;
5493	state.route_model = route_model;
5494
5495	let persistence_result = (\|\| -> Result<()> {
5496	self.store.save_turn(&turn)?;
5497	current_thread.latest_turn_id = Some(turn_id.clone());
5498	current_thread.updated_at = now;
5499	self.store.save_thread(&current_thread)
5500	})();
5501	if let Err(persistence_error) = persistence_result {
5502	let cleanup_error = self.cleanup_unaccepted_turn_records(&turn_id, None).err();
5503	state.active_turn = None;
5504	state.route_identity = previous_active_route.0;
5505	state.route_model = previous_active_route.1;
5506	return match cleanup_error {
5507	None => Err(anyhow!("Failed to persist compaction: {persistence_error}")),
5508	Some(cleanup_error) => Err(anyhow!(
5509	"Failed to persist compaction: {persistence_error}; cleanup also failed: {cleanup_error}"
5510	)),
5511	};
5512	}
5513
5514	self.register_runtime_usage_sink(&turn_id);
5515	let policy = RuntimePolicyProjection::from_persisted(
5516	&current_thread.mode,
5517	current_thread.permission_posture.as_deref(),
5518	current_thread.auto_approve,
5519	);
5520	engine.publish_turn_authority(
5521	policy.mode,
5522	current_thread.allow_shell,
5523	current_thread.trust_mode,
5524	policy.auto_approve(),
5525	policy.permission,
5526	configured_sandbox_mode,
5527	);
5528	let _sender = permit.send(op);
5529	touch_lru(&mut active.lru, thread_id);
5530	self.spawn_claimed_turn_monitor(
5531	turn.clone(),
5532	None,
5533	engine.clone(),
5534	ClaimedTurnKind::Compaction,
5535	)
5536	};
5537
5538	acceptance_rx
5539	.await
5540	.map_err(\|_\| anyhow!("Compaction lifecycle task ended before acknowledgement"))?
5541	.map_err(anyhow::Error::msg)
5542	}
5543
5544	#[cfg(test)]
5545	pub fn events_since(
5546	&self,
5547	thread_id: &str,
5548	since_seq: Option<u64>,
5549	) -> Result<Vec<RuntimeEventRecord>> {
5550	self.store.events_since(thread_id, since_seq)
5551	}
5552
5553	pub(crate) async fn events_since_async(
5554	&self,
5555	thread_id: &str,
5556	since_seq: Option<u64>,
5557	) -> Result<Vec<RuntimeEventRecord>> {
5558	let store = self.store.clone();
5559	let thread_id = thread_id.to_string();
5560	tokio::task::spawn_blocking(move \|\| store.events_since(&thread_id, since_seq))
5561	.await
5562	.context("Runtime event history task failed")?
5563	}
5564
5565	pub(crate) async fn replay_events(
5566	&self,
5567	thread_id: &str,
5568	since_seq: Option<u64>,
5569	tail_limit: Option<usize>,
5570	) -> Result<RuntimeEventReplay> {
5571	if tail_limit.is_some_and(\|limit\| limit > MAX_RUNTIME_EVENT_REPLAY_TAIL) {
5572	bail!("Runtime event replay_limit cannot exceed {MAX_RUNTIME_EVENT_REPLAY_TAIL}");
5573	}
5574	let (base_tx, base_rx) = oneshot::channel();
5575	let (batch_tx, batches) = mpsc::channel(2);
5576	let store = self.store.clone();
5577	let thread_id = thread_id.to_string();
5578	tokio::task::spawn_blocking(move \|\| {
5579	store.publish_event_replay(&thread_id, since_seq, tail_limit, base_tx, batch_tx);
5580	});
5581	let base_seq = base_rx
5582	.await
5583	.context("Runtime event replay worker ended before initialization")?
5584	.map_err(anyhow::Error::msg)?;
5585	Ok(RuntimeEventReplay { base_seq, batches })
5586	}
5587
5588	async fn ensure_engine_loaded(&self, thread_hint: &ThreadRecord) -> Result<EngineHandle> {
5589	{
5590	let mut active = self.active.lock().await;
5591	if let Some(engine) = active
5592	.engines
5593	.get(thread_hint.id.as_str())
5594	.map(\|state\| state.engine.clone())
5595	{
5596	touch_lru(&mut active.lru, &thread_hint.id);
5597	return Ok(engine);
5598	}
5599	}
5600
5601	// Only one cache-miss build may run at a time. Recheck after taking
5602	// the build lock because another caller may already have won.
5603	let _engine_load = self.engine_load.lock().await;
5604	loop {
5605	{
5606	let mut active = self.active.lock().await;
5607	if let Some(engine) = active
5608	.engines
5609	.get(thread_hint.id.as_str())
5610	.map(\|state\| state.engine.clone())
5611	{
5612	touch_lru(&mut active.lru, &thread_hint.id);
5613	return Ok(engine);
5614	}
5615	}
5616	let thread = {
5617	let _thread_mutation = self.store.thread_mutation.lock();
5618	self.store
5619	.load_thread(&thread_hint.id)
5620	.with_context(\|\| format!("Thread not found: {}", thread_hint.id))?
5621	};
5622
5623	// Snapshot and prepare the concrete provider route once so the engine,
5624	// route limits, compaction budget, and restored session all agree.
5625	let base_config = self.read_config().clone();
5626	let route = self.resolved_route_for_thread(&base_config, &thread)?;
5627	let provider = route.identity.provider;
5628	let route_identity = route.identity;
5629	let route_model = route.model;
5630	let route_limits = known_route_limits(route.candidate.limits());
5631	let cfg = route.config;
5632
5633	// Resolve the provider-route-aware auto-compaction default unless the
5634	// user persisted an explicit preference.
5635	let settings = crate::settings::Settings::load().unwrap_or_default();
5636	let compaction = runtime_compaction_config(
5637	provider,
5638	&route_model,
5639	route_limits,
5640	settings.auto_compact,
5641	crate::settings::Settings::auto_compact_explicitly_configured(),
5642	settings.auto_compact_threshold_percent,
5643	);
5644	let network_policy = cfg.network.clone().map(\|toml_cfg\| {
5645	crate::network_policy::NetworkPolicyDecider::with_default_audit(
5646	toml_cfg.into_runtime(),
5647	)
5648	});
5649	let lsp_config = cfg
5650	.lsp
5651	.clone()
5652	.map(crate::config::LspConfigToml::into_runtime);
5653	let max_subagents = cfg
5654	.max_subagents_for_provider(provider)
5655	.clamp(1, MAX_SUBAGENTS);
5656	let engine_cfg = EngineConfig {
5657	model: route_model.clone(),
5658	active_route_limits: route_limits,
5659	workspace: thread.workspace.clone(),
5660	plugin_registry: self
5661	.plugin_registry
5662	.as_ref()
5663	.map(\|registry\| registry.rediscover_for_workspace(&thread.workspace)),
5664	allow_shell: thread.allow_shell,
5665	trust_mode: thread.trust_mode,
5666	notes_path: cfg.notes_path(),
5667	mcp_config_path: cfg.mcp_config_path(),
5668	skills_dir: cfg.skills_dir(),
5669	skills_scan_codewhale_only: cfg.skills_config().scan_codewhale_only(),
5670	instructions: cfg
5671	.instructions_paths()
5672	.into_iter()
5673	.map(Into::into)
5674	.collect(),
5675	project_context_pack_enabled: cfg.project_context_pack_enabled(),
5676	translation_enabled: false,
5677	max_steps: 100,
5678	max_subagents,
5679	max_admitted_subagents: cfg
5680	.max_admitted_subagents_for_provider(provider)
5681	.max(max_subagents),
5682	launch_concurrency: cfg.launch_concurrency_for_provider(provider),
5683	subagents_enabled: cfg.subagents_enabled_for_provider(provider),
5684	features: cfg.features(),
5685	auto_review_policy: cfg.auto_review_policy(),
5686	compaction,
5687	todos: new_shared_todo_list(),
5688	plan_state: new_shared_plan_state(),
5689	goal_state: crate::tools::goal::new_shared_goal_state(),
5690	max_spawn_depth: cfg.subagent_max_spawn_depth_for_provider(provider),
5691	subagent_token_budget: cfg.subagent_token_budget_for_provider(provider),
5692	network_policy,
5693	snapshots_enabled: cfg.snapshots_config().enabled,
5694	snapshots_max_workspace_bytes: cfg
5695	.snapshots_config()
5696	.max_workspace_gb
5697	.saturating_mul(1024 * 1024 * 1024),
5698	lsp_config,
5699	runtime_services: crate::tools::spec::RuntimeToolServices {
5700	task_manager: self.task_manager.lock().clone(),
5701	automations: self.automations.lock().clone(),
5702	task_data_dir: Some(self.manager_cfg.task_data_dir.clone()),
5703	active_task_id: thread.task_id.clone(),
5704	active_thread_id: Some(thread.id.clone()),
5705	dynamic_tool_executor: Some(Arc::new(self.clone())),
5706	work: None,
5707	shell_manager: None,
5708	hook_executor: None,
5709	handle_store: crate::tools::handle::new_shared_handle_store(),
5710	rlm_sessions: crate::rlm::session::new_shared_rlm_session_store(),
5711	},
5712	subagent_model_overrides: cfg.subagent_model_overrides(),
5713	fleet_roster: Arc::new(crate::fleet::roster::FleetRoster::load(
5714	&cfg.fleet_config(),
5715	&thread.workspace,
5716	)),
5717	subagent_api_timeout: std::time::Duration::from_secs(
5718	cfg.subagent_api_timeout_secs_for_provider(provider),
5719	),
5720	stream_chunk_timeout: std::time::Duration::from_secs(
5721	cfg.stream_chunk_timeout_secs(),
5722	),
5723	subagent_heartbeat_timeout: std::time::Duration::from_secs(
5724	cfg.subagent_heartbeat_timeout_secs_for_provider(provider),
5725	),
5726	prefer_bwrap: cfg.prefer_bwrap.unwrap_or(false),
5727	memory_enabled: cfg.memory_enabled(),
5728	memory_path: cfg.memory_path(),
5729	speech_output_dir: cfg.speech_output_dir(),
5730	vision_config: cfg.vision_model_config(),
5731	strict_tool_mode: cfg.strict_tool_mode.unwrap_or(false),
5732	goal_objective: None,
5733	goal_token_budget: None,
5734	goal_status: crate::tools::goal::GoalStatus::Active,
5735	goal_max_continuations: cfg.goal_max_continuations(),
5736	allowed_tools: None,
5737	disallowed_tools: None,
5738	max_tool_calls: None,
5739	hook_executor: None,
5740	locale_tag: crate::localization::resolve_locale(&settings.locale)
5741	.tag()
5742	.to_string(),
5743	workshop: cfg.workshop.clone(),
5744	search_provider: cfg.search_provider(),
5745	search_api_key: cfg.search.as_ref().and_then(\|s\| s.api_key.clone()),
5746	search_base_url: cfg.search.as_ref().and_then(\|s\| s.base_url.clone()),
5747	tools_always_load: cfg.tools_always_load(),
5748	tools: cfg.tools.clone(),
5749	verbosity: cfg.verbosity.clone(),
5750	workspace_follow_symlinks: settings.workspace_follow_symlinks,
5751	exec_policy_engine: cfg.exec_policy_engine.clone(),
5752	terminal_chrome_enabled: false,
5753	advisor_config: cfg
5754	.advisor
5755	.as_ref()
5756	.map(crate::tools::subagent::AdvisorConfig::from_toml)
5757	.unwrap_or_else(crate::tools::subagent::AdvisorConfig::disabled),
5758	};
5759
5760	let engine = spawn_engine_with_authoritative_route_config(
5761	engine_cfg,
5762	&cfg,
5763	Arc::clone(&self.config),
5764	);
5765
5766	// When the thread has an associated session, load the full message history
5767	// (including thinking/tool blocks) from the session file. This preserves
5768	// process information that `reconstruct_messages_from_turns` would lose.
5769	let session_messages = if let Some(ref sid) = thread.session_id {
5770	match crate::session_manager::default_sessions_dir() {
5771	Ok(sessions_dir) => {
5772	match crate::session_manager::SessionManager::new(sessions_dir) {
5773	Ok(manager) => match manager.load_session(sid) {
5774	Ok(session) => session.messages,
5775	Err(e) => {
5776	tracing::warn!(
5777	"Failed to load session {} for thread {}: {e}; falling back to turn reconstruction",
5778	sid,
5779	thread.id
5780	);
5781	let turns = self.store.list_turns_for_thread(&thread.id)?;
5782	self.reconstruct_messages_from_turns(&turns)?
5783	}
5784	},
5785	Err(e) => {
5786	tracing::warn!(
5787	"Failed to open sessions dir: {e}; falling back to turn reconstruction"
5788	);
5789	let turns = self.store.list_turns_for_thread(&thread.id)?;
5790	self.reconstruct_messages_from_turns(&turns)?
5791	}
5792	}
5793	}
5794	Err(e) => {
5795	tracing::warn!(
5796	"Failed to resolve sessions dir: {e}; falling back to turn reconstruction"
5797	);
5798	let turns = self.store.list_turns_for_thread(&thread.id)?;
5799	self.reconstruct_messages_from_turns(&turns)?
5800	}
5801	}
5802	} else {
5803	let turns = self.store.list_turns_for_thread(&thread.id)?;
5804	self.reconstruct_messages_from_turns(&turns)?
5805	};
5806	let sys_prompt = thread
5807	.system_prompt
5808	.as_ref()
5809	.map(\|s\| SystemPrompt::Text(s.clone()));
5810	if !session_messages.is_empty() \|\| sys_prompt.is_some() {
5811	engine
5812	.send(Op::SyncSession {
5813	session_id: thread.session_id.clone(),
5814	messages: session_messages,
5815	system_prompt: sys_prompt,
5816	system_prompt_override: thread.system_prompt.is_some(),
5817	model: route_model.clone(),
5818	workspace: thread.workspace.clone(),
5819	mode: RuntimePolicyProjection::from_persisted(
5820	&thread.mode,
5821	thread.permission_posture.as_deref(),
5822	thread.auto_approve,
5823	)
5824	.mode,
5825	})
5826	.await
5827	.map_err(\|e\| anyhow!("Failed to sync thread session: {e}"))?;
5828	}
5829
5830	let mut active = self.active.lock().await;
5831	if let Some(winner) = active
5832	.engines
5833	.get(&thread.id)
5834	.map(\|state\| state.engine.clone())
5835	{
5836	touch_lru(&mut active.lru, &thread.id);
5837	drop(active);
5838	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
5839	let _ = engine.try_send(Op::Shutdown);
5840	return Ok(winner);
5841	}
5842
5843	// Atomically compare the record used for construction with the latest
5844	// durable record while holding the same active -> thread lock order as
5845	// updates. A concurrent workspace/model/session/policy change makes
5846	// this engine stale; discard it and rebuild from the new snapshot.
5847	let thread_mutation = self.store.thread_mutation.lock();
5848	let record_is_current = self.store.load_thread(&thread.id)? == thread;
5849	if !record_is_current {
5850	drop(thread_mutation);
5851	drop(active);
5852	engine.cancel_with_reason(crate::core::engine::CancelReason::Internal);
5853	let _ = engine.try_send(Op::Shutdown);
5854	continue;
5855	}
5856
5857	let evicted = enforce_lru_capacity(&mut active, self.manager_cfg.max_active_threads);
5858	active.engines.insert(
5859	thread.id.clone(),
5860	ActiveThreadState {
5861	engine: engine.clone(),
5862	active_turn: None,
5863	route_identity,
5864	route_model,
5865	client_preflight_required: true,
5866	},
5867	);
5868	touch_lru(&mut active.lru, &thread.id);
5869	drop(thread_mutation);
5870	drop(active);
5871	for handle in evicted {
5872	let _ = handle.send(Op::Shutdown).await;
5873	}
5874	return Ok(engine);
5875	}
5876	}
5877
5878	/// Get the engine handle for a thread, loading it if necessary.
5879	/// Public wrapper around the private `ensure_engine_loaded`.
5880	pub async fn get_engine(&self, thread_id: &str) -> Result<EngineHandle> {
5881	let thread = self.get_thread(thread_id).await?;
5882	self.ensure_engine_loaded(&thread).await
5883	}
5884
5885	fn reconstruct_messages_from_turns(&self, turns: &[TurnRecord]) -> Result<Vec<Message>> {
5886	let mut messages = Vec::new();
5887	for turn in turns {
5888	let stored_items = self.store.list_items_for_turn(&turn.id)?;
5889	let items = if turn.item_ids.is_empty() {
5890	stored_items
5891	} else {
5892	let mut by_id: HashMap<String, TurnItemRecord> = stored_items
5893	.iter()
5894	.cloned()
5895	.map(\|item\| (item.id.clone(), item))
5896	.collect();
5897	let mut ordered = Vec::new();
5898	for item_id in &turn.item_ids {
5899	if let Some(item) = by_id.remove(item_id) {
5900	ordered.push(item);
5901	}
5902	}
5903	for item in stored_items {
5904	if by_id.contains_key(&item.id) {
5905	ordered.push(item);
5906	}
5907	}
5908	ordered
5909	};
5910
5911	let mut assistant_blocks: Vec<ContentBlock> = Vec::new();
5912	let mut user_blocks: Vec<ContentBlock> = Vec::new();
5913	let flush_assistant = \|blocks: &mut Vec<ContentBlock>, msgs: &mut Vec<Message>\| {
5914	if !blocks.is_empty() {
5915	msgs.push(Message {
5916	role: "assistant".to_string(),
5917	content: std::mem::take(blocks),
5918	});
5919	}
5920	};
5921	let flush_user = \|blocks: &mut Vec<ContentBlock>, msgs: &mut Vec<Message>\| {
5922	if !blocks.is_empty() {
5923	msgs.push(Message {
5924	role: "user".to_string(),
5925	content: std::mem::take(blocks),
5926	});
5927	}
5928	};
5929	for item in items {
5930	match item.kind {
5931	TurnItemKind::UserMessage => {
5932	flush_assistant(&mut assistant_blocks, &mut messages);
5933	let text = item.detail.unwrap_or(item.summary);
5934	if !text.trim().is_empty() {
5935	user_blocks.push(ContentBlock::Text {
5936	text,
5937	cache_control: None,
5938	});
5939	}
5940	}
5941	TurnItemKind::AgentMessage => {
5942	flush_user(&mut user_blocks, &mut messages);
5943	let text = item.detail.unwrap_or(item.summary);
5944	if !text.trim().is_empty() {
5945	assistant_blocks.push(ContentBlock::Text {
5946	text,
5947	cache_control: None,
5948	});
5949	}
5950	}
5951	TurnItemKind::AgentReasoning => {
5952	flush_user(&mut user_blocks, &mut messages);
5953	let thinking = item.detail.unwrap_or(item.summary);
5954	if !thinking.trim().is_empty() {
5955	assistant_blocks.push(ContentBlock::Thinking {
5956	thinking,
5957	signature: None,
5958	});
5959	}
5960	}
5961	TurnItemKind::ToolCall => {
5962	let meta = item.metadata.as_ref();
5963	let is_tool_result = meta.and_then(\|m\| m.get("tool_result_for")).is_some();
5964	if is_tool_result {
5965	flush_assistant(&mut assistant_blocks, &mut messages);
5966	let tool_use_id = meta
5967	.and_then(\|m\| m.get("tool_result_for"))
5968	.and_then(\|v\| v.as_str())
5969	.unwrap_or("")
5970	.to_string();
5971	let content = item.detail.unwrap_or_default();
5972	let is_error = meta
5973	.and_then(\|m\| m.get("is_error"))
5974	.and_then(\|v\| v.as_bool())
5975	.unwrap_or(false);
5976	let content_blocks = meta
5977	.and_then(\|m\| m.get("content_blocks"))
5978	.and_then(\|v\| v.as_array())
5979	.cloned();
5980	user_blocks.push(ContentBlock::ToolResult {
5981	tool_use_id,
5982	content,
5983	is_error: if is_error { Some(true) } else { None },
5984	content_blocks,
5985	});
5986	} else {
5987	flush_user(&mut user_blocks, &mut messages);
5988	let tool_use_id = meta
5989	.and_then(\|m\| m.get("tool_use_id"))
5990	.and_then(\|v\| v.as_str())
5991	.unwrap_or("")
5992	.to_string();
5993	let tool_name = meta
5994	.and_then(\|m\| m.get("tool_name"))
5995	.and_then(\|v\| v.as_str())
5996	.unwrap_or("")
5997	.to_string();
5998	let input_str = item.detail.unwrap_or_default();
5999	let input: serde_json::Value =
6000	serde_json::from_str(&input_str).unwrap_or(serde_json::Value::Null);
6001	assistant_blocks.push(ContentBlock::ToolUse {
6002	id: tool_use_id,
6003	name: tool_name,
6004	input,
6005	caller: None,
6006	});
6007	}
6008	}
6009	_ => {}
6010	}
6011	}
6012	flush_assistant(&mut assistant_blocks, &mut messages);
6013	flush_user(&mut user_blocks, &mut messages);
6014	}
6015	Ok(messages)
6016	}
6017
6018	fn append_routed_usage_to_turn(
6019	&self,
6020	turn_id: &str,
6021	source_id: &str,
6022	usage: EffectiveRouteUsage,
6023	) -> Result<()> {
6024	let _turn_mutation = self.store.turn_mutation.lock();
6025	let mut turn = self.store.load_turn(turn_id)?;
6026	if append_routed_usage_record(&mut turn, source_id, usage) {
6027	self.store.save_turn(&turn)?;
6028	}
6029	Ok(())
6030	}
6031
6032	fn register_runtime_usage_sink(&self, turn_id: &str) {
6033	let store = self.store.clone();
6034	let sink_turn_id = turn_id.to_string();
6035	crate::cost_status::register_runtime_usage_sink(
6036	turn_id,
6037	Arc::new(move \|record: RuntimeUsageRecord\| {
6038	let _turn_mutation = store.turn_mutation.lock();
6039	let Ok(mut turn) = store.load_turn(&sink_turn_id) else {
6040	return false;
6041	};
6042	if !append_routed_usage_record(&mut turn, &record.source_id, record.usage) {
6043	return true;
6044	}
6045	store.save_turn(&turn).is_ok()
6046	}),
6047	);
6048	}
6049
6050	async fn monitor_turn(
6051	&self,
6052	thread_id: String,
6053	turn_id: String,
6054	engine: EngineHandle,
6055	) -> Result<()> {
6056	let mut current_message_item: Option<TurnItemRecord> = None;
6057	let mut current_reasoning_item: Option<TurnItemRecord> = None;
6058	let mut tool_items: HashMap<String, String> = HashMap::new();
6059	let mut compaction_items: HashMap<String, String> = HashMap::new();
6060	let mut turn_usage: Option<Usage> = None;
6061	let mut turn_status: Option<RuntimeTurnStatus> = None;
6062	let mut turn_error: Option<String> = None;
6063	let mut saw_engine_activity = false;
6064	let mut saw_turn_started = false;
6065	let mut engine_turn_id: Option<String> = None;
6066	let mut pending_event: Option<EngineEvent> = None;
6067	let mut event_channel_closed = false;
6068
6069	loop {
6070	let event = if let Some(event) = pending_event.take() {
6071	Some(event)
6072	} else if event_channel_closed {
6073	None
6074	} else {
6075	let mut rx = engine.rx_event.write().await;
6076	rx.recv().await
6077	};
6078	let Some(event) = event else {
6079	if self
6080	.is_interrupt_requested(&thread_id, &turn_id)
6081	.await
6082	.unwrap_or(false)
6083	{
6084	turn_status = Some(RuntimeTurnStatus::Interrupted);
6085	break;
6086	}
6087	bail!("engine event channel closed before turn {turn_id} completed");
6088	};
6089
6090	// SyncSession and configuration operations emit control status
6091	// receipts on the same channel before SendMessage is processed.
6092	// They belong to engine setup, not to the next claimed turn.
6093	if !saw_turn_started
6094	&& matches!(
6095	&event,
6096	EngineEvent::Status { .. }
6097	\| EngineEvent::SessionUpdated { .. }
6098	\| EngineEvent::AgentList { .. }
6099	\| EngineEvent::AgentSpawned { .. }
6100	\| EngineEvent::AgentProgress { .. }
6101	\| EngineEvent::AgentComplete { .. }
6102	\| EngineEvent::SubAgentMailbox { .. }
6103	)
6104	{
6105	continue;
6106	}
6107
6108	// Engine configuration and session synchronization can emit
6109	// Status/SessionUpdated events before a turn is claimed. Those
6110	// control-plane receipts share the engine channel, but they are
6111	// not model output and must not make an otherwise empty turn look
6112	// successful. Count only events that carry turn-scoped work or
6113	// user-visible output.
6114	if matches!(
6115	&event,
6116	EngineEvent::MessageStarted { .. }
6117	\| EngineEvent::MessageDelta { .. }
6118	\| EngineEvent::MessageComplete { .. }
6119	\| EngineEvent::ThinkingStarted { .. }
6120	\| EngineEvent::ThinkingDelta { .. }
6121	\| EngineEvent::ThinkingComplete { .. }
6122	\| EngineEvent::ToolCallStarted { .. }
6123	\| EngineEvent::ToolCallComplete { .. }
6124	\| EngineEvent::CompactionStarted { .. }
6125	\| EngineEvent::CompactionCompleted { .. }
6126	\| EngineEvent::CompactionFailed { .. }
6127	\| EngineEvent::AgentSpawned { .. }
6128	\| EngineEvent::AgentProgress { .. }
6129	\| EngineEvent::AgentComplete { .. }
6130	\| EngineEvent::SubAgentMailbox { .. }
6131	\| EngineEvent::ApprovalRequired { .. }
6132	\| EngineEvent::ElevationRequired { .. }
6133	\| EngineEvent::UserInputRequired { .. }
6134	\| EngineEvent::Error { .. }
6135	) {
6136	saw_engine_activity = true;
6137	}
6138
6139	match event {
6140	EngineEvent::TurnStarted {
6141	turn_id: started_turn_id,
6142	created_at,
6143	route,
6144	} => {
6145	saw_turn_started = true;
6146	engine_turn_id = Some(started_turn_id);
6147	{
6148	let _turn_mutation = self.store.turn_mutation.lock();
6149	let mut turn = self.store.load_turn(&turn_id)?;
6150	turn.started_at = Some(created_at);
6151	// A lifecycle start carries no billing envelope, so
6152	// there is nothing to persist yet. The dispatch event
6153	// below is the only writer of effective-route columns.
6154	if let Some(route) = route
6155	.as_ref()
6156	.and_then(crate::core::events::TurnRoute::cost_envelope)
6157	{
6158	turn.persist_effective_route(&route);
6159	}
6160	self.store.save_turn(&turn)?;
6161	}
6162	self.emit_event(
6163	&thread_id,
6164	Some(&turn_id),
6165	None,
6166	"turn.lifecycle",
6167	json!({ "status": "in_progress" }),
6168	)
6169	.await?;
6170	}
6171	EngineEvent::RouteDispatched {
6172	turn_id: dispatched_turn_id,
6173	route,
6174	} => {
6175	if engine_turn_id
6176	.as_deref()
6177	.is_some_and(\|started\| started == dispatched_turn_id)
6178	{
6179	let _turn_mutation = self.store.turn_mutation.lock();
6180	let mut turn = self.store.load_turn(&turn_id)?;
6181	if let Some(envelope) = route.cost_envelope() {
6182	turn.persist_effective_route(&envelope);
6183	}
6184	self.store.save_turn(&turn)?;
6185	}
6186	}
6187	EngineEvent::MessageStarted { .. } => {
6188	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
6189	let item = TurnItemRecord {
6190	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6191	id: item_id.clone(),
6192	turn_id: turn_id.clone(),
6193	kind: TurnItemKind::AgentMessage,
6194	status: TurnItemLifecycleStatus::InProgress,
6195	summary: String::new(),
6196	detail: Some(String::new()),
6197	metadata: None,
6198	artifact_refs: Vec::new(),
6199	started_at: Some(Utc::now()),
6200	ended_at: None,
6201	};
6202	self.store.save_item(&item)?;
6203	self.attach_item_to_turn(&turn_id, &item.id)?;
6204	self.emit_event(
6205	&thread_id,
6206	Some(&turn_id),
6207	Some(&item_id),
6208	"item.started",
6209	json!({ "item": item.clone() }),
6210	)
6211	.await?;
6212	current_message_item = Some(item);
6213	}
6214	EngineEvent::MessageDelta { content, .. } => {
6215	let batch =
6216	coalesce_stream_delta(&engine, StreamDeltaKind::Message, content).await;
6217	pending_event = batch.pending_event;
6218	event_channel_closed \|= batch.channel_closed;
6219	let content = batch.content;
6220	if let Some(item) = current_message_item.as_mut() {
6221	let text = item.detail.get_or_insert_default();
6222	text.push_str(&content);
6223	// Materialize the prefix before sequencing its delta.
6224	// A snapshot whose cursor includes this event must not
6225	// still observe the empty item saved at MessageStarted,
6226	// and restart recovery must retain the partial output.
6227	item.summary = summarize_text(text, SUMMARY_LIMIT);
6228	let projection_lock = self.projection_lock(&thread_id);
6229	let _projection = projection_lock.lock().await;
6230	self.save_streaming_item(item).await?;
6231	self.emit_event(
6232	&thread_id,
6233	Some(&turn_id),
6234	Some(&item.id),
6235	"item.delta",
6236	json!({ "delta": content, "kind": "agent_message" }),
6237	)
6238	.await?;
6239	}
6240	}
6241	EngineEvent::MessageComplete { .. } => {
6242	if let Some(mut item) = current_message_item.take() {
6243	item.status = TurnItemLifecycleStatus::Completed;
6244	item.summary = summarize_text(
6245	item.detail.as_deref().unwrap_or_default(),
6246	SUMMARY_LIMIT,
6247	);
6248	item.ended_at = Some(Utc::now());
6249	self.save_streaming_item(&item).await?;
6250	self.emit_event(
6251	&thread_id,
6252	Some(&turn_id),
6253	Some(&item.id),
6254	"item.completed",
6255	json!({ "item": item }),
6256	)
6257	.await?;
6258	}
6259	}
6260	EngineEvent::ThinkingStarted { .. } => {
6261	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
6262	let item = TurnItemRecord {
6263	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6264	id: item_id.clone(),
6265	turn_id: turn_id.clone(),
6266	kind: TurnItemKind::AgentReasoning,
6267	status: TurnItemLifecycleStatus::InProgress,
6268	summary: String::new(),
6269	detail: Some(String::new()),
6270	metadata: None,
6271	artifact_refs: Vec::new(),
6272	started_at: Some(Utc::now()),
6273	ended_at: None,
6274	};
6275	self.store.save_item(&item)?;
6276	self.attach_item_to_turn(&turn_id, &item.id)?;
6277	self.emit_event(
6278	&thread_id,
6279	Some(&turn_id),
6280	Some(&item_id),
6281	"item.started",
6282	json!({ "item": item.clone() }),
6283	)
6284	.await?;
6285	current_reasoning_item = Some(item);
6286	}
6287	EngineEvent::ThinkingDelta { content, .. } => {
6288	let batch =
6289	coalesce_stream_delta(&engine, StreamDeltaKind::Reasoning, content).await;
6290	pending_event = batch.pending_event;
6291	event_channel_closed \|= batch.channel_closed;
6292	let content = batch.content;
6293	if let Some(item) = current_reasoning_item.as_mut() {
6294	let text = item.detail.get_or_insert_default();
6295	text.push_str(&content);
6296	item.summary = summarize_text(text, SUMMARY_LIMIT);
6297	let projection_lock = self.projection_lock(&thread_id);
6298	let _projection = projection_lock.lock().await;
6299	self.save_streaming_item(item).await?;
6300	self.emit_event(
6301	&thread_id,
6302	Some(&turn_id),
6303	Some(&item.id),
6304	"item.delta",
6305	json!({ "delta": content, "kind": "agent_reasoning" }),
6306	)
6307	.await?;
6308	}
6309	}
6310	EngineEvent::ThinkingComplete { .. } => {
6311	if let Some(mut item) = current_reasoning_item.take() {
6312	item.status = TurnItemLifecycleStatus::Completed;
6313	item.summary = summarize_text(
6314	item.detail.as_deref().unwrap_or_default(),
6315	SUMMARY_LIMIT,
6316	);
6317	item.ended_at = Some(Utc::now());
6318	self.save_streaming_item(&item).await?;
6319	self.emit_event(
6320	&thread_id,
6321	Some(&turn_id),
6322	Some(&item.id),
6323	"item.completed",
6324	json!({ "item": item }),
6325	)
6326	.await?;
6327	}
6328	}
6329	EngineEvent::ToolCallStarted { id, name, input } => {
6330	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
6331	tool_items.insert(id.clone(), item_id.clone());
6332	let kind = tool_kind_for_name(&name);
6333	let summary = summarize_text(&format!("{name} started"), SUMMARY_LIMIT);
6334	let item = TurnItemRecord {
6335	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6336	id: item_id.clone(),
6337	turn_id: turn_id.clone(),
6338	kind,
6339	status: TurnItemLifecycleStatus::InProgress,
6340	summary,
6341	detail: Some(serde_json::to_string(&input).unwrap_or_default()),
6342	metadata: None,
6343	artifact_refs: Vec::new(),
6344	started_at: Some(Utc::now()),
6345	ended_at: None,
6346	};
6347	self.store.save_item(&item)?;
6348	self.attach_item_to_turn(&turn_id, &item.id)?;
6349	self.emit_event(
6350	&thread_id,
6351	Some(&turn_id),
6352	Some(&item_id),
6353	"item.started",
6354	json!({ "item": item, "tool": { "id": id, "name": name, "input": input } }),
6355	)
6356	.await?;
6357	}
6358	EngineEvent::ToolCallComplete { id, name, result } => {
6359	if let Ok(output) = &result
6360	&& let Some(metadata) = output.metadata.as_ref()
6361	&& let Some(route) =
6362	crate::cost_status::child_route_envelope_from_metadata(metadata)
6363	&& let Some(usage) = crate::cost_status::child_usage_from_metadata(metadata)
6364	{
6365	let source = format!("tool:{id}");
6366	self.append_routed_usage_to_turn(
6367	&turn_id,
6368	&source,
6369	EffectiveRouteUsage { route, usage },
6370	)?;
6371	}
6372	if let Some(item_id) = tool_items.remove(&id) {
6373	let mut item = self.store.load_item(&item_id)?;
6374	let now = Utc::now();
6375	item.ended_at = Some(now);
6376	match result {
6377	Ok(output) => {
6378	item.status = if output.success {
6379	TurnItemLifecycleStatus::Completed
6380	} else {
6381	TurnItemLifecycleStatus::Failed
6382	};
6383	if name == REQUEST_USER_INPUT_TOOL_NAME {
6384	// The engine must return the structured
6385	// answers to the model, but Runtime
6386	// receipts are durable and fan out to UI
6387	// clients. Persist only a machine-readable
6388	// redaction marker, never answer labels or
6389	// free-text values.
6390	item.summary = REDACTED_USER_INPUT_RECEIPT.to_string();
6391	item.detail = Some(REDACTED_USER_INPUT_RECEIPT.to_string());
6392	item.metadata = Some(json!({
6393	"tool_call_id": id,
6394	"tool_name": REQUEST_USER_INPUT_TOOL_NAME,
6395	"response_redacted": true,
6396	}));
6397	} else {
6398	item.summary = summarize_text(
6399	&format!("{name}: {}", output.content),
6400	SUMMARY_LIMIT,
6401	);
6402	item.detail = Some(output.content.clone());
6403	item.metadata = output.metadata.clone();
6404	}
6405	}
6406	Err(err) => {
6407	item.status = TurnItemLifecycleStatus::Failed;
6408	item.summary =
6409	summarize_text(&format!("{name} failed: {err}"), SUMMARY_LIMIT);
6410	item.detail = Some(err.to_string());
6411	}
6412	}
6413	self.store.save_item(&item)?;
6414	self.emit_event(
6415	&thread_id,
6416	Some(&turn_id),
6417	Some(&item_id),
6418	if item.status == TurnItemLifecycleStatus::Completed {
6419	"item.completed"
6420	} else {
6421	"item.failed"
6422	},
6423	json!({ "item": item }),
6424	)
6425	.await?;
6426	}
6427	}
6428	EngineEvent::SubAgentMailbox {
6429	turn_id: mailbox_turn_id,
6430	message:
6431	crate::tools::subagent::MailboxMessage::TokenUsage {
6432	source_id,
6433	route,
6434	usage,
6435	..
6436	},
6437	..
6438	} => {
6439	let belongs_to_turn = engine_turn_id
6440	.as_deref()
6441	.is_some_and(\|started\| started == mailbox_turn_id);
6442	if belongs_to_turn {
6443	self.append_routed_usage_to_turn(
6444	&turn_id,
6445	&source_id,
6446	EffectiveRouteUsage { route, usage },
6447	)?;
6448	}
6449	}
6450	EngineEvent::CompactionStarted { id, auto, message } => {
6451	let item_id = format!("item_{}", &Uuid::new_v4().to_string()[..8]);
6452	compaction_items.insert(id.clone(), item_id.clone());
6453	let item = TurnItemRecord {
6454	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6455	id: item_id.clone(),
6456	turn_id: turn_id.clone(),
6457	kind: TurnItemKind::ContextCompaction,
6458	status: TurnItemLifecycleStatus::InProgress,
6459	summary: summarize_text(&message, SUMMARY_LIMIT),
6460	detail: Some(message.clone()),
6461	metadata: None,
6462	artifact_refs: Vec::new(),
6463	started_at: Some(Utc::now()),
6464	ended_at: None,
6465	};
6466	self.store.save_item(&item)?;
6467	self.attach_item_to_turn(&turn_id, &item.id)?;
6468	self.emit_event(
6469	&thread_id,
6470	Some(&turn_id),
6471	Some(&item_id),
6472	"item.started",
6473	json!({ "item": item, "auto": auto }),
6474	)
6475	.await?;
6476	}
6477	EngineEvent::CompactionCompleted {
6478	id,
6479	auto,
6480	message,
6481	messages_before,
6482	messages_after,
6483	summary_prompt,
6484	} => {
6485	// Persist the summary into the thread record so engine
6486	// reloads (LRU eviction / restart) restore it: reload
6487	// passes the record prompt through SyncSession, where
6488	// `extract_compaction_summary_prompt` picks the summary
6489	// back up. Without this the summary lives only in engine
6490	// memory and silently dies with the engine.
6491	if let Some(summary) =
6492	summary_prompt.as_deref().filter(\|s\| !s.trim().is_empty())
6493	{
6494	let persist_summary = (\|\| -> Result<()> {
6495	let _thread_mutation = self.store.thread_mutation.lock();
6496	let mut thread = self.store.load_thread(&thread_id)?;
6497	let merged =
6498	merge_summary_into_prompt(thread.system_prompt.as_deref(), summary);
6499	if thread.system_prompt.as_deref() != Some(merged.as_str()) {
6500	thread.system_prompt = Some(merged);
6501	thread.updated_at = Utc::now();
6502	self.store.save_thread(&thread)?;
6503	}
6504	Ok(())
6505	})();
6506	if let Err(e) = persist_summary {
6507	tracing::warn!(
6508	thread_id = %thread_id,
6509	"Failed to persist compaction summary to thread record: {e}"
6510	);
6511	}
6512	}
6513	if let Some(item_id) = compaction_items.remove(&id) {
6514	let mut item = self.store.load_item(&item_id)?;
6515	item.status = TurnItemLifecycleStatus::Completed;
6516	item.summary = summarize_text(&message, SUMMARY_LIMIT);
6517	item.detail = Some(message);
6518	item.ended_at = Some(Utc::now());
6519	self.store.save_item(&item)?;
6520	self.emit_event(
6521	&thread_id,
6522	Some(&turn_id),
6523	Some(&item_id),
6524	"item.completed",
6525	json!({
6526	"item": item,
6527	"auto": auto,
6528	"messages_before": messages_before,
6529	"messages_after": messages_after,
6530	}),
6531	)
6532	.await?;
6533	}
6534	}
6535	EngineEvent::CompactionFailed { id, auto, message } => {
6536	if let Some(item_id) = compaction_items.remove(&id) {
6537	let mut item = self.store.load_item(&item_id)?;
6538	item.status = TurnItemLifecycleStatus::Failed;
6539	item.summary = summarize_text(&message, SUMMARY_LIMIT);
6540	item.detail = Some(message);
6541	item.ended_at = Some(Utc::now());
6542	self.store.save_item(&item)?;
6543	self.emit_event(
6544	&thread_id,
6545	Some(&turn_id),
6546	Some(&item_id),
6547	"item.failed",
6548	json!({ "item": item, "auto": auto }),
6549	)
6550	.await?;
6551	}
6552	}
6553	EngineEvent::AgentSpawned { id, prompt, .. } => {
6554	let message = format!(
6555	"Sub-agent {id} spawned: {}",
6556	summarize_text(&prompt, SUMMARY_LIMIT)
6557	);
6558	let item = TurnItemRecord {
6559	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6560	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6561	turn_id: turn_id.clone(),
6562	kind: TurnItemKind::Status,
6563	status: TurnItemLifecycleStatus::Completed,
6564	summary: summarize_text(&message, SUMMARY_LIMIT),
6565	detail: Some(message),
6566	metadata: None,
6567	artifact_refs: Vec::new(),
6568	started_at: Some(Utc::now()),
6569	ended_at: Some(Utc::now()),
6570	};
6571	self.store.save_item(&item)?;
6572	self.attach_item_to_turn(&turn_id, &item.id)?;
6573	self.emit_event(
6574	&thread_id,
6575	Some(&turn_id),
6576	Some(&item.id),
6577	"agent.spawned",
6578	json!({ "item": item, "agent_id": id }),
6579	)
6580	.await?;
6581	}
6582	EngineEvent::AgentProgress { id, status, .. } => {
6583	let message = format!("Sub-agent {id}: {status}");
6584	let item = TurnItemRecord {
6585	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6586	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6587	turn_id: turn_id.clone(),
6588	kind: TurnItemKind::Status,
6589	status: TurnItemLifecycleStatus::Completed,
6590	summary: summarize_text(&message, SUMMARY_LIMIT),
6591	detail: Some(message),
6592	metadata: None,
6593	artifact_refs: Vec::new(),
6594	started_at: Some(Utc::now()),
6595	ended_at: Some(Utc::now()),
6596	};
6597	self.store.save_item(&item)?;
6598	self.attach_item_to_turn(&turn_id, &item.id)?;
6599	self.emit_event(
6600	&thread_id,
6601	Some(&turn_id),
6602	Some(&item.id),
6603	"agent.progress",
6604	json!({ "item": item, "agent_id": id }),
6605	)
6606	.await?;
6607	}
6608	EngineEvent::AgentComplete { id, result } => {
6609	let message = format!(
6610	"Sub-agent {id} completed: {}",
6611	summarize_text(&result, SUMMARY_LIMIT)
6612	);
6613	let item = TurnItemRecord {
6614	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6615	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6616	turn_id: turn_id.clone(),
6617	kind: TurnItemKind::Status,
6618	status: TurnItemLifecycleStatus::Completed,
6619	summary: summarize_text(&message, SUMMARY_LIMIT),
6620	detail: Some(message),
6621	metadata: None,
6622	artifact_refs: Vec::new(),
6623	started_at: Some(Utc::now()),
6624	ended_at: Some(Utc::now()),
6625	};
6626	self.store.save_item(&item)?;
6627	self.attach_item_to_turn(&turn_id, &item.id)?;
6628	self.emit_event(
6629	&thread_id,
6630	Some(&turn_id),
6631	Some(&item.id),
6632	"agent.completed",
6633	json!({ "item": item, "agent_id": id }),
6634	)
6635	.await?;
6636	}
6637	EngineEvent::AgentList { agents, .. } => {
6638	let running = agents
6639	.iter()
6640	.filter(\|agent\| matches!(agent.status, SubAgentStatus::Running))
6641	.count();
6642	let interrupted = agents
6643	.iter()
6644	.filter(\|agent\| matches!(agent.status, SubAgentStatus::Interrupted(_)))
6645	.count();
6646	let completed = agents
6647	.iter()
6648	.filter(\|agent\| matches!(agent.status, SubAgentStatus::Completed))
6649	.count();
6650	let message = format!(
6651	"Sub-agent list refreshed: {} total ({running} running, {interrupted} interrupted, {completed} completed)",
6652	agents.len()
6653	);
6654	let item = TurnItemRecord {
6655	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6656	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6657	turn_id: turn_id.clone(),
6658	kind: TurnItemKind::Status,
6659	status: TurnItemLifecycleStatus::Completed,
6660	summary: summarize_text(&message, SUMMARY_LIMIT),
6661	detail: Some(message),
6662	metadata: None,
6663	artifact_refs: Vec::new(),
6664	started_at: Some(Utc::now()),
6665	ended_at: Some(Utc::now()),
6666	};
6667	self.store.save_item(&item)?;
6668	self.attach_item_to_turn(&turn_id, &item.id)?;
6669	self.emit_event(
6670	&thread_id,
6671	Some(&turn_id),
6672	Some(&item.id),
6673	"agent.list",
6674	json!({ "item": item, "agents": agents }),
6675	)
6676	.await?;
6677	}
6678	EngineEvent::ApprovalRequired {
6679	id,
6680	tool_name,
6681	description,
6682	intent_summary,
6683	..
6684	} => {
6685	let Some(authority) = self
6686	.active_turn_authority(&thread_id, &turn_id, &engine)
6687	.await
6688	else {
6689	let _ = engine.deny_tool_call(&id).await;
6690	continue;
6691	};
6692	let auto_approve = authority.auto_approve;
6693	let trust_mode = authority.trust_mode;
6694	let approval_mode = authority.approval_mode;
6695
6696	let pending_request = PendingApprovalRequest {
6697	id: id.clone(),
6698	turn_id: turn_id.clone(),
6699	tool_name: tool_name.clone(),
6700	description: description.clone(),
6701	intent_summary: intent_summary.clone(),
6702	};
6703
6704	if auto_approve {
6705	self.emit_event(
6706	&thread_id,
6707	Some(&turn_id),
6708	None,
6709	"approval.required",
6710	json!({
6711	"id": id,
6712	"approval_id": id,
6713	"tool_name": tool_name,
6714	"description": description,
6715	"intent_summary": intent_summary,
6716	}),
6717	)
6718	.await?;
6719	let auto_decision =
6720	Self::approval_decision(auto_approve, trust_mode, false);
6721	let (dec_str, approved) = match auto_decision {
6722	RuntimeApprovalDecision::ApproveTool => ("allow", true),
6723	RuntimeApprovalDecision::DenyTool
6724	\| RuntimeApprovalDecision::RetryWithFullAccess => ("deny", false),
6725	};
6726	// Emit approval.decided so external clients (GUI)
6727	// know the approval was resolved automatically and
6728	// can clear any pending approval UI. Without this
6729	// event the GUI would show a frozen approval dialog
6730	// that never receives approval.decided.
6731	self.emit_event(
6732	&thread_id,
6733	Some(&turn_id),
6734	None,
6735	"approval.decided",
6736	json!({
6737	"approval_id": id,
6738	"decision": dec_str,
6739	"remember": false,
6740	"auto": true,
6741	}),
6742	)
6743	.await
6744	.ok();
6745	if approved {
6746	let _ = engine.approve_tool_call(id).await;
6747	} else {
6748	let _ = engine.deny_tool_call(id).await;
6749	}
6750	continue;
6751	}
6752
6753	// Auto-Review never opens an approval modal. The engine
6754	// resolves gated tools under Auto itself, so reaching
6755	// this branch means a host injected the event directly:
6756	// fail closed (the audit trail stays authoritative)
6757	// instead of pausing the turn.
6758	if approval_mode == crate::tui::approval::ApprovalMode::Auto {
6759	self.emit_event(
6760	&thread_id,
6761	Some(&turn_id),
6762	None,
6763	"approval.decided",
6764	json!({
6765	"approval_id": id,
6766	"decision": "deny",
6767	"remember": false,
6768	"auto": true,
6769	"posture": "auto_review",
6770	}),
6771	)
6772	.await
6773	.ok();
6774	let _ = engine.deny_tool_call(id).await;
6775	continue;
6776	}
6777
6778	// Register before sequencing the event. A snapshot racing
6779	// this branch therefore either contains the request or
6780	// subscribes from an older cursor that will replay it.
6781	let projection_lock = self.projection_lock(&thread_id);
6782	let projection = projection_lock.lock().await;
6783	let rx = self.register_pending_approval(&thread_id, pending_request);
6784	if let Err(err) = self
6785	.emit_event(
6786	&thread_id,
6787	Some(&turn_id),
6788	None,
6789	"approval.required",
6790	json!({
6791	"id": id,
6792	"approval_id": id,
6793	"tool_name": tool_name,
6794	"description": description,
6795	"intent_summary": intent_summary,
6796	}),
6797	)
6798	.await
6799	{
6800	self.cancel_pending_approval(&id);
6801	drop(projection);
6802	let _ = engine.deny_tool_call(&id).await;
6803	return Err(err);
6804	}
6805	drop(projection);
6806	let approval_timeout = approval_decision_timeout();
6807	match tokio::time::timeout(approval_timeout, rx).await {
6808	Ok(Ok(ExternalApprovalDecision::Allow { remember })) => {
6809	if remember {
6810	self.remember_thread_auto_approve(&thread_id).await;
6811	}
6812	self.emit_event(
6813	&thread_id,
6814	Some(&turn_id),
6815	None,
6816	"approval.decided",
6817	json!({
6818	"approval_id": id,
6819	"decision": "allow",
6820	"remember": remember,
6821	}),
6822	)
6823	.await
6824	.ok();
6825	let _ = engine.approve_tool_call(id).await;
6826	}
6827	Ok(Ok(ExternalApprovalDecision::Deny { remember })) => {
6828	self.emit_event(
6829	&thread_id,
6830	Some(&turn_id),
6831	None,
6832	"approval.decided",
6833	json!({
6834	"approval_id": id,
6835	"decision": "deny",
6836	"remember": remember,
6837	}),
6838	)
6839	.await
6840	.ok();
6841	let _ = engine.deny_tool_call(id).await;
6842	}
6843	Ok(Err(_recv_err)) => {
6844	self.cancel_pending_approval(&id);
6845	let _ = engine.deny_tool_call(id).await;
6846	}
6847	Err(_timeout) => {
6848	self.cancel_pending_approval(&id);
6849	self.emit_event(
6850	&thread_id,
6851	Some(&turn_id),
6852	None,
6853	"approval.timeout",
6854	json!({
6855	"approval_id": id,
6856	"timeout_secs": approval_timeout.as_secs(),
6857	}),
6858	)
6859	.await
6860	.ok();
6861	self.emit_event(
6862	&thread_id,
6863	Some(&turn_id),
6864	None,
6865	"approval.decided",
6866	json!({
6867	"approval_id": id,
6868	"decision": "deny",
6869	"remember": false,
6870	"timeout": true,
6871	}),
6872	)
6873	.await
6874	.ok();
6875	let _ = engine.deny_tool_call(id).await;
6876	}
6877	}
6878	}
6879	EngineEvent::ElevationRequired {
6880	tool_id,
6881	tool_name,
6882	denial_reason,
6883	..
6884	} => {
6885	self.emit_event(
6886	&thread_id,
6887	Some(&turn_id),
6888	None,
6889	"sandbox.denied",
6890	json!({
6891	"tool_id": tool_id,
6892	"tool_name": tool_name,
6893	"reason": denial_reason,
6894	}),
6895	)
6896	.await?;
6897	let authority = self
6898	.active_turn_authority(&thread_id, &turn_id, &engine)
6899	.await
6900	.unwrap_or(crate::core::engine::RuntimePermissionAuthority {
6901	auto_approve: false,
6902	trust_mode: false,
6903	approval_mode: crate::tui::approval::ApprovalMode::Suggest,
6904	});
6905	let auto_approve = authority.auto_approve;
6906	let trust_mode = authority.trust_mode;
6907	match Self::approval_decision(auto_approve, trust_mode, true) {
6908	RuntimeApprovalDecision::RetryWithFullAccess => {
6909	let _ = engine
6910	.retry_tool_with_policy(
6911	tool_id,
6912	crate::sandbox::SandboxPolicy::DangerFullAccess,
6913	)
6914	.await;
6915	}
6916	RuntimeApprovalDecision::ApproveTool
6917	\| RuntimeApprovalDecision::DenyTool => {
6918	let _ = engine.deny_tool_call(tool_id).await;
6919	}
6920	}
6921	}
6922	EngineEvent::UserInputRequired { id, request } => {
6923	let projection_lock = self.projection_lock(&thread_id);
6924	let projection = projection_lock.lock().await;
6925	self.register_pending_user_input(
6926	&thread_id,
6927	PendingUserInputRequest {
6928	id: id.clone(),
6929	turn_id: turn_id.clone(),
6930	request: request.clone(),
6931	},
6932	);
6933	if let Err(err) = self
6934	.emit_event(
6935	&thread_id,
6936	Some(&turn_id),
6937	None,
6938	"user_input.required",
6939	json!({
6940	"id": id,
6941	"request": request,
6942	}),
6943	)
6944	.await
6945	{
6946	self.discard_pending_user_input_registration(&thread_id, &id);
6947	drop(projection);
6948	let _ = engine.cancel_user_input(&id).await;
6949	return Err(err);
6950	}
6951	drop(projection);
6952	}
6953	EngineEvent::Status { message } => {
6954	let item = TurnItemRecord {
6955	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6956	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6957	turn_id: turn_id.clone(),
6958	kind: TurnItemKind::Status,
6959	status: TurnItemLifecycleStatus::Completed,
6960	summary: summarize_text(&message, SUMMARY_LIMIT),
6961	detail: Some(message.clone()),
6962	metadata: None,
6963	artifact_refs: Vec::new(),
6964	started_at: Some(Utc::now()),
6965	ended_at: Some(Utc::now()),
6966	};
6967	self.store.save_item(&item)?;
6968	self.attach_item_to_turn(&turn_id, &item.id)?;
6969	self.emit_event(
6970	&thread_id,
6971	Some(&turn_id),
6972	Some(&item.id),
6973	"item.completed",
6974	json!({ "item": item }),
6975	)
6976	.await?;
6977	}
6978	EngineEvent::Error { envelope, .. } => {
6979	turn_status = Some(RuntimeTurnStatus::Failed);
6980	turn_error = Some(envelope.message.clone());
6981	let message = envelope.message.clone();
6982	let item = TurnItemRecord {
6983	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
6984	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
6985	turn_id: turn_id.clone(),
6986	kind: TurnItemKind::Error,
6987	status: TurnItemLifecycleStatus::Failed,
6988	summary: summarize_text(&message, SUMMARY_LIMIT),
6989	detail: Some(message),
6990	metadata: None,
6991	artifact_refs: Vec::new(),
6992	started_at: Some(Utc::now()),
6993	ended_at: Some(Utc::now()),
6994	};
6995	self.store.save_item(&item)?;
6996	self.attach_item_to_turn(&turn_id, &item.id)?;
6997	self.emit_event(
6998	&thread_id,
6999	Some(&turn_id),
7000	Some(&item.id),
7001	"item.failed",
7002	json!({ "item": item }),
7003	)
7004	.await?;
7005	}
7006	EngineEvent::TurnComplete {
7007	usage,
7008	status,
7009	error,
7010	..
7011	} => {
7012	turn_usage = Some(usage);
7013	let reported_status = match status {
7014	TurnOutcomeStatus::Completed => RuntimeTurnStatus::Completed,
7015	TurnOutcomeStatus::Interrupted => RuntimeTurnStatus::Interrupted,
7016	TurnOutcomeStatus::Failed => RuntimeTurnStatus::Failed,
7017	};
7018	// Some engines emit a categorized Error followed by their
7019	// generic TurnComplete(Completed) cleanup receipt. Keep
7020	// the error authoritative instead of silently converting
7021	// a failed turn back to success.
7022	turn_status = Some(
7023	if turn_status == Some(RuntimeTurnStatus::Failed)
7024	&& reported_status == RuntimeTurnStatus::Completed
7025	{
7026	RuntimeTurnStatus::Failed
7027	} else {
7028	reported_status
7029	},
7030	);
7031	if let Some(err) = error {
7032	turn_error = Some(err);
7033	}
7034	break;
7035	}
7036	_ => {}
7037	}
7038	}
7039
7040	let mut turn_status = turn_status
7041	.expect("turn monitor exits normally only after assigning a terminal status");
7042
7043	if self
7044	.is_interrupt_requested(&thread_id, &turn_id)
7045	.await
7046	.unwrap_or(false)
7047	{
7048	turn_status = RuntimeTurnStatus::Interrupted;
7049	}
7050
7051	if let Some(mut item) = current_message_item.take() {
7052	if turn_status == RuntimeTurnStatus::Interrupted {
7053	item.status = TurnItemLifecycleStatus::Interrupted;
7054	} else {
7055	item.status = TurnItemLifecycleStatus::Completed;
7056	}
7057	item.summary =
7058	summarize_text(item.detail.as_deref().unwrap_or_default(), SUMMARY_LIMIT);
7059	item.ended_at = Some(Utc::now());
7060	self.save_streaming_item(&item).await?;
7061	self.emit_event(
7062	&thread_id,
7063	Some(&turn_id),
7064	Some(&item.id),
7065	if item.status == TurnItemLifecycleStatus::Interrupted {
7066	"item.interrupted"
7067	} else {
7068	"item.completed"
7069	},
7070	json!({ "item": item }),
7071	)
7072	.await?;
7073	}
7074
7075	if let Some(mut item) = current_reasoning_item.take() {
7076	if turn_status == RuntimeTurnStatus::Interrupted {
7077	item.status = TurnItemLifecycleStatus::Interrupted;
7078	} else {
7079	item.status = TurnItemLifecycleStatus::Completed;
7080	}
7081	item.summary =
7082	summarize_text(item.detail.as_deref().unwrap_or_default(), SUMMARY_LIMIT);
7083	item.ended_at = Some(Utc::now());
7084	self.save_streaming_item(&item).await?;
7085	self.emit_event(
7086	&thread_id,
7087	Some(&turn_id),
7088	Some(&item.id),
7089	if item.status == TurnItemLifecycleStatus::Interrupted {
7090	"item.interrupted"
7091	} else {
7092	"item.completed"
7093	},
7094	json!({ "item": item }),
7095	)
7096	.await?;
7097	}
7098
7099	if turn_status == RuntimeTurnStatus::Completed && !saw_engine_activity {
7100	turn_status = RuntimeTurnStatus::Failed;
7101	turn_error = Some(EMPTY_TURN_REASON.to_string());
7102	let item = TurnItemRecord {
7103	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
7104	id: format!("item_{}", &Uuid::new_v4().to_string()[..8]),
7105	turn_id: turn_id.clone(),
7106	kind: TurnItemKind::Error,
7107	status: TurnItemLifecycleStatus::Failed,
7108	summary: EMPTY_TURN_REASON.to_string(),
7109	detail: Some(EMPTY_TURN_REASON.to_string()),
7110	metadata: None,
7111	artifact_refs: Vec::new(),
7112	started_at: Some(Utc::now()),
7113	ended_at: Some(Utc::now()),
7114	};
7115	self.store.save_item(&item)?;
7116	self.attach_item_to_turn(&turn_id, &item.id)?;
7117	self.emit_event(
7118	&thread_id,
7119	Some(&turn_id),
7120	Some(&item.id),
7121	"item.failed",
7122	json!({ "item": item }),
7123	)
7124	.await?;
7125	}
7126
7127	let ended_at = Utc::now();
7128	crate::cost_status::finish_runtime_usage_owner(&turn_id);
7129	let background_usage = crate::cost_status::take_runtime_usage(&turn_id);
7130	let turn = {
7131	let _turn_mutation = self.store.turn_mutation.lock();
7132	let mut turn = self.store.load_turn(&turn_id)?;
7133	turn.status = turn_status;
7134	turn.ended_at = Some(ended_at);
7135	turn.duration_ms = turn.started_at.map(\|start\| duration_ms(start, ended_at));
7136	turn.usage = turn_usage;
7137	for record in background_usage.records {
7138	append_routed_usage_record(&mut turn, &record.source_id, record.usage);
7139	}
7140	turn.routed_usage_dropped_records = turn
7141	.routed_usage_dropped_records
7142	.saturating_add(background_usage.dropped_records);
7143	turn.error = turn_error;
7144	turn
7145	};
7146
7147	// A terminal turn can no longer answer an outstanding prompt. Commit
7148	// each cancellation while the request remains snapshot-authoritative,
7149	// then remove and notify the engine before publishing completion.
7150	self.settle_user_inputs_for_terminal_turn(&thread_id, &turn_id, Some(engine.clone()))
7151	.await?;
7152
7153	self.settle_dynamic_tools_for_terminal_turn(&thread_id, &turn_id)
7154	.await?;
7155
7156	// Publish the terminal projection as one snapshot boundary. The
7157	// duplicate scan is offloaded while this guard is held, so public
7158	// readers cannot observe a terminal record before its receipt and
7159	// active-claim cleanup are ordered.
7160	let projection_lock = self.projection_lock(&thread_id);
7161	let _projection = projection_lock.lock().await;
7162	{
7163	let _turn_mutation = self.store.turn_mutation.lock();
7164	self.store.save_turn(&turn)?;
7165	}
7166	{
7167	let _thread_mutation = self.store.thread_mutation.lock();
7168	let mut thread = self.store.load_thread(&thread_id)?;
7169	thread.latest_turn_id = Some(turn_id.clone());
7170	thread.updated_at = Utc::now();
7171	self.store.save_thread(&thread)?;
7172	}
7173	self.emit_turn_completed_if_missing(&turn, false).await?;
7174
7175	{
7176	let mut active = self.active.lock().await;
7177	if let Some(state) = active.engines.get_mut(&thread_id)
7178	&& state
7179	.active_turn
7180	.as_ref()
7181	.is_some_and(\|t\| t.turn_id == turn_id)
7182	{
7183	state.active_turn = None;
7184	}
7185	touch_lru(&mut active.lru, &thread_id);
7186	}
7187
7188	Ok(())
7189	}
7190
7191	fn attach_item_to_turn(&self, turn_id: &str, item_id: &str) -> Result<()> {
7192	let _turn_mutation = self.store.turn_mutation.lock();
7193	let mut turn = self.store.load_turn(turn_id)?;
7194	if !turn.item_ids.iter().any(\|id\| id == item_id) {
7195	turn.item_ids.push(item_id.to_string());
7196	self.store.save_turn(&turn)?;
7197	}
7198	Ok(())
7199	}
7200
7201	async fn is_interrupt_requested(&self, thread_id: &str, turn_id: &str) -> Result<bool> {
7202	let active = self.active.lock().await;
7203	let Some(state) = active.engines.get(thread_id) else {
7204	return Ok(false);
7205	};
7206	let Some(turn) = state.active_turn.as_ref() else {
7207	return Ok(false);
7208	};
7209	Ok(turn.turn_id == turn_id && turn.interrupt_requested)
7210	}
7211
7212	async fn active_turn_authority(
7213	&self,
7214	thread_id: &str,
7215	turn_id: &str,
7216	engine: &EngineHandle,
7217	) -> Option<crate::core::engine::RuntimePermissionAuthority> {
7218	let active = self.active.lock().await;
7219	let state = active.engines.get(thread_id)?;
7220	let turn = state.active_turn.as_ref()?;
7221	if turn.turn_id != turn_id {
7222	return None;
7223	}
7224	Some(engine.runtime_permission_authority())
7225	}
7226
7227	#[cfg(test)]
7228	async fn active_turn_flags(&self, thread_id: &str, turn_id: &str) -> Option<(bool, bool)> {
7229	let active = self.active.lock().await;
7230	let state = active.engines.get(thread_id)?;
7231	let turn = state.active_turn.as_ref()?;
7232	if turn.turn_id != turn_id {
7233	return None;
7234	}
7235	let authority = state.engine.runtime_permission_authority();
7236	Some((authority.auto_approve, authority.trust_mode))
7237	}
7238
7239	async fn active_turn_id(&self, thread_id: &str) -> Option<String> {
7240	let active = self.active.lock().await;
7241	active
7242	.engines
7243	.get(thread_id)?
7244	.active_turn
7245	.as_ref()
7246	.map(\|turn\| turn.turn_id.clone())
7247	}
7248
7249	fn approval_decision(
7250	auto_approve: bool,
7251	trust_mode: bool,
7252	requires_full_access: bool,
7253	) -> RuntimeApprovalDecision {
7254	if !auto_approve {
7255	return RuntimeApprovalDecision::DenyTool;
7256	}
7257	if requires_full_access {
7258	if trust_mode {
7259	RuntimeApprovalDecision::RetryWithFullAccess
7260	} else {
7261	RuntimeApprovalDecision::DenyTool
7262	}
7263	} else {
7264	RuntimeApprovalDecision::ApproveTool
7265	}
7266	}
7267
7268	fn recover_interrupted_state(&self) -> Result<()> {
7269	let now = Utc::now();
7270	let mut threads = self
7271	.store
7272	.list_threads()?
7273	.into_iter()
7274	.map(\|thread\| (thread.id.clone(), thread))
7275	.collect::<HashMap<_, _>>();
7276	let mut turns_by_thread: HashMap<String, Vec<TurnRecord>> = HashMap::new();
7277	let mut changed_threads = HashSet::new();
7278
7279	// First terminalize interrupted candidates. Keep every terminal turn
7280	// in the same one-pass grouping so already-terminal records whose
7281	// completion append failed are reconciled too.
7282	for mut turn in self.store.list_all_turns()? {
7283	let mut thread_changed = false;
7284	if matches!(
7285	turn.status,
7286	RuntimeTurnStatus::Queued \| RuntimeTurnStatus::InProgress
7287	) {
7288	turn.status = RuntimeTurnStatus::Interrupted;
7289	turn.error = Some(RUNTIME_RESTART_REASON.to_string());
7290	turn.ended_at = Some(now);
7291	if let Some(started_at) = turn.started_at {
7292	let elapsed = now.signed_duration_since(started_at);
7293	turn.duration_ms = Some(elapsed.num_milliseconds().max(0) as u64);
7294	}
7295	self.store.save_turn(&turn)?;
7296
7297	for item_id in &turn.item_ids {
7298	let mut item = self.store.load_item(item_id)?;
7299	if matches!(
7300	item.status,
7301	TurnItemLifecycleStatus::Queued \| TurnItemLifecycleStatus::InProgress
7302	) {
7303	item.status = TurnItemLifecycleStatus::Interrupted;
7304	item.ended_at = Some(now);
7305	self.store.save_item(&item)?;
7306	}
7307	}
7308
7309	thread_changed = true;
7310	}
7311	if thread_changed && let Some(thread) = threads.get_mut(&turn.thread_id) {
7312	thread.updated_at = now;
7313	changed_threads.insert(thread.id.clone());
7314	}
7315	if matches!(
7316	turn.status,
7317	RuntimeTurnStatus::Completed
7318	\| RuntimeTurnStatus::Failed
7319	\| RuntimeTurnStatus::Interrupted
7320	\| RuntimeTurnStatus::Canceled
7321	) {
7322	turns_by_thread
7323	.entry(turn.thread_id.clone())
7324	.or_default()
7325	.push(turn);
7326	}
7327	}
7328
7329	for thread_id in changed_threads {
7330	if let Some(thread) = threads.get(&thread_id) {
7331	self.store.save_thread(thread)?;
7332	}
7333	}
7334
7335	let mut recovery_receipts: HashMap<String, Vec<RecoveredTurnReceipt>> = HashMap::new();
7336	for (thread_id, mut turns) in turns_by_thread {
7337	let events = self.store.events_since(&thread_id, None)?;
7338	let completed_turns = events
7339	.iter()
7340	.filter(\|event\| event.event == "turn.completed")
7341	.filter_map(\|event\| event.turn_id.clone())
7342	.collect::<HashSet<_>>();
7343	let terminal_calls = events
7344	.iter()
7345	.filter(\|event\| {
7346	matches!(
7347	event.event.as_str(),
7348	"tool_call.resolved" \| "tool_call.canceled" \| "tool_call.timeout"
7349	)
7350	})
7351	.filter_map(\|event\| {
7352	let turn_id = event.turn_id.as_deref()?;
7353	let call_id = event.payload.get("call_id")?.as_str()?;
7354	Some((turn_id.to_string(), call_id.to_string()))
7355	})
7356	.collect::<HashSet<_>>();
7357	let mut requests_by_turn: HashMap<String, Vec<DynamicToolCallParams>> = HashMap::new();
7358	for event in events
7359	.iter()
7360	.filter(\|event\| event.event == "tool_call.requested")
7361	{
7362	let Ok(params) =
7363	serde_json::from_value::<DynamicToolCallParams>(event.payload.clone())
7364	else {
7365	tracing::warn!(
7366	thread_id,
7367	seq = event.seq,
7368	"Ignoring malformed dynamic-tool request during Runtime recovery"
7369	);
7370	continue;
7371	};
7372	if params.thread_id == thread_id
7373	&& !terminal_calls.contains(&(params.turn_id.clone(), params.call_id.clone()))
7374	{
7375	requests_by_turn
7376	.entry(params.turn_id.clone())
7377	.or_default()
7378	.push(params);
7379	}
7380	}
7381
7382	turns.sort_by_key(\|turn\| turn.created_at);
7383	for turn in turns {
7384	let unresolved_dynamic_tools =
7385	requests_by_turn.remove(&turn.id).unwrap_or_default();
7386	if completed_turns.contains(&turn.id) && unresolved_dynamic_tools.is_empty() {
7387	continue;
7388	}
7389	recovery_receipts
7390	.entry(thread_id.clone())
7391	.or_default()
7392	.push(RecoveredTurnReceipt {
7393	unresolved_dynamic_tools,
7394	turn,
7395	});
7396	}
7397	}
7398
7399	*self.recovery_receipts.lock() = recovery_receipts;
7400
7401	Ok(())
7402	}
7403
7404	#[cfg(test)]
7405	pub(crate) async fn install_test_engine(
7406	&self,
7407	thread_id: &str,
7408	engine: EngineHandle,
7409	) -> Result<()> {
7410	let thread = self.get_thread(thread_id).await?;
7411	let config = self.read_config().clone();
7412	let route = self.resolved_route_for_thread(&config, &thread)?;
7413	let mut active = self.active.lock().await;
7414	active.engines.insert(
7415	thread_id.to_string(),
7416	ActiveThreadState {
7417	engine,
7418	active_turn: None,
7419	route_identity: route.identity,
7420	route_model: route.model,
7421	client_preflight_required: false,
7422	},
7423	);
7424	touch_lru(&mut active.lru, thread_id);
7425	Ok(())
7426	}
7427	}
7428
7429	fn dynamic_tool_result_text(content: &[DynamicToolCallContent]) -> String {
7430	content
7431	.iter()
7432	.map(\|item\| match item {
7433	DynamicToolCallContent::InputText { text } => text.clone(),
7434	DynamicToolCallContent::InputImage { image_url } => format!("[image] {image_url}"),
7435	})
7436	.collect::<Vec<_>>()
7437	.join("\n")
7438	}
7439
7440	fn dynamic_tool_result_to_tool_result(
7441	result: DynamicToolCallResult,
7442	) -> crate::tools::spec::ToolResult {
7443	let text = dynamic_tool_result_text(&result.content);
7444	if result.success {
7445	crate::tools::spec::ToolResult::success(text)
7446	} else {
7447	crate::tools::spec::ToolResult::error(if text.is_empty() {
7448	"dynamic tool failed".to_string()
7449	} else {
7450	text
7451	})
7452	}
7453	}
7454
7455	fn dynamic_tool_terminal_payload(
7456	params: &DynamicToolCallParams,
7457	status: &str,
7458	success: Option<bool>,
7459	reason: Option<&str>,
7460	) -> Value {
7461	let mut payload = json!({
7462	"thread_id": params.thread_id,
7463	"turn_id": params.turn_id,
7464	"call_id": params.call_id,
7465	"status": status,
7466	});
7467	if let Some(object) = payload.as_object_mut() {
7468	if let Some(success) = success {
7469	object.insert("success".to_string(), json!(success));
7470	}
7471	if let Some(reason) = reason {
7472	object.insert("reason".to_string(), json!(reason));
7473	}
7474	}
7475	payload
7476	}
7477
7478	#[async_trait::async_trait]
7479	impl crate::tools::spec::DynamicToolExecutor for RuntimeThreadManager {
7480	async fn execute_dynamic_tool(
7481	&self,
7482	thread_id: Option<String>,
7483	namespace: Option<String>,
7484	name: String,
7485	input: Value,
7486	) -> std::result::Result<crate::tools::spec::ToolResult, crate::tools::spec::ToolError> {
7487	let thread_id = thread_id.ok_or_else(\|\| {
7488	crate::tools::spec::ToolError::not_available(format!(
7489	"runtime dynamic tool '{name}' has no active thread"
7490	))
7491	})?;
7492	let turn_id = self.active_turn_id(&thread_id).await.ok_or_else(\|\| {
7493	crate::tools::spec::ToolError::not_available(format!(
7494	"runtime dynamic tool '{name}' has no active turn"
7495	))
7496	})?;
7497	let call_id = format!("call_{}", &Uuid::new_v4().to_string()[..8]);
7498	let params = DynamicToolCallParams {
7499	thread_id: thread_id.clone(),
7500	turn_id: turn_id.clone(),
7501	call_id: call_id.clone(),
7502	namespace,
7503	tool: name.clone(),
7504	arguments: input,
7505	};
7506	let projection_lock = self.projection_lock(&thread_id);
7507	let projection = projection_lock.lock().await;
7508	let mut rx = self
7509	.register_pending_dynamic_tool(params.clone())
7510	.map_err(\|err\| crate::tools::spec::ToolError::execution_failed(err.to_string()))?;
7511	if let Err(err) = self
7512	.emit_event(
7513	&thread_id,
7514	Some(&turn_id),
7515	None,
7516	"tool_call.requested",
7517	json!(&params),
7518	)
7519	.await
7520	{
7521	self.remove_pending_dynamic_tool(&thread_id, &turn_id, &call_id);
7522	drop(projection);
7523	return Err(crate::tools::spec::ToolError::execution_failed(format!(
7524	"failed to emit runtime dynamic tool request for '{name}': {err}"
7525	)));
7526	}
7527	drop(projection);
7528
7529	let result_timeout = dynamic_tool_result_timeout();
7530	match tokio::time::timeout(result_timeout, &mut rx).await {
7531	Ok(Ok(result)) => Ok(dynamic_tool_result_to_tool_result(result)),
7532	Ok(Err(_recv_err)) => Err(crate::tools::spec::ToolError::execution_failed(format!(
7533	"runtime dynamic tool '{name}' result channel closed"
7534	))),
7535	Err(_timeout) => {
7536	let mut settlement_progress = match self
7537	.claim_pending_dynamic_tool(&thread_id, &turn_id, &call_id)
7538	{
7539	PendingDynamicToolClaim::Claimed(claim) => {
7540	self.settle_dynamic_tool_timeout(claim, result_timeout)
7541	.await
7542	.map_err(\|err\| {
7543	crate::tools::spec::ToolError::execution_failed(err.to_string())
7544	})?;
7545	return Err(crate::tools::spec::ToolError::Timeout {
7546	seconds: result_timeout.as_secs(),
7547	});
7548	}
7549	PendingDynamicToolClaim::Settling(progress) => progress,
7550	PendingDynamicToolClaim::Indeterminate => {
7551	return Err(crate::tools::spec::ToolError::execution_failed(format!(
7552	"runtime dynamic tool '{name}' has an indeterminate terminal receipt"
7553	)));
7554	}
7555	PendingDynamicToolClaim::Missing => {
7556	return match rx.await {
7557	Ok(result) => Ok(dynamic_tool_result_to_tool_result(result)),
7558	Err(_recv_err) => Err(crate::tools::spec::ToolError::execution_failed(
7559	format!("runtime dynamic tool '{name}' result channel closed"),
7560	)),
7561	};
7562	}
7563	};
7564
7565	// A result or turn cancellation claimed the call just before
7566	// the timer fired. Preserve that winner. Its supervised task
7567	// notifies this watcher on either durable completion or
7568	// rollback, so a panic/persistence error cannot strand this
7569	// executor in an unbounded `rx.await`.
7570	loop {
7571	tokio::select! {
7572	received = &mut rx => {
7573	return match received {
7574	Ok(result) => Ok(dynamic_tool_result_to_tool_result(result)),
7575	Err(_recv_err) => Err(
7576	crate::tools::spec::ToolError::execution_failed(format!(
7577	"runtime dynamic tool '{name}' result channel closed"
7578	)),
7579	),
7580	};
7581	}
7582	_ = settlement_progress.changed() => {
7583	match self.claim_pending_dynamic_tool(
7584	&thread_id,
7585	&turn_id,
7586	&call_id,
7587	) {
7588	PendingDynamicToolClaim::Claimed(claim) => {
7589	self.settle_dynamic_tool_timeout(claim, result_timeout)
7590	.await
7591	.map_err(\|err\| {
7592	crate::tools::spec::ToolError::execution_failed(
7593	err.to_string(),
7594	)
7595	})?;
7596	return Err(crate::tools::spec::ToolError::Timeout {
7597	seconds: result_timeout.as_secs(),
7598	});
7599	}
7600	PendingDynamicToolClaim::Settling(progress) => {
7601	settlement_progress = progress;
7602	}
7603	PendingDynamicToolClaim::Indeterminate => {
7604	return Err(
7605	crate::tools::spec::ToolError::execution_failed(format!(
7606	"runtime dynamic tool '{name}' has an indeterminate terminal receipt"
7607	)),
7608	);
7609	}
7610	PendingDynamicToolClaim::Missing => {
7611	return match rx.await {
7612	Ok(result) => {
7613	Ok(dynamic_tool_result_to_tool_result(result))
7614	}
7615	Err(_recv_err) => Err(
7616	crate::tools::spec::ToolError::execution_failed(
7617	format!(
7618	"runtime dynamic tool '{name}' result channel closed"
7619	),
7620	),
7621	),
7622	};
7623	}
7624	}
7625	}
7626	}
7627	}
7628	}
7629	}
7630	}
7631	}
7632
7633	fn touch_lru(lru: &mut VecDeque<String>, thread_id: &str) {
7634	if let Some(idx) = lru.iter().position(\|id\| id == thread_id) {
7635	lru.remove(idx);
7636	}
7637	lru.push_back(thread_id.to_string());
7638	}
7639
7640	fn enforce_lru_capacity(
7641	active: &mut ActiveThreads,
7642	max_active_threads: usize,
7643	) -> Vec<EngineHandle> {
7644	let mut evicted = Vec::new();
7645	if max_active_threads == 0 \|\| active.engines.len() < max_active_threads {
7646	return evicted;
7647	}
7648	let protected = active
7649	.engines
7650	.iter()
7651	.filter_map(\|(thread_id, state)\| {
7652	if state.active_turn.is_some() {
7653	Some(thread_id.clone())
7654	} else {
7655	None
7656	}
7657	})
7658	.collect::<HashSet<_>>();
7659
7660	let scan_limit = active.lru.len();
7661	for _ in 0..scan_limit {
7662	let Some(candidate) = active.lru.pop_front() else {
7663	break;
7664	};
7665	if protected.contains(&candidate) {
7666	active.lru.push_back(candidate);
7667	continue;
7668	}
7669	if let Some(state) = active.engines.remove(&candidate) {
7670	evicted.push(state.engine);
7671	}
7672	break;
7673	}
7674	evicted
7675	}
7676
7677	/// Merge per-request compatibility inputs with a thread's canonical policy.
7678	/// A mode-only edit must preserve the effective posture of a legacy record
7679	/// even when that record predates `permission_posture`.
7680	fn runtime_policy_with_overrides(
7681	thread: &ThreadRecord,
7682	mode: Option<&str>,
7683	permission_posture: Option<&str>,
7684	auto_approve: Option<bool>,
7685	) -> Result<RuntimePolicyProjection> {
7686	let requested_mode = mode.unwrap_or(&thread.mode);
7687	let legacy_bypass_mode = mode.is_some_and(\|mode\| {
7688	matches!(
7689	mode.trim().to_ascii_lowercase().as_str(),
7690	"yolo" \| "4" \| "bypass" \| "bypass-permissions" \| "bypasspermissions"
7691	)
7692	});
7693	let inherited = RuntimePolicyProjection::from_persisted(
7694	&thread.mode,
7695	thread.permission_posture.as_deref(),
7696	thread.auto_approve,
7697	);
7698	let requested_permission = match permission_posture {
7699	Some(explicit) => Some(explicit),
7700	None if auto_approve.is_some() \|\| legacy_bypass_mode => None,
7701	None => Some(inherited.permission_wire()),
7702	};
7703	RuntimePolicyProjection::from_request(requested_mode, requested_permission, auto_approve)
7704	}
7705
7706	/// Compatibility parser retained for focused Runtime tests.
7707	#[cfg(test)]
7708	fn parse_mode_opt(mode: &str) -> Option<AppMode> {
7709	crate::runtime_policy::parse_runtime_mode(mode)
7710	}
7711
7712	#[cfg(test)]
7713	fn parse_mode(mode: &str) -> AppMode {
7714	parse_mode_opt(mode).unwrap_or(AppMode::Agent)
7715	}
7716
7717	fn tool_kind_for_name(name: &str) -> TurnItemKind {
7718	let lower = name.to_ascii_lowercase();
7719	if lower == "exec_shell" \|\| lower == "exec_shell_wait" \|\| lower == "exec_shell_interact" {
7720	return TurnItemKind::CommandExecution;
7721	}
7722	if lower.contains("patch") \|\| lower.contains("write") \|\| lower.contains("edit") {
7723	return TurnItemKind::FileChange;
7724	}
7725	TurnItemKind::ToolCall
7726	}
7727
7728	/// One sub-agent rebind hint extracted from a thread's persisted event
7729	/// timeline (issue #128). When the TUI resumes a session that was
7730	/// mid-fanout, the in-transcript card stack is empty — these hints let the
7731	/// UI know which agent_ids were live (or recently terminal) so it can
7732	/// reconstruct the matching `DelegateCard` / `FanoutCard` placeholders
7733	/// before fresh mailbox envelopes arrive on a re-attached engine.
7734	///
7735	/// The helper is the testable contract here — actual TUI wire-up to the
7736	/// resume flow is a follow-up; the runtime API consumer (`runtime_api.rs`)
7737	/// can already call `resume_thread_with_agent_rebind` to drive it.
7738	#[derive(Debug, Clone, PartialEq, Eq)]
7739	#[allow(dead_code)] // consumed by #128 follow-up TUI resume wiring; tested here.
7740	pub struct AgentRebindHint {
7741	pub agent_id: String,
7742	pub status: AgentRebindStatus,
7743	}
7744
7745	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7746	#[allow(dead_code)]
7747	pub enum AgentRebindStatus {
7748	Spawned,
7749	InProgress,
7750	Completed,
7751	}
7752
7753	/// Collapse a chronologically ordered slice of `RuntimeEventRecord` into
7754	/// the latest known status per `agent_id`. Drops entries that aren't in
7755	/// the `agent.*` family. Cards built from these hints are immediately
7756	/// open to mutation by subsequent live mailbox envelopes (each envelope's
7757	/// `agent_id` matches one already in the rebind map).
7758	#[must_use]
7759	#[allow(dead_code)]
7760	pub fn collect_agent_rebind_hints(events: &[RuntimeEventRecord]) -> Vec<AgentRebindHint> {
7761	use std::collections::BTreeMap;
7762	let mut latest: BTreeMap<String, AgentRebindStatus> = BTreeMap::new();
7763	for event in events {
7764	let id = match event.payload.get("agent_id").and_then(\|v\| v.as_str()) {
7765	Some(id) => id.to_string(),
7766	None => continue,
7767	};
7768	let next_status = match event.event.as_str() {
7769	"agent.spawned" => Some(AgentRebindStatus::Spawned),
7770	"agent.progress" => Some(AgentRebindStatus::InProgress),
7771	"agent.completed" => Some(AgentRebindStatus::Completed),
7772	_ => None,
7773	};
7774	if let Some(status) = next_status {
7775	// Don't downgrade Completed → InProgress on out-of-order events.
7776	let entry = latest.entry(id).or_insert(status);
7777	if !matches!(*entry, AgentRebindStatus::Completed) {
7778	*entry = status;
7779	}
7780	}
7781	}
7782	latest
7783	.into_iter()
7784	.map(\|(agent_id, status)\| AgentRebindHint { agent_id, status })
7785	.collect()
7786	}
7787
7788	pub fn summarize_text(text: &str, limit: usize) -> String {
7789	let take = limit.saturating_sub(3);
7790	let mut count = 0;
7791	let mut out = String::new();
7792	for ch in text.chars() {
7793	if count >= take {
7794	out.push_str("...");
7795	return out;
7796	}
7797	if ch.is_control() && ch != '\n' && ch != '\t' {
7798	continue;
7799	}
7800	out.push(ch);
7801	count += 1;
7802	}
7803	out
7804	}
7805
7806	fn duration_ms(start: DateTime<Utc>, end: DateTime<Utc>) -> u64 {
7807	let millis = (end - start).num_milliseconds();
7808	if millis.is_negative() {
7809	0
7810	} else {
7811	u64::try_from(millis).unwrap_or(u64::MAX)
7812	}
7813	}
7814
7815	fn panic_payload_message(payload: &(dyn std::any::Any + Send)) -> String {
7816	if let Some(message) = payload.downcast_ref::<&str>() {
7817	(*message).to_string()
7818	} else if let Some(message) = payload.downcast_ref::<String>() {
7819	message.clone()
7820	} else {
7821	"unknown panic payload".to_string()
7822	}
7823	}
7824
7825	fn checked_runtime_store_root(root: PathBuf) -> Result<PathBuf> {
7826	if root.as_os_str().is_empty() {
7827	bail!("Runtime store root cannot be empty");
7828	}
7829	if root
7830	.components()
7831	.any(\|component\| matches!(component, Component::ParentDir))
7832	{
7833	bail!("Runtime store root cannot contain '..' components");
7834	}
7835	let absolute = if root.is_absolute() {
7836	root
7837	} else {
7838	std::env::current_dir()
7839	.context("failed to resolve current directory for runtime store")?
7840	.join(root)
7841	};
7842	match absolute.canonicalize() {
7843	Ok(path) => Ok(path),
7844	Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
7845	Ok(normalize_path_components(&absolute))
7846	}
7847	Err(err) => Err(err).with_context(\|\| {
7848	format!(
7849	"Failed to resolve runtime store root {}",
7850	absolute.display()
7851	)
7852	}),
7853	}
7854	}
7855
7856	fn checked_existing_runtime_store_dir(path: &Path) -> Result<PathBuf> {
7857	reject_symlinked_store_dir(path)?;
7858	path.canonicalize()
7859	.with_context(\|\| format!("Failed to resolve {}", path.display()))
7860	}
7861
7862	fn normalize_path_components(path: &Path) -> PathBuf {
7863	let mut normalized = PathBuf::new();
7864	for component in path.components() {
7865	match component {
7866	Component::Prefix(_) \| Component::RootDir => normalized.push(component.as_os_str()),
7867	Component::CurDir => {}
7868	Component::ParentDir => {
7869	normalized.pop();
7870	}
7871	Component::Normal(part) => normalized.push(part),
7872	}
7873	}
7874	if normalized.as_os_str().is_empty() {
7875	PathBuf::from(".")
7876	} else {
7877	normalized
7878	}
7879	}
7880
7881	fn reject_symlinked_store_file(path: &Path) -> Result<()> {
7882	let Ok(metadata) = fs::symlink_metadata(path) else {
7883	return Ok(());
7884	};
7885	if metadata.file_type().is_symlink() {
7886	bail!(
7887	"Runtime store file must not be a symlink: {}",
7888	path.display()
7889	);
7890	}
7891	Ok(())
7892	}
7893
7894	fn open_runtime_store_file(
7895	path: &Path,
7896	purpose: &str,
7897	configure: impl FnOnce(&mut OpenOptions),
7898	) -> Result<File> {
7899	reject_symlinked_store_file(path)?;
7900	let mut options = OpenOptions::new();
7901	configure(&mut options);
7902	#[cfg(unix)]
7903	{
7904	use std::os::unix::fs::OpenOptionsExt as _;
7905	options
7906	.mode(0o600)
7907	.custom_flags(libc::O_NOFOLLOW \| libc::O_CLOEXEC);
7908	}
7909	#[cfg(windows)]
7910	{
7911	use std::os::windows::fs::OpenOptionsExt as _;
7912	use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT;
7913	options.custom_flags(FILE_FLAG_OPEN_REPARSE_POINT);
7914	}
7915	let file = options
7916	.open(path)
7917	.with_context(\|\| format!("Failed to open {purpose} {}", path.display()))?;
7918	runtime_store_file_identity(&file)
7919	.with_context(\|\| format!("Invalid {purpose} {}", path.display()))?;
7920	Ok(file)
7921	}
7922
7923	#[cfg(unix)]
7924	fn runtime_store_file_identity(file: &File) -> Result<(u64, u64)> {
7925	use std::os::unix::fs::MetadataExt as _;
7926
7927	let metadata = file.metadata()?;
7928	anyhow::ensure!(
7929	metadata.is_file() && metadata.nlink() == 1,
7930	"not one regular file"
7931	);
7932	Ok((metadata.dev(), metadata.ino()))
7933	}
7934
7935	#[cfg(windows)]
7936	fn runtime_store_file_identity(file: &File) -> Result<(u64, u64)> {
7937	use std::os::windows::fs::MetadataExt as _;
7938	use std::os::windows::io::AsRawHandle as _;
7939	use windows_sys::Win32::Storage::FileSystem::{
7940	BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_REPARSE_POINT, GetFileInformationByHandle,
7941	};
7942
7943	let metadata = file.metadata()?;
7944	let safe = metadata.is_file() && metadata.file_attributes() & FILE_ATTRIBUTE_REPARSE_POINT == 0;
7945	anyhow::ensure!(safe, "not a regular non-reparse file");
7946	let mut info = BY_HANDLE_FILE_INFORMATION::default();
7947	// SAFETY: the handle and writable output remain valid for the call.
7948	if unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut info) } == 0 {
7949	return Err(std::io::Error::last_os_error()).context("Inspect Runtime store file identity");
7950	}
7951	anyhow::ensure!(info.nNumberOfLinks == 1, "has multiple filesystem links");
7952	Ok((
7953	u64::from(info.dwVolumeSerialNumber),
7954	(u64::from(info.nFileIndexHigh) << 32) \| u64::from(info.nFileIndexLow),
7955	))
7956	}
7957
7958	#[cfg(all(not(unix), not(windows)))]
7959	fn runtime_store_file_identity(file: &File) -> Result<(u64, u64)> {
7960	anyhow::ensure!(file.metadata()?.is_file(), "must be a regular file");
7961	Ok((0, 0))
7962	}
7963
7964	fn validate_same_runtime_store_file_handles(
7965	first: &File,
7966	second: &File,
7967	path: &Path,
7968	) -> Result<()> {
7969	let first = runtime_store_file_identity(first)?;
7970	let second = runtime_store_file_identity(second)?;
7971	anyhow::ensure!(
7972	first == second,
7973	"Runtime event file changed: {}",
7974	path.display()
7975	);
7976	Ok(())
7977	}
7978
7979	fn wait_for_event_lock(started: Instant, timeout: Duration) -> Result<()> {
7980	let elapsed = started.elapsed();
7981	if elapsed >= timeout {
7982	return Err(anyhow!(RuntimeEventLockTimeout(timeout)));
7983	}
7984	std::thread::sleep(EVENT_TRANSACTION_LOCK_POLL.min(timeout - elapsed));
7985	Ok(())
7986	}
7987
7988	fn rollback_failed_event_append_handle(rollback_file: &File, original_len: u64) -> Result<()> {
7989	rollback_file
7990	.set_len(original_len)
7991	.context("Failed to roll back Runtime event")?;
7992	rollback_file
7993	.sync_all()
7994	.context("Failed to sync Runtime event rollback")
7995	}
7996
7997	fn reject_symlinked_store_dir(path: &Path) -> Result<()> {
7998	let Ok(metadata) = fs::symlink_metadata(path) else {
7999	return Ok(());
8000	};
8001	if metadata.file_type().is_symlink() {
8002	bail!(
8003	"Runtime store directory must not be a symlink: {}",
8004	path.display()
8005	);
8006	}
8007	if !metadata.is_dir() {
8008	bail!("Runtime store path must be a directory: {}", path.display());
8009	}
8010	Ok(())
8011	}
8012
8013	fn ensure_runtime_store_dir(path: &Path) -> Result<()> {
8014	fs::create_dir_all(path).with_context(\|\| format!("Failed to create {}", path.display()))?;
8015	reject_symlinked_store_dir(path)
8016	}
8017
8018	fn read_complete_event(
8019	reader: &mut impl BufRead,
8020	path: &Path,
8021	) -> Result<Option<RuntimeEventRecord>> {
8022	loop {
8023	let mut line = String::new();
8024	if reader.read_line(&mut line)? == 0 {
8025	return Ok(None);
8026	}
8027	// A concurrent append can be visible before write_all finishes. The
8028	// subscribed broadcast path will deliver that event after its durable
8029	// append completes, so stop at an unterminated live tail instead of
8030	// misclassifying it as durable corruption. Store startup separately
8031	// truncates an unterminated tail left by a dead process.
8032	if !line.ends_with('\n') {
8033	return Ok(None);
8034	}
8035	if line.trim().is_empty() {
8036	continue;
8037	}
8038	let event = serde_json::from_str(&line)
8039	.with_context(\|\| format!("Failed to parse event line in {}", path.display()))?;
8040	return Ok(Some(event));
8041	}
8042	}
8043
8044	/// Remove only an unterminated final JSONL fragment left by a process or
8045	/// machine stopping before the append's newline commit marker. This includes
8046	/// an otherwise valid JSON object whose delimiter never reached disk: without
8047	/// the newline, the append did not commit. A newline-terminated bad record is
8048	/// not crash debris we can identify safely, so normal replay keeps rejecting
8049	/// it instead of silently discarding durable data.
8050	fn repair_torn_event_log_tails(events_dir: &Path) -> Result<()> {
8051	let events_dir = checked_existing_runtime_store_dir(events_dir)?;
8052	for entry in fs::read_dir(&events_dir)
8053	.with_context(\|\| format!("Failed to read {}", events_dir.display()))?
8054	{
8055	let entry = entry?;
8056	let path = entry.path();
8057	if path
8058	.extension()
8059	.is_none_or(\|extension\| extension != "jsonl")
8060	{
8061	continue;
8062	}
8063	if !entry
8064	.file_type()
8065	.with_context(\|\| format!("Failed to inspect {}", path.display()))?
8066	.is_file()
8067	{
8068	continue;
8069	}
8070	repair_torn_event_log_tail(&path)?;
8071	}
8072	Ok(())
8073	}
8074
8075	fn repair_torn_event_log_tail(path: &Path) -> Result<()> {
8076	if !path.exists() {
8077	return Ok(());
8078	}
8079	let mut file = open_runtime_store_file(path, "Runtime event tail recovery", \|options\| {
8080	options.read(true).write(true);
8081	})?;
8082	let len = file
8083	.metadata()
8084	.with_context(\|\| format!("Failed to inspect {}", path.display()))?
8085	.len();
8086	if len == 0 {
8087	return Ok(());
8088	}
8089
8090	file.seek(SeekFrom::End(-1))?;
8091	let mut last = [0_u8; 1];
8092	file.read_exact(&mut last)?;
8093	if last[0] == b'\n' {
8094	return Ok(());
8095	}
8096
8097	let mut search_end = len;
8098	let mut truncate_at = 0_u64;
8099	let mut buffer = [0_u8; 8 * 1024];
8100	let buffer_len = u64::try_from(buffer.len()).expect("event recovery buffer fits u64");
8101	while search_end > 0 {
8102	let chunk_len = usize::try_from(search_end.min(buffer_len))
8103	.expect("event recovery chunk length fits usize");
8104	let chunk_len_u64 = u64::try_from(chunk_len).expect("event recovery chunk length fits u64");
8105	let chunk_start = search_end - chunk_len_u64;
8106	file.seek(SeekFrom::Start(chunk_start))?;
8107	file.read_exact(&mut buffer[..chunk_len])?;
8108	if let Some(index) = buffer[..chunk_len].iter().rposition(\|byte\| *byte == b'\n') {
8109	truncate_at = chunk_start
8110	+ u64::try_from(index).expect("event recovery newline index fits u64")
8111	+ 1;
8112	break;
8113	}
8114	search_end = chunk_start;
8115	}
8116
8117	file.set_len(truncate_at)
8118	.with_context(\|\| format!("Failed to truncate torn tail in {}", path.display()))?;
8119	file.sync_all()
8120	.with_context(\|\| format!("Failed to sync repaired {}", path.display()))?;
8121	tracing::warn!(
8122	path = %path.display(),
8123	removed_bytes = len.saturating_sub(truncate_at),
8124	"Recovered an unterminated Runtime event-log tail"
8125	);
8126	Ok(())
8127	}
8128
8129	fn read_store_file(path: &Path) -> Result<String> {
8130	reject_symlinked_store_file(path)?;
8131	fs::read_to_string(path).with_context(\|\| format!("Failed to read {}", path.display()))
8132	}
8133
8134	fn load_runtime_store_state(path: &Path) -> Result<RuntimeStoreState> {
8135	let file = open_runtime_store_file(path, "Runtime store state", \|options\| {
8136	options.read(true);
8137	})?;
8138	serde_json::from_reader(file).with_context(\|\| format!("Failed to parse {}", path.display()))
8139	}
8140
8141	fn write_json_atomic<T: Serialize>(path: &Path, value: &T) -> Result<()> {
8142	if let Some(parent) = path.parent() {
8143	fs::create_dir_all(parent)
8144	.with_context(\|\| format!("Failed to create directory {}", parent.display()))?;
8145	}
8146	reject_symlinked_store_file(path)?;
8147	let payload = serde_json::to_string_pretty(value)?;
8148	crate::utils::write_atomic(path, payload.as_bytes())
8149	.with_context(\|\| format!("Failed to write {}", path.display()))
8150	}
8151
8152	fn remove_file_if_exists(path: &Path) -> Result<()> {
8153	reject_symlinked_store_file(path)?;
8154	match fs::remove_file(path) {
8155	Ok(()) => Ok(()),
8156	Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
8157	Err(err) => Err(err).with_context(\|\| format!("Failed to remove {}", path.display())),
8158	}
8159	}
8160
8161	#[cfg(test)]
8162	mod tests;
8163

8163 lines RUST