返回 CodeWhale
mailbox.rs
根目录 / crates / tui / src / tools / subagent / mailbox.rs
1 //! Mailbox abstraction for sub-agent runtime coordination.
2 //!
3 //! Monotonic sequence numbers give every consumer a consistent ordering even
4 //! when multiple subscribers (e.g. UI card + parent agent) drain
5 //! independently; close-as-cancel lets a single signal both stop new mail and
6 //! propagate cancellation through nested children.
7
8 use std::collections::VecDeque;
9 use std::sync::Arc;
10 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11 #[cfg(test)]
12 use std::time::Duration;
13
14 use serde::{Deserialize, Serialize};
15 use tokio::sync::{mpsc, watch};
16 use tokio_util::sync::CancellationToken;
17
18 #[cfg(test)]
19 use crate::config::ApiProvider;
20 use crate::models::Usage;
21 use crate::tools::todo::TodoListSnapshot;
22
23 use super::FleetRole;
24
25 /// Stable, structured progress envelope shared across the sub-agent surface.
26 ///
27 /// Tracks the lifecycle of a single agent (identified by `agent_id`) end to
28 /// end: spawn, per-step progress, tool execution, completion / failure /
29 /// cancellation, and parent → child topology so consumers can render trees.
30 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
31 #[serde(tag = "kind", rename_all = "snake_case")]
32 pub enum MailboxMessage {
33 /// Agent has been started (background task is running).
34 Started {
35 agent_id: String,
36 agent_type: String,
37 },
38 /// Free-form human-readable progress (mirrors `Event::AgentProgress`).
39 Progress { agent_id: String, status: String },
40 /// A tool call inside the agent has started.
41 ToolCallStarted {
42 agent_id: String,
43 tool_name: String,
44 step: u32,
45 },
46 /// A tool call inside the agent has finished.
47 ToolCallCompleted {
48 agent_id: String,
49 tool_name: String,
50 step: u32,
51 ok: bool,
52 },
53 /// A child agent was spawned by this agent.
54 ChildSpawned { parent_id: String, child_id: String },
55 /// Agent completed successfully (carries the summary line shown in the
56 /// transcript; full result is still available through the transcript handle).
57 Completed { agent_id: String, summary: String },
58 /// Agent failed with the carried error message.
59 Failed { agent_id: String, error: String },
60 /// Agent was interrupted (e.g. API timeout) with a continuable
61 /// checkpoint; the worker is parked waiting for continuation input.
62 Interrupted { agent_id: String, reason: String },
63 /// Cancellation propagated to this agent.
64 Cancelled { agent_id: String },
65 /// This agent's **own** bounded To-do snapshot (#4810).
66 ///
67 /// Published by the agent that owns the ledger, from its private list, so a
68 /// consumer keyed on `agent_id` can never attribute a parent's or a
69 /// sibling's work to this agent. Emitted only when the snapshot actually
70 /// changes; the payload is the canonical [`TodoListSnapshot`], not a second
71 /// ledger.
72 WorkState {
73 agent_id: String,
74 /// Absent in older persisted payloads, which predate per-agent Work
75 /// state; those deserialize to an empty (no work stated) snapshot.
76 #[serde(default)]
77 todo: TodoListSnapshot,
78 },
79 /// Incremental token usage from a sub-agent's API call.
80 /// Published after each turn so the parent's cost counter updates live.
81 TokenUsage {
82 agent_id: String,
83 /// Stable identity of the provider response. Runtime accounting uses
84 /// this across direct durability, mailbox replay, and restart dedupe.
85 source_id: String,
86 /// Immutable provider/model/billing evidence captured before the
87 /// child request was sent.
88 route: crate::cost_status::EffectiveRouteEnvelope,
89 /// Provider usage payload, including cache-hit/cache-miss fields.
90 usage: Usage,
91 },
92 }
93
94 impl MailboxMessage {
95 /// `agent_id` of the message subject (for `ChildSpawned` this is the
96 /// child, since that's the new lifecycle being announced).
97 #[must_use]
98 pub fn agent_id(&self) -> &str {
99 match self {
100 Self::Started { agent_id, .. }
101 | Self::Progress { agent_id, .. }
102 | Self::ToolCallStarted { agent_id, .. }
103 | Self::ToolCallCompleted { agent_id, .. }
104 | Self::Completed { agent_id, .. }
105 | Self::Failed { agent_id, .. }
106 | Self::Interrupted { agent_id, .. }
107 | Self::Cancelled { agent_id }
108 | Self::WorkState { agent_id, .. }
109 | Self::TokenUsage { agent_id, .. } => agent_id,
110 Self::ChildSpawned { child_id, .. } => child_id,
111 }
112 }
113
114 pub(crate) fn started(agent_id: impl Into<String>, agent_type: FleetRole) -> Self {
115 Self::Started {
116 agent_id: agent_id.into(),
117 agent_type: agent_type.as_str().to_string(),
118 }
119 }
120
121 pub(crate) fn progress(agent_id: impl Into<String>, status: impl Into<String>) -> Self {
122 Self::Progress {
123 agent_id: agent_id.into(),
124 status: status.into(),
125 }
126 }
127
128 pub(crate) fn work_state(agent_id: impl Into<String>, todo: TodoListSnapshot) -> Self {
129 Self::WorkState {
130 agent_id: agent_id.into(),
131 todo,
132 }
133 }
134
135 pub(crate) fn token_usage(
136 agent_id: impl Into<String>,
137 source_id: impl Into<String>,
138 route: crate::cost_status::EffectiveRouteEnvelope,
139 usage: Usage,
140 ) -> Self {
141 Self::TokenUsage {
142 agent_id: agent_id.into(),
143 source_id: source_id.into(),
144 route,
145 usage,
146 }
147 }
148 }
149
150 /// One delivery: a sequence number plus the message. The sequence is
151 /// monotonic across the entire mailbox (not per-agent) so a single ordering
152 /// is well-defined even when multiple sub-agents share one mailbox.
153 #[derive(Debug, Clone, PartialEq, Eq)]
154 pub struct MailboxEnvelope {
155 pub seq: u64,
156 pub message: MailboxMessage,
157 }
158
159 /// Sender side of the mailbox.
160 ///
161 /// Cheaply cloneable (everything inside is `Arc`/atomic). Cloning a
162 /// `Mailbox` shares the same delivery channel, sequence counter, watch
163 /// notifier, and close/cancel state — so a child runtime that clones its
164 /// parent's `Mailbox` participates in the same stream.
165 #[derive(Clone)]
166 pub struct Mailbox {
167 inner: Arc<MailboxInner>,
168 }
169
170 struct MailboxInner {
171 tx: mpsc::UnboundedSender<MailboxEnvelope>,
172 next_seq: AtomicU64,
173 seq_tx: watch::Sender<u64>,
174 closed: AtomicBool,
175 /// Linearizes publication against turn-end sealing. Without this gate a
176 /// producer could observe `closed = false`, lose the race to the turn
177 /// completion barrier, and enqueue usage after `TurnComplete`.
178 send_gate: std::sync::Mutex<()>,
179 #[cfg(test)]
180 cancel_token: CancellationToken,
181 }
182
183 /// Receiver side of the mailbox. Not `Clone` — only the original creator
184 /// can drain. Use `Mailbox::subscribe()` for fanout (UI cards + parent both
185 /// observing the same stream).
186 pub struct MailboxReceiver {
187 rx: mpsc::UnboundedReceiver<MailboxEnvelope>,
188 pending: VecDeque<MailboxEnvelope>,
189 }
190
191 impl Mailbox {
192 /// Create a new mailbox bound to the given cancellation token. Closing
193 /// the mailbox (or dropping the last sender) cancels this token. Runtimes
194 /// that derive from the same token observe that cancellation; detached
195 /// background `agent` sessions use their own runtime token.
196 #[must_use]
197 pub fn new(cancel_token: CancellationToken) -> (Self, MailboxReceiver) {
198 #[cfg(not(test))]
199 let _ = cancel_token;
200 let (tx, rx) = mpsc::unbounded_channel();
201 let (seq_tx, _) = watch::channel(0);
202 let inner = MailboxInner {
203 tx,
204 next_seq: AtomicU64::new(0),
205 seq_tx,
206 closed: AtomicBool::new(false),
207 send_gate: std::sync::Mutex::new(()),
208 #[cfg(test)]
209 cancel_token,
210 };
211 (
212 Self {
213 inner: Arc::new(inner),
214 },
215 MailboxReceiver {
216 rx,
217 pending: VecDeque::new(),
218 },
219 )
220 }
221
222 /// Subscribe to seq-bump notifications. Each `recv()` returns when the
223 /// sequence counter advances, signaling new mail without copying it —
224 /// the consumer then calls `drain` (or `recv_one` on its own receiver).
225 /// Multiple subscribers may exist; this is the fanout primitive.
226 #[cfg(test)]
227 #[must_use]
228 pub fn subscribe(&self) -> watch::Receiver<u64> {
229 self.inner.seq_tx.subscribe()
230 }
231
232 /// Send a message; returns `Some(seq)` on success, `None` if the
233 /// mailbox is already closed (callers should treat this as "the
234 /// receiver is gone, stop publishing").
235 pub fn send(&self, message: MailboxMessage) -> Option<u64> {
236 let _send_gate = self
237 .inner
238 .send_gate
239 .lock()
240 .unwrap_or_else(|error| error.into_inner());
241 if self.inner.closed.load(Ordering::Acquire) {
242 return None;
243 }
244 let seq = self.inner.next_seq.fetch_add(1, Ordering::Relaxed) + 1;
245 let envelope = MailboxEnvelope { seq, message };
246 if self.inner.tx.send(envelope).is_err() {
247 return None;
248 }
249 let _ = self.inner.seq_tx.send_replace(seq);
250 Some(seq)
251 }
252
253 /// Stop publication for this turn without cancelling detached workers.
254 ///
255 /// The engine seals, drains, and awaits the mailbox before it emits
256 /// `TurnComplete`. The send gate makes this a hard ordering boundary:
257 /// once this returns, every accepted envelope is already in the receiver
258 /// and no later worker message can attach itself to the completed turn.
259 pub(crate) fn seal(&self) {
260 let _send_gate = self
261 .inner
262 .send_gate
263 .lock()
264 .unwrap_or_else(|error| error.into_inner());
265 self.inner.closed.store(true, Ordering::Release);
266 }
267
268 /// Whether the mailbox has been closed.
269 #[cfg(test)]
270 #[must_use]
271 pub fn is_closed(&self) -> bool {
272 self.inner.closed.load(Ordering::Acquire)
273 }
274
275 /// Close the mailbox AND cancel the bound cancellation token.
276 ///
277 /// "Close-as-cancel": there's no useful state where the consumer is gone
278 /// but producers bound to this mailbox token should keep publishing.
279 /// Closing cancels the bound token; directly derived `child_runtime()`
280 /// children observe it, while detached `agent` sessions rely on their
281 /// own explicit cancellation.
282 #[cfg(test)]
283 pub fn close(&self) {
284 let was_closed = self.inner.closed.load(Ordering::Acquire);
285 self.seal();
286 if !was_closed {
287 self.inner.cancel_token.cancel();
288 }
289 }
290 }
291
292 impl MailboxReceiver {
293 #[cfg(test)]
294 fn sync_pending(&mut self) {
295 while let Ok(env) = self.rx.try_recv() {
296 self.pending.push_back(env);
297 }
298 }
299
300 /// Whether any envelopes are buffered (or arrived since last check).
301 #[cfg(test)]
302 pub fn has_pending(&mut self) -> bool {
303 self.sync_pending();
304 !self.pending.is_empty()
305 }
306
307 /// Drain all currently available envelopes, in delivery order.
308 #[cfg(test)]
309 pub fn drain(&mut self) -> Vec<MailboxEnvelope> {
310 self.sync_pending();
311 self.pending.drain(..).collect()
312 }
313
314 /// Await the next envelope, with backpressure-aware blocking. Returns
315 /// `None` when every sender has been dropped and the buffer is drained.
316 pub async fn recv(&mut self) -> Option<MailboxEnvelope> {
317 if let Some(env) = self.pending.pop_front() {
318 return Some(env);
319 }
320 self.rx.recv().await
321 }
322
323 /// Drain all envelopes accepted before a mailbox was sealed.
324 pub(crate) fn drain_available(&mut self) -> Vec<MailboxEnvelope> {
325 while let Ok(envelope) = self.rx.try_recv() {
326 self.pending.push_back(envelope);
327 }
328 self.pending.drain(..).collect()
329 }
330
331 /// Awaits the next envelope with a timeout. Useful in tests.
332 #[cfg(test)]
333 pub async fn recv_timeout(&mut self, timeout: Duration) -> Option<MailboxEnvelope> {
334 tokio::time::timeout(timeout, self.recv())
335 .await
336 .ok()
337 .flatten()
338 }
339 }
340
341 #[cfg(test)]
342 mod tests {
343 use super::*;
344 use tokio::time::Duration;
345
346 fn open() -> (Mailbox, MailboxReceiver, CancellationToken) {
347 let token = CancellationToken::new();
348 let (mb, rx) = Mailbox::new(token.clone());
349 (mb, rx, token)
350 }
351
352 fn test_route(
353 provider: ApiProvider,
354 model: &str,
355 ) -> crate::cost_status::EffectiveRouteEnvelope {
356 crate::cost_status::EffectiveRouteEnvelope::capture(
357 None,
358 provider,
359 provider.as_str(),
360 model,
361 Some(provider.default_base_url()),
362 chrono::Utc::now(),
363 )
364 }
365
366 #[tokio::test]
367 async fn mailbox_assigns_monotonic_sequence_numbers() {
368 let (mb, _rx, _tok) = open();
369 let s1 = mb
370 .send(MailboxMessage::progress("a", "one"))
371 .expect("seq 1");
372 let s2 = mb
373 .send(MailboxMessage::progress("a", "two"))
374 .expect("seq 2");
375 let s3 = mb
376 .send(MailboxMessage::progress("b", "three"))
377 .expect("seq 3");
378 assert_eq!(s1, 1);
379 assert_eq!(s2, 2);
380 assert_eq!(s3, 3);
381 assert!(s2 > s1 && s3 > s2);
382 }
383
384 #[tokio::test]
385 async fn mailbox_drains_in_delivery_order() {
386 let (mb, mut rx, _tok) = open();
387 mb.send(MailboxMessage::progress("a", "first"));
388 mb.send(MailboxMessage::progress("a", "second"));
389 mb.send(MailboxMessage::Completed {
390 agent_id: "a".into(),
391 summary: "done".into(),
392 });
393 let drained = rx.drain();
394 assert_eq!(drained.len(), 3);
395 assert_eq!(drained[0].seq, 1);
396 assert_eq!(drained[1].seq, 2);
397 assert_eq!(drained[2].seq, 3);
398 assert!(matches!(
399 drained[0].message,
400 MailboxMessage::Progress { .. }
401 ));
402 assert!(matches!(
403 drained[2].message,
404 MailboxMessage::Completed { .. }
405 ));
406 assert!(!rx.has_pending());
407 }
408
409 #[tokio::test]
410 async fn subscribers_receive_seq_bumps_for_backpressure() {
411 let (mb, _rx, _tok) = open();
412 let mut sub_a = mb.subscribe();
413 let mut sub_b = mb.subscribe();
414 // Initial state: both at 0.
415 assert_eq!(*sub_a.borrow(), 0);
416 assert_eq!(*sub_b.borrow(), 0);
417
418 mb.send(MailboxMessage::progress("x", "tick"));
419 sub_a.changed().await.expect("subscriber a sees bump");
420 sub_b.changed().await.expect("subscriber b sees bump");
421 assert_eq!(*sub_a.borrow(), 1);
422 assert_eq!(*sub_b.borrow(), 1);
423
424 // A second send updates both subscribers' watch values too — even
425 // though they share a single watch channel, fanout is N-to-many.
426 mb.send(MailboxMessage::progress("x", "tick2"));
427 sub_a.changed().await.expect("a sees second bump");
428 assert_eq!(*sub_a.borrow(), 2);
429 }
430
431 #[tokio::test]
432 async fn close_cancels_bound_token_and_blocks_further_sends() {
433 let (mb, _rx, token) = open();
434 assert!(!token.is_cancelled());
435 mb.send(MailboxMessage::progress("a", "before close"));
436 mb.close();
437 assert!(token.is_cancelled(), "close-as-cancel: token must fire");
438 assert!(mb.is_closed());
439 // Further sends are no-ops, returning None instead of poisoning seq.
440 assert!(
441 mb.send(MailboxMessage::progress("a", "after close"))
442 .is_none()
443 );
444 }
445
446 #[test]
447 fn turn_end_seal_forms_a_flush_barrier_without_cancelling_worker() {
448 let (mb, mut rx, token) = open();
449 assert_eq!(
450 mb.send(MailboxMessage::progress("a", "accepted before barrier")),
451 Some(1)
452 );
453 mb.seal();
454
455 assert!(!token.is_cancelled(), "detached worker is not cancelled");
456 assert!(
457 mb.send(MailboxMessage::progress("a", "too late")).is_none(),
458 "no event may be accepted after the completion barrier"
459 );
460 let drained = rx.drain_available();
461 assert_eq!(drained.len(), 1);
462 assert_eq!(drained[0].seq, 1);
463 }
464
465 #[tokio::test]
466 async fn close_propagates_to_child_tokens_across_max_spawn_depth() {
467 // Mirror the runtime: root → child → grandchild (default depth 3).
468 let root = CancellationToken::new();
469 let child = root.child_token();
470 let grandchild = child.child_token();
471 let (mb, _rx) = Mailbox::new(root.clone());
472
473 assert!(!child.is_cancelled());
474 assert!(!grandchild.is_cancelled());
475 mb.close();
476 assert!(child.is_cancelled(), "child inherits root close");
477 assert!(
478 grandchild.is_cancelled(),
479 "grandchild inherits too — covers default max_spawn_depth = 3"
480 );
481 }
482
483 #[tokio::test]
484 async fn recv_returns_envelope_then_none_after_close_and_drop() {
485 let (mb, mut rx, _tok) = open();
486 mb.send(MailboxMessage::progress("a", "queued"));
487 let env = rx.recv().await.expect("buffered envelope");
488 assert_eq!(env.seq, 1);
489
490 // After closing AND dropping the sender, recv must yield None.
491 mb.close();
492 drop(mb);
493 let next = rx.recv_timeout(Duration::from_millis(100)).await;
494 assert!(next.is_none(), "drained + dropped → recv yields None");
495 }
496
497 #[tokio::test]
498 async fn cloned_mailbox_shares_sequence_and_close_state() {
499 let (mb, mut rx, token) = open();
500 let mb_clone = mb.clone();
501 let s1 = mb
502 .send(MailboxMessage::progress("a", "from original"))
503 .unwrap();
504 let s2 = mb_clone
505 .send(MailboxMessage::progress("a", "from clone"))
506 .unwrap();
507 assert_eq!(s1, 1);
508 assert_eq!(s2, 2, "clones share the seq counter");
509
510 let drained = rx.drain();
511 assert_eq!(drained.len(), 2);
512
513 // Closing through one clone closes them all (the AtomicBool is shared).
514 mb_clone.close();
515 assert!(mb.is_closed());
516 assert!(token.is_cancelled());
517 }
518
519 #[test]
520 fn work_state_payload_round_trips_and_tolerates_a_missing_snapshot() {
521 use crate::tools::todo::{TodoItem, TodoStatus};
522
523 let message = MailboxMessage::work_state(
524 "agent_child",
525 TodoListSnapshot {
526 items: vec![TodoItem {
527 id: 2,
528 content: "write the projection".to_string(),
529 status: TodoStatus::InProgress,
530 }],
531 completion_pct: 50,
532 in_progress_id: Some(2),
533 },
534 );
535 let encoded = serde_json::to_string(&message).expect("encode");
536 let decoded: MailboxMessage = serde_json::from_str(&encoded).expect("decode");
537 assert_eq!(decoded, message);
538
539 // An older payload that predates per-agent Work state decodes to an
540 // empty snapshot rather than failing the whole stream.
541 let legacy: MailboxMessage =
542 serde_json::from_str(r#"{"kind":"work_state","agent_id":"agent_old"}"#)
543 .expect("legacy");
544 assert_eq!(legacy.agent_id(), "agent_old");
545 match legacy {
546 MailboxMessage::WorkState { todo, .. } => assert!(todo.is_empty()),
547 other => panic!("expected work state, got {other:?}"),
548 }
549
550 // Every pre-existing variant still decodes unchanged.
551 let started: MailboxMessage =
552 serde_json::from_str(r#"{"kind":"started","agent_id":"a","agent_type":"worker"}"#)
553 .expect("started");
554 assert_eq!(started.agent_id(), "a");
555 }
556
557 #[tokio::test]
558 async fn agent_id_is_extractable_from_every_variant() {
559 let cases: Vec<(MailboxMessage, &str)> = vec![
560 (MailboxMessage::started("a1", FleetRole::Worker), "a1"),
561 (MailboxMessage::progress("a2", "x"), "a2"),
562 (
563 MailboxMessage::ToolCallStarted {
564 agent_id: "a3".into(),
565 tool_name: "read_file".into(),
566 step: 1,
567 },
568 "a3",
569 ),
570 (
571 MailboxMessage::ToolCallCompleted {
572 agent_id: "a4".into(),
573 tool_name: "read_file".into(),
574 step: 1,
575 ok: true,
576 },
577 "a4",
578 ),
579 (
580 MailboxMessage::ChildSpawned {
581 parent_id: "parent".into(),
582 child_id: "a5".into(),
583 },
584 "a5",
585 ),
586 (
587 MailboxMessage::Completed {
588 agent_id: "a6".into(),
589 summary: "done".into(),
590 },
591 "a6",
592 ),
593 (
594 MailboxMessage::Failed {
595 agent_id: "a7".into(),
596 error: "boom".into(),
597 },
598 "a7",
599 ),
600 (
601 MailboxMessage::Cancelled {
602 agent_id: "a8".into(),
603 },
604 "a8",
605 ),
606 (
607 MailboxMessage::Interrupted {
608 agent_id: "a10".into(),
609 reason: "API call timed out".into(),
610 },
611 "a10",
612 ),
613 (
614 MailboxMessage::work_state("a11", TodoListSnapshot::default()),
615 "a11",
616 ),
617 (
618 MailboxMessage::TokenUsage {
619 agent_id: "a9".into(),
620 source_id: "response-a9".into(),
621 route: test_route(ApiProvider::Deepseek, "deepseek-v4-flash"),
622 usage: Usage {
623 input_tokens: 100,
624 output_tokens: 50,
625 ..Default::default()
626 },
627 },
628 "a9",
629 ),
630 ];
631 for (msg, expected) in cases {
632 assert_eq!(msg.agent_id(), expected, "extract failed for {msg:?}");
633 }
634 }
635
636 #[test]
637 fn token_usage_serde_round_trip_preserves_immutable_route_evidence() {
638 let route = crate::cost_status::EffectiveRouteEnvelope {
639 provider: ApiProvider::Moonshot,
640 provider_identity: "kimi-membership".to_string(),
641 model: "k3".to_string(),
642 billing_surface: Some(crate::pricing::MOONSHOT_KIMI_CODE_BILLING_SURFACE.to_string()),
643 endpoint_fingerprint: Some("a".repeat(64)),
644 billing_mode: crate::cost_status::RouteBillingMode::Subscription,
645 dispatched_at: chrono::DateTime::<chrono::Utc>::from_timestamp(1_234, 0)
646 .expect("timestamp"),
647 };
648 let message =
649 MailboxMessage::token_usage("agent-k3", "response-k3", route, Usage::default());
650 let json = serde_json::to_string(&message).expect("serialize token usage");
651 let restored: MailboxMessage =
652 serde_json::from_str(&json).expect("deserialize token usage");
653 assert_eq!(restored, message);
654 }
655 }
656
656 lines RUST