From 716bf97db8eb58149d29712dfbe7b811876beb6e Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:09:34 -0700 Subject: [PATCH 1/8] Record when a sub-agent's outstanding tool call started A worker inside one long tool call emits no events until the result lands, so the session store's silence clock cannot tell a wedged reactor from a ten-minute test run. Stamping the start of the outstanding call gives the surfaces above the fact that separates them. --- src/subagent/session-store.test.ts | 41 +++++++++++++++++++++++++ src/subagent/session-store.ts | 47 ++++++++++++++++++++++++----- src/tui-opentui/runner-host.test.ts | 1 + 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/src/subagent/session-store.test.ts b/src/subagent/session-store.test.ts index bcc37d3c7..367fcbb70 100644 --- a/src/subagent/session-store.test.ts +++ b/src/subagent/session-store.test.ts @@ -104,3 +104,44 @@ describe("session-store snapshot caching", () => { expect(snapshot?.entries).toEqual([]); }); }); + + +describe("outstanding tool clock", () => { + // A worker inside one long tool call emits nothing until the result lands. + // Without a start clock for that call, silence is indistinguishable from a + // wedged reactor, and a whole fleet running shell commands reads as stalled. + test("tool.start stamps the clock and tool.done clears it", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + + clock = 5_000; + store.appendEvent(session.id, { + type: "tool.start", + seq: 1, + data: { call: { id: "call-1", name: "run_shell", arguments: {} } }, + } as unknown as ReactorEmittedEvent); + expect(store.get(session.id)?.currentToolName).toBe("run_shell"); + expect(store.get(session.id)?.currentToolStartedAt).toBe(5_000); + + clock = 95_000; + store.appendEvent(session.id, { + type: "tool.done", + seq: 2, + data: { result: { callId: "call-1", content: "ok", isError: false } }, + } as unknown as ReactorEmittedEvent); + expect(store.get(session.id)?.currentToolName).toBeNull(); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); + + test("a terminal transition never leaves a tool clock outstanding", () => { + const store = createSubAgentSessionStore(); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + store.appendEvent(session.id, startCall(1, "call-1", "grep")); + expect(store.get(session.id)?.currentToolStartedAt).not.toBeNull(); + + store.complete(session.id, "report"); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); +}); diff --git a/src/subagent/session-store.ts b/src/subagent/session-store.ts index 6fd8e25b6..5868a27fa 100644 --- a/src/subagent/session-store.ts +++ b/src/subagent/session-store.ts @@ -25,6 +25,11 @@ export type SubAgentSession = { status: SubAgentSessionStatus; toolNames: string[]; currentToolName: string | null; + // Clock of when the in-flight tool call began executing, or null when no + // tool is outstanding. A worker inside one long tool emits no events for the + // whole execution, so silence alone cannot tell "wedged" from "running a + // ten-minute test suite". This is the fact that separates them. + currentToolStartedAt: number | null; entries: SubAgentTranscriptEntry[]; startedAt: number; // Clock of the last event this session recorded (a stream token, a tool @@ -95,6 +100,28 @@ function defaultCreateId(): string { return `subagent-${nextId}`; } +/** + * Name and clock move together so no caller can leave a tool name outstanding + * with a stale (or absent) start time — the pair is what the UI reads to tell + * a long tool call from silence with no explanation. + */ +function setCurrentTool( + session: SubAgentSession, + name: string | null, + nowMs: number, + restartClock = false, +): void { + if (name === null) { + session.currentToolName = null; + session.currentToolStartedAt = null; + return; + } + if (restartClock || session.currentToolName !== name || session.currentToolStartedAt === null) { + session.currentToolStartedAt = nowMs; + } + session.currentToolName = name; +} + function capText(text: string, max: number): string { if (text.length <= max) return text; return text.slice(0, max); @@ -164,7 +191,7 @@ export function createSubAgentSessionStore( session.status = "cancelled"; session.finishedAt = now(); session.lastActivityAt = now(); - session.currentToolName = null; + setCurrentTool(session, null, now()); session.error = reason; pushEntry(session, { kind: "report", @@ -268,6 +295,7 @@ export function createSubAgentSessionStore( status: "running", toolNames: [], currentToolName: null, + currentToolStartedAt: null, entries: [], startedAt: now(), lastActivityAt: now(), @@ -309,7 +337,7 @@ export function createSubAgentSessionStore( const data = event.data as { name?: unknown; callId?: unknown }; const name = typeof data.name === "string" ? data.name : "tool"; const callId = typeof data.callId === "string" ? data.callId : `${name}-${session.entries.length}`; - session.currentToolName = name; + setCurrentTool(session, name, now()); if (!session.toolNames.includes(name)) session.toolNames.push(name); pushEntry(session, { kind: "tool", callId, name, arguments: "" }); return; @@ -344,14 +372,14 @@ export function createSubAgentSessionStore( if (callId !== null && entry.callId !== callId) continue; if (name !== null) entry.name = name; if (args !== null && args.length > 0) entry.arguments = args; - session.currentToolName = entry.name; + setCurrentTool(session, entry.name, now()); return; } // No matching start — record a complete tool entry. if (name !== null) { const idForEntry = callId ?? `${name}-${session.entries.length}`; if (!session.toolNames.includes(name)) session.toolNames.push(name); - session.currentToolName = name; + setCurrentTool(session, name, now()); pushEntry(session, { kind: "tool", callId: idForEntry, @@ -367,7 +395,9 @@ export function createSubAgentSessionStore( const call = (event as { data?: { call?: { name?: unknown; id?: unknown } } }).data?.call; const name = typeof call?.name === "string" ? call.name : null; if (name === null) return; - session.currentToolName = name; + // Execution start, not argument streaming: restart the clock so the + // elapsed figure beside the tool name is time spent running it. + setCurrentTool(session, name, now(), true); if (!session.toolNames.includes(name)) session.toolNames.push(name); return; } @@ -388,7 +418,7 @@ export function createSubAgentSessionStore( const content = capText(stringifyUnknown(result.content ?? ""), maxEntryChars); const isError = result.isError === true; pushEntry(session, { kind: "tool_result", callId, name, content, isError }); - session.currentToolName = null; + setCurrentTool(session, null, now()); return; } default: @@ -404,7 +434,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "done"; session.finishedAt = now(); - session.currentToolName = null; + setCurrentTool(session, null, now()); session.report = report; pushEntry(session, { kind: "report", content: capText(report, maxEntryChars) }); cancelHandles.delete(id); @@ -417,7 +447,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "failed"; session.finishedAt = now(); - session.currentToolName = null; + setCurrentTool(session, null, now()); session.error = error; pushEntry(session, { kind: "report", @@ -475,6 +505,7 @@ function cloneSession(session: SubAgentSession): SubAgentSession { status: session.status, toolNames: [...session.toolNames], currentToolName: session.currentToolName, + currentToolStartedAt: session.currentToolStartedAt, entries: session.entries.map(cloneEntry), startedAt: session.startedAt, lastActivityAt: session.lastActivityAt, diff --git a/src/tui-opentui/runner-host.test.ts b/src/tui-opentui/runner-host.test.ts index ba5fc8adc..966491ee5 100644 --- a/src/tui-opentui/runner-host.test.ts +++ b/src/tui-opentui/runner-host.test.ts @@ -47,6 +47,7 @@ function session(over: Partial): SubAgentSession { status: "running", toolNames: [], currentToolName: null, + currentToolStartedAt: null, entries: [], startedAt: 0, lastActivityAt: 0, From 4647b5a721b3787508246797124174648f8a6d93 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:09:41 -0700 Subject: [PATCH 2/8] Stop reading a long tool call as a stalled sub-agent Every lane of a fleet running shell commands flipped to stalled in lockstep while all of them were working, because silence was the only input. A lane is now stalled only when nothing outstanding explains the silence, and the clock beside each state is the clock that justifies it rather than the worker's unrelated lifetime. --- src/tui-opentui/agent-progress.test.ts | 131 +++++++++++++++++++++++-- src/tui-opentui/agent-progress.ts | 120 ++++++++++++++++++++-- src/tui-opentui/chrome-state.test.ts | 22 +++-- src/tui-opentui/chrome-state.ts | 41 +++++++- src/tui-opentui/runner-host.ts | 1 + src/tui/runner.ts | 1 + 6 files changed, 292 insertions(+), 24 deletions(-) diff --git a/src/tui-opentui/agent-progress.test.ts b/src/tui-opentui/agent-progress.test.ts index aefdcb69b..d0fe2eff1 100644 --- a/src/tui-opentui/agent-progress.test.ts +++ b/src/tui-opentui/agent-progress.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from "bun:test" -import { agentProgress, clockLabel } from "./agent-progress" +import { + agentProgress, + clockLabel, + fleetLabel, + fleetProgress, + laneState, +} from "./agent-progress" describe("clockLabel", () => { test("formats sub-minute and multi-minute elapsed as m:ss", () => { @@ -13,6 +19,7 @@ describe("agentProgress", () => { const base = { status: "running" as const, currentToolName: "grep", + currentToolStartedAt: null, startedAt: 0, lastActivityAt: 0, } @@ -25,7 +32,12 @@ describe("agentProgress", () => { test("a running session reports elapsed time and its current tool", () => { const progress = agentProgress({ ...base, lastActivityAt: 42_000 }, 42_000) - expect(progress).toEqual({ stat: "0:42 · grep", working: true, stalled: false }) + expect(progress).toEqual({ + stat: "0:42 · grep", + state: "working", + working: true, + stalled: false, + }) }) test("a running session with no current tool reports elapsed time alone", () => { @@ -33,12 +45,45 @@ describe("agentProgress", () => { { ...base, currentToolName: null, lastActivityAt: 42_000 }, 42_000, ) - expect(progress).toEqual({ stat: "0:42", working: true, stalled: false }) + expect(progress).toEqual({ + stat: "0:42", + state: "working", + working: true, + stalled: false, + }) }) - test("silence past the stall window flips working to stalled", () => { - const progress = agentProgress({ ...base, lastActivityAt: 0 }, 31_000, 30_000) - expect(progress).toEqual({ stat: "0:31 · grep", working: false, stalled: true }) + test("silence with no tool outstanding is a stall, and the clock shown is the silence", () => { + const progress = agentProgress( + { ...base, currentToolName: null, lastActivityAt: 0 }, + 31_000, + 30_000, + ) + expect(progress).toEqual({ + stat: "0:31 · quiet 0:31", + state: "stalled", + working: false, + stalled: true, + }) + }) + + // The defect the whole surface turned on: a worker inside one long tool call + // emits nothing for the entire execution, so every lane of a fleet running + // e.g. a test suite flipped to "stalled" simultaneously while working fine. + test("silence inside an outstanding tool call is not a stall", () => { + const progress = agentProgress( + { + ...base, + currentToolName: "run_shell", + currentToolStartedAt: 1_000, + lastActivityAt: 1_000, + }, + 91_000, + 30_000, + ) + expect(progress?.state).toBe("in_tool") + expect(progress?.stalled).toBe(false) + expect(progress?.stat).toBe("1:31 · run_shell 1:30") }) test("recent activity keeps a long-running session marked working", () => { @@ -47,3 +92,77 @@ describe("agentProgress", () => { expect(progress?.stalled).toBe(false) }) }) + +describe("laneState", () => { + const running = { + status: "running" as const, + currentToolName: null, + currentToolStartedAt: null, + startedAt: 0, + lastActivityAt: 0, + } + + test("names the three lanes a running worker can be in", () => { + expect(laneState({ ...running, lastActivityAt: 1_000 }, 2_000, 30_000)).toBe("working") + expect(laneState(running, 60_000, 30_000)).toBe("stalled") + expect( + laneState( + { ...running, currentToolName: "run_shell", currentToolStartedAt: 0 }, + 60_000, + 30_000, + ), + ).toBe("in_tool") + }) +}) + +describe("fleetProgress", () => { + const lane = (over: Partial[0]>) => ({ + status: "running" as const, + currentToolName: null, + currentToolStartedAt: null, + startedAt: 0, + lastActivityAt: 0, + ...over, + }) + + test("counts only running lanes, bucketed by their single lane state", () => { + const fleet = fleetProgress( + [ + lane({ lastActivityAt: 59_000 }), + lane({ currentToolName: "run_shell", currentToolStartedAt: 0 }), + lane({}), + lane({ status: "done" }), + ], + 60_000, + 30_000, + ) + expect(fleet).toEqual({ running: 3, working: 1, inTool: 1, stalled: 1 }) + }) + + test("no sub-agents leaves the fleet empty", () => { + expect(fleetProgress([], 1_000)).toEqual({ + running: 0, + working: 0, + inTool: 0, + stalled: 0, + }) + }) +}) + +describe("fleetLabel", () => { + test("is null with nothing running so the single-agent case is untouched", () => { + expect(fleetLabel({ running: 0, working: 0, inTool: 0, stalled: 0 })).toBeNull() + }) + + test("names the stalled count when any lane is stuck", () => { + expect(fleetLabel({ running: 6, working: 4, inTool: 0, stalled: 2 })).toBe( + "6 agents · 2 stalled", + ) + }) + + test("says when the whole fleet is inside tool calls", () => { + expect(fleetLabel({ running: 3, working: 0, inTool: 3, stalled: 0 })).toBe( + "3 agents · in tools", + ) + }) +}) diff --git a/src/tui-opentui/agent-progress.ts b/src/tui-opentui/agent-progress.ts index dcea4ef7a..4fefc499f 100644 --- a/src/tui-opentui/agent-progress.ts +++ b/src/tui-opentui/agent-progress.ts @@ -1,27 +1,46 @@ /** - * Live progress for a dispatched sub-agent's pending row in the transcript. + * Live progress for a dispatched sub-agent's pending row in the transcript, + * and the fleet-level roll-up of those same lanes. * * A "task" tool call renders as one row for its whole lifetime (see * `runtime-bridge.ts`'s `syncAgentProgress`). While the call is outstanding * this fills in what a bare pending mark cannot say: how long the worker has * been running, what it is doing right now, and whether it has gone quiet * long enough to look hung rather than merely slow. + * + * Lane state and the fleet roll-up live in this one file on purpose. "Stalled" + * has exactly one definition — `laneState` below — and the fleet summary + * consumes it rather than re-deriving staleness from raw timestamps. */ /** Minimal session shape this module reads — avoids a hard dep on the store. */ export type AgentProgressSession = { readonly status: "running" | "done" | "failed" | "cancelled"; readonly currentToolName: string | null; + /** When the outstanding tool call began, or null when none is in flight. */ + readonly currentToolStartedAt?: number | null; readonly startedAt: number; readonly lastActivityAt: number; }; +/** + * What a lane is actually doing, as opposed to how long it has been alive. + * + * `in_tool` is the state that makes the surface honest: a worker inside one + * long tool call emits nothing for the whole execution, so silence on its own + * cannot tell a wedged reactor from a ten-minute test run. A lane only reads + * `stalled` when it has gone quiet with no tool outstanding to explain it — + * that is the case an operator can act on. + */ +export type LaneState = "working" | "in_tool" | "stalled"; + export type AgentProgress = { /** Dim trailer painted after the row's subject, e.g. "0:42 · grep". */ readonly stat: string; - /** True while the worker has reported activity within the stall window. */ + readonly state: LaneState; + /** True while the worker is making visible progress. */ readonly working: boolean; - /** True once silence has run longer than the stall window. */ + /** True once silence has run longer than the stall window with nothing to explain it. */ readonly stalled: boolean; }; @@ -36,9 +55,32 @@ export function clockLabel(ms: number): string { return `${minutes}:${String(seconds).padStart(2, "0")}`; } +/** + * The single definition of what a lane is doing. Every other surface — the + * transcript trailer, the agents panel, the top-level indicator — reads this + * result rather than comparing timestamps itself. + */ +export function laneState( + session: AgentProgressSession, + nowMs: number, + stallMs: number = DEFAULT_STALL_MS, +): LaneState { + if (nowMs - session.lastActivityAt < stallMs) return "working"; + const toolStartedAt = session.currentToolStartedAt; + if (session.currentToolName !== null && toolStartedAt !== null && toolStartedAt !== undefined) { + return "in_tool"; + } + return "stalled"; +} + /** * Progress for a running session's pending row, or null once it has finished — * a terminal session resolves its row through the tool-result path instead. + * + * The number beside the state word always explains that word: a healthy lane + * shows its lifetime, a lane stuck in one tool shows how long that tool has + * been running, and a silent lane shows how long it has been silent. Reading + * a lifetime clock next to "stalled" tells an operator nothing about the stall. */ export function agentProgress( session: AgentProgressSession, @@ -48,10 +90,74 @@ export function agentProgress( if (session.status !== "running") return null; const elapsed = clockLabel(nowMs - session.startedAt); const tool = session.currentToolName; - const stalled = nowMs - session.lastActivityAt >= stallMs; + const hasTool = tool !== null && tool.length > 0; + const state = laneState(session, nowMs, stallMs); + + const base = hasTool ? `${elapsed} · ${tool}` : elapsed; + const stat = + state === "in_tool" && session.currentToolStartedAt != null + ? `${base} ${clockLabel(nowMs - session.currentToolStartedAt)}` + : state === "stalled" + ? `${base} · quiet ${clockLabel(nowMs - session.lastActivityAt)}` + : base; + return { - stat: tool !== null && tool.length > 0 ? `${elapsed} · ${tool}` : elapsed, - working: !stalled, - stalled, + stat, + state, + working: state !== "stalled", + stalled: state === "stalled", }; } + +/** + * What the whole fleet is doing, rolled up from the per-lane states above. + * + * The top-level indicator otherwise reports the parent's own activity, and at + * fleet scale the parent is almost always just awaiting children — so it reads + * "working" permanently, including while every lane is stuck. Counting lanes + * here is what lets the indicator speak for the fleet instead. + */ +export type FleetProgress = { + readonly running: number; + readonly working: number; + readonly inTool: number; + readonly stalled: number; +}; + +export function fleetProgress( + sessions: readonly AgentProgressSession[], + nowMs: number, + stallMs: number = DEFAULT_STALL_MS, +): FleetProgress { + let working = 0; + let inTool = 0; + let stalled = 0; + for (const session of sessions) { + if (session.status !== "running") continue; + switch (laneState(session, nowMs, stallMs)) { + case "working": + working += 1; + break; + case "in_tool": + inTool += 1; + break; + case "stalled": + stalled += 1; + break; + } + } + return { running: working + inTool + stalled, working, inTool, stalled }; +} + +/** + * Compact fleet summary for the status ticker, or null with no live lanes — + * with no sub-agents running the indicator must behave exactly as it does for + * a plain single-agent turn. + */ +export function fleetLabel(fleet: FleetProgress): string | null { + if (fleet.running === 0) return null; + const parts = [`${fleet.running} agents`]; + if (fleet.stalled > 0) parts.push(`${fleet.stalled} stalled`); + else if (fleet.inTool === fleet.running) parts.push("in tools"); + return parts.join(" · "); +} diff --git a/src/tui-opentui/chrome-state.test.ts b/src/tui-opentui/chrome-state.test.ts index cd408b21a..e44a21747 100644 --- a/src/tui-opentui/chrome-state.test.ts +++ b/src/tui-opentui/chrome-state.test.ts @@ -164,6 +164,7 @@ describe("formatAgentsPanel", () => { NOW, ) expect(rows).toEqual([ + { label: "2 agents", tail: "", stalled: false }, { label: "b: two", tail: " · 0:02", stalled: false }, { label: "a: one", tail: " · 0:01", stalled: false }, ]) @@ -197,7 +198,7 @@ describe("formatAgentsPanel", () => { NOW, ) expect(rows).toEqual([ - { label: "a: quiet worker", tail: " · 1:00 · stalled", stalled: true }, + { label: "a: quiet worker", tail: " · 1:00 · quiet 0:40 · stalled", stalled: true }, ]) }) @@ -210,8 +211,10 @@ describe("formatAgentsPanel", () => { lastActivityAt: NOW, })) const rows = formatAgentsPanel(running, undefined, NOW, 5) - expect(rows).toHaveLength(6) - expect(rows?.[5]).toEqual({ label: "+3 more", tail: "", stalled: false }) + // Fleet summary, five lanes, then the fold-away row. + expect(rows?.[0]).toEqual({ label: "8 agents", tail: "", stalled: false }) + expect(rows).toHaveLength(7) + expect(rows?.[6]).toEqual({ label: "+3 more", tail: "", stalled: false }) }) test("observe empty id+desc hides", () => { @@ -238,12 +241,12 @@ describe("formatAgentsPanel", () => { const frame2 = frame1.map((a) => (a.agentId === "b" ? { ...a, lastActivityAt: NOW + 200 } : a)) const rowsAfter = formatAgentsPanel(frame2, undefined, NOW + 200) - expect(rowsBefore?.map((r) => r.label.split(":")[0])).toEqual( - rowsAfter?.map((r) => r.label.split(":")[0]), - ) + const lanes = (rows: readonly { label: string }[] | null) => + rows?.slice(1).map((r) => r.label.split(":")[0]) + expect(lanes(rowsBefore)).toEqual(lanes(rowsAfter)) // Sanity: presentation order is oldest-started first (a, b, c), matching // the tiebreak-free startedAt sort. - expect(rowsBefore?.map((r) => r.label.split(":")[0])).toEqual(["a", "b", "c"]) + expect(lanes(rowsBefore)).toEqual(["a", "b", "c"]) }) test("a stalled agent stays visible over newer agents when the fan-out is truncated", () => { @@ -267,8 +270,9 @@ describe("formatAgentsPanel", () => { const rows = formatAgentsPanel([...newest, stalled], undefined, NOW, 5) expect(rows?.some((r) => r.label.includes("quiet"))).toBe(true) expect(rows?.some((r) => r.stalled)).toBe(true) - expect(rows).toHaveLength(6) - expect(rows?.[5]).toEqual({ label: "+1 more", tail: "", stalled: false }) + expect(rows).toHaveLength(7) + expect(rows?.[0]).toEqual({ label: "6 agents · 1 stalled", tail: "", stalled: true }) + expect(rows?.[6]).toEqual({ label: "+1 more", tail: "", stalled: false }) }) }) diff --git a/src/tui-opentui/chrome-state.ts b/src/tui-opentui/chrome-state.ts index 3289b1bf6..68a70b83d 100644 --- a/src/tui-opentui/chrome-state.ts +++ b/src/tui-opentui/chrome-state.ts @@ -22,7 +22,12 @@ * stale. Observe mode can override the agents line via `state.observe`. */ -import { agentProgress, DEFAULT_STALL_MS } from "./agent-progress.js" +import { + agentProgress, + fleetLabel, + fleetProgress, + DEFAULT_STALL_MS, +} from "./agent-progress.js" import { AGENTS_PANEL_MAX_VISIBLE, TASKS_PANEL_MAX_VISIBLE } from "./geometry/zones.js" import type { ChromeZoneContent } from "./shell.js" @@ -37,6 +42,8 @@ export type ChromeAgentSession = { readonly startedAt?: number /** Clock of the worker's last reported activity; feeds stalled detection. */ readonly lastActivityAt?: number + /** Clock the outstanding tool call began; separates a long tool from silence. */ + readonly currentToolStartedAt?: number | null } /** Lightweight task row: title + status, as written by the task tool. */ @@ -193,6 +200,26 @@ export function formatAgentsPanel( ) const rows = presented.map((s) => formatAgentRow(s, nowMs, stallMs)) if (hidden > 0) rows.push({ label: `+${hidden} more`, tail: "", stalled: false }) + + // Fleet roll-up first: past a couple of lanes an operator reads the summary, + // not six individual rows, and any row folded into "+N more" is otherwise + // invisible. Counted from the same lane states the rows below are rendered + // from, so the header can never disagree with them. + const fleet = fleetProgress( + running.map((s) => ({ + status: "running" as const, + currentToolName: s.currentToolName ?? null, + currentToolStartedAt: s.currentToolStartedAt ?? null, + startedAt: s.startedAt ?? 0, + lastActivityAt: s.lastActivityAt ?? s.startedAt ?? 0, + })), + nowMs, + stallMs, + ) + const summary = fleetLabel(fleet) + if (summary !== null && running.length > 1) { + rows.unshift({ label: summary, tail: "", stalled: fleet.stalled > 0 }) + } return rows } @@ -214,6 +241,7 @@ function formatAgentRow(session: ChromeAgentSession, nowMs: number, stallMs: num { status: "running", currentToolName: session.currentToolName ?? null, + currentToolStartedAt: session.currentToolStartedAt ?? null, startedAt: session.startedAt, lastActivityAt: session.lastActivityAt ?? session.startedAt, }, @@ -255,7 +283,12 @@ export function annotateAgentTools( agents: agents.map((a) => { if (a.status !== "running") return a const tool = toolByDescription.get(a.description) - return tool === undefined ? a : { ...a, currentToolName: tool } + if (tool === undefined) return a + // The overlay renames the tool but must not invent a start clock for it. + // Only the store observes a call ending, so letting a progress ping + // supply a clock would keep a finished lane reading as busy forever — + // exactly the true stalls this surface exists to show. + return { ...a, currentToolName: tool } }), } } @@ -280,6 +313,7 @@ export type ChromeSessionAgent = { readonly description: string readonly status: "running" | "done" | "failed" | "cancelled" readonly currentToolName?: string | null + readonly currentToolStartedAt?: number | null readonly startedAt?: number readonly lastActivityAt?: number } @@ -340,6 +374,9 @@ function mapSessionAgents( ...(a.currentToolName !== undefined ? { currentToolName: a.currentToolName } : {}), + ...(a.currentToolStartedAt !== undefined + ? { currentToolStartedAt: a.currentToolStartedAt } + : {}), ...(a.startedAt !== undefined ? { startedAt: a.startedAt } : {}), ...(a.lastActivityAt !== undefined ? { lastActivityAt: a.lastActivityAt } diff --git a/src/tui-opentui/runner-host.ts b/src/tui-opentui/runner-host.ts index d1ff977cf..a8fd66df0 100644 --- a/src/tui-opentui/runner-host.ts +++ b/src/tui-opentui/runner-host.ts @@ -272,6 +272,7 @@ export async function mountRunnerHost(deps: RunnerHostDeps): Promise id: s.id, status: s.status, currentToolName: s.currentToolName, + currentToolStartedAt: s.currentToolStartedAt, startedAt: s.startedAt, lastActivityAt: s.lastActivityAt, })), diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 54dbd300d..d5ff008ec 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -2036,6 +2036,7 @@ export async function runTUI(initialConfig: Config): Promise { description: s.description, status: s.status, currentToolName: s.currentToolName, + currentToolStartedAt: s.currentToolStartedAt, startedAt: s.startedAt, lastActivityAt: s.lastActivityAt, })), From 2d1465cbbd1313e01b8d4a16b3912b6553694ec3 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:09:48 -0700 Subject: [PATCH 3/8] Report the fleet in the activity indicator, not the idle parent At fleet scale the parent is almost always just awaiting children, so the indicator read working permanently, including while every lane was stuck. It now consumes the per-lane state rather than deriving a second one, and falls back to the parent's own clock when nothing is running. --- src/tui-opentui/runtime-bridge.ts | 19 ++++++- src/tui-opentui/session-chrome.test.ts | 77 +++++++++++++++++++++++--- src/tui-opentui/session-chrome.ts | 13 +++++ 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/src/tui-opentui/runtime-bridge.ts b/src/tui-opentui/runtime-bridge.ts index 35433db3b..e9762c46a 100644 --- a/src/tui-opentui/runtime-bridge.ts +++ b/src/tui-opentui/runtime-bridge.ts @@ -70,7 +70,11 @@ import { } from "./tool-rows.js" import type { StreamRow } from "./stream.js" import { advanceRevealChars, flattenReasoningText, type Thought } from "./thinking.js" -import { agentProgress, type AgentProgressSession } from "./agent-progress.js" +import { + agentProgress, + fleetProgress, + type AgentProgressSession, +} from "./agent-progress.js" /** Tool name a sub-agent dispatch call carries — its row gets live progress. */ const TASK_TOOL_NAME = "task" @@ -336,6 +340,12 @@ type BridgeBag = { * find the handful that are sub-agent dispatches. */ taskCallIds: Set + /** + * Last sub-agent session list the host synced. Retained rather than consumed + * and dropped because the status ticker recomputes fleet state at paint time + * on the animation tick, not only when a worker happens to emit an event. + */ + agentSessions: readonly TaskProgressSession[] /** * Row index where the inference attempt in progress began, or null when no * boundary is armed. The mapper decides when to mark, clear and roll back; @@ -727,6 +737,7 @@ export function attachSessionBridge( toolRows: new Map(), lastToolRow: -1, taskCallIds: new Set(), + agentSessions: [], attemptRow: null, turnThinking: null, } @@ -805,7 +816,8 @@ export function attachSessionBridge( currentToolName: turn.currentToolName, streamingType: turn.streamingType, } - const label = resolveTurnLabel(input, isStalled) + const fleet = fleetProgress(bag.agentSessions, nowMs) + const label = resolveTurnLabel(input, isStalled, fleet) if (label === undefined) { // The bottom-left status slot rides the same re-entry as the landing // mark, so it crossfades between phases without a timer of its own. @@ -822,7 +834,7 @@ export function attachSessionBridge( applyCadence(bag.turn.quota !== null ? frozenTickMs : null) return } - const rampPhase = resolveRampPhase(input, isStalled) + const rampPhase = resolveRampPhase(input, isStalled, fleet) const stalledFor = stalledForMs(nowMs, rampPhase === "stalled") setLockupFrame(shell, { nowMs, @@ -1066,6 +1078,7 @@ export function attachSessionBridge( }, syncAgentProgress: (sessions) => { if (bag.disposed) return + bag.agentSessions = sessions syncAgentProgress(shell, bag, sessions, now()) }, dispose: () => { diff --git a/src/tui-opentui/session-chrome.test.ts b/src/tui-opentui/session-chrome.test.ts index 58fcbc203..015e4fb98 100644 --- a/src/tui-opentui/session-chrome.test.ts +++ b/src/tui-opentui/session-chrome.test.ts @@ -47,6 +47,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, false, + null, ) expect(label).not.toBe(currentToolName) expect(ACTIVITY_STATES).toContain(label!) @@ -62,6 +63,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, true, + null, ) expect(label).toBe("stalled") expect(ACTIVITY_STATES).toContain(label!) @@ -76,6 +78,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, false, + null, ) expect(label).toBe("waiting") expect(label).not.toBe("working") @@ -94,6 +97,7 @@ describe("resolveTurnLabel", () => { streamingType: null, }, false, + null, ), ).toBeUndefined() }) @@ -108,6 +112,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("waiting") }) @@ -122,6 +127,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("stopping") }) @@ -136,6 +142,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("researching") }) @@ -147,13 +154,13 @@ describe("resolveTurnLabel", () => { currentToolName: null, } expect( - resolveTurnLabel({ ...base, streamingType: "thinking" }, false), + resolveTurnLabel({ ...base, streamingType: "thinking" }, false, null), ).toBe("thinking") expect( - resolveTurnLabel({ ...base, streamingType: "text" }, false), + resolveTurnLabel({ ...base, streamingType: "text" }, false, null), ).toBe("working") expect( - resolveTurnLabel({ ...base, streamingType: null }, false), + resolveTurnLabel({ ...base, streamingType: null }, false, null), ).toBe("working") }) }) @@ -166,27 +173,27 @@ describe("resolveRampPhase", () => { } test("blocked gate freezes the ramp", () => { - expect(resolveRampPhase({ ...base, status: "blocked" }, false)).toBe("blocked") + expect(resolveRampPhase({ ...base, status: "blocked" }, false, null)).toBe("blocked") }) test("done fills the ramp", () => { - expect(resolveRampPhase({ ...base, status: "done" }, false)).toBe("done") + expect(resolveRampPhase({ ...base, status: "done" }, false, null)).toBe("done") }) test("everything else is working", () => { - expect(resolveRampPhase({ ...base, status: "running" }, false)).toBe("working") - expect(resolveRampPhase({ ...base, status: "stopping" }, false)).toBe("working") + expect(resolveRampPhase({ ...base, status: "running" }, false, null)).toBe("working") + expect(resolveRampPhase({ ...base, status: "stopping" }, false, null)).toBe("working") }) test("a stalled running turn paints stalled, not working", () => { expect( - resolveRampPhase({ ...base, status: "running" }, true), + resolveRampPhase({ ...base, status: "running" }, true, null), ).toBe("stalled") }) test("a blocked gate beats stalled — waiting on you outranks silence", () => { expect( - resolveRampPhase({ ...base, status: "blocked" }, true), + resolveRampPhase({ ...base, status: "blocked" }, true, null), ).toBe("blocked") }) }) @@ -242,3 +249,55 @@ describe("sendFailureText", () => { expect(classifySendFailureMessage("connection reset by peer")).toBe("error") }) }) + + +describe("fleet state in the top-level indicator", () => { + const parentAwaitingChildren = { + isProcessing: true, + status: "running" as const, + currentToolName: "task", + streamingType: "tool" as const, + } + const fleet = (running: number, stalled: number) => ({ + running, + working: running - stalled, + inTool: 0, + stalled, + }) + + test("a healthy fleet reads as orchestrating, not the parent's own tool", () => { + const label = resolveTurnLabel(parentAwaitingChildren, false, fleet(6, 0)) + expect(label).toBe("orchestrating") + expect(ACTIVITY_STATES).toContain(label!) + }) + + test("a stalled lane surfaces at the top level instead of staying on its row", () => { + expect(resolveTurnLabel(parentAwaitingChildren, false, fleet(6, 1))).toBe("stalled") + expect(resolveRampPhase(parentAwaitingChildren, false, fleet(6, 1))).toBe("stalled") + }) + + // The parent is idle by design while children run, so its own stall clock + // firing says nothing about whether the session is progressing. + test("live lanes outrank the parent's own stall clock", () => { + expect(resolveTurnLabel(parentAwaitingChildren, true, fleet(6, 0))).toBe("orchestrating") + expect(resolveRampPhase(parentAwaitingChildren, true, fleet(6, 0))).toBe("working") + }) + + test("with no sub-agents running the single-agent case is unchanged", () => { + const none = fleet(0, 0) + expect(resolveTurnLabel(parentAwaitingChildren, false, none)).toBe( + resolveTurnLabel(parentAwaitingChildren, false, null), + ) + expect(resolveTurnLabel(parentAwaitingChildren, true, none)).toBe("stalled") + expect(resolveRampPhase(parentAwaitingChildren, true, none)).toBe("stalled") + }) + + test("a blocked gate still outranks the fleet", () => { + expect( + resolveTurnLabel({ ...parentAwaitingChildren, status: "blocked" }, false, fleet(6, 3)), + ).toBe("waiting") + expect( + resolveTurnLabel({ ...parentAwaitingChildren, status: "stopping" }, false, fleet(6, 3)), + ).toBe("stopping") + }) +}) diff --git a/src/tui-opentui/session-chrome.ts b/src/tui-opentui/session-chrome.ts index e2bec6236..6a283ecf8 100644 --- a/src/tui-opentui/session-chrome.ts +++ b/src/tui-opentui/session-chrome.ts @@ -6,6 +6,7 @@ */ import type { Telemetry } from "../telemetry/index.js" +import type { FleetProgress } from "./agent-progress.js" import type { RampPhase } from "./ramp.js" /** Agent lifecycle status the progress label reads (mirrors the stream state). */ @@ -39,6 +40,7 @@ export const ACTIVITY_STATES = [ "building", "working", "waiting", + "orchestrating", "stalled", "stopping", ] as const @@ -91,12 +93,19 @@ function activityStateForTool(name: string | null): ActivityState { export function resolveTurnLabel( input: TurnLabelInput, isStalled: boolean, + fleet: FleetProgress | null, ): ActivityState | undefined { if (!input.isProcessing) return undefined if (input.status === "blocked") return "waiting" if (input.status === "stopping" || input.status === "stopped") { return "stopping" } + // Live lanes outrank the parent's own stall clock: while sub-agents run the + // parent is idle by design, so its silence says nothing about the session. + // The fleet is the thing actually working, so it is the thing reported. + if (fleet !== null && fleet.running > 0) { + return fleet.stalled > 0 ? "stalled" : "orchestrating" + } if (isStalled) return "stalled" if (input.currentToolName !== null) return activityStateForTool(input.currentToolName) if (input.streamingType === "thinking") return "thinking" @@ -112,9 +121,13 @@ export function resolveTurnLabel( export function resolveRampPhase( input: TurnLabelInput, isStalled: boolean, + fleet: FleetProgress | null, ): RampPhase { if (input.status === "blocked") return "blocked" if (input.status === "done") return "done" + if (fleet !== null && fleet.running > 0) { + return fleet.stalled > 0 ? "stalled" : "working" + } if (isStalled) return "stalled" return "working" } From 1d6c35b4b88bf289ab2beb5dd210c2b98294c645 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:09:49 -0700 Subject: [PATCH 4/8] Document lane states and the fleet roll-up --- docs/TUI.md | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/TUI.md b/docs/TUI.md index dd96f02d8..2f7663d39 100644 --- a/docs/TUI.md +++ b/docs/TUI.md @@ -57,8 +57,8 @@ activity word — never the raw tool, MCP server, or plugin identifier that is actually executing. `resolveTurnLabel` (`src/tui-opentui/session-chrome.ts`) maps execution onto the closed set `ACTIVITY_STATES` exported from that module (`thinking`, `planning`, `researching`, `building`, `working`, -`waiting`, `stalled`, `stopping`); that export is the source of truth for -what the slot can say, not this list. It is led by a single density cell +`waiting`, `orchestrating`, `stalled`, `stopping`); that export is the source +of truth for what the slot can say, not this list. It is led by a single density cell (`rampPulse`, `src/tui-opentui/ramp.ts`). The cell, not the word, is what says whether the session is healthy, and it carries four states: @@ -78,6 +78,18 @@ printed identically, so the only way to tell them apart was to wait. waiting on something outside itself — and are told apart by motion: `blocked` holds perfectly still, which is the signal that the session is waiting on *you*. +While sub-agents are running, the slot reports the *fleet*, not the parent. +`resolveTurnLabel` and `resolveRampPhase` take a `FleetProgress` roll-up and +rank it above the parent's own stall clock: with live lanes the parent is +idle by design, so its silence says nothing about whether the session is +progressing, and reporting it was how a session with every lane wedged still +read as `working`. A fleet with no stalled lane reads `orchestrating`; one +stalled lane makes the whole indicator read `stalled`, which is the state that +should pull an operator's eye to the panel. A blocked gate and a stopping turn +still outrank the fleet. With zero running sub-agents the roll-up is empty and +every path through both functions behaves exactly as it does for a plain +single-agent turn. + The stall phase is driven by the watchdog's own silence clock (`stallLevel`, `src/tui-opentui/stall-watchdog.ts`), so the indicator and the abort can never disagree about which runs are stuck. It arms at @@ -158,13 +170,37 @@ fail a test as well as the type checker. ## The live agents panel The `agents` chrome zone renders a standing panel above the transcript, one -row per currently-running sub-agent — not a count. Each row reads +row per currently-running sub-agent. Each row reads `agentId: description · elapsed · tool`, sourced from the same `agentProgress()` clock/tool/stall computation used to trail a task row in the transcript (`src/tui-opentui/agent-progress.ts`); the panel does not compute -progress a second way. A worker silent past the stall window (`DEFAULT_STALL_MS`) -gets a `· stalled` suffix so it reads distinct from one still working, without -relying on color alone. +progress a second way. Past one running agent the panel is led by a fleet +summary row (`N agents`, plus `· N stalled` or `· in tools`), counted from the +same lane states the rows below render, so header and rows can never disagree. + +`laneState()` is the single definition of what a lane is doing, and every +surface consumes it rather than comparing timestamps itself. It returns one of +three states: + +| Lane state | Means | Row reads | +|---|---|---| +| `working` | activity within `DEFAULT_STALL_MS` | `· 2:34 · grep` | +| `in_tool` | silent, but a tool call is outstanding | `· 2:34 · run_shell 1:30` | +| `stalled` | silent with nothing outstanding to explain it | `· 2:34 · quiet 0:45 · stalled` | + +`in_tool` is what makes the surface honest. A worker inside one long tool call +emits no events for the entire execution, so silence alone cannot separate a +wedged reactor from a ten-minute test run — and it did not: a fleet whose lanes +were all running shell commands flipped to `stalled` in lockstep while every +one of them was working. `currentToolStartedAt` on the sub-agent session store +(`src/subagent/session-store.ts`) is the fact that separates them; only the +store sets it, because only the store observes a call ending. + +The number beside a lane's state always explains that state. A healthy lane +shows its lifetime; a lane stuck in one tool also shows how long that tool has +run; a stalled lane also shows how long it has been silent. Reading a lifetime +clock next to the word `stalled` was the original defect — the number the +operator watched climbing was unrelated to the word beside it. The panel is bounded to `AGENTS_PANEL_MAX_VISIBLE` rows (`src/tui-opentui/geometry/zones.ts`); a larger fan-out degrades to a trailing From 293506acf84fcea94f4165f2209853ab22e4ea8f Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:31:33 -0700 Subject: [PATCH 5/8] Track a sub-agent's outstanding tool calls by call id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reactor runs parallel calls concurrently, so one scalar clock could not hold them: a fast grep finishing beside a ten-minute shell command retired the shell command's clock, and thirty seconds later that lane read as stalled while working perfectly. A result carrying an id that was never seen to start now retires nothing, and the lane reports the oldest live call — the one that explains the longest silence. --- src/subagent/session-store.test.ts | 76 ++++++++++++++++++++ src/subagent/session-store.ts | 103 ++++++++++++++++++++-------- src/tui-opentui/runner-host.test.ts | 1 + 3 files changed, 153 insertions(+), 27 deletions(-) diff --git a/src/subagent/session-store.test.ts b/src/subagent/session-store.test.ts index 367fcbb70..3b5356612 100644 --- a/src/subagent/session-store.test.ts +++ b/src/subagent/session-store.test.ts @@ -145,3 +145,79 @@ describe("outstanding tool clock", () => { expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); }); }); + + +describe("parallel tool calls", () => { + const toolStart = (callId: string, name: string) => + ({ + type: "tool.start", + seq: 1, + data: { call: { id: callId, name, arguments: {} } }, + }) as unknown as ReactorEmittedEvent; + const toolDone = (callId: string) => + ({ + type: "tool.done", + seq: 2, + data: { result: { callId, content: "ok", isError: false } }, + }) as unknown as ReactorEmittedEvent; + + // The reactor runs parallel calls concurrently. A fast sibling finishing must + // not retire the clock of a long call still executing, or the lane reads as + // silent-for-no-reason thirty seconds later while it is working perfectly. + test("a fast sibling completing leaves a long call's clock outstanding", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("slow", "run_shell")); + clock = 2_000; + store.appendEvent(session.id, toolStart("fast", "grep")); + clock = 3_000; + store.appendEvent(session.id, toolDone("fast")); + + const stored = store.get(session.id); + expect(stored?.currentToolName).toBe("run_shell"); + expect(stored?.currentToolStartedAt).toBe(1_000); + }); + + test("a completion bearing an unknown call id retires nothing", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("slow", "run_shell")); + clock = 4_000; + store.appendEvent(session.id, toolDone("never-started")); + + expect(store.get(session.id)?.currentToolStartedAt).toBe(1_000); + }); + + // The oldest live call is the one that explains the longest silence, so it is + // the one the lane reports. + test("the reported call is the oldest still outstanding", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("first", "run_shell")); + clock = 2_000; + store.appendEvent(session.id, toolStart("second", "grep")); + expect(store.get(session.id)?.currentToolName).toBe("run_shell"); + + clock = 3_000; + store.appendEvent(session.id, toolDone("first")); + const stored = store.get(session.id); + expect(stored?.currentToolName).toBe("grep"); + expect(stored?.currentToolStartedAt).toBe(2_000); + }); + + test("the last completion retires the clock entirely", () => { + const store = createSubAgentSessionStore(); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + store.appendEvent(session.id, toolStart("only", "run_shell")); + store.appendEvent(session.id, toolDone("only")); + + expect(store.get(session.id)?.currentToolName).toBeNull(); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); +}); diff --git a/src/subagent/session-store.ts b/src/subagent/session-store.ts index 5868a27fa..0ffad525a 100644 --- a/src/subagent/session-store.ts +++ b/src/subagent/session-store.ts @@ -17,6 +17,12 @@ export type SubAgentTranscriptEntry = | { kind: "tool_result"; callId: string; name: string; content: string; isError: boolean } | { kind: "report"; content: string }; +export type OutstandingToolCall = { + callId: string; + name: string; + startedAt: number; +}; + export type SubAgentSession = { id: string; description: string; @@ -24,12 +30,21 @@ export type SubAgentSession = { brief: string; status: SubAgentSessionStatus; toolNames: string[]; + // Name and start clock of the OLDEST outstanding call — the one that + // explains the longest silence. Both are derived from `outstandingTools`; + // never assign them directly. Null when nothing is in flight. + // + // A worker inside one long tool emits no events for the whole execution, so + // silence alone cannot tell "wedged" from "running a ten-minute test suite". + // The start clock is the fact that separates them. currentToolName: string | null; - // Clock of when the in-flight tool call began executing, or null when no - // tool is outstanding. A worker inside one long tool emits no events for the - // whole execution, so silence alone cannot tell "wedged" from "running a - // ten-minute test suite". This is the fact that separates them. currentToolStartedAt: number | null; + // Calls the reactor has started and not yet reported a result for. The + // reactor runs parallel calls concurrently, so this cannot collapse to one + // scalar: a fast grep finishing beside a ten-minute shell command would + // otherwise retire the shell command's clock and the lane would read as + // stalled while working perfectly. + outstandingTools: OutstandingToolCall[]; entries: SubAgentTranscriptEntry[]; startedAt: number; // Clock of the last event this session recorded (a stream token, a tool @@ -101,25 +116,53 @@ function defaultCreateId(): string { } /** - * Name and clock move together so no caller can leave a tool name outstanding - * with a stale (or absent) start time — the pair is what the UI reads to tell - * a long tool call from silence with no explanation. + * The one place the displayed pair is produced, so a name can never be shown + * beside another call's clock. Called after every change to `outstandingTools`. + */ +function syncCurrentTool(session: SubAgentSession): void { + let oldest: OutstandingToolCall | undefined; + for (const call of session.outstandingTools) { + if (oldest === undefined || call.startedAt < oldest.startedAt) oldest = call; + } + session.currentToolName = oldest?.name ?? null; + session.currentToolStartedAt = oldest?.startedAt ?? null; +} + +/** + * `restartClock` marks the execution boundary: argument streaming already + * registered the call, and the figure worth showing is time spent running it. */ -function setCurrentTool( +function beginToolCall( session: SubAgentSession, - name: string | null, + callId: string, + name: string, nowMs: number, restartClock = false, ): void { - if (name === null) { - session.currentToolName = null; - session.currentToolStartedAt = null; - return; + const existing = session.outstandingTools.find((c) => c.callId === callId); + if (existing !== undefined) { + existing.name = name; + if (restartClock) existing.startedAt = nowMs; + } else { + session.outstandingTools.push({ callId, name, startedAt: nowMs }); } - if (restartClock || session.currentToolName !== name || session.currentToolStartedAt === null) { - session.currentToolStartedAt = nowMs; - } - session.currentToolName = name; + syncCurrentTool(session); +} + +/** + * Retires exactly the call that finished. A result carrying an id we never saw + * start retires nothing, rather than silently clearing a live sibling's clock. + */ +function endToolCall(session: SubAgentSession, callId: string): void { + const index = session.outstandingTools.findIndex((c) => c.callId === callId); + if (index === -1) return; + session.outstandingTools.splice(index, 1); + syncCurrentTool(session); +} + +function clearToolCalls(session: SubAgentSession): void { + session.outstandingTools.length = 0; + syncCurrentTool(session); } function capText(text: string, max: number): string { @@ -191,7 +234,7 @@ export function createSubAgentSessionStore( session.status = "cancelled"; session.finishedAt = now(); session.lastActivityAt = now(); - setCurrentTool(session, null, now()); + clearToolCalls(session); session.error = reason; pushEntry(session, { kind: "report", @@ -296,6 +339,7 @@ export function createSubAgentSessionStore( toolNames: [], currentToolName: null, currentToolStartedAt: null, + outstandingTools: [], entries: [], startedAt: now(), lastActivityAt: now(), @@ -337,7 +381,7 @@ export function createSubAgentSessionStore( const data = event.data as { name?: unknown; callId?: unknown }; const name = typeof data.name === "string" ? data.name : "tool"; const callId = typeof data.callId === "string" ? data.callId : `${name}-${session.entries.length}`; - setCurrentTool(session, name, now()); + beginToolCall(session, callId, name, now()); if (!session.toolNames.includes(name)) session.toolNames.push(name); pushEntry(session, { kind: "tool", callId, name, arguments: "" }); return; @@ -372,14 +416,16 @@ export function createSubAgentSessionStore( if (callId !== null && entry.callId !== callId) continue; if (name !== null) entry.name = name; if (args !== null && args.length > 0) entry.arguments = args; - setCurrentTool(session, entry.name, now()); + // Arguments finished streaming; the call itself is still in + // flight, so this renames it rather than restarting its clock. + beginToolCall(session, entry.callId, entry.name, now()); return; } // No matching start — record a complete tool entry. if (name !== null) { const idForEntry = callId ?? `${name}-${session.entries.length}`; if (!session.toolNames.includes(name)) session.toolNames.push(name); - setCurrentTool(session, name, now()); + beginToolCall(session, idForEntry, name, now()); pushEntry(session, { kind: "tool", callId: idForEntry, @@ -395,9 +441,11 @@ export function createSubAgentSessionStore( const call = (event as { data?: { call?: { name?: unknown; id?: unknown } } }).data?.call; const name = typeof call?.name === "string" ? call.name : null; if (name === null) return; - // Execution start, not argument streaming: restart the clock so the - // elapsed figure beside the tool name is time spent running it. - setCurrentTool(session, name, now(), true); + const callId = typeof call?.id === "string" ? call.id : null; + // Without an id there is no way to tell which of several parallel + // calls this starts, and guessing would retime the wrong one. The + // inference-side start already registered it, so leave it alone. + if (callId !== null) beginToolCall(session, callId, name, now(), true); if (!session.toolNames.includes(name)) session.toolNames.push(name); return; } @@ -418,7 +466,7 @@ export function createSubAgentSessionStore( const content = capText(stringifyUnknown(result.content ?? ""), maxEntryChars); const isError = result.isError === true; pushEntry(session, { kind: "tool_result", callId, name, content, isError }); - setCurrentTool(session, null, now()); + endToolCall(session, callId); return; } default: @@ -434,7 +482,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "done"; session.finishedAt = now(); - setCurrentTool(session, null, now()); + clearToolCalls(session); session.report = report; pushEntry(session, { kind: "report", content: capText(report, maxEntryChars) }); cancelHandles.delete(id); @@ -447,7 +495,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "failed"; session.finishedAt = now(); - setCurrentTool(session, null, now()); + clearToolCalls(session); session.error = error; pushEntry(session, { kind: "report", @@ -506,6 +554,7 @@ function cloneSession(session: SubAgentSession): SubAgentSession { toolNames: [...session.toolNames], currentToolName: session.currentToolName, currentToolStartedAt: session.currentToolStartedAt, + outstandingTools: session.outstandingTools.map((c) => ({ ...c })), entries: session.entries.map(cloneEntry), startedAt: session.startedAt, lastActivityAt: session.lastActivityAt, diff --git a/src/tui-opentui/runner-host.test.ts b/src/tui-opentui/runner-host.test.ts index 966491ee5..957e90ea4 100644 --- a/src/tui-opentui/runner-host.test.ts +++ b/src/tui-opentui/runner-host.test.ts @@ -48,6 +48,7 @@ function session(over: Partial): SubAgentSession { toolNames: [], currentToolName: null, currentToolStartedAt: null, + outstandingTools: [], entries: [], startedAt: 0, lastActivityAt: 0, From 99f07842d6c129bba6e74160cb906b6c4440c1d4 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:31:38 -0700 Subject: [PATCH 6/8] Reserve a row for the fleet summary in the agents zone The zone was sized for the lanes plus the fold-away trailer. Adding the summary on top pushed the trailer past the reservation, so it was clipped at exactly the fan-out where it is the only thing reporting the hidden lanes. --- src/tui-opentui/geometry.test.ts | 9 +++++---- src/tui-opentui/geometry/zones.ts | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/tui-opentui/geometry.test.ts b/src/tui-opentui/geometry.test.ts index 8179feb29..a58bc38db 100644 --- a/src/tui-opentui/geometry.test.ts +++ b/src/tui-opentui/geometry.test.ts @@ -100,8 +100,8 @@ describe("resolveGeometry — 80×24 idle floor", () => { describe("resolveGeometry — agents panel", () => { test("N running agents request N rows, bounded by the zone max", () => { - for (let n = 0; n <= AGENTS_PANEL_MAX_VISIBLE + 3; n++) { - const requested = Math.min(n, AGENTS_PANEL_MAX_VISIBLE + 1); + for (let n = 0; n <= AGENTS_PANEL_MAX_VISIBLE + 4; n++) { + const requested = Math.min(n, ZONE_REGISTRY.agents.max); const layout = idle80x24({ visibility: { agents: n } }); expect(layout.heights.agents).toBe(requested); } @@ -116,11 +116,12 @@ describe("resolveGeometry — agents panel", () => { test("a large fan-out never grows the zone past its bounded max", () => { const layout = idle80x24({ visibility: { agents: 50 } }); expect(layout.heights.agents).toBe(ZONE_REGISTRY.agents.max); - expect(layout.heights.agents).toBe(AGENTS_PANEL_MAX_VISIBLE + 1); + // Fleet summary + the visible lanes + the "+N more" trailer. + expect(layout.heights.agents).toBe(AGENTS_PANEL_MAX_VISIBLE + 2); }); test("a bounded agents panel never eats the transcript floor", () => { - const layout = idle80x24({ visibility: { agents: AGENTS_PANEL_MAX_VISIBLE + 1 } }); + const layout = idle80x24({ visibility: { agents: ZONE_REGISTRY.agents.max } }); expect(layout.transcriptHeight).toBeGreaterThanOrEqual(layout.transcriptFloor); }); diff --git a/src/tui-opentui/geometry/zones.ts b/src/tui-opentui/geometry/zones.ts index 34b6b8ce7..49906b5ee 100644 --- a/src/tui-opentui/geometry/zones.ts +++ b/src/tui-opentui/geometry/zones.ts @@ -84,12 +84,14 @@ export const ZONE_REGISTRY: { readonly [K in ZoneId]: ZoneDeclaration } = { idleDefault: 0, alwaysOn: false, }, - // One row per running agent (bounded by AGENTS_PANEL_MAX_VISIBLE) plus an - // optional trailing "+N more" row. + // A leading fleet-summary row, one row per running agent (bounded by + // AGENTS_PANEL_MAX_VISIBLE), then an optional trailing "+N more" row. All + // three must fit: clipping the last one drops the fold-away count at exactly + // the fan-out size where it is the only thing reporting the hidden lanes. agents: { id: "agents", min: 0, - max: AGENTS_PANEL_MAX_VISIBLE + 1, + max: AGENTS_PANEL_MAX_VISIBLE + 2, idleDefault: 0, alwaysOn: false, }, From bc63d45feeeb0fdc6aa60cf7756f6ddbfe030e95 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:31:48 -0700 Subject: [PATCH 7/8] Stop a lane's state being lost or overstated in transit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ways the state could lie. The tool clock was optional on every type between the store and a surface, so a mapper that dropped it still compiled and silently reclassified a busy lane as stalled — which is how it shipped broken once, caught only by running a fleet; required makes that a compile error. The tool annotation overwrote a live call's name while keeping the previous call's clock, painting one tool beside another's elapsed time; it now fills gaps only. And an outstanding call was believed indefinitely, so a wedged build or a shell blocked on stdin read as busy forever and never reached the fleet stall count. --- src/tui-opentui/agent-progress.test.ts | 28 +++++++ src/tui-opentui/agent-progress.ts | 37 ++++++++- src/tui-opentui/chrome-state.test.ts | 96 +++++++++++++++++++++--- src/tui-opentui/chrome-state.ts | 23 +++--- src/tui-opentui/demo.ts | 1 + src/tui-opentui/product-host.test.ts | 1 + src/tui-opentui/runtime-bridge.test.ts | 1 + src/tui-opentui/runtime-channels.test.ts | 4 +- 8 files changed, 165 insertions(+), 26 deletions(-) diff --git a/src/tui-opentui/agent-progress.test.ts b/src/tui-opentui/agent-progress.test.ts index d0fe2eff1..30f53e554 100644 --- a/src/tui-opentui/agent-progress.test.ts +++ b/src/tui-opentui/agent-progress.test.ts @@ -5,6 +5,7 @@ import { fleetLabel, fleetProgress, laneState, + IN_TOOL_STALL_MS, } from "./agent-progress" describe("clockLabel", () => { @@ -166,3 +167,30 @@ describe("fleetLabel", () => { ) }) }) + + +describe("the in-tool bound", () => { + const wedged = { + status: "running" as const, + currentToolName: "run_shell", + currentToolStartedAt: 0, + startedAt: 0, + lastActivityAt: 0, + } + + // in_tool must not be terminal, or a wedged build reads as busy forever and + // never reaches the fleet stall count. + test("a call outstanding past the bound escalates to stalled", () => { + expect(laneState(wedged, IN_TOOL_STALL_MS - 1_000)).toBe("in_tool") + expect(laneState(wedged, IN_TOOL_STALL_MS + 1_000)).toBe("stalled") + }) + + test("an escalated lane counts toward the fleet stall count", () => { + expect(fleetProgress([wedged], IN_TOOL_STALL_MS + 1_000)).toEqual({ + running: 1, + working: 0, + inTool: 0, + stalled: 1, + }) + }) +}) diff --git a/src/tui-opentui/agent-progress.ts b/src/tui-opentui/agent-progress.ts index 4fefc499f..42359dce8 100644 --- a/src/tui-opentui/agent-progress.ts +++ b/src/tui-opentui/agent-progress.ts @@ -17,8 +17,13 @@ export type AgentProgressSession = { readonly status: "running" | "done" | "failed" | "cancelled"; readonly currentToolName: string | null; - /** When the outstanding tool call began, or null when none is in flight. */ - readonly currentToolStartedAt?: number | null; + /** + * When the oldest outstanding tool call began, or null when none is in + * flight. Required, not optional: every hop from the store to a surface is a + * chance to drop it, and a dropped field would silently reclassify a busy + * lane as stalled. A compile error is a better guard than a test. + */ + readonly currentToolStartedAt: number | null; readonly startedAt: number; readonly lastActivityAt: number; }; @@ -47,6 +52,25 @@ export type AgentProgress = { /** Silence after which a running worker reads as hung rather than thinking. */ export const DEFAULT_STALL_MS = 30_000; +/** + * Second, far longer bound: how long one tool call may stay outstanding before + * the lane reads as stalled anyway. + * + * Without it `in_tool` would be terminal — a wedged build, a shell blocked on + * stdin, or a deadlocked child would read as busy forever and never reach the + * fleet stall count, trading a false-positive storm for a false negative on the + * failure operators most need to see. It is deliberately generous: real test + * suites and builds run for minutes, and crying stall over those is the defect + * this surface was fixed to remove. + * + * It also backstops calls that never report a result at all. The reactor's + * approval-suspend path emits no completion, so a before-tool extension + * returning suspend would leave a call outstanding permanently. Nothing + * registers such an extension today, but this bound means it degrades to a + * late stall rather than a lane that never stops looking busy. + */ +export const IN_TOOL_STALL_MS = 600_000; + /** "m:ss" — compact enough to sit in a row's dim trailer alongside a tool name. */ export function clockLabel(ms: number): string { const totalSeconds = Math.max(0, Math.floor(ms / 1000)); @@ -64,10 +88,15 @@ export function laneState( session: AgentProgressSession, nowMs: number, stallMs: number = DEFAULT_STALL_MS, + inToolStallMs: number = IN_TOOL_STALL_MS, ): LaneState { if (nowMs - session.lastActivityAt < stallMs) return "working"; const toolStartedAt = session.currentToolStartedAt; - if (session.currentToolName !== null && toolStartedAt !== null && toolStartedAt !== undefined) { + if ( + session.currentToolName !== null && + toolStartedAt !== null && + nowMs - toolStartedAt < inToolStallMs + ) { return "in_tool"; } return "stalled"; @@ -95,7 +124,7 @@ export function agentProgress( const base = hasTool ? `${elapsed} · ${tool}` : elapsed; const stat = - state === "in_tool" && session.currentToolStartedAt != null + state === "in_tool" && session.currentToolStartedAt !== null ? `${base} ${clockLabel(nowMs - session.currentToolStartedAt)}` : state === "stalled" ? `${base} · quiet ${clockLabel(nowMs - session.lastActivityAt)}` diff --git a/src/tui-opentui/chrome-state.test.ts b/src/tui-opentui/chrome-state.test.ts index e44a21747..6959cc2c3 100644 --- a/src/tui-opentui/chrome-state.test.ts +++ b/src/tui-opentui/chrome-state.test.ts @@ -7,6 +7,7 @@ import { formatTasksPanel, type ChromeLiveState, } from "./chrome-state" +import { agentProgress, laneState } from "./agent-progress" const NOW = 1_000_000 @@ -46,6 +47,7 @@ describe("formatChromeZones", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map setChromeZones callers", status: "running", currentToolName: "grep", @@ -54,6 +56,7 @@ describe("formatChromeZones", () => { }, { agentId: "general", + currentToolStartedAt: null, description: "write tests", status: "done", }, @@ -80,6 +83,7 @@ describe("formatChromeZones", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", }, @@ -157,8 +161,8 @@ describe("formatAgentsPanel", () => { test("one row per running agent, oldest-started first", () => { const rows = formatAgentsPanel( [ - { agentId: "a", description: "one", status: "running", startedAt: NOW - 1_000, lastActivityAt: NOW }, - { agentId: "b", description: "two", status: "running", startedAt: NOW - 2_000, lastActivityAt: NOW }, + { agentId: "a", description: "one", status: "running", currentToolStartedAt: null, startedAt: NOW - 1_000, lastActivityAt: NOW }, + { agentId: "b", description: "two", status: "running", currentToolStartedAt: null, startedAt: NOW - 2_000, lastActivityAt: NOW }, ], undefined, NOW, @@ -174,8 +178,8 @@ describe("formatAgentsPanel", () => { expect( formatAgentsPanel( [ - { agentId: "a", description: "x", status: "done" }, - { agentId: "b", description: "y", status: "failed" }, + { agentId: "a", description: "x", status: "done", currentToolStartedAt: null }, + { agentId: "b", description: "y", status: "failed", currentToolStartedAt: null }, ], undefined, NOW, @@ -188,6 +192,7 @@ describe("formatAgentsPanel", () => { [ { agentId: "a", + currentToolStartedAt: null, description: "quiet worker", status: "running", startedAt: NOW - 60_000, @@ -205,6 +210,7 @@ describe("formatAgentsPanel", () => { test("bounds fan-out to maxVisible plus a +N more row", () => { const running = Array.from({ length: 8 }, (_, i) => ({ agentId: `agent-${i}`, + currentToolStartedAt: null, description: "working", status: "running" as const, startedAt: NOW, @@ -229,9 +235,9 @@ describe("formatAgentsPanel", () => { // keying the visible row order on it would reshuffle the panel every // time any agent made progress — unreadable at a busy 200ms repaint. const frame1 = [ - { agentId: "b", description: "second", status: "running" as const, startedAt: NOW - 1_000, lastActivityAt: NOW - 1_000 }, - { agentId: "a", description: "first", status: "running" as const, startedAt: NOW - 2_000, lastActivityAt: NOW - 2_000 }, - { agentId: "c", description: "third", status: "running" as const, startedAt: NOW - 500, lastActivityAt: NOW - 500 }, + { agentId: "b", description: "second", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 1_000, lastActivityAt: NOW - 1_000 }, + { agentId: "a", description: "first", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 2_000, lastActivityAt: NOW - 2_000 }, + { agentId: "c", description: "third", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 500, lastActivityAt: NOW - 500 }, ] const rowsBefore = formatAgentsPanel(frame1, undefined, NOW) @@ -255,6 +261,7 @@ describe("formatAgentsPanel", () => { // to need attention is exactly the one that gets folded into "+N more". const newest = Array.from({ length: 5 }, (_, i) => ({ agentId: `fresh-${i}`, + currentToolStartedAt: null, description: "just started", status: "running" as const, startedAt: NOW, @@ -262,6 +269,7 @@ describe("formatAgentsPanel", () => { })) const stalled = { agentId: "quiet", + currentToolStartedAt: null, description: "gone silent", status: "running" as const, startedAt: NOW - 300_000, @@ -286,6 +294,7 @@ describe("chromeFromSession", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", @@ -300,6 +309,7 @@ describe("chromeFromSession", () => { expect(state.agents).toEqual([ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", @@ -324,6 +334,7 @@ describe("chromeFromSession", () => { id: "sess-1", description: "write tests", status: "done", + currentToolStartedAt: null, }, ], }) @@ -349,8 +360,8 @@ describe("chromeFromSession", () => { describe("annotateAgentTools", () => { const state: ChromeLiveState = { agents: [ - { agentId: "explore", description: "map callers", status: "running" }, - { agentId: "review", description: "map callers", status: "done" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, + { agentId: "review", description: "map callers", status: "done", currentToolStartedAt: null }, ], } @@ -367,3 +378,70 @@ describe("annotateAgentTools", () => { expect(next.agents?.[0]?.currentToolName).toBeUndefined() }) }) + + +describe("lane state survives the mapping hops", () => { + // The panel and the transcript trailer reach laneState by different routes. + // A hop that drops currentToolStartedAt silently reclassifies a busy lane as + // stalled — which is exactly how this shipped broken once, caught only by + // running it. The types make the drop a compile error; this proves the two + // routes still agree on a live example. + const inTool = { + id: "sess-1", + agentId: "worker", + description: "sleep 150", + status: "running" as const, + currentToolName: "run_shell", + currentToolStartedAt: NOW - 90_000, + startedAt: NOW - 100_000, + lastActivityAt: NOW - 90_000, + } + + test("the panel and the transcript row agree that the lane is in a tool", () => { + expect(laneState(inTool, NOW)).toBe("in_tool") + + const rows = formatAgentsPanel( + chromeFromSession({ agents: [inTool] }).agents, + undefined, + NOW, + ) + expect(rows?.[0]?.stalled).toBe(false) + expect(rows?.[0]?.tail).toContain("run_shell 1:30") + expect(rows?.[0]?.tail).not.toContain("stalled") + + expect(agentProgress(inTool, NOW)?.stat).toContain("run_shell 1:30") + }) + + test("a genuinely silent lane still reads stalled through the same hops", () => { + const silent = { ...inTool, currentToolName: null, currentToolStartedAt: null } + expect(laneState(silent, NOW)).toBe("stalled") + + const rows = formatAgentsPanel( + chromeFromSession({ agents: [silent] }).agents, + undefined, + NOW, + ) + expect(rows?.[0]?.stalled).toBe(true) + }) + + // A progress ping renames the tool but carries no clock of its own, so it + // must not override a call the store is already timing. + test("the tool annotation never repaints a live call with another name", () => { + const annotated = annotateAgentTools( + { agents: [inTool] }, + new Map([["sleep 150", "grep"]]), + ) + expect(annotated.agents?.[0]?.currentToolName).toBe("run_shell") + expect(annotated.agents?.[0]?.currentToolStartedAt).toBe(NOW - 90_000) + }) + + test("the tool annotation still fills a gap when no call is outstanding", () => { + const idle = { ...inTool, currentToolName: null, currentToolStartedAt: null } + const annotated = annotateAgentTools( + { agents: [idle] }, + new Map([["sleep 150", "grep"]]), + ) + expect(annotated.agents?.[0]?.currentToolName).toBe("grep") + expect(annotated.agents?.[0]?.currentToolStartedAt).toBeNull() + }) +}) diff --git a/src/tui-opentui/chrome-state.ts b/src/tui-opentui/chrome-state.ts index 68a70b83d..710e7e7dc 100644 --- a/src/tui-opentui/chrome-state.ts +++ b/src/tui-opentui/chrome-state.ts @@ -42,8 +42,8 @@ export type ChromeAgentSession = { readonly startedAt?: number /** Clock of the worker's last reported activity; feeds stalled detection. */ readonly lastActivityAt?: number - /** Clock the outstanding tool call began; separates a long tool from silence. */ - readonly currentToolStartedAt?: number | null + /** Clock the oldest outstanding tool call began; separates a long tool from silence. */ + readonly currentToolStartedAt: number | null } /** Lightweight task row: title + status, as written by the task tool. */ @@ -241,7 +241,7 @@ function formatAgentRow(session: ChromeAgentSession, nowMs: number, stallMs: num { status: "running", currentToolName: session.currentToolName ?? null, - currentToolStartedAt: session.currentToolStartedAt ?? null, + currentToolStartedAt: session.currentToolStartedAt, startedAt: session.startedAt, lastActivityAt: session.lastActivityAt ?? session.startedAt, }, @@ -284,10 +284,13 @@ export function annotateAgentTools( if (a.status !== "running") return a const tool = toolByDescription.get(a.description) if (tool === undefined) return a - // The overlay renames the tool but must not invent a start clock for it. - // Only the store observes a call ending, so letting a progress ping - // supply a clock would keep a finished lane reading as busy forever — - // exactly the true stalls this surface exists to show. + // Gap-fill only. When the store has a call outstanding it owns both the + // name and the clock, and overriding just the name would paint one + // tool's identifier beside another tool's elapsed time. A progress ping + // also cannot supply a clock of its own — only the store observes a call + // ending, so a ping-sourced clock would keep a finished lane reading + // busy forever, hiding exactly the stalls this surface exists to show. + if (a.currentToolStartedAt !== null) return a return { ...a, currentToolName: tool } }), } @@ -313,7 +316,7 @@ export type ChromeSessionAgent = { readonly description: string readonly status: "running" | "done" | "failed" | "cancelled" readonly currentToolName?: string | null - readonly currentToolStartedAt?: number | null + readonly currentToolStartedAt: number | null readonly startedAt?: number readonly lastActivityAt?: number } @@ -374,9 +377,7 @@ function mapSessionAgents( ...(a.currentToolName !== undefined ? { currentToolName: a.currentToolName } : {}), - ...(a.currentToolStartedAt !== undefined - ? { currentToolStartedAt: a.currentToolStartedAt } - : {}), + currentToolStartedAt: a.currentToolStartedAt, ...(a.startedAt !== undefined ? { startedAt: a.startedAt } : {}), ...(a.lastActivityAt !== undefined ? { lastActivityAt: a.lastActivityAt } diff --git a/src/tui-opentui/demo.ts b/src/tui-opentui/demo.ts index 96ca9d38a..7ef566be8 100644 --- a/src/tui-opentui/demo.ts +++ b/src/tui-opentui/demo.ts @@ -298,6 +298,7 @@ renderer.keyInput.on("keypress", (key: KeyEvent) => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", diff --git a/src/tui-opentui/product-host.test.ts b/src/tui-opentui/product-host.test.ts index c5d6fd118..7097cd4b2 100644 --- a/src/tui-opentui/product-host.test.ts +++ b/src/tui-opentui/product-host.test.ts @@ -286,6 +286,7 @@ describe("mountProductHost", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", startedAt: now - 59_000, diff --git a/src/tui-opentui/runtime-bridge.test.ts b/src/tui-opentui/runtime-bridge.test.ts index e50fc7e5f..e9809da1d 100644 --- a/src/tui-opentui/runtime-bridge.test.ts +++ b/src/tui-opentui/runtime-bridge.test.ts @@ -651,6 +651,7 @@ describe("syncAgentProgress", () => { id: "task-1", status: "running", currentToolName: "grep", + currentToolStartedAt: null, startedAt: 0, lastActivityAt: 0, ...over, diff --git a/src/tui-opentui/runtime-channels.test.ts b/src/tui-opentui/runtime-channels.test.ts index 1c8daeb59..f38de2155 100644 --- a/src/tui-opentui/runtime-channels.test.ts +++ b/src/tui-opentui/runtime-channels.test.ts @@ -177,7 +177,7 @@ describe("subagent.progress channel", () => { const { host, emitter, frame, cleanup } = await mountHeadless({ chrome: { agents: [ - { agentId: "explore", description: "map callers", status: "running" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, ], }, }) @@ -205,7 +205,7 @@ describe("subagent.progress channel", () => { }) host.setChrome({ agents: [ - { agentId: "explore", description: "map callers", status: "running" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, ], }) expect(await frame()).toContain("map callers · grep") From c3e79b92f991736dc364c370f9300f82aed80080 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 18:31:48 -0700 Subject: [PATCH 8/8] Document the tool clock's ownership and the bound on in-tool --- docs/TUI.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/TUI.md b/docs/TUI.md index 2f7663d39..46ca2ae25 100644 --- a/docs/TUI.md +++ b/docs/TUI.md @@ -177,6 +177,10 @@ transcript (`src/tui-opentui/agent-progress.ts`); the panel does not compute progress a second way. Past one running agent the panel is led by a fleet summary row (`N agents`, plus `· N stalled` or `· in tools`), counted from the same lane states the rows below render, so header and rows can never disagree. +The zone reserves `AGENTS_PANEL_MAX_VISIBLE + 2` rows to hold that summary, the +lanes, and the `+N more` trailer together — clipping the last of the three +would drop the fold-away count at exactly the fan-out where it is the only +thing reporting the hidden lanes. `laneState()` is the single definition of what a lane is doing, and every surface consumes it rather than comparing timestamps itself. It returns one of @@ -185,7 +189,7 @@ three states: | Lane state | Means | Row reads | |---|---|---| | `working` | activity within `DEFAULT_STALL_MS` | `· 2:34 · grep` | -| `in_tool` | silent, but a tool call is outstanding | `· 2:34 · run_shell 1:30` | +| `in_tool` | silent, but a tool call is outstanding and under `IN_TOOL_STALL_MS` | `· 2:34 · run_shell 1:30` | | `stalled` | silent with nothing outstanding to explain it | `· 2:34 · quiet 0:45 · stalled` | `in_tool` is what makes the surface honest. A worker inside one long tool call @@ -196,6 +200,32 @@ one of them was working. `currentToolStartedAt` on the sub-agent session store (`src/subagent/session-store.ts`) is the fact that separates them; only the store sets it, because only the store observes a call ending. +The store keys outstanding calls by call id (`outstandingTools`) and reports +the oldest live one — the call that explains the longest silence. It cannot +collapse to a single scalar: the reactor runs parallel calls concurrently, so a +fast grep finishing beside a ten-minute shell command would retire the shell +command's clock and reproduce the original defect on one lane. A result whose +call id was never seen to start retires nothing. + +`currentToolStartedAt` is a **required** field on every type between the store +and a surface. There are four hand-written mapping hops on the live path, and +a hop that drops it silently reclassifies a busy lane as stalled — which is how +this shipped broken once, caught only by running a real fleet. Required makes +that a compile error rather than a misclassification; `chrome-state.test.ts` +also asserts the panel and the transcript row agree on a live example. + +`in_tool` is bounded, not terminal. A call outstanding longer than +`IN_TOOL_STALL_MS` (10 minutes) escalates to `stalled` regardless, so a wedged +build, a shell blocked on stdin, or a deadlocked child eventually surfaces +instead of reading as busy forever. **Within that window those failures are +genuinely invisible to the stall signal** — the honest trade for not crying +stall over every real test suite. The per-row tool clock climbing is the signal +a human can read in the meantime, which is why the row shows it. The same bound +backstops calls that never report a result at all: the reactor's +approval-suspend path emits no completion, so a before-tool extension returning +suspend would otherwise leave a call outstanding permanently. Nothing registers +such an extension today. + The number beside a lane's state always explains that state. A healthy lane shows its lifetime; a lane stuck in one tool also shows how long that tool has run; a stalled lane also shows how long it has been silent. Reading a lifetime