diff --git a/docs/TUI.md b/docs/TUI.md index dd96f02d8..46ca2ae25 100644 --- a/docs/TUI.md +++ b/docs/TUI.md @@ -57,8 +57,8 @@ activity word — never the raw tool, MCP server, or plugin identifier that is actually executing. `resolveTurnLabel` (`src/tui-opentui/session-chrome.ts`) maps execution onto the closed set `ACTIVITY_STATES` exported from that module (`thinking`, `planning`, `researching`, `building`, `working`, -`waiting`, `stalled`, `stopping`); that export is the source of truth for -what the slot can say, not this list. It is led by a single density cell +`waiting`, `orchestrating`, `stalled`, `stopping`); that export is the source +of truth for what the slot can say, not this list. It is led by a single density cell (`rampPulse`, `src/tui-opentui/ramp.ts`). The cell, not the word, is what says whether the session is healthy, and it carries four states: @@ -78,6 +78,18 @@ printed identically, so the only way to tell them apart was to wait. waiting on something outside itself — and are told apart by motion: `blocked` holds perfectly still, which is the signal that the session is waiting on *you*. +While sub-agents are running, the slot reports the *fleet*, not the parent. +`resolveTurnLabel` and `resolveRampPhase` take a `FleetProgress` roll-up and +rank it above the parent's own stall clock: with live lanes the parent is +idle by design, so its silence says nothing about whether the session is +progressing, and reporting it was how a session with every lane wedged still +read as `working`. A fleet with no stalled lane reads `orchestrating`; one +stalled lane makes the whole indicator read `stalled`, which is the state that +should pull an operator's eye to the panel. A blocked gate and a stopping turn +still outrank the fleet. With zero running sub-agents the roll-up is empty and +every path through both functions behaves exactly as it does for a plain +single-agent turn. + The stall phase is driven by the watchdog's own silence clock (`stallLevel`, `src/tui-opentui/stall-watchdog.ts`), so the indicator and the abort can never disagree about which runs are stuck. It arms at @@ -158,13 +170,67 @@ fail a test as well as the type checker. ## The live agents panel The `agents` chrome zone renders a standing panel above the transcript, one -row per currently-running sub-agent — not a count. Each row reads +row per currently-running sub-agent. Each row reads `agentId: description · elapsed · tool`, sourced from the same `agentProgress()` clock/tool/stall computation used to trail a task row in the transcript (`src/tui-opentui/agent-progress.ts`); the panel does not compute -progress a second way. A worker silent past the stall window (`DEFAULT_STALL_MS`) -gets a `· stalled` suffix so it reads distinct from one still working, without -relying on color alone. +progress a second way. Past one running agent the panel is led by a fleet +summary row (`N agents`, plus `· N stalled` or `· in tools`), counted from the +same lane states the rows below render, so header and rows can never disagree. +The zone reserves `AGENTS_PANEL_MAX_VISIBLE + 2` rows to hold that summary, the +lanes, and the `+N more` trailer together — clipping the last of the three +would drop the fold-away count at exactly the fan-out where it is the only +thing reporting the hidden lanes. + +`laneState()` is the single definition of what a lane is doing, and every +surface consumes it rather than comparing timestamps itself. It returns one of +three states: + +| Lane state | Means | Row reads | +|---|---|---| +| `working` | activity within `DEFAULT_STALL_MS` | `· 2:34 · grep` | +| `in_tool` | silent, but a tool call is outstanding and under `IN_TOOL_STALL_MS` | `· 2:34 · run_shell 1:30` | +| `stalled` | silent with nothing outstanding to explain it | `· 2:34 · quiet 0:45 · stalled` | + +`in_tool` is what makes the surface honest. A worker inside one long tool call +emits no events for the entire execution, so silence alone cannot separate a +wedged reactor from a ten-minute test run — and it did not: a fleet whose lanes +were all running shell commands flipped to `stalled` in lockstep while every +one of them was working. `currentToolStartedAt` on the sub-agent session store +(`src/subagent/session-store.ts`) is the fact that separates them; only the +store sets it, because only the store observes a call ending. + +The store keys outstanding calls by call id (`outstandingTools`) and reports +the oldest live one — the call that explains the longest silence. It cannot +collapse to a single scalar: the reactor runs parallel calls concurrently, so a +fast grep finishing beside a ten-minute shell command would retire the shell +command's clock and reproduce the original defect on one lane. A result whose +call id was never seen to start retires nothing. + +`currentToolStartedAt` is a **required** field on every type between the store +and a surface. There are four hand-written mapping hops on the live path, and +a hop that drops it silently reclassifies a busy lane as stalled — which is how +this shipped broken once, caught only by running a real fleet. Required makes +that a compile error rather than a misclassification; `chrome-state.test.ts` +also asserts the panel and the transcript row agree on a live example. + +`in_tool` is bounded, not terminal. A call outstanding longer than +`IN_TOOL_STALL_MS` (10 minutes) escalates to `stalled` regardless, so a wedged +build, a shell blocked on stdin, or a deadlocked child eventually surfaces +instead of reading as busy forever. **Within that window those failures are +genuinely invisible to the stall signal** — the honest trade for not crying +stall over every real test suite. The per-row tool clock climbing is the signal +a human can read in the meantime, which is why the row shows it. The same bound +backstops calls that never report a result at all: the reactor's +approval-suspend path emits no completion, so a before-tool extension returning +suspend would otherwise leave a call outstanding permanently. Nothing registers +such an extension today. + +The number beside a lane's state always explains that state. A healthy lane +shows its lifetime; a lane stuck in one tool also shows how long that tool has +run; a stalled lane also shows how long it has been silent. Reading a lifetime +clock next to the word `stalled` was the original defect — the number the +operator watched climbing was unrelated to the word beside it. The panel is bounded to `AGENTS_PANEL_MAX_VISIBLE` rows (`src/tui-opentui/geometry/zones.ts`); a larger fan-out degrades to a trailing diff --git a/src/subagent/session-store.test.ts b/src/subagent/session-store.test.ts index bcc37d3c7..3b5356612 100644 --- a/src/subagent/session-store.test.ts +++ b/src/subagent/session-store.test.ts @@ -104,3 +104,120 @@ describe("session-store snapshot caching", () => { expect(snapshot?.entries).toEqual([]); }); }); + + +describe("outstanding tool clock", () => { + // A worker inside one long tool call emits nothing until the result lands. + // Without a start clock for that call, silence is indistinguishable from a + // wedged reactor, and a whole fleet running shell commands reads as stalled. + test("tool.start stamps the clock and tool.done clears it", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + + clock = 5_000; + store.appendEvent(session.id, { + type: "tool.start", + seq: 1, + data: { call: { id: "call-1", name: "run_shell", arguments: {} } }, + } as unknown as ReactorEmittedEvent); + expect(store.get(session.id)?.currentToolName).toBe("run_shell"); + expect(store.get(session.id)?.currentToolStartedAt).toBe(5_000); + + clock = 95_000; + store.appendEvent(session.id, { + type: "tool.done", + seq: 2, + data: { result: { callId: "call-1", content: "ok", isError: false } }, + } as unknown as ReactorEmittedEvent); + expect(store.get(session.id)?.currentToolName).toBeNull(); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); + + test("a terminal transition never leaves a tool clock outstanding", () => { + const store = createSubAgentSessionStore(); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + store.appendEvent(session.id, startCall(1, "call-1", "grep")); + expect(store.get(session.id)?.currentToolStartedAt).not.toBeNull(); + + store.complete(session.id, "report"); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); +}); + + +describe("parallel tool calls", () => { + const toolStart = (callId: string, name: string) => + ({ + type: "tool.start", + seq: 1, + data: { call: { id: callId, name, arguments: {} } }, + }) as unknown as ReactorEmittedEvent; + const toolDone = (callId: string) => + ({ + type: "tool.done", + seq: 2, + data: { result: { callId, content: "ok", isError: false } }, + }) as unknown as ReactorEmittedEvent; + + // The reactor runs parallel calls concurrently. A fast sibling finishing must + // not retire the clock of a long call still executing, or the lane reads as + // silent-for-no-reason thirty seconds later while it is working perfectly. + test("a fast sibling completing leaves a long call's clock outstanding", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("slow", "run_shell")); + clock = 2_000; + store.appendEvent(session.id, toolStart("fast", "grep")); + clock = 3_000; + store.appendEvent(session.id, toolDone("fast")); + + const stored = store.get(session.id); + expect(stored?.currentToolName).toBe("run_shell"); + expect(stored?.currentToolStartedAt).toBe(1_000); + }); + + test("a completion bearing an unknown call id retires nothing", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("slow", "run_shell")); + clock = 4_000; + store.appendEvent(session.id, toolDone("never-started")); + + expect(store.get(session.id)?.currentToolStartedAt).toBe(1_000); + }); + + // The oldest live call is the one that explains the longest silence, so it is + // the one the lane reports. + test("the reported call is the oldest still outstanding", () => { + let clock = 1_000; + const store = createSubAgentSessionStore({ now: () => clock }); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + + store.appendEvent(session.id, toolStart("first", "run_shell")); + clock = 2_000; + store.appendEvent(session.id, toolStart("second", "grep")); + expect(store.get(session.id)?.currentToolName).toBe("run_shell"); + + clock = 3_000; + store.appendEvent(session.id, toolDone("first")); + const stored = store.get(session.id); + expect(stored?.currentToolName).toBe("grep"); + expect(stored?.currentToolStartedAt).toBe(2_000); + }); + + test("the last completion retires the clock entirely", () => { + const store = createSubAgentSessionStore(); + const session = store.start({ description: "d", agentId: "a", brief: "b" }); + store.appendEvent(session.id, toolStart("only", "run_shell")); + store.appendEvent(session.id, toolDone("only")); + + expect(store.get(session.id)?.currentToolName).toBeNull(); + expect(store.get(session.id)?.currentToolStartedAt).toBeNull(); + }); +}); diff --git a/src/subagent/session-store.ts b/src/subagent/session-store.ts index 6fd8e25b6..0ffad525a 100644 --- a/src/subagent/session-store.ts +++ b/src/subagent/session-store.ts @@ -17,6 +17,12 @@ export type SubAgentTranscriptEntry = | { kind: "tool_result"; callId: string; name: string; content: string; isError: boolean } | { kind: "report"; content: string }; +export type OutstandingToolCall = { + callId: string; + name: string; + startedAt: number; +}; + export type SubAgentSession = { id: string; description: string; @@ -24,7 +30,21 @@ export type SubAgentSession = { brief: string; status: SubAgentSessionStatus; toolNames: string[]; + // Name and start clock of the OLDEST outstanding call — the one that + // explains the longest silence. Both are derived from `outstandingTools`; + // never assign them directly. Null when nothing is in flight. + // + // A worker inside one long tool emits no events for the whole execution, so + // silence alone cannot tell "wedged" from "running a ten-minute test suite". + // The start clock is the fact that separates them. currentToolName: string | null; + currentToolStartedAt: number | null; + // Calls the reactor has started and not yet reported a result for. The + // reactor runs parallel calls concurrently, so this cannot collapse to one + // scalar: a fast grep finishing beside a ten-minute shell command would + // otherwise retire the shell command's clock and the lane would read as + // stalled while working perfectly. + outstandingTools: OutstandingToolCall[]; entries: SubAgentTranscriptEntry[]; startedAt: number; // Clock of the last event this session recorded (a stream token, a tool @@ -95,6 +115,56 @@ function defaultCreateId(): string { return `subagent-${nextId}`; } +/** + * The one place the displayed pair is produced, so a name can never be shown + * beside another call's clock. Called after every change to `outstandingTools`. + */ +function syncCurrentTool(session: SubAgentSession): void { + let oldest: OutstandingToolCall | undefined; + for (const call of session.outstandingTools) { + if (oldest === undefined || call.startedAt < oldest.startedAt) oldest = call; + } + session.currentToolName = oldest?.name ?? null; + session.currentToolStartedAt = oldest?.startedAt ?? null; +} + +/** + * `restartClock` marks the execution boundary: argument streaming already + * registered the call, and the figure worth showing is time spent running it. + */ +function beginToolCall( + session: SubAgentSession, + callId: string, + name: string, + nowMs: number, + restartClock = false, +): void { + const existing = session.outstandingTools.find((c) => c.callId === callId); + if (existing !== undefined) { + existing.name = name; + if (restartClock) existing.startedAt = nowMs; + } else { + session.outstandingTools.push({ callId, name, startedAt: nowMs }); + } + syncCurrentTool(session); +} + +/** + * Retires exactly the call that finished. A result carrying an id we never saw + * start retires nothing, rather than silently clearing a live sibling's clock. + */ +function endToolCall(session: SubAgentSession, callId: string): void { + const index = session.outstandingTools.findIndex((c) => c.callId === callId); + if (index === -1) return; + session.outstandingTools.splice(index, 1); + syncCurrentTool(session); +} + +function clearToolCalls(session: SubAgentSession): void { + session.outstandingTools.length = 0; + syncCurrentTool(session); +} + function capText(text: string, max: number): string { if (text.length <= max) return text; return text.slice(0, max); @@ -164,7 +234,7 @@ export function createSubAgentSessionStore( session.status = "cancelled"; session.finishedAt = now(); session.lastActivityAt = now(); - session.currentToolName = null; + clearToolCalls(session); session.error = reason; pushEntry(session, { kind: "report", @@ -268,6 +338,8 @@ export function createSubAgentSessionStore( status: "running", toolNames: [], currentToolName: null, + currentToolStartedAt: null, + outstandingTools: [], entries: [], startedAt: now(), lastActivityAt: now(), @@ -309,7 +381,7 @@ export function createSubAgentSessionStore( const data = event.data as { name?: unknown; callId?: unknown }; const name = typeof data.name === "string" ? data.name : "tool"; const callId = typeof data.callId === "string" ? data.callId : `${name}-${session.entries.length}`; - session.currentToolName = name; + beginToolCall(session, callId, name, now()); if (!session.toolNames.includes(name)) session.toolNames.push(name); pushEntry(session, { kind: "tool", callId, name, arguments: "" }); return; @@ -344,14 +416,16 @@ export function createSubAgentSessionStore( if (callId !== null && entry.callId !== callId) continue; if (name !== null) entry.name = name; if (args !== null && args.length > 0) entry.arguments = args; - session.currentToolName = entry.name; + // Arguments finished streaming; the call itself is still in + // flight, so this renames it rather than restarting its clock. + beginToolCall(session, entry.callId, entry.name, now()); return; } // No matching start — record a complete tool entry. if (name !== null) { const idForEntry = callId ?? `${name}-${session.entries.length}`; if (!session.toolNames.includes(name)) session.toolNames.push(name); - session.currentToolName = name; + beginToolCall(session, idForEntry, name, now()); pushEntry(session, { kind: "tool", callId: idForEntry, @@ -367,7 +441,11 @@ export function createSubAgentSessionStore( const call = (event as { data?: { call?: { name?: unknown; id?: unknown } } }).data?.call; const name = typeof call?.name === "string" ? call.name : null; if (name === null) return; - session.currentToolName = name; + const callId = typeof call?.id === "string" ? call.id : null; + // Without an id there is no way to tell which of several parallel + // calls this starts, and guessing would retime the wrong one. The + // inference-side start already registered it, so leave it alone. + if (callId !== null) beginToolCall(session, callId, name, now(), true); if (!session.toolNames.includes(name)) session.toolNames.push(name); return; } @@ -388,7 +466,7 @@ export function createSubAgentSessionStore( const content = capText(stringifyUnknown(result.content ?? ""), maxEntryChars); const isError = result.isError === true; pushEntry(session, { kind: "tool_result", callId, name, content, isError }); - session.currentToolName = null; + endToolCall(session, callId); return; } default: @@ -404,7 +482,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "done"; session.finishedAt = now(); - session.currentToolName = null; + clearToolCalls(session); session.report = report; pushEntry(session, { kind: "report", content: capText(report, maxEntryChars) }); cancelHandles.delete(id); @@ -417,7 +495,7 @@ export function createSubAgentSessionStore( if (session.status !== "running") return; session.status = "failed"; session.finishedAt = now(); - session.currentToolName = null; + clearToolCalls(session); session.error = error; pushEntry(session, { kind: "report", @@ -475,6 +553,8 @@ function cloneSession(session: SubAgentSession): SubAgentSession { status: session.status, toolNames: [...session.toolNames], currentToolName: session.currentToolName, + currentToolStartedAt: session.currentToolStartedAt, + outstandingTools: session.outstandingTools.map((c) => ({ ...c })), entries: session.entries.map(cloneEntry), startedAt: session.startedAt, lastActivityAt: session.lastActivityAt, diff --git a/src/tui-opentui/agent-progress.test.ts b/src/tui-opentui/agent-progress.test.ts index aefdcb69b..30f53e554 100644 --- a/src/tui-opentui/agent-progress.test.ts +++ b/src/tui-opentui/agent-progress.test.ts @@ -1,5 +1,12 @@ import { describe, expect, test } from "bun:test" -import { agentProgress, clockLabel } from "./agent-progress" +import { + agentProgress, + clockLabel, + fleetLabel, + fleetProgress, + laneState, + IN_TOOL_STALL_MS, +} from "./agent-progress" describe("clockLabel", () => { test("formats sub-minute and multi-minute elapsed as m:ss", () => { @@ -13,6 +20,7 @@ describe("agentProgress", () => { const base = { status: "running" as const, currentToolName: "grep", + currentToolStartedAt: null, startedAt: 0, lastActivityAt: 0, } @@ -25,7 +33,12 @@ describe("agentProgress", () => { test("a running session reports elapsed time and its current tool", () => { const progress = agentProgress({ ...base, lastActivityAt: 42_000 }, 42_000) - expect(progress).toEqual({ stat: "0:42 · grep", working: true, stalled: false }) + expect(progress).toEqual({ + stat: "0:42 · grep", + state: "working", + working: true, + stalled: false, + }) }) test("a running session with no current tool reports elapsed time alone", () => { @@ -33,12 +46,45 @@ describe("agentProgress", () => { { ...base, currentToolName: null, lastActivityAt: 42_000 }, 42_000, ) - expect(progress).toEqual({ stat: "0:42", working: true, stalled: false }) + expect(progress).toEqual({ + stat: "0:42", + state: "working", + working: true, + stalled: false, + }) }) - test("silence past the stall window flips working to stalled", () => { - const progress = agentProgress({ ...base, lastActivityAt: 0 }, 31_000, 30_000) - expect(progress).toEqual({ stat: "0:31 · grep", working: false, stalled: true }) + test("silence with no tool outstanding is a stall, and the clock shown is the silence", () => { + const progress = agentProgress( + { ...base, currentToolName: null, lastActivityAt: 0 }, + 31_000, + 30_000, + ) + expect(progress).toEqual({ + stat: "0:31 · quiet 0:31", + state: "stalled", + working: false, + stalled: true, + }) + }) + + // The defect the whole surface turned on: a worker inside one long tool call + // emits nothing for the entire execution, so every lane of a fleet running + // e.g. a test suite flipped to "stalled" simultaneously while working fine. + test("silence inside an outstanding tool call is not a stall", () => { + const progress = agentProgress( + { + ...base, + currentToolName: "run_shell", + currentToolStartedAt: 1_000, + lastActivityAt: 1_000, + }, + 91_000, + 30_000, + ) + expect(progress?.state).toBe("in_tool") + expect(progress?.stalled).toBe(false) + expect(progress?.stat).toBe("1:31 · run_shell 1:30") }) test("recent activity keeps a long-running session marked working", () => { @@ -47,3 +93,104 @@ describe("agentProgress", () => { expect(progress?.stalled).toBe(false) }) }) + +describe("laneState", () => { + const running = { + status: "running" as const, + currentToolName: null, + currentToolStartedAt: null, + startedAt: 0, + lastActivityAt: 0, + } + + test("names the three lanes a running worker can be in", () => { + expect(laneState({ ...running, lastActivityAt: 1_000 }, 2_000, 30_000)).toBe("working") + expect(laneState(running, 60_000, 30_000)).toBe("stalled") + expect( + laneState( + { ...running, currentToolName: "run_shell", currentToolStartedAt: 0 }, + 60_000, + 30_000, + ), + ).toBe("in_tool") + }) +}) + +describe("fleetProgress", () => { + const lane = (over: Partial[0]>) => ({ + status: "running" as const, + currentToolName: null, + currentToolStartedAt: null, + startedAt: 0, + lastActivityAt: 0, + ...over, + }) + + test("counts only running lanes, bucketed by their single lane state", () => { + const fleet = fleetProgress( + [ + lane({ lastActivityAt: 59_000 }), + lane({ currentToolName: "run_shell", currentToolStartedAt: 0 }), + lane({}), + lane({ status: "done" }), + ], + 60_000, + 30_000, + ) + expect(fleet).toEqual({ running: 3, working: 1, inTool: 1, stalled: 1 }) + }) + + test("no sub-agents leaves the fleet empty", () => { + expect(fleetProgress([], 1_000)).toEqual({ + running: 0, + working: 0, + inTool: 0, + stalled: 0, + }) + }) +}) + +describe("fleetLabel", () => { + test("is null with nothing running so the single-agent case is untouched", () => { + expect(fleetLabel({ running: 0, working: 0, inTool: 0, stalled: 0 })).toBeNull() + }) + + test("names the stalled count when any lane is stuck", () => { + expect(fleetLabel({ running: 6, working: 4, inTool: 0, stalled: 2 })).toBe( + "6 agents · 2 stalled", + ) + }) + + test("says when the whole fleet is inside tool calls", () => { + expect(fleetLabel({ running: 3, working: 0, inTool: 3, stalled: 0 })).toBe( + "3 agents · in tools", + ) + }) +}) + + +describe("the in-tool bound", () => { + const wedged = { + status: "running" as const, + currentToolName: "run_shell", + currentToolStartedAt: 0, + startedAt: 0, + lastActivityAt: 0, + } + + // in_tool must not be terminal, or a wedged build reads as busy forever and + // never reaches the fleet stall count. + test("a call outstanding past the bound escalates to stalled", () => { + expect(laneState(wedged, IN_TOOL_STALL_MS - 1_000)).toBe("in_tool") + expect(laneState(wedged, IN_TOOL_STALL_MS + 1_000)).toBe("stalled") + }) + + test("an escalated lane counts toward the fleet stall count", () => { + expect(fleetProgress([wedged], IN_TOOL_STALL_MS + 1_000)).toEqual({ + running: 1, + working: 0, + inTool: 0, + stalled: 1, + }) + }) +}) diff --git a/src/tui-opentui/agent-progress.ts b/src/tui-opentui/agent-progress.ts index dcea4ef7a..42359dce8 100644 --- a/src/tui-opentui/agent-progress.ts +++ b/src/tui-opentui/agent-progress.ts @@ -1,33 +1,76 @@ /** - * Live progress for a dispatched sub-agent's pending row in the transcript. + * Live progress for a dispatched sub-agent's pending row in the transcript, + * and the fleet-level roll-up of those same lanes. * * A "task" tool call renders as one row for its whole lifetime (see * `runtime-bridge.ts`'s `syncAgentProgress`). While the call is outstanding * this fills in what a bare pending mark cannot say: how long the worker has * been running, what it is doing right now, and whether it has gone quiet * long enough to look hung rather than merely slow. + * + * Lane state and the fleet roll-up live in this one file on purpose. "Stalled" + * has exactly one definition — `laneState` below — and the fleet summary + * consumes it rather than re-deriving staleness from raw timestamps. */ /** Minimal session shape this module reads — avoids a hard dep on the store. */ export type AgentProgressSession = { readonly status: "running" | "done" | "failed" | "cancelled"; readonly currentToolName: string | null; + /** + * When the oldest outstanding tool call began, or null when none is in + * flight. Required, not optional: every hop from the store to a surface is a + * chance to drop it, and a dropped field would silently reclassify a busy + * lane as stalled. A compile error is a better guard than a test. + */ + readonly currentToolStartedAt: number | null; readonly startedAt: number; readonly lastActivityAt: number; }; +/** + * What a lane is actually doing, as opposed to how long it has been alive. + * + * `in_tool` is the state that makes the surface honest: a worker inside one + * long tool call emits nothing for the whole execution, so silence on its own + * cannot tell a wedged reactor from a ten-minute test run. A lane only reads + * `stalled` when it has gone quiet with no tool outstanding to explain it — + * that is the case an operator can act on. + */ +export type LaneState = "working" | "in_tool" | "stalled"; + export type AgentProgress = { /** Dim trailer painted after the row's subject, e.g. "0:42 · grep". */ readonly stat: string; - /** True while the worker has reported activity within the stall window. */ + readonly state: LaneState; + /** True while the worker is making visible progress. */ readonly working: boolean; - /** True once silence has run longer than the stall window. */ + /** True once silence has run longer than the stall window with nothing to explain it. */ readonly stalled: boolean; }; /** Silence after which a running worker reads as hung rather than thinking. */ export const DEFAULT_STALL_MS = 30_000; +/** + * Second, far longer bound: how long one tool call may stay outstanding before + * the lane reads as stalled anyway. + * + * Without it `in_tool` would be terminal — a wedged build, a shell blocked on + * stdin, or a deadlocked child would read as busy forever and never reach the + * fleet stall count, trading a false-positive storm for a false negative on the + * failure operators most need to see. It is deliberately generous: real test + * suites and builds run for minutes, and crying stall over those is the defect + * this surface was fixed to remove. + * + * It also backstops calls that never report a result at all. The reactor's + * approval-suspend path emits no completion, so a before-tool extension + * returning suspend would leave a call outstanding permanently. Nothing + * registers such an extension today, but this bound means it degrades to a + * late stall rather than a lane that never stops looking busy. + */ +export const IN_TOOL_STALL_MS = 600_000; + /** "m:ss" — compact enough to sit in a row's dim trailer alongside a tool name. */ export function clockLabel(ms: number): string { const totalSeconds = Math.max(0, Math.floor(ms / 1000)); @@ -36,9 +79,37 @@ export function clockLabel(ms: number): string { return `${minutes}:${String(seconds).padStart(2, "0")}`; } +/** + * The single definition of what a lane is doing. Every other surface — the + * transcript trailer, the agents panel, the top-level indicator — reads this + * result rather than comparing timestamps itself. + */ +export function laneState( + session: AgentProgressSession, + nowMs: number, + stallMs: number = DEFAULT_STALL_MS, + inToolStallMs: number = IN_TOOL_STALL_MS, +): LaneState { + if (nowMs - session.lastActivityAt < stallMs) return "working"; + const toolStartedAt = session.currentToolStartedAt; + if ( + session.currentToolName !== null && + toolStartedAt !== null && + nowMs - toolStartedAt < inToolStallMs + ) { + return "in_tool"; + } + return "stalled"; +} + /** * Progress for a running session's pending row, or null once it has finished — * a terminal session resolves its row through the tool-result path instead. + * + * The number beside the state word always explains that word: a healthy lane + * shows its lifetime, a lane stuck in one tool shows how long that tool has + * been running, and a silent lane shows how long it has been silent. Reading + * a lifetime clock next to "stalled" tells an operator nothing about the stall. */ export function agentProgress( session: AgentProgressSession, @@ -48,10 +119,74 @@ export function agentProgress( if (session.status !== "running") return null; const elapsed = clockLabel(nowMs - session.startedAt); const tool = session.currentToolName; - const stalled = nowMs - session.lastActivityAt >= stallMs; + const hasTool = tool !== null && tool.length > 0; + const state = laneState(session, nowMs, stallMs); + + const base = hasTool ? `${elapsed} · ${tool}` : elapsed; + const stat = + state === "in_tool" && session.currentToolStartedAt !== null + ? `${base} ${clockLabel(nowMs - session.currentToolStartedAt)}` + : state === "stalled" + ? `${base} · quiet ${clockLabel(nowMs - session.lastActivityAt)}` + : base; + return { - stat: tool !== null && tool.length > 0 ? `${elapsed} · ${tool}` : elapsed, - working: !stalled, - stalled, + stat, + state, + working: state !== "stalled", + stalled: state === "stalled", }; } + +/** + * What the whole fleet is doing, rolled up from the per-lane states above. + * + * The top-level indicator otherwise reports the parent's own activity, and at + * fleet scale the parent is almost always just awaiting children — so it reads + * "working" permanently, including while every lane is stuck. Counting lanes + * here is what lets the indicator speak for the fleet instead. + */ +export type FleetProgress = { + readonly running: number; + readonly working: number; + readonly inTool: number; + readonly stalled: number; +}; + +export function fleetProgress( + sessions: readonly AgentProgressSession[], + nowMs: number, + stallMs: number = DEFAULT_STALL_MS, +): FleetProgress { + let working = 0; + let inTool = 0; + let stalled = 0; + for (const session of sessions) { + if (session.status !== "running") continue; + switch (laneState(session, nowMs, stallMs)) { + case "working": + working += 1; + break; + case "in_tool": + inTool += 1; + break; + case "stalled": + stalled += 1; + break; + } + } + return { running: working + inTool + stalled, working, inTool, stalled }; +} + +/** + * Compact fleet summary for the status ticker, or null with no live lanes — + * with no sub-agents running the indicator must behave exactly as it does for + * a plain single-agent turn. + */ +export function fleetLabel(fleet: FleetProgress): string | null { + if (fleet.running === 0) return null; + const parts = [`${fleet.running} agents`]; + if (fleet.stalled > 0) parts.push(`${fleet.stalled} stalled`); + else if (fleet.inTool === fleet.running) parts.push("in tools"); + return parts.join(" · "); +} diff --git a/src/tui-opentui/chrome-state.test.ts b/src/tui-opentui/chrome-state.test.ts index cd408b21a..6959cc2c3 100644 --- a/src/tui-opentui/chrome-state.test.ts +++ b/src/tui-opentui/chrome-state.test.ts @@ -7,6 +7,7 @@ import { formatTasksPanel, type ChromeLiveState, } from "./chrome-state" +import { agentProgress, laneState } from "./agent-progress" const NOW = 1_000_000 @@ -46,6 +47,7 @@ describe("formatChromeZones", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map setChromeZones callers", status: "running", currentToolName: "grep", @@ -54,6 +56,7 @@ describe("formatChromeZones", () => { }, { agentId: "general", + currentToolStartedAt: null, description: "write tests", status: "done", }, @@ -80,6 +83,7 @@ describe("formatChromeZones", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", }, @@ -157,13 +161,14 @@ describe("formatAgentsPanel", () => { test("one row per running agent, oldest-started first", () => { const rows = formatAgentsPanel( [ - { agentId: "a", description: "one", status: "running", startedAt: NOW - 1_000, lastActivityAt: NOW }, - { agentId: "b", description: "two", status: "running", startedAt: NOW - 2_000, lastActivityAt: NOW }, + { agentId: "a", description: "one", status: "running", currentToolStartedAt: null, startedAt: NOW - 1_000, lastActivityAt: NOW }, + { agentId: "b", description: "two", status: "running", currentToolStartedAt: null, startedAt: NOW - 2_000, lastActivityAt: NOW }, ], undefined, NOW, ) expect(rows).toEqual([ + { label: "2 agents", tail: "", stalled: false }, { label: "b: two", tail: " · 0:02", stalled: false }, { label: "a: one", tail: " · 0:01", stalled: false }, ]) @@ -173,8 +178,8 @@ describe("formatAgentsPanel", () => { expect( formatAgentsPanel( [ - { agentId: "a", description: "x", status: "done" }, - { agentId: "b", description: "y", status: "failed" }, + { agentId: "a", description: "x", status: "done", currentToolStartedAt: null }, + { agentId: "b", description: "y", status: "failed", currentToolStartedAt: null }, ], undefined, NOW, @@ -187,6 +192,7 @@ describe("formatAgentsPanel", () => { [ { agentId: "a", + currentToolStartedAt: null, description: "quiet worker", status: "running", startedAt: NOW - 60_000, @@ -197,21 +203,24 @@ describe("formatAgentsPanel", () => { NOW, ) expect(rows).toEqual([ - { label: "a: quiet worker", tail: " · 1:00 · stalled", stalled: true }, + { label: "a: quiet worker", tail: " · 1:00 · quiet 0:40 · stalled", stalled: true }, ]) }) test("bounds fan-out to maxVisible plus a +N more row", () => { const running = Array.from({ length: 8 }, (_, i) => ({ agentId: `agent-${i}`, + currentToolStartedAt: null, description: "working", status: "running" as const, startedAt: NOW, lastActivityAt: NOW, })) const rows = formatAgentsPanel(running, undefined, NOW, 5) - expect(rows).toHaveLength(6) - expect(rows?.[5]).toEqual({ label: "+3 more", tail: "", stalled: false }) + // Fleet summary, five lanes, then the fold-away row. + expect(rows?.[0]).toEqual({ label: "8 agents", tail: "", stalled: false }) + expect(rows).toHaveLength(7) + expect(rows?.[6]).toEqual({ label: "+3 more", tail: "", stalled: false }) }) test("observe empty id+desc hides", () => { @@ -226,9 +235,9 @@ describe("formatAgentsPanel", () => { // keying the visible row order on it would reshuffle the panel every // time any agent made progress — unreadable at a busy 200ms repaint. const frame1 = [ - { agentId: "b", description: "second", status: "running" as const, startedAt: NOW - 1_000, lastActivityAt: NOW - 1_000 }, - { agentId: "a", description: "first", status: "running" as const, startedAt: NOW - 2_000, lastActivityAt: NOW - 2_000 }, - { agentId: "c", description: "third", status: "running" as const, startedAt: NOW - 500, lastActivityAt: NOW - 500 }, + { agentId: "b", description: "second", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 1_000, lastActivityAt: NOW - 1_000 }, + { agentId: "a", description: "first", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 2_000, lastActivityAt: NOW - 2_000 }, + { agentId: "c", description: "third", status: "running" as const, currentToolStartedAt: null, startedAt: NOW - 500, lastActivityAt: NOW - 500 }, ] const rowsBefore = formatAgentsPanel(frame1, undefined, NOW) @@ -238,12 +247,12 @@ describe("formatAgentsPanel", () => { const frame2 = frame1.map((a) => (a.agentId === "b" ? { ...a, lastActivityAt: NOW + 200 } : a)) const rowsAfter = formatAgentsPanel(frame2, undefined, NOW + 200) - expect(rowsBefore?.map((r) => r.label.split(":")[0])).toEqual( - rowsAfter?.map((r) => r.label.split(":")[0]), - ) + const lanes = (rows: readonly { label: string }[] | null) => + rows?.slice(1).map((r) => r.label.split(":")[0]) + expect(lanes(rowsBefore)).toEqual(lanes(rowsAfter)) // Sanity: presentation order is oldest-started first (a, b, c), matching // the tiebreak-free startedAt sort. - expect(rowsBefore?.map((r) => r.label.split(":")[0])).toEqual(["a", "b", "c"]) + expect(lanes(rowsBefore)).toEqual(["a", "b", "c"]) }) test("a stalled agent stays visible over newer agents when the fan-out is truncated", () => { @@ -252,6 +261,7 @@ describe("formatAgentsPanel", () => { // to need attention is exactly the one that gets folded into "+N more". const newest = Array.from({ length: 5 }, (_, i) => ({ agentId: `fresh-${i}`, + currentToolStartedAt: null, description: "just started", status: "running" as const, startedAt: NOW, @@ -259,6 +269,7 @@ describe("formatAgentsPanel", () => { })) const stalled = { agentId: "quiet", + currentToolStartedAt: null, description: "gone silent", status: "running" as const, startedAt: NOW - 300_000, @@ -267,8 +278,9 @@ describe("formatAgentsPanel", () => { const rows = formatAgentsPanel([...newest, stalled], undefined, NOW, 5) expect(rows?.some((r) => r.label.includes("quiet"))).toBe(true) expect(rows?.some((r) => r.stalled)).toBe(true) - expect(rows).toHaveLength(6) - expect(rows?.[5]).toEqual({ label: "+1 more", tail: "", stalled: false }) + expect(rows).toHaveLength(7) + expect(rows?.[0]).toEqual({ label: "6 agents · 1 stalled", tail: "", stalled: true }) + expect(rows?.[6]).toEqual({ label: "+1 more", tail: "", stalled: false }) }) }) @@ -282,6 +294,7 @@ describe("chromeFromSession", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", @@ -296,6 +309,7 @@ describe("chromeFromSession", () => { expect(state.agents).toEqual([ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", @@ -320,6 +334,7 @@ describe("chromeFromSession", () => { id: "sess-1", description: "write tests", status: "done", + currentToolStartedAt: null, }, ], }) @@ -345,8 +360,8 @@ describe("chromeFromSession", () => { describe("annotateAgentTools", () => { const state: ChromeLiveState = { agents: [ - { agentId: "explore", description: "map callers", status: "running" }, - { agentId: "review", description: "map callers", status: "done" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, + { agentId: "review", description: "map callers", status: "done", currentToolStartedAt: null }, ], } @@ -363,3 +378,70 @@ describe("annotateAgentTools", () => { expect(next.agents?.[0]?.currentToolName).toBeUndefined() }) }) + + +describe("lane state survives the mapping hops", () => { + // The panel and the transcript trailer reach laneState by different routes. + // A hop that drops currentToolStartedAt silently reclassifies a busy lane as + // stalled — which is exactly how this shipped broken once, caught only by + // running it. The types make the drop a compile error; this proves the two + // routes still agree on a live example. + const inTool = { + id: "sess-1", + agentId: "worker", + description: "sleep 150", + status: "running" as const, + currentToolName: "run_shell", + currentToolStartedAt: NOW - 90_000, + startedAt: NOW - 100_000, + lastActivityAt: NOW - 90_000, + } + + test("the panel and the transcript row agree that the lane is in a tool", () => { + expect(laneState(inTool, NOW)).toBe("in_tool") + + const rows = formatAgentsPanel( + chromeFromSession({ agents: [inTool] }).agents, + undefined, + NOW, + ) + expect(rows?.[0]?.stalled).toBe(false) + expect(rows?.[0]?.tail).toContain("run_shell 1:30") + expect(rows?.[0]?.tail).not.toContain("stalled") + + expect(agentProgress(inTool, NOW)?.stat).toContain("run_shell 1:30") + }) + + test("a genuinely silent lane still reads stalled through the same hops", () => { + const silent = { ...inTool, currentToolName: null, currentToolStartedAt: null } + expect(laneState(silent, NOW)).toBe("stalled") + + const rows = formatAgentsPanel( + chromeFromSession({ agents: [silent] }).agents, + undefined, + NOW, + ) + expect(rows?.[0]?.stalled).toBe(true) + }) + + // A progress ping renames the tool but carries no clock of its own, so it + // must not override a call the store is already timing. + test("the tool annotation never repaints a live call with another name", () => { + const annotated = annotateAgentTools( + { agents: [inTool] }, + new Map([["sleep 150", "grep"]]), + ) + expect(annotated.agents?.[0]?.currentToolName).toBe("run_shell") + expect(annotated.agents?.[0]?.currentToolStartedAt).toBe(NOW - 90_000) + }) + + test("the tool annotation still fills a gap when no call is outstanding", () => { + const idle = { ...inTool, currentToolName: null, currentToolStartedAt: null } + const annotated = annotateAgentTools( + { agents: [idle] }, + new Map([["sleep 150", "grep"]]), + ) + expect(annotated.agents?.[0]?.currentToolName).toBe("grep") + expect(annotated.agents?.[0]?.currentToolStartedAt).toBeNull() + }) +}) diff --git a/src/tui-opentui/chrome-state.ts b/src/tui-opentui/chrome-state.ts index 3289b1bf6..710e7e7dc 100644 --- a/src/tui-opentui/chrome-state.ts +++ b/src/tui-opentui/chrome-state.ts @@ -22,7 +22,12 @@ * stale. Observe mode can override the agents line via `state.observe`. */ -import { agentProgress, DEFAULT_STALL_MS } from "./agent-progress.js" +import { + agentProgress, + fleetLabel, + fleetProgress, + DEFAULT_STALL_MS, +} from "./agent-progress.js" import { AGENTS_PANEL_MAX_VISIBLE, TASKS_PANEL_MAX_VISIBLE } from "./geometry/zones.js" import type { ChromeZoneContent } from "./shell.js" @@ -37,6 +42,8 @@ export type ChromeAgentSession = { readonly startedAt?: number /** Clock of the worker's last reported activity; feeds stalled detection. */ readonly lastActivityAt?: number + /** Clock the oldest outstanding tool call began; separates a long tool from silence. */ + readonly currentToolStartedAt: number | null } /** Lightweight task row: title + status, as written by the task tool. */ @@ -193,6 +200,26 @@ export function formatAgentsPanel( ) const rows = presented.map((s) => formatAgentRow(s, nowMs, stallMs)) if (hidden > 0) rows.push({ label: `+${hidden} more`, tail: "", stalled: false }) + + // Fleet roll-up first: past a couple of lanes an operator reads the summary, + // not six individual rows, and any row folded into "+N more" is otherwise + // invisible. Counted from the same lane states the rows below are rendered + // from, so the header can never disagree with them. + const fleet = fleetProgress( + running.map((s) => ({ + status: "running" as const, + currentToolName: s.currentToolName ?? null, + currentToolStartedAt: s.currentToolStartedAt ?? null, + startedAt: s.startedAt ?? 0, + lastActivityAt: s.lastActivityAt ?? s.startedAt ?? 0, + })), + nowMs, + stallMs, + ) + const summary = fleetLabel(fleet) + if (summary !== null && running.length > 1) { + rows.unshift({ label: summary, tail: "", stalled: fleet.stalled > 0 }) + } return rows } @@ -214,6 +241,7 @@ function formatAgentRow(session: ChromeAgentSession, nowMs: number, stallMs: num { status: "running", currentToolName: session.currentToolName ?? null, + currentToolStartedAt: session.currentToolStartedAt, startedAt: session.startedAt, lastActivityAt: session.lastActivityAt ?? session.startedAt, }, @@ -255,7 +283,15 @@ export function annotateAgentTools( agents: agents.map((a) => { if (a.status !== "running") return a const tool = toolByDescription.get(a.description) - return tool === undefined ? a : { ...a, currentToolName: tool } + if (tool === undefined) return a + // Gap-fill only. When the store has a call outstanding it owns both the + // name and the clock, and overriding just the name would paint one + // tool's identifier beside another tool's elapsed time. A progress ping + // also cannot supply a clock of its own — only the store observes a call + // ending, so a ping-sourced clock would keep a finished lane reading + // busy forever, hiding exactly the stalls this surface exists to show. + if (a.currentToolStartedAt !== null) return a + return { ...a, currentToolName: tool } }), } } @@ -280,6 +316,7 @@ export type ChromeSessionAgent = { readonly description: string readonly status: "running" | "done" | "failed" | "cancelled" readonly currentToolName?: string | null + readonly currentToolStartedAt: number | null readonly startedAt?: number readonly lastActivityAt?: number } @@ -340,6 +377,7 @@ function mapSessionAgents( ...(a.currentToolName !== undefined ? { currentToolName: a.currentToolName } : {}), + currentToolStartedAt: a.currentToolStartedAt, ...(a.startedAt !== undefined ? { startedAt: a.startedAt } : {}), ...(a.lastActivityAt !== undefined ? { lastActivityAt: a.lastActivityAt } diff --git a/src/tui-opentui/demo.ts b/src/tui-opentui/demo.ts index 96ca9d38a..7ef566be8 100644 --- a/src/tui-opentui/demo.ts +++ b/src/tui-opentui/demo.ts @@ -298,6 +298,7 @@ renderer.keyInput.on("keypress", (key: KeyEvent) => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", currentToolName: "grep", diff --git a/src/tui-opentui/geometry.test.ts b/src/tui-opentui/geometry.test.ts index 8179feb29..a58bc38db 100644 --- a/src/tui-opentui/geometry.test.ts +++ b/src/tui-opentui/geometry.test.ts @@ -100,8 +100,8 @@ describe("resolveGeometry — 80×24 idle floor", () => { describe("resolveGeometry — agents panel", () => { test("N running agents request N rows, bounded by the zone max", () => { - for (let n = 0; n <= AGENTS_PANEL_MAX_VISIBLE + 3; n++) { - const requested = Math.min(n, AGENTS_PANEL_MAX_VISIBLE + 1); + for (let n = 0; n <= AGENTS_PANEL_MAX_VISIBLE + 4; n++) { + const requested = Math.min(n, ZONE_REGISTRY.agents.max); const layout = idle80x24({ visibility: { agents: n } }); expect(layout.heights.agents).toBe(requested); } @@ -116,11 +116,12 @@ describe("resolveGeometry — agents panel", () => { test("a large fan-out never grows the zone past its bounded max", () => { const layout = idle80x24({ visibility: { agents: 50 } }); expect(layout.heights.agents).toBe(ZONE_REGISTRY.agents.max); - expect(layout.heights.agents).toBe(AGENTS_PANEL_MAX_VISIBLE + 1); + // Fleet summary + the visible lanes + the "+N more" trailer. + expect(layout.heights.agents).toBe(AGENTS_PANEL_MAX_VISIBLE + 2); }); test("a bounded agents panel never eats the transcript floor", () => { - const layout = idle80x24({ visibility: { agents: AGENTS_PANEL_MAX_VISIBLE + 1 } }); + const layout = idle80x24({ visibility: { agents: ZONE_REGISTRY.agents.max } }); expect(layout.transcriptHeight).toBeGreaterThanOrEqual(layout.transcriptFloor); }); diff --git a/src/tui-opentui/geometry/zones.ts b/src/tui-opentui/geometry/zones.ts index 34b6b8ce7..49906b5ee 100644 --- a/src/tui-opentui/geometry/zones.ts +++ b/src/tui-opentui/geometry/zones.ts @@ -84,12 +84,14 @@ export const ZONE_REGISTRY: { readonly [K in ZoneId]: ZoneDeclaration } = { idleDefault: 0, alwaysOn: false, }, - // One row per running agent (bounded by AGENTS_PANEL_MAX_VISIBLE) plus an - // optional trailing "+N more" row. + // A leading fleet-summary row, one row per running agent (bounded by + // AGENTS_PANEL_MAX_VISIBLE), then an optional trailing "+N more" row. All + // three must fit: clipping the last one drops the fold-away count at exactly + // the fan-out size where it is the only thing reporting the hidden lanes. agents: { id: "agents", min: 0, - max: AGENTS_PANEL_MAX_VISIBLE + 1, + max: AGENTS_PANEL_MAX_VISIBLE + 2, idleDefault: 0, alwaysOn: false, }, diff --git a/src/tui-opentui/product-host.test.ts b/src/tui-opentui/product-host.test.ts index c5d6fd118..7097cd4b2 100644 --- a/src/tui-opentui/product-host.test.ts +++ b/src/tui-opentui/product-host.test.ts @@ -286,6 +286,7 @@ describe("mountProductHost", () => { agents: [ { agentId: "explore", + currentToolStartedAt: null, description: "map callers", status: "running", startedAt: now - 59_000, diff --git a/src/tui-opentui/runner-host.test.ts b/src/tui-opentui/runner-host.test.ts index ba5fc8adc..957e90ea4 100644 --- a/src/tui-opentui/runner-host.test.ts +++ b/src/tui-opentui/runner-host.test.ts @@ -47,6 +47,8 @@ function session(over: Partial): SubAgentSession { status: "running", toolNames: [], currentToolName: null, + currentToolStartedAt: null, + outstandingTools: [], entries: [], startedAt: 0, lastActivityAt: 0, diff --git a/src/tui-opentui/runner-host.ts b/src/tui-opentui/runner-host.ts index d1ff977cf..a8fd66df0 100644 --- a/src/tui-opentui/runner-host.ts +++ b/src/tui-opentui/runner-host.ts @@ -272,6 +272,7 @@ export async function mountRunnerHost(deps: RunnerHostDeps): Promise id: s.id, status: s.status, currentToolName: s.currentToolName, + currentToolStartedAt: s.currentToolStartedAt, startedAt: s.startedAt, lastActivityAt: s.lastActivityAt, })), diff --git a/src/tui-opentui/runtime-bridge.test.ts b/src/tui-opentui/runtime-bridge.test.ts index e50fc7e5f..e9809da1d 100644 --- a/src/tui-opentui/runtime-bridge.test.ts +++ b/src/tui-opentui/runtime-bridge.test.ts @@ -651,6 +651,7 @@ describe("syncAgentProgress", () => { id: "task-1", status: "running", currentToolName: "grep", + currentToolStartedAt: null, startedAt: 0, lastActivityAt: 0, ...over, diff --git a/src/tui-opentui/runtime-bridge.ts b/src/tui-opentui/runtime-bridge.ts index 35433db3b..e9762c46a 100644 --- a/src/tui-opentui/runtime-bridge.ts +++ b/src/tui-opentui/runtime-bridge.ts @@ -70,7 +70,11 @@ import { } from "./tool-rows.js" import type { StreamRow } from "./stream.js" import { advanceRevealChars, flattenReasoningText, type Thought } from "./thinking.js" -import { agentProgress, type AgentProgressSession } from "./agent-progress.js" +import { + agentProgress, + fleetProgress, + type AgentProgressSession, +} from "./agent-progress.js" /** Tool name a sub-agent dispatch call carries — its row gets live progress. */ const TASK_TOOL_NAME = "task" @@ -336,6 +340,12 @@ type BridgeBag = { * find the handful that are sub-agent dispatches. */ taskCallIds: Set + /** + * Last sub-agent session list the host synced. Retained rather than consumed + * and dropped because the status ticker recomputes fleet state at paint time + * on the animation tick, not only when a worker happens to emit an event. + */ + agentSessions: readonly TaskProgressSession[] /** * Row index where the inference attempt in progress began, or null when no * boundary is armed. The mapper decides when to mark, clear and roll back; @@ -727,6 +737,7 @@ export function attachSessionBridge( toolRows: new Map(), lastToolRow: -1, taskCallIds: new Set(), + agentSessions: [], attemptRow: null, turnThinking: null, } @@ -805,7 +816,8 @@ export function attachSessionBridge( currentToolName: turn.currentToolName, streamingType: turn.streamingType, } - const label = resolveTurnLabel(input, isStalled) + const fleet = fleetProgress(bag.agentSessions, nowMs) + const label = resolveTurnLabel(input, isStalled, fleet) if (label === undefined) { // The bottom-left status slot rides the same re-entry as the landing // mark, so it crossfades between phases without a timer of its own. @@ -822,7 +834,7 @@ export function attachSessionBridge( applyCadence(bag.turn.quota !== null ? frozenTickMs : null) return } - const rampPhase = resolveRampPhase(input, isStalled) + const rampPhase = resolveRampPhase(input, isStalled, fleet) const stalledFor = stalledForMs(nowMs, rampPhase === "stalled") setLockupFrame(shell, { nowMs, @@ -1066,6 +1078,7 @@ export function attachSessionBridge( }, syncAgentProgress: (sessions) => { if (bag.disposed) return + bag.agentSessions = sessions syncAgentProgress(shell, bag, sessions, now()) }, dispose: () => { diff --git a/src/tui-opentui/runtime-channels.test.ts b/src/tui-opentui/runtime-channels.test.ts index 1c8daeb59..f38de2155 100644 --- a/src/tui-opentui/runtime-channels.test.ts +++ b/src/tui-opentui/runtime-channels.test.ts @@ -177,7 +177,7 @@ describe("subagent.progress channel", () => { const { host, emitter, frame, cleanup } = await mountHeadless({ chrome: { agents: [ - { agentId: "explore", description: "map callers", status: "running" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, ], }, }) @@ -205,7 +205,7 @@ describe("subagent.progress channel", () => { }) host.setChrome({ agents: [ - { agentId: "explore", description: "map callers", status: "running" }, + { agentId: "explore", description: "map callers", status: "running", currentToolStartedAt: null }, ], }) expect(await frame()).toContain("map callers · grep") diff --git a/src/tui-opentui/session-chrome.test.ts b/src/tui-opentui/session-chrome.test.ts index 58fcbc203..015e4fb98 100644 --- a/src/tui-opentui/session-chrome.test.ts +++ b/src/tui-opentui/session-chrome.test.ts @@ -47,6 +47,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, false, + null, ) expect(label).not.toBe(currentToolName) expect(ACTIVITY_STATES).toContain(label!) @@ -62,6 +63,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, true, + null, ) expect(label).toBe("stalled") expect(ACTIVITY_STATES).toContain(label!) @@ -76,6 +78,7 @@ describe("resolveTurnLabel closed-set guarantee", () => { streamingType: "tool", }, false, + null, ) expect(label).toBe("waiting") expect(label).not.toBe("working") @@ -94,6 +97,7 @@ describe("resolveTurnLabel", () => { streamingType: null, }, false, + null, ), ).toBeUndefined() }) @@ -108,6 +112,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("waiting") }) @@ -122,6 +127,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("stopping") }) @@ -136,6 +142,7 @@ describe("resolveTurnLabel", () => { streamingType: "tool", }, false, + null, ), ).toBe("researching") }) @@ -147,13 +154,13 @@ describe("resolveTurnLabel", () => { currentToolName: null, } expect( - resolveTurnLabel({ ...base, streamingType: "thinking" }, false), + resolveTurnLabel({ ...base, streamingType: "thinking" }, false, null), ).toBe("thinking") expect( - resolveTurnLabel({ ...base, streamingType: "text" }, false), + resolveTurnLabel({ ...base, streamingType: "text" }, false, null), ).toBe("working") expect( - resolveTurnLabel({ ...base, streamingType: null }, false), + resolveTurnLabel({ ...base, streamingType: null }, false, null), ).toBe("working") }) }) @@ -166,27 +173,27 @@ describe("resolveRampPhase", () => { } test("blocked gate freezes the ramp", () => { - expect(resolveRampPhase({ ...base, status: "blocked" }, false)).toBe("blocked") + expect(resolveRampPhase({ ...base, status: "blocked" }, false, null)).toBe("blocked") }) test("done fills the ramp", () => { - expect(resolveRampPhase({ ...base, status: "done" }, false)).toBe("done") + expect(resolveRampPhase({ ...base, status: "done" }, false, null)).toBe("done") }) test("everything else is working", () => { - expect(resolveRampPhase({ ...base, status: "running" }, false)).toBe("working") - expect(resolveRampPhase({ ...base, status: "stopping" }, false)).toBe("working") + expect(resolveRampPhase({ ...base, status: "running" }, false, null)).toBe("working") + expect(resolveRampPhase({ ...base, status: "stopping" }, false, null)).toBe("working") }) test("a stalled running turn paints stalled, not working", () => { expect( - resolveRampPhase({ ...base, status: "running" }, true), + resolveRampPhase({ ...base, status: "running" }, true, null), ).toBe("stalled") }) test("a blocked gate beats stalled — waiting on you outranks silence", () => { expect( - resolveRampPhase({ ...base, status: "blocked" }, true), + resolveRampPhase({ ...base, status: "blocked" }, true, null), ).toBe("blocked") }) }) @@ -242,3 +249,55 @@ describe("sendFailureText", () => { expect(classifySendFailureMessage("connection reset by peer")).toBe("error") }) }) + + +describe("fleet state in the top-level indicator", () => { + const parentAwaitingChildren = { + isProcessing: true, + status: "running" as const, + currentToolName: "task", + streamingType: "tool" as const, + } + const fleet = (running: number, stalled: number) => ({ + running, + working: running - stalled, + inTool: 0, + stalled, + }) + + test("a healthy fleet reads as orchestrating, not the parent's own tool", () => { + const label = resolveTurnLabel(parentAwaitingChildren, false, fleet(6, 0)) + expect(label).toBe("orchestrating") + expect(ACTIVITY_STATES).toContain(label!) + }) + + test("a stalled lane surfaces at the top level instead of staying on its row", () => { + expect(resolveTurnLabel(parentAwaitingChildren, false, fleet(6, 1))).toBe("stalled") + expect(resolveRampPhase(parentAwaitingChildren, false, fleet(6, 1))).toBe("stalled") + }) + + // The parent is idle by design while children run, so its own stall clock + // firing says nothing about whether the session is progressing. + test("live lanes outrank the parent's own stall clock", () => { + expect(resolveTurnLabel(parentAwaitingChildren, true, fleet(6, 0))).toBe("orchestrating") + expect(resolveRampPhase(parentAwaitingChildren, true, fleet(6, 0))).toBe("working") + }) + + test("with no sub-agents running the single-agent case is unchanged", () => { + const none = fleet(0, 0) + expect(resolveTurnLabel(parentAwaitingChildren, false, none)).toBe( + resolveTurnLabel(parentAwaitingChildren, false, null), + ) + expect(resolveTurnLabel(parentAwaitingChildren, true, none)).toBe("stalled") + expect(resolveRampPhase(parentAwaitingChildren, true, none)).toBe("stalled") + }) + + test("a blocked gate still outranks the fleet", () => { + expect( + resolveTurnLabel({ ...parentAwaitingChildren, status: "blocked" }, false, fleet(6, 3)), + ).toBe("waiting") + expect( + resolveTurnLabel({ ...parentAwaitingChildren, status: "stopping" }, false, fleet(6, 3)), + ).toBe("stopping") + }) +}) diff --git a/src/tui-opentui/session-chrome.ts b/src/tui-opentui/session-chrome.ts index e2bec6236..6a283ecf8 100644 --- a/src/tui-opentui/session-chrome.ts +++ b/src/tui-opentui/session-chrome.ts @@ -6,6 +6,7 @@ */ import type { Telemetry } from "../telemetry/index.js" +import type { FleetProgress } from "./agent-progress.js" import type { RampPhase } from "./ramp.js" /** Agent lifecycle status the progress label reads (mirrors the stream state). */ @@ -39,6 +40,7 @@ export const ACTIVITY_STATES = [ "building", "working", "waiting", + "orchestrating", "stalled", "stopping", ] as const @@ -91,12 +93,19 @@ function activityStateForTool(name: string | null): ActivityState { export function resolveTurnLabel( input: TurnLabelInput, isStalled: boolean, + fleet: FleetProgress | null, ): ActivityState | undefined { if (!input.isProcessing) return undefined if (input.status === "blocked") return "waiting" if (input.status === "stopping" || input.status === "stopped") { return "stopping" } + // Live lanes outrank the parent's own stall clock: while sub-agents run the + // parent is idle by design, so its silence says nothing about the session. + // The fleet is the thing actually working, so it is the thing reported. + if (fleet !== null && fleet.running > 0) { + return fleet.stalled > 0 ? "stalled" : "orchestrating" + } if (isStalled) return "stalled" if (input.currentToolName !== null) return activityStateForTool(input.currentToolName) if (input.streamingType === "thinking") return "thinking" @@ -112,9 +121,13 @@ export function resolveTurnLabel( export function resolveRampPhase( input: TurnLabelInput, isStalled: boolean, + fleet: FleetProgress | null, ): RampPhase { if (input.status === "blocked") return "blocked" if (input.status === "done") return "done" + if (fleet !== null && fleet.running > 0) { + return fleet.stalled > 0 ? "stalled" : "working" + } if (isStalled) return "stalled" return "working" } diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 54dbd300d..d5ff008ec 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -2036,6 +2036,7 @@ export async function runTUI(initialConfig: Config): Promise { description: s.description, status: s.status, currentToolName: s.currentToolName, + currentToolStartedAt: s.currentToolStartedAt, startedAt: s.startedAt, lastActivityAt: s.lastActivityAt, })),