Add read_agent_trace fleet verb (orchestrator tiers only)

CHANGELOG.md

@@ -39,6 +39,22 @@ parallel copies under `docs/` or `scripts/notes/`. At cut time: rename
   own), so the two layers no longer multiply. Attempt counts are now logged
   on each recovery so retry storms are visible in traces.
 
+- **`read_agent_trace` lets an orchestrator inspect a worker's on-disk trace
+  directly**, so a cancelled or interrupted worker's completed work is no
+  longer invisible just because its in-memory session record is gone. Reads
+  turns, tool calls, and tool errors straight from the worker's
+  `turns.jsonl`, tolerating a partially written or malformed line without
+  failing. Every response is bounded on four independent axes — turn window,
+  entry count, per-entry characters, and total output characters (the first
+  three multiply, so a total-output ceiling caps them together) — each with
+  a hard maximum the caller cannot exceed, and a truncated response says
+  exactly what was left out and how to page for the rest. A Tier 2 nested
+  orchestrator can only read its own descendants' traces, enforced by
+  reusing `SubAgentSessionStore`'s existing parentSessionId chain
+  (`assertCanTargetAgent`'s first live call site); leaf directors never see
+  the tool at all. `progress_note` for leaf workers is a separate,
+  not-yet-implemented follow-up.
+
 ### Fixed
 
 - **Interrupting a turn no longer risks a startup crash.** If an interrupt hit

src/agent/tools.ts

@@ -46,6 +46,7 @@ import { createWebSearchTool, disposeWebSearchClients } from "../tools/web-searc
 import { createUseSkillTool } from "./use-skill.js";
 import { createToolIndex, createToolSearchTool } from "./tool-search.js";
 import { createSearchAgentsTool } from "./agent-search.js";
+import { createReadAgentTraceTool } from "../subagent/trace-tool.js";
 import {
   createCodexToolProxies,
   type CodexRunManageTasks,
@@ -309,6 +310,11 @@ export async function createAgentToolset(args: AgentToolsetArgs): Promise<AgentT
                 }),
               ]
             : []),
+          // Tier 1: the primary session is always an orchestrator and may
+          // target any worker (assertCanTargetAgent's rule), so no authority
+          // context is passed here — omitting it is treated as unrestricted,
+          // matching Tier 1's actual authority.
+          createReadAgentTraceTool(args.subAgent.getWorkdirBase),
         ]
       : []),
     stringTool({

src/subagent/authority.ts

@@ -5,9 +5,10 @@
  * in a prompt. This module owns two checks:
  *
  *  - assertTierMayMountFleetVerb: a Tier 3 leaf may never mount a fleet verb
- *    (today: task, search_agents; the spawn_agent/wait_agents/list_agents/
- *    send_input/interrupt_agent/close_agent/resume_agent/read_agent_trace
- *    verbs land in later child issues against this same gate).
+ *    (today: task, search_agents, read_agent_trace; the spawn_agent/
+ *    wait_agents/list_agents/send_input/interrupt_agent/close_agent/
+ *    resume_agent/followup_task verbs land in later child issues against
+ *    this same gate).
  *  - assertCanTargetAgent: a Tier 2 nested orchestrator may act only on its
  *    own descendants, never a sibling or anything above it in the tree.
  *    Tier 1 (the primary orchestrator) may target anyone. Callers pass the

src/subagent/run.ts

@@ -105,6 +105,7 @@ import {
 } from "./stop-policy.js";
 import { SubAgentDirector } from "./nudge-director.js";
 import { assertTierMayMountFleetVerb } from "./authority.js";
+import { createReadAgentTraceTool } from "./trace-tool.js";
 import {
   abortError,
   createSubAgentSpawnRegistryPlugin,
@@ -435,7 +436,7 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
       // AgentProfile with orchestrator: true from mounting task/search_agents
       // just because it is outside the closed director set.
       const tier = params.orchestratorTier ?? "leaf";
-      for (const verb of ["task", "search_agents"]) {
+      for (const verb of ["task", "search_agents", "read_agent_trace"]) {
         assertTierMayMountFleetVerb(tier, verb);
       }
       if (params.nestedDispatch === undefined) {
@@ -481,6 +482,19 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
               }),
             ]
           : []),
+        // Every worker at every nesting depth is created under the same root
+        // workdirBase (nestedDispatch.getWorkdirBase is threaded through
+        // unchanged, never rebound to this worker's own dir), so the trace
+        // reader's search root is that same function. Descendant-only
+        // scoping is enforced inside the tool via assertCanTargetAgent,
+        // reusing the fleet nodes SubAgentSessionStore already tracks and
+        // this worker's own store id (params.id) — not the disk layout,
+        // which is intentionally flat across the whole fleet.
+        createReadAgentTraceTool(nd.getWorkdirBase, {
+          actorId: params.id,
+          tier,
+          getNodes: () => nd.sessions?.list() ?? [],
+        }),
       ];
     }
 
@@ -578,7 +592,14 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
       },
     });
 
-    const workdir = join(params.workdirBase, "subagents", generateSessionId());
+    // Reuse the caller's session-store id as the on-disk directory name
+    // when it is safe as a path segment, so read_agent_trace's descendant
+    // check can walk the same parentSessionId chain SubAgentSessionStore
+    // already tracks instead of needing a second, disk-only identity
+    // scheme.
+    const safeRequestedId =
+      params.id !== undefined && /^[A-Za-z0-9_-]+$/.test(params.id) ? params.id : undefined;
+    const workdir = join(params.workdirBase, "subagents", safeRequestedId ?? generateSessionId());
     await mkdir(workdir, { recursive: true });
     // One record per stop/nudge, with its measured value beside its threshold,
     // written into this leaf's own trace dir (CL-6938).

src/subagent/task-tool.ts

@@ -804,6 +804,10 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool {
             ...sandbox,
             cwd: worktreeCwd ?? deps.cwd,
             workdirBase: deps.getWorkdirBase(),
+            // Same id as the SubAgentSessionStore record so read_agent_trace's
+            // descendant check (authority.ts assertCanTargetAgent) can reuse the
+            // store's parentSessionId chain instead of a second identity scheme.
+            ...(session !== undefined ? { id: session.id } : {}),
             provider,
             ...(settings !== undefined ? { settings } : {}),
             ...(catalog !== undefined ? { catalog } : {}),

src/subagent/trace-reader.test.ts

@@ -0,0 +1,250 @@
+import { describe, test, expect } from "bun:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  AgentTraceNotFoundError,
+  findAgentTraceDir,
+  listUniqueSubdirs,
+  readAgentTrace,
+  MAX_TRACE_ENTRY_LIMIT,
+  MAX_TRACE_TOTAL_CHARS,
+  MAX_TRACE_TURN_WINDOW,
+} from "./trace-reader.js";
+
+function tempDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), "trace-reader-"));
+}
+
+function writeTurns(dir: string, turns: unknown[]): void {
+  fs.mkdirSync(dir, { recursive: true });
+  const text = turns.map((t) => JSON.stringify(t)).join("\n") + (turns.length > 0 ? "\n" : "");
+  fs.writeFileSync(path.join(dir, "turns.jsonl"), text);
+}
+
+describe("listUniqueSubdirs", () => {
+  test("a directory containing latest plus its target enumerates the session exactly once", async () => {
+    const root = tempDir();
+    const real = path.join(root, "01234567-89ab-7def-8123-456789abcdef");
+    fs.mkdirSync(real);
+    fs.symlinkSync(path.basename(real), path.join(root, "latest"));
+
+    const entries = await listUniqueSubdirs(root);
+    expect(entries).toHaveLength(1);
+    expect(entries[0]!.path).toBe(fs.realpathSync(real));
+  });
+
+  test("two distinct real directories are both listed", async () => {
+    const root = tempDir();
+    fs.mkdirSync(path.join(root, "a"));
+    fs.mkdirSync(path.join(root, "b"));
+    const entries = await listUniqueSubdirs(root);
+    expect(entries).toHaveLength(2);
+  });
+
+  test("a broken symlink is skipped, not thrown", async () => {
+    const root = tempDir();
+    fs.symlinkSync(path.join(root, "does-not-exist"), path.join(root, "dangling"));
+    const entries = await listUniqueSubdirs(root);
+    expect(entries).toHaveLength(0);
+  });
+
+  test("missing directory returns empty rather than throwing", async () => {
+    const entries = await listUniqueSubdirs(path.join(tempDir(), "nope"));
+    expect(entries).toHaveLength(0);
+  });
+});
+
+describe("findAgentTraceDir", () => {
+  test("finds a direct child under root/subagents", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "child-1");
+    writeTurns(childDir, []);
+    const found = await findAgentTraceDir(root, "child-1");
+    expect(found).toBe(fs.realpathSync(childDir));
+  });
+
+  test("finds a nested descendant several levels deep", async () => {
+    const root = tempDir();
+    const grandchildDir = path.join(root, "subagents", "child-1", "subagents", "grandchild-1");
+    writeTurns(grandchildDir, []);
+    const found = await findAgentTraceDir(root, "grandchild-1");
+    expect(found).toBe(fs.realpathSync(grandchildDir));
+  });
+
+  test("returns null for an unknown id", async () => {
+    const root = tempDir();
+    writeTurns(path.join(root, "subagents", "child-1"), []);
+    const found = await findAgentTraceDir(root, "does-not-exist");
+    expect(found).toBeNull();
+  });
+
+  test("is not confused by a latest symlink alongside the real worker dir", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "child-1");
+    writeTurns(childDir, []);
+    fs.symlinkSync("child-1", path.join(root, "subagents", "latest"));
+    const found = await findAgentTraceDir(root, "child-1");
+    expect(found).toBe(fs.realpathSync(childDir));
+  });
+});
+
+describe("readAgentTrace", () => {
+  test("throws a clean error for a missing target", async () => {
+    const root = tempDir();
+    await expect(readAgentTrace(root, "ghost")).rejects.toBeInstanceOf(AgentTraceNotFoundError);
+  });
+
+  test("reads turns, tool calls, and tool errors", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    writeTurns(childDir, [
+      { role: "user", content: [{ type: "text", text: "do the thing" }] },
+      {
+        role: "assistant",
+        content: [{ type: "tool_call", id: "call-1", name: "run_shell", arguments: { cmd: "ls" } }],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            callId: "call-1",
+            content: [{ type: "text", text: "boom" }],
+            isError: true,
+          },
+        ],
+      },
+    ]);
+
+    const result = await readAgentTrace(root, "worker-1");
+    expect(result.totalTurns).toBe(3);
+    expect(result.entries.map((e) => e.kind)).toEqual(["text", "tool_call", "error"]);
+    expect(result.entries[2]!.isError).toBe(true);
+    expect(result.omitted).toBeNull();
+  });
+
+  test("skips a malformed trailing line instead of throwing", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    fs.mkdirSync(childDir, { recursive: true });
+    const good = JSON.stringify({ role: "user", content: [{ type: "text", text: "hi" }] });
+    fs.writeFileSync(path.join(childDir, "turns.jsonl"), `${good}\n{"role":"assistant","cont`);
+
+    const result = await readAgentTrace(root, "worker-1");
+    expect(result.totalTurns).toBe(1);
+    expect(result.parseWarnings).toBe(1);
+    expect(result.entries).toHaveLength(1);
+  });
+
+  test("bounds the entry count to the requested limit and reports omission", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    const turns = Array.from({ length: 5 }, (_, i) => ({
+      role: "assistant",
+      content: [{ type: "text", text: `turn ${i}` }],
+    }));
+    writeTurns(childDir, turns);
+
+    const result = await readAgentTrace(root, "worker-1", { limit: 2 });
+    expect(result.entries).toHaveLength(2);
+    expect(result.entriesTruncated).toBe(true);
+    expect(result.omitted).not.toBeNull();
+    expect(result.omitted!.hint.length).toBeGreaterThan(0);
+  });
+
+  test("never exceeds the total-output character cap regardless of entry/window caps", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    const turns = Array.from({ length: 600 }, (_, i) => ({
+      role: "assistant",
+      content: [{ type: "text", text: `turn ${i} `.repeat(1000) }], // ~5,000 chars each
+    }));
+    writeTurns(childDir, turns);
+
+    const result = await readAgentTrace(root, "worker-1", {
+      fromTurn: 0,
+      toTurn: 600,
+      limit: MAX_TRACE_ENTRY_LIMIT,
+    });
+    const totalChars = result.entries.reduce((sum, e) => sum + e.content.length, 0);
+    expect(totalChars).toBeLessThanOrEqual(MAX_TRACE_TOTAL_CHARS);
+    expect(result.entriesTruncated).toBe(true);
+    expect(result.omitted).not.toBeNull();
+    expect(result.omitted!.reason).toContain("total output cap");
+  });
+
+  test("never exceeds the hard entry-limit cap regardless of requested limit", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    const turns = Array.from({ length: 10 }, (_, i) => ({
+      role: "assistant",
+      content: [{ type: "text", text: `turn ${i}` }],
+    }));
+    writeTurns(childDir, turns);
+
+    const result = await readAgentTrace(root, "worker-1", { limit: 1_000_000 });
+    expect(result.entries.length).toBeLessThanOrEqual(MAX_TRACE_ENTRY_LIMIT);
+  });
+
+  test("never exceeds the hard turn-window cap regardless of requested range", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    const turns = Array.from({ length: 500 }, (_, i) => ({
+      role: "assistant",
+      content: [{ type: "text", text: `turn ${i}` }],
+    }));
+    writeTurns(childDir, turns);
+
+    const result = await readAgentTrace(root, "worker-1", {
+      fromTurn: 0,
+      toTurn: 500,
+      limit: MAX_TRACE_ENTRY_LIMIT,
+    });
+    expect(result.toTurn - result.fromTurn).toBeLessThanOrEqual(MAX_TRACE_TURN_WINDOW);
+  });
+
+  test("filters entries by kind", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    writeTurns(childDir, [
+      {
+        role: "assistant",
+        content: [
+          { type: "thinking", thinking: "hmm" },
+          { type: "text", text: "hello" },
+        ],
+      },
+    ]);
+
+    const result = await readAgentTrace(root, "worker-1", { kinds: ["text"] });
+    expect(result.entries.map((e) => e.kind)).toEqual(["text"]);
+  });
+
+  test("truncates an oversized entry body and marks it truncated", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    writeTurns(childDir, [
+      { role: "assistant", content: [{ type: "text", text: "x".repeat(10_000) }] },
+    ]);
+
+    const result = await readAgentTrace(root, "worker-1");
+    expect(result.entries[0]!.truncated).toBe(true);
+    expect(result.entries[0]!.content.length).toBeLessThan(10_000);
+  });
+
+  test("a partially written trace (worker still running) reads what exists so far", async () => {
+    const root = tempDir();
+    const childDir = path.join(root, "subagents", "worker-1");
+    fs.mkdirSync(childDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(childDir, "turns.jsonl"),
+      `${JSON.stringify({ role: "user", content: [{ type: "text", text: "go" }] })}\n`,
+    );
+
+    const result = await readAgentTrace(root, "worker-1");
+    expect(result.totalTurns).toBe(1);
+    expect(result.entries).toHaveLength(1);
+  });
+});