diff --git a/src/main/json-claude-attachments.ts b/src/main/json-claude-attachments.ts index 01dbb898f..d80de4779 100644 --- a/src/main/json-claude-attachments.ts +++ b/src/main/json-claude-attachments.ts @@ -13,7 +13,7 @@ import { writeFileSync, mkdirSync, existsSync, readFileSync, statSync } from 'fs' import { tmpdir } from 'os' import { join } from 'path' -import { randomUUID } from 'crypto' +import { createHash, randomUUID } from 'crypto' const ATTACHMENT_DIR = join(tmpdir(), 'harness-attachments') @@ -59,3 +59,21 @@ export function writeAttachmentImage( writeFileSync(path, Buffer.from(base64Data, 'base64'), { mode: 0o600 }) return path } + +/** Same as writeAttachmentImage but keyed by content hash, so extracting + * the same image twice yields the same path and writes once. Tool-result + * images (browser screenshots) need this: resuming a session replays the + * whole transcript through the extractor, and uuid names would leak a + * fresh copy of every screenshot on every resume. */ +export function writeResultImage(base64Data: string, mediaType: string): string { + if (!existsSync(ATTACHMENT_DIR)) { + mkdirSync(ATTACHMENT_DIR, { recursive: true, mode: 0o700 }) + } + const ext = EXT_BY_MEDIA_TYPE[mediaType.toLowerCase()] || 'bin' + const hash = createHash('sha256').update(base64Data).digest('hex').slice(0, 32) + const path = join(ATTACHMENT_DIR, `result-${hash}.${ext}`) + if (!existsSync(path)) { + writeFileSync(path, Buffer.from(base64Data, 'base64'), { mode: 0o600 }) + } + return path +} diff --git a/src/main/json-claude-manager-fork.test.ts b/src/main/json-claude-manager-fork.test.ts index bead13e96..6e666cc63 100644 --- a/src/main/json-claude-manager-fork.test.ts +++ b/src/main/json-claude-manager-fork.test.ts @@ -39,6 +39,7 @@ vi.mock('child_process', () => ({ import { Store } from './store' import { JsonClaudeManager } from './json-claude-manager' +import type { JsonClaudeMessageBlock } from '../shared/state/json-claude' function transcriptDir(worktreePath: string): string { return join(tmpHome, '.claude', 'projects', worktreePath.replace(/[^a-zA-Z0-9]/g, '-')) @@ -366,3 +367,113 @@ describe('JsonClaudeManager.seedFromTranscript — mid-turn messages', () => { expect(entries[0].text).toBe('hello') }) }) + +// Browser-screenshot tool results arrive as Anthropic-shaped image +// blocks. The extractor spills them to disk and keeps a path, so the +// renderer can show a thumbnail without megabytes of base64 riding +// through every state event. +describe('JsonClaudeManager.seedFromTranscript — tool_result images', () => { + // 1x1 red JPEG. + const JPEG_B64 = + '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAIAQEAAD8AKp//2Q==' + const written: string[] = [] + + beforeEach(() => { + tmpHome = mkdtempSync(join(tmpdir(), 'harness-seed-img-')) + }) + + afterEach(() => { + rmSync(tmpHome, { recursive: true, force: true }) + for (const p of written) rmSync(p, { force: true }) + written.length = 0 + vi.clearAllMocks() + }) + + function seedWithScreenshot(sessionId: string, worktree: string): JsonClaudeMessageBlock { + const dir = transcriptDir(worktree) + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, `${sessionId}.jsonl`), + [ + { type: 'user', sessionId, message: { content: 'shot it' } }, + { + type: 'assistant', + sessionId, + message: { + id: 'msg_a', + content: [ + { + type: 'tool_use', + id: 'tu-shot', + name: 'mcp__ness-control__screenshot_tab' + } + ] + } + }, + { + type: 'user', + sessionId, + message: { + content: [ + { + type: 'tool_result', + tool_use_id: 'tu-shot', + content: [ + { type: 'text', text: 'took a shot' }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/jpeg', data: JPEG_B64 } + } + ] + } + ] + } + } + ] + .map((l) => JSON.stringify(l)) + .join('\n') + '\n', + 'utf8' + ) + const store = new Store() + store.dispatch({ + type: 'jsonClaude/sessionStarted', + payload: { sessionId, worktreePath: worktree } + }) + makeManager(store).seedFromTranscript(sessionId, worktree) + const entries = store.getSnapshot().state.jsonClaude.sessions[sessionId].entries + const resultEntry = entries.find((e) => e.kind === 'tool_result')! + const block = resultEntry.blocks![0] + for (const img of block.images ?? []) written.push(img.path) + return block + } + + it('spills the image to disk and keeps the text separate', () => { + const block = seedWithScreenshot( + '77777777-7777-7777-7777-777777777777', + '/tmp/wt-seed-shot' + ) + + expect(block.images).toHaveLength(1) + expect(block.images![0].mediaType).toBe('image/jpeg') + expect(existsSync(block.images![0].path)).toBe(true) + // The base64 never lands in state — only the path. + expect(block.content).toBe('took a shot') + expect(JSON.stringify(block)).not.toContain(JPEG_B64) + // The bytes on disk round-trip. + expect(readFileSync(block.images![0].path).toString('base64')).toBe(JPEG_B64) + }) + + it('reuses one file when the same image is extracted twice', () => { + // Resuming a session replays the whole transcript through the + // extractor; uuid-named files would leak a copy on every resume. + const first = seedWithScreenshot( + '88888888-8888-8888-8888-888888888888', + '/tmp/wt-seed-shot-a' + ) + const second = seedWithScreenshot( + '99999999-9999-9999-9999-999999999999', + '/tmp/wt-seed-shot-b' + ) + expect(second.images![0].path).toBe(first.images![0].path) + }) +}) diff --git a/src/main/json-claude-manager.ts b/src/main/json-claude-manager.ts index 63510db9c..aec0b1427 100644 --- a/src/main/json-claude-manager.ts +++ b/src/main/json-claude-manager.ts @@ -26,11 +26,13 @@ import { isPackaged, resolveBundledMcpScript } from './paths' import type { Store } from './store' import type { JsonClaudeChatEntry, + JsonClaudeImageRef, JsonClaudeMessageBlock, JsonClaudePermissionMode, JsonClaudeSessionState } from '../shared/state/json-claude' import { parseAutomatedMessage } from '../shared/state/json-claude' +import { writeResultImage } from './json-claude-attachments' import type { ClaudeLaunchSettings } from './claude-launch' import { log } from './debug' import { shellQuote } from './shell-quote' @@ -475,7 +477,8 @@ export class JsonClaudeManager { type: 'tool_result', toolUseId: r.toolUseId, content: r.content, - isError: r.isError + isError: r.isError, + ...(r.images ? { images: r.images } : {}) } ] }) @@ -1714,7 +1717,8 @@ export class JsonClaudeManager { sessionId: instance.sessionId, toolUseId: r.toolUseId, content: r.content, - isError: r.isError + isError: r.isError, + ...(r.images ? { images: r.images } : {}) } }) } @@ -2277,19 +2281,43 @@ function extractAssistantBlocks(ev: Record): JsonClaudeMessageB return out } -function extractToolResults( - ev: Record -): Array<{ toolUseId: string; content: string; isError: boolean }> { +interface ExtractedToolResult { + toolUseId: string + content: string + isError: boolean + images?: JsonClaudeImageRef[] +} + +function extractToolResults(ev: Record): ExtractedToolResult[] { const message = ev['message'] as { content?: unknown } | undefined const content = message?.content if (!Array.isArray(content)) return [] return extractToolResultsFromArray(content) } -function extractToolResultsFromArray( - content: unknown[] -): Array<{ toolUseId: string; content: string; isError: boolean }> { - const out: Array<{ toolUseId: string; content: string; isError: boolean }> = [] +/** MCP tools that return images (browser screenshots) surface them as + * Anthropic-shaped blocks: {type:'image', source:{type:'base64', + * media_type, data}}. Spill the bytes to a temp file and keep only the + * path — a PNG screenshot is megabytes of base64, and everything in a + * state event gets re-broadcast to every connected client. */ +function extractResultImage(part: Record): JsonClaudeImageRef | null { + if (part['type'] !== 'image') return null + const source = part['source'] + if (!source || typeof source !== 'object') return null + const s = source as Record + const data = s['data'] + const mediaType = s['media_type'] + if (typeof data !== 'string' || !data) return null + if (typeof mediaType !== 'string' || !mediaType.startsWith('image/')) return null + try { + return { path: writeResultImage(data, mediaType), mediaType } + } catch { + return null + } +} + +function extractToolResultsFromArray(content: unknown[]): ExtractedToolResult[] { + const out: ExtractedToolResult[] = [] for (const raw of content) { if (!raw || typeof raw !== 'object') continue const b = raw as Record @@ -2297,23 +2325,31 @@ function extractToolResultsFromArray( const id = typeof b['tool_use_id'] === 'string' ? (b['tool_use_id'] as string) : '' if (!id) continue const rawContent = b['content'] + const images: JsonClaudeImageRef[] = [] const text = typeof rawContent === 'string' ? rawContent : Array.isArray(rawContent) ? rawContent .map((p) => { - if (typeof p === 'object' && p && 'text' in (p as Record)) { - return String((p as Record)['text']) + if (!p || typeof p !== 'object') return '' + const part = p as Record + const img = extractResultImage(part) + if (img) { + images.push(img) + return '' } + if ('text' in part) return String(part['text']) return '' }) + .filter((s) => s !== '') .join('\n') : JSON.stringify(rawContent) out.push({ toolUseId: id, content: text, - isError: Boolean(b['is_error']) + isError: Boolean(b['is_error']), + ...(images.length > 0 ? { images } : {}) }) } return out diff --git a/src/renderer/components/JsonModeChat.tsx b/src/renderer/components/JsonModeChat.tsx index ef7b90c62..677170ffe 100644 --- a/src/renderer/components/JsonModeChat.tsx +++ b/src/renderer/components/JsonModeChat.tsx @@ -40,7 +40,7 @@ import { useJsonClaudeApprovals } from '../hooks/useJsonClaudeApprovals' import { JsonClaudeApprovalCard } from './JsonClaudeApprovalCard' import { JsonClaudeQuestionCard } from './JsonClaudeQuestionCard' import { Tooltip } from './Tooltip' -import { dispatchToolCard, ToolCardChrome } from './json-mode-cards' +import { dispatchToolCard, ToolCardChrome, type ToolResultView } from './json-mode-cards' import { NessIcon } from './json-mode-cards/tool-icons' import { ToolGroup } from './json-mode-cards/ToolGroup' import { TaskCard } from './json-mode-cards/TaskCard' @@ -292,6 +292,10 @@ interface RenderedRow { toolName?: string hasError?: boolean hasPendingApproval?: boolean + /** This row's tool returned an image (a browser screenshot). Bubbles + * up to ToolGroup so the group opens far enough to show it — a + * screenshot behind two collapsed chevrons may as well not be there. */ + hasImages?: boolean /** Marks this row as a thinking card. Lives in the 'tool' bucket so * it groups with adjacent tool_use rows (thinking + tools are both * agent work between user-facing replies), but ToolGroup counts it @@ -855,7 +859,7 @@ function AutomatedTurnCard({ } interface RenderContext { - resultsByToolUseId: Map + resultsByToolUseId: Map childrenByParentToolUseId: Map approvalCard: (toolUseId: string | undefined) => ReactNode pendingToolUseIds: Set @@ -1129,6 +1133,7 @@ function renderEntries( type: 'tool', toolName: block.name, hasError: !!result?.isError, + hasImages: !!result?.images && result.images.length > 0, hasPendingApproval: (!!block.id && ctx.pendingToolUseIds.has(block.id)) || subAgentDescendantHasPendingApproval, @@ -1681,17 +1686,15 @@ export function JsonModeChat({ sessionId, worktreePath, mode = 'awake' }: JsonMo // tool_use_id → tool_result lookup built once over the full // entries array (results live in top-level tool_result entries // even when their corresponding tool_use was a sub-agent's call). - const resultsByToolUseId = new Map< - string, - { content: string; isError: boolean } - >() + const resultsByToolUseId = new Map() for (const entry of deferredEntries) { if (entry.kind !== 'tool_result' || !entry.blocks) continue for (const b of entry.blocks) { if (b.type === 'tool_result' && b.toolUseId) { resultsByToolUseId.set(b.toolUseId, { content: b.content || '', - isError: !!b.isError + isError: !!b.isError, + ...(b.images && b.images.length > 0 ? { images: b.images } : {}) }) } } diff --git a/src/renderer/components/JsonModeChatImageThumb.tsx b/src/renderer/components/JsonModeChatImageThumb.tsx index 6e0b65526..10ea1944d 100644 --- a/src/renderer/components/JsonModeChatImageThumb.tsx +++ b/src/renderer/components/JsonModeChatImageThumb.tsx @@ -32,9 +32,18 @@ function fetchImage(path: string, mediaType: string): Promise { interface Props { path: string mediaType: string + /** 'square' crops to a 64px tile — right for pasted attachments, where + * the thumbnail is an affordance rather than something to read. + * 'wide' keeps the aspect ratio at 128px tall, for browser + * screenshots where a centre-crop would throw away the page. */ + shape?: 'square' | 'wide' } -export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element { +export function JsonModeChatImageThumb({ + path, + mediaType, + shape = 'square' +}: Props): JSX.Element { const [dataUrl, setDataUrl] = useState( CACHE.has(path) ? CACHE.get(path)! : null ) @@ -69,11 +78,16 @@ export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element }, [showFull]) const name = path.split('/').pop() || path + const boxClass = shape === 'wide' ? 'h-32 w-48' : 'h-16 w-16' + const imgClass = + shape === 'wide' + ? 'h-32 w-auto max-w-full object-contain bg-app' + : 'h-16 w-16 object-cover' if (pending) { return (
) @@ -81,7 +95,7 @@ export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element if (!dataUrl) { return (
@@ -99,7 +113,7 @@ export function JsonModeChatImageThumb({ path, mediaType }: Props): JSX.Element {name} {showFull && ( diff --git a/src/renderer/components/json-mode-cards/GenericToolCard.tsx b/src/renderer/components/json-mode-cards/GenericToolCard.tsx index 73597172e..02bf9b3f9 100644 --- a/src/renderer/components/json-mode-cards/GenericToolCard.tsx +++ b/src/renderer/components/json-mode-cards/GenericToolCard.tsx @@ -8,12 +8,14 @@ import { } from './index' import { ArgsBlock, CompactArgs } from './ArgsDisplay' import { HighlightedText } from '../JsonModeChatFind' +import { JsonModeChatImageThumb } from '../JsonModeChatImageThumb' export function GenericToolCard({ block, result, autoApproved, sessionAllowed }: ToolCardProps): JSX.Element { const brand = isNessControl(block.name) const display = getToolDisplay(block.name) const args = extractArgs(block.input) const hasArgs = args.length > 0 + const images = result?.images ?? [] return ( 0} > - {block.input && ( - 
-          
-
+ {images.length > 0 && ( +
+ {images.map((img) => ( + + ))} +
)} {hasArgs && } - {result && ( + {result && result.content.trim() !== '' && ( 
 !r.isThinking && isNessControl(r.toolName)
   )
-  // Auto-expand only for pending approvals — those need user action.
-  // Errors get a header badge but stay collapsed; user can drill in.
-  const wasAutoExpandedRef = useRef(hasPending)
-  const [expanded, setExpanded] = useState(hasPending)
-  useEffect(() => {
-    if (hasPending && !expanded) {
-      wasAutoExpandedRef.current = true
-      setExpanded(true)
-    } else if (!hasPending && wasAutoExpandedRef.current && expanded) {
-      wasAutoExpandedRef.current = false
-      setExpanded(false)
-    }
-  }, [hasPending, expanded])
+  // Auto-expand for pending approvals (they need user action) and for
+  // screenshots (they're the payload, not a detail). Errors get a header
+  // badge but stay collapsed; user can drill in.
+  //
+  // null means "no explicit choice yet, follow autoExpand" — so a group
+  // opens when an approval lands and closes again once it resolves,
+  // while a click pins it either way. Tracking the user's choice as its
+  // own state (rather than reverting via an effect) is what lets a
+  // screenshot group be collapsed at all: hasImages never goes back to
+  // false, so an effect-driven revert would immediately re-open it.
+  const autoExpand = hasPending || rows.some((r) => r.hasImages)
+  const [userChoice, setUserChoice] = useState(null)
+  const expanded = userChoice ?? autoExpand
 
   const toolRows = rows.filter((r) => !r.isThinking)
   const thinkingCount = rows.length - toolRows.length
@@ -81,10 +85,7 @@ export function ToolGroup({ rows }: { rows: ToolGroupRow[] }): JSX.Element {
       {anyBrand && 
}