diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 89f88ec8e..d0ba232f2 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -1266,6 +1266,35 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run. Dropping downloadId and filePath is +// therefore not just prompt economy: filePath is a CDP-backed read of any local +// path into an untrusted page's input, and compact omits the full-tier guidance +// that exists to stop the model inventing one. Deleting the keys rather than +// rebuilding `parameters` keeps any future base parameter reaching compact. +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; + +function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; + return { + ...tool, + function: { + ...tool.function, + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + parameters: { + ...tool.function.parameters, + properties: { + ...properties, + selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Never guess an id.' }, + }, + required: ['selector'], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1285,7 +1314,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1698,10 +1729,16 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', + // get_interactive_elements is the only tool that returns a verified, unique + // CSS selector for a file input, so upload_file's ambiguous-selector + // recovery is built on it. It stays in the compact catalog for as long as + // upload_file does; without it an ambiguous selector is unrecoverable. + 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1734,6 +1771,7 @@ TOOLS — use ONLY these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images. +- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. PREFERRED. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1747,6 +1785,7 @@ TOOLS — use ONLY these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch a URL for its content. +- upload_file({selector, attachmentId}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - done({summary, outcome}): Signal success, partial progress, or a failed blocker. diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 966bb4942..f0b807922 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -989,10 +989,16 @@ export const COMPACT_TOOL_NAMES = new Set([ 'get_accessibility_tree', 'read_page', 'scroll', 'get_window_info', 'extract_data', 'get_selection', 'find_text', + // get_interactive_elements is the only tool that returns a verified, unique + // CSS selector for a file input, so upload_file's ambiguous-selector + // recovery is built on it. It stays in the compact catalog for as long as + // upload_file does; without it an ambiguous selector is unrecoverable. + 'get_interactive_elements', 'click_ax', 'set_checked', 'type_ax', 'set_field', 'click', 'type_text', 'press_keys', 'navigate', 'new_tab', 'wait_for_element', 'fetch_url', + 'upload_file', 'scratchpad_write', 'progress_update', 'progress_read', 'clarify', 'done', ]); @@ -1114,6 +1120,33 @@ const WATCH_BEEP_TOOL = { }, }; +// Compact has no download tools, so the only file it can legitimately reach is +// the one the user attached to this run, or one the user picks themselves. +// Deleting the keys rather than rebuilding `parameters` keeps any future base +// parameter reaching compact. Firefox never had filePath (no CDP). +const COMPACT_UPLOAD_HIDDEN_PARAMS = ['downloadId', 'filePath']; + +function compactUploadFileTool(tool) { + const properties = { ...tool.function.parameters.properties }; + for (const key of COMPACT_UPLOAD_HIDDEN_PARAMS) delete properties[key]; + return { + ...tool, + function: { + ...tool.function, + description: 'Attach a user-provided file directly to an existing file input without clicking the page upload control. Use attachmentId from the current user-attachment notice, or omit it to open WebBrain\'s file picker. This proves only local page attachment, not remote upload or submission. Use the exact selector for the intended input; never guess a generic input[type="file"] selector when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If the widget creates its input lazily, make one guarded click on its add-files control, then retry upload_file with the exact selector.', + parameters: { + ...tool.function.parameters, + properties: { + ...properties, + selector: { type: 'string', description: 'Exact CSS selector for the intended file input.' }, + attachmentId: { type: 'string', description: 'Opaque id from the current user-attachment notice. Omit only to ask the user through WebBrain\'s file picker; never guess an id.' }, + }, + required: ['selector'], + }, + }, + }; +} + /** * Get tools filtered by mode. * @@ -1132,7 +1165,9 @@ export function getToolsForMode(mode, opts = {}) { } else if (devCompactBlocked) { base = []; } else if (tier === 'compact') { - base = AGENT_TOOLS.filter(t => COMPACT_TOOL_NAMES.has(t.function.name)); + base = AGENT_TOOLS + .filter(t => COMPACT_TOOL_NAMES.has(t.function.name)) + .map(t => (t.function.name === 'upload_file' ? compactUploadFileTool(t) : t)); } else if (tier === 'mid') { base = AGENT_TOOLS.filter(t => MID_TOOL_NAMES.has(t.function.name)); } else { @@ -1217,6 +1252,7 @@ TOOLS - use only these: - get_window_info: Read window/viewport size. - scroll: Scroll up/down. - extract_data: Get tables, headings, images, or links. +- get_interactive_elements: List interactive elements with exact CSS selectors. Use it when you need a selector rather than a ref_id — above all to find the intended file input before upload_file, and to recover after an ambiguous upload selector. - click_ax({ref_id}): Click by ref_id from the tree. Preferred. - set_checked({ref_id, checked}): Idempotently set and verify a native checkbox. Never toggle checkboxes repeatedly with click_ax. - type_ax({ref_id, text}): Type into a field by ref_id. @@ -1230,6 +1266,7 @@ TOOLS - use only these: - new_tab({url}): Open a URL in a background tab for user reference. It does not activate or retarget the current run, so never use it as a site-permission workaround. - wait_for_element({selector}): Wait for an element to appear. - fetch_url({url}): Fetch other URLs for reading only; do not use it to re-read the active tab. +- upload_file({selector, attachmentId?}): Attach a user-provided file directly to an existing file input; do not click the page upload control first. Use the current attachmentId or omit it for WebBrain's picker. Use the exact selector from get_interactive_elements and never guess generic input[type="file"] when multiple inputs exist. If the selector is ambiguous, call get_interactive_elements and retry with the exact selector it returns. If the widget creates its input lazily, make one guarded initializer click, re-read the page, then retry with the exact selector. Verify the page shows the attachment before submitting. - scratchpad_write({text}): Save notes that persist across steps. - progress_update({items}) / progress_read({status}): Structured progress ledger for the active repeated item/action task. On GitHub stargazers, only "Follow USER" buttons are follow targets when following is allowed by the task; "Unfollow USER" means skip/already followed unless the ledger shows acted. - clarify({question, options?}): Ask the user only when materially blocked or ambiguous. Unanswered clarifies auto-select options[0] after timeout (source=timeout is not user approval for high-risk steps; source=auto Instant is intentional auto-approve). diff --git a/test/run.js b/test/run.js index 8c24ea4a2..22985dfec 100644 --- a/test/run.js +++ b/test/run.js @@ -12785,6 +12785,7 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => [...compactNames].sort(), ); assert.ok(compactNamesActual.includes('done'), `[${label}] compact mode must keep done`); + assert.ok(compactNamesActual.includes('upload_file'), `[${label}] compact mode must expose upload_file`); for (const excluded of ['resize_window', 'download_social_media', 'solve_captcha']) { assert.equal(compactNamesActual.includes(excluded), false, `[${label}] compact mode must omit ${excluded}`); } @@ -12792,6 +12793,101 @@ test('getToolsForMode: compact mode restricts act tools in both browsers', () => } }); +test('compact Act exposes a direct-upload-only file workflow in both browsers', () => { + for (const [label, getTools, prompt] of [ + ['chrome', getToolsForModeCh, SYSTEM_PROMPT_ACT_COMPACT_CH], + ['firefox', getToolsForModeFx, SYSTEM_PROMPT_ACT_COMPACT_FX], + ]) { + const askNames = getTools('ask').map(tool => tool.function.name); + const compactTools = getTools('act', { tier: 'compact' }); + const compactNames = compactTools.map(tool => tool.function.name); + const upload = compactTools.find(tool => tool.function.name === 'upload_file'); + const fullUpload = getTools('act').find(tool => tool.function.name === 'upload_file'); + + assert.ok(upload, `[${label}] compact Act must expose upload_file`); + assert.equal(askNames.includes('upload_file'), false, `[${label}] Ask must remain read-only`); + for (const unavailable of [ + 'download_files', + 'download_resource_from_page', + 'list_downloads', + 'read_downloaded_file', + ]) { + assert.equal(compactNames.includes(unavailable), false, `[${label}] compact Act exposed ${unavailable}`); + } + assert.match(upload.function.description, /directly to an existing file input/i); + assert.match(upload.function.description, /without clicking the page upload control/i); + assert.match(upload.function.description, /exact selector/i); + assert.match(upload.function.description, /one guarded click/i); + assert.doesNotMatch(upload.function.description, /download_files|list_downloads|downloadId/i); + assert.ok(upload.function.parameters.properties.attachmentId, `[${label}] compact upload must accept attachmentId`); + assert.ok(fullUpload.function.parameters.properties.downloadId, `[${label}] full upload must retain downloadId`); + + // Compact can reach only the file the user attached to this run: it has no + // download tools to produce a downloadId, and no way to learn a local path + // that the model did not invent. + for (const hidden of ['downloadId', 'filePath']) { + assert.equal( + upload.function.parameters.properties[hidden], + undefined, + `[${label}] compact upload must hide ${hidden}`, + ); + } + assert.deepEqual( + Object.keys(upload.function.parameters.properties).sort(), + ['attachmentId', 'selector'], + `[${label}] compact upload must expose exactly selector + attachmentId`, + ); + + // An ambiguous selector latches upload_file until get_interactive_elements + // returns a verified file-input selector, so the recovery tool has to be in + // the catalog the model is given. + assert.ok( + compactNames.includes('get_interactive_elements'), + `[${label}] compact Act must expose upload_file's ambiguity-recovery tool`, + ); + assert.match(upload.function.description, /get_interactive_elements/); + + // Assert on the prompt's own upload_file bullet rather than on character + // distances, so rewording the neighbouring bullets cannot break these. + const uploadLine = prompt.split('\n').find(line => line.startsWith('- upload_file(')); + assert.ok(uploadLine, `[${label}] compact prompt must document upload_file`); + assert.match(uploadLine, /do not click the page upload control/i); + assert.match(uploadLine, /exact selector/i); + assert.match(uploadLine, /one guarded initializer click/i); + assert.match(uploadLine, /get_interactive_elements/); + assert.ok( + prompt.split('\n').some(line => line.startsWith('- get_interactive_elements')), + `[${label}] compact prompt must document the recovery tool it points at`, + ); + assert.doesNotMatch(prompt, /download_files|list_downloads|downloadId/i); + + if (label === 'firefox') { + assert.match(upload.function.description, /WebBrain's file picker/i); + } else { + assert.doesNotMatch(uploadLine, /filePath/); + } + } +}); + +test('every tier exposing upload_file also exposes its ambiguity-recovery tool', () => { + // upload_file latches on an ambiguous selector and only a + // get_interactive_elements response carrying a verified file-input selector + // clears it (Agent._clearUploadSelectorRecoveryAfterInspection). A tier that + // ships upload_file without it can never recover: the handler keeps + // returning recoveryRequired:'get_interactive_elements' and the model has no + // way to satisfy it until the page is replaced. + for (const [label, getTools] of [['chrome', getToolsForModeCh], ['firefox', getToolsForModeFx]]) { + for (const tier of ['compact', 'mid', 'full']) { + const names = new Set(getTools('act', { tier }).map(tool => tool.function.name)); + if (!names.has('upload_file')) continue; + assert.ok( + names.has('get_interactive_elements'), + `[${label}] act/${tier} exposes upload_file without get_interactive_elements, so an ambiguous selector is unrecoverable`, + ); + } + } +}); + test('getToolsForMode: mode/tier redesign exposes the intended normal and Dev tools', () => { for (const [label, getTools] of [ ['chrome', getToolsForModeCh], @@ -50221,7 +50317,7 @@ test('user attachment upload guidance follows the active tier tool catalog', () ]); for (const [mode, tier, shouldAdvertiseUpload] of [ - ['act', 'compact', false], + ['act', 'compact', true], ['act', 'mid', true], ['act', 'full', true], ['ask', 'full', false], @@ -50776,6 +50872,20 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i true, ); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false); + + // The retry the recovery exists to enable: once the inspection has supplied + // a unique selector, the corrected upload must actually dispatch. Without + // this the latch could clear and still leave uploads wedged. + selectorMatches = ['input-501']; + const recoveredRetry = await agent.executeTool(42, 'upload_file', { + selector: 'input[type=file]:not([accept])', + downloadId: 9123, + }); + assert.equal(recoveredRetry.success, true, 'a corrected selector must upload after recovery'); + assert.equal(recoveredRetry.file, realPath); + assert.equal(recoveredRetry.attachmentState, 'input_attached'); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'a successful retry must leave the latch clear'); + agent._uploadSelectorRecoveryRequired.set(42, 2); agent._clearRunLoopState(42); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'run cleanup must clear upload recovery'); @@ -50784,8 +50894,8 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'navigation cleanup must clear upload recovery'); assert.deepEqual( releasedGroups, - ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4'], - 'early upload failures must release selector handles', + ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4', 'upload-query-5'], + 'early upload failures and the post-recovery retry must release selector handles', ); } finally { if (originalChrome === undefined) delete globalThis.chrome;