From 25ff5a9c8ad1c35f7f92f5674f0bbdc1ec542c12 Mon Sep 17 00:00:00 2001 From: Barack Sokullu Date: Tue, 4 Aug 2026 15:36:25 +0300 Subject: [PATCH 1/2] Fix ambiguous file uploads --- src/chrome/src/agent/adapters.js | 10 ++ src/chrome/src/agent/agent.js | 70 ++++++++++- src/chrome/src/agent/tools.js | 4 +- src/chrome/src/content/content.js | 72 ++++++++++- src/firefox/src/agent/adapters.js | 10 ++ src/firefox/src/agent/agent.js | 95 ++++++++++++++- src/firefox/src/agent/tools.js | 4 +- src/firefox/src/content/content.js | 72 ++++++++++- test/fixtures/run.mjs | 101 +++++++++++++++- test/run.js | 184 ++++++++++++++++++++++++++++- 10 files changed, 601 insertions(+), 21 deletions(-) diff --git a/src/chrome/src/agent/adapters.js b/src/chrome/src/agent/adapters.js index 26bc145eb..4c8e1e9c0 100644 --- a/src/chrome/src/agent/adapters.js +++ b/src/chrome/src/agent/adapters.js @@ -15780,6 +15780,16 @@ const ADAPTERS = [ - Releases live at ///-/releases/new. Tag must exist or be created via "Create tag" inline. - Merge requests have a "Merge" button that may be disabled until pipelines pass; check the pipeline status before clicking. - The sidebar collapses on narrow viewports — scroll horizontally or expand it before clicking sidebar items.`, + }, + { + name: 'huggingface', + category: 'general', + matches: (url) => /^https?:\/\/(?:www\.)?huggingface\.co(?:[/?#]|$)/i.test(url), + notes: ` +- Repository upload routes expose two file inputs. Use \`input[type="file"]:not([accept])\` for repository files; \`input[type="file"][accept*="image"]\` belongs to the extended-description editor and does not stage a repository file. +- When the repository input already exists, call \`upload_file\` directly; do not click "Upload file(s)" or the drop zone first. +- A filename chip, generated commit summary, and enabled "Commit changes" button mean the file is staged only. Click "Commit changes", wait, and verify the file under "Files and versions" before reporting upload success. +- For model-card media, commit the asset first, then edit README Markdown to reference it, commit README, and verify the rendered model card.`, }, { name: 'mozilla-addons-developer', diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 1cf1c02a2..eca1587d8 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -414,6 +414,7 @@ export class Agent extends LoopDetector { this._lastClickProgress = new Map(); // tabId -> { ident, snapshot } this._clickAxCdpFallbacks = new Map(); // tabId -> Set(documentToken|ref_id), one trusted fallback per document target this._lastAxScopes = new Map(); // tabId -> { documentToken, pageUrl }, captured by the latest AX read + this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1664,6 +1665,7 @@ export class Agent extends LoopDetector { */ _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); + this._uploadSelectorRecoveryRequired.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1675,6 +1677,12 @@ export class Agent extends LoopDetector { } } + _clearUploadSelectorRecoveryAfterInspection(tabId, name, response) { + if (name !== 'get_interactive_elements' || !Array.isArray(response)) return false; + this._uploadSelectorRecoveryRequired.delete(tabId); + return true; + } + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -1715,6 +1723,7 @@ export class Agent extends LoopDetector { && this._isSuccessfulExecutionEvidence(result) && result?.noProgress !== true && result?.verified !== false + && result?.remoteStateVerified !== false && result?.inconclusive !== true; } @@ -3588,7 +3597,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ) { return 'action_failed'; } - if (toolResult?.inconclusive || toolResult?.verified === false) { + if ( + toolResult?.inconclusive + || toolResult?.verified === false + || toolResult?.remoteStateVerified === false + ) { return 'action_unverified'; } if ( @@ -13036,7 +13049,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } case 'upload_file': { if (parsed.success === false) return `upload failed: ${this._truncate(parsed.error || '', 110)}`; - if (parsed.attached) return `uploaded ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`; + if (parsed.remoteStateVerified === false) { + const localState = parsed.attachmentState === 'page_consumed' + ? 'page consumed attachment' + : 'file attached to input'; + return `${localState} (remote submission unverified)`; + } + if (parsed.attached) return `attached ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`; return parsed.verified === false ? `upload sent (unverified)` : `uploaded ${this._truncate(parsed.file || '', 70)}`; } case 'new_tab': { @@ -16650,6 +16669,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { args = args || {}; + if (this._uploadSelectorRecoveryRequired.has(tabId)) { + return { + success: false, + dispatched: false, + noDispatch: true, + ambiguous: true, + matchCount: Number(this._uploadSelectorRecoveryRequired.get(tabId)) || 0, + recoveryRequired: 'get_interactive_elements', + error: 'A previous upload selector matched multiple file inputs. Call get_interactive_elements now and use the exact selector returned on the intended file-input record before retrying upload_file; do not guess another selector variant.', + }; + } // Accept a downloadId as an alternative to filePath. After context // compaction the model often can't recall the exact on-disk path, but the // small integer id (returned by download_files/list_downloads and @@ -16697,9 +16727,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d return { success: false, error: `File input not found for selector "${args.selector}". Re-inspect the page with get_interactive_elements or get_accessibility_tree to find the real (some upload widgets hide it until you click their "add files" button first).` }; } if (objectIds.length > 1) { + this._uploadSelectorRecoveryRequired.set(tabId, objectIds.length); return { success: false, - error: `Selector "${args.selector}" matched ${objectIds.length} elements across the document and open shadow roots. Use an exact, unique selector for the intended ; do not use a generic input[type=file] selector when multiple inputs exist.`, + dispatched: false, + noDispatch: true, + ambiguous: true, + matchCount: objectIds.length, + recoveryRequired: 'get_interactive_elements', + error: `Selector "${args.selector}" matched ${objectIds.length} elements across the document and open shadow roots. Call get_interactive_elements and use the exact, unique selector returned on the intended file-input record; do not guess another selector variant.`, }; } @@ -16766,7 +16802,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // success for the model to confirm against the page, NOT a hard // failure: the old hard failure made the model loop, re-uploading a // file that was already attached and clobbering the page. - return { success: true, file: args.filePath, verified: false, note: `The file input is empty after upload — this usually means an async uploader (e.g. a GitHub release attachment) already consumed the file. Confirm "${basename}" now appears attached via get_accessibility_tree before re-uploading; only retry if it is genuinely missing (and if so, re-check the path with list_downloads).` }; + return { + success: true, + file: args.filePath, + verified: false, + attachmentState: 'page_consumed', + remoteStateVerified: false, + note: `The page consumed the file input, but upload_file does not prove a remote upload or form submission. Confirm "${basename}" appears attached via get_accessibility_tree, then submit/commit the page when the task requires it. Only retry if the file is genuinely missing.`, + }; } const attached = files.find(f => f.name === basename) || files[files.length - 1] || null; // readable === false means the bytes couldn't be read — the path is @@ -16777,7 +16820,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (attached.readable === false) { return { success: false, dispatched: true, error: `"${args.filePath}" could not be read — it almost certainly does not exist at that path. Confirm the absolute path (use list_downloads to see where files were actually saved) and retry.` }; } - return { success: true, file: args.filePath, attached: { name: attached.name, size: attached.size } }; + return { + success: true, + file: args.filePath, + attached: { name: attached.name, size: attached.size }, + verified: false, + attachmentState: 'input_attached', + remoteStateVerified: false, + }; } // Could not read the FileList back. If the probe confirmed the path is @@ -16788,7 +16838,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (!pathConfirmed) { return { success: false, dispatched: true, error: `Could not confirm "${basename}" uploaded: the input.files list was unreadable and the local path "${args.filePath}" was not validated. Check whether "${basename}" appears attached via get_accessibility_tree — if it does, you're done; if not, re-check the path with list_downloads and the selector, then retry.` }; } - return { success: true, file: args.filePath, verified: false, note: 'Attachment could not be verified (the input.files list was unreadable), but the local path validated as readable. If the file does not appear attached on the page, re-check the selector.' }; + return { + success: true, + file: args.filePath, + verified: false, + attachmentState: 'page_consumed', + remoteStateVerified: false, + note: 'The local file was readable and the page handled the attachment event, but upload_file could not read the resulting FileList. This does not prove a remote upload or form submission; verify the page state and submit/commit when required.', + }; } catch (e) { return { success: false, dispatched: uploadDispatched, error: `Upload failed: ${e.message}` }; } finally { @@ -18560,6 +18617,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'read_page') { response = applyReadPageWindow(response, args); } + this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); return response; } finally { clickAxSideEffectWatch?.stop(); diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js index 1aa044667..1af602e25 100644 --- a/src/chrome/src/agent/tools.js +++ b/src/chrome/src/agent/tools.js @@ -923,7 +923,7 @@ export const AGENT_TOOLS = [ type: 'function', function: { name: 'upload_file', - description: 'Upload a file directly to an existing file input without opening the page or OS file-picker dialog. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide EITHER downloadId (preferred — the id from download_files/list_downloads; you do not need to recall the path) OR filePath (absolute local path). Never guess a downloadId. If both are accidentally provided, a valid downloadId is preferred; if that id cannot resolve, the supplied filePath is used as a fallback. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. The file must exist on the local filesystem.', + description: 'Attach a file directly to an existing file input without opening the page or OS file-picker dialog. This only proves that the page input received or consumed the file; it does NOT prove a remote upload, form submission, or repository commit. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide EITHER downloadId (preferred — the id from download_files/list_downloads; you do not need to recall the path) OR filePath (absolute local path). Never guess a downloadId. If both are accidentally provided, a valid downloadId is preferred; if that id cannot resolve, the supplied filePath is used as a fallback. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. The file must exist on the local filesystem.', parameters: { type: 'object', properties: { @@ -1616,7 +1616,7 @@ CLICKING — read this: - For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing. - If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector. - You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`. -- FILE UPLOADS: when the page already has an \`\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. Find the exact selector and call \`upload_file({selector, downloadId})\` directly; it attaches the file without opening a native dialog. Exception: if \`upload_file\` reports that no input exists because the widget creates it lazily, make one guarded click on the widget's add-files control to initialize it. A blocked-picker result may return the new exact selector; retry \`upload_file\` with that selector. Never substitute a generic \`input[type="file"]\` selector when multiple file inputs exist. +- FILE UPLOADS: when the page already has an \`\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. Call \`get_interactive_elements\` when needed and use the exact \`selector\` returned on the intended file-input record, then call \`upload_file({selector, downloadId})\`. \`attachmentState\` proves only local input attachment/page consumption; it does NOT prove a remote upload or submit. Verify the filename/status in the page, then activate and verify the required Submit/Commit control. If \`upload_file\` reports an ambiguous selector, a fresh \`get_interactive_elements\` call is required before retrying. Exception: if no input exists because the widget creates it lazily, make one guarded click on its add-files control to initialize it. - Order of preference: 1. \`click_ax({ref_id: "ref_N"})\` — ref_id from get_accessibility_tree. Most reliable; carries role+name so you always know what you're clicking, and ref_ids are stable across calls. 2. \`click({text: "..."})\` — visible button/link text. Good fallback if the tree didn't surface the element cleanly. diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index b9ceb3057..caa18d08e 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -2854,10 +2854,73 @@ return queryInteractiveFull().map(c => c.el); } + function _cssString(value) { + return String(value || '') + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\0/g, '\uFFFD') + .replace(/[\n\r\f]/g, ch => `\\${ch.codePointAt(0).toString(16)} `); + } + + function _deepSelectorMatches(selector) { + const matches = []; + const visit = (root) => { + root.querySelectorAll(selector).forEach(el => matches.push(el)); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot); + }); + }; + try { visit(document); } catch { return []; } + return matches; + } + + function _fileInputPath(el) { + const parts = []; + for (let node = el; node && node.nodeType === 1 && parts.length < 10; node = node.parentElement) { + let part = String(node.tagName || '').toLowerCase(); + if (!part) break; + if (node.id) { + try { part += `#${CSS.escape(node.id)}`; } catch {} + parts.unshift(part); + break; + } + const parent = node.parentElement; + if (parent) { + const sameTag = Array.from(parent.children).filter(child => child.tagName === node.tagName); + if (sameTag.length > 1) part += `:nth-of-type(${sameTag.indexOf(node) + 1})`; + } + parts.unshift(part); + } + return parts.join(' > '); + } + + function _uniqueFileInputSelector(el) { + if (!(el instanceof HTMLInputElement) || el.type !== 'file') return ''; + const candidates = []; + if (el.id) { + try { candidates.push(`#${CSS.escape(el.id)}`); } catch {} + } + if (el.name) candidates.push(`input[type="file"][name="${_cssString(el.name)}"]`); + const accept = el.getAttribute('accept'); + const acceptPart = accept == null + ? ':not([accept])' + : `[accept="${_cssString(accept)}"]`; + const multiplePart = el.hasAttribute('multiple') ? '[multiple]' : ':not([multiple])'; + candidates.push(`input[type="file"]${acceptPart}${multiplePart}`); + candidates.push(`input[type="file"]${acceptPart}`); + const path = _fileInputPath(el); + if (path) candidates.push(path); + for (const selector of candidates) { + const matches = _deepSelectorMatches(selector); + if (matches.length === 1 && matches[0] === el) return selector; + } + return ''; + } + function getInteractiveElementsFull() { return queryInteractiveFull().map((c, i) => { const el = c.el; - return { + const result = { index: i, tag: el.tagName.toLowerCase(), type: el.type || '', @@ -2869,6 +2932,13 @@ rect: { x: Math.round(c.rect.x), y: Math.round(c.rect.y), w: Math.round(c.rect.width), h: Math.round(c.rect.height) }, inShadowDOM: c.inShadow, }; + if (el instanceof HTMLInputElement && el.type === 'file') { + const selector = _uniqueFileInputSelector(el); + result.accept = el.getAttribute('accept'); + result.multiple = el.hasAttribute('multiple'); + if (selector) result.selector = selector; + } + return result; }); } diff --git a/src/firefox/src/agent/adapters.js b/src/firefox/src/agent/adapters.js index 5c2112af9..6ca67ee1f 100644 --- a/src/firefox/src/agent/adapters.js +++ b/src/firefox/src/agent/adapters.js @@ -15779,6 +15779,16 @@ const ADAPTERS = [ - Releases live at ///-/releases/new. Tag must exist or be created via "Create tag" inline. - Merge requests have a "Merge" button that may be disabled until pipelines pass; check the pipeline status before clicking. - The sidebar collapses on narrow viewports — scroll horizontally or expand it before clicking sidebar items.`, + }, + { + name: 'huggingface', + category: 'general', + matches: (url) => /^https?:\/\/(?:www\.)?huggingface\.co(?:[/?#]|$)/i.test(url), + notes: ` +- Repository upload routes expose two file inputs. Use \`input[type="file"]:not([accept])\` for repository files; \`input[type="file"][accept*="image"]\` belongs to the extended-description editor and does not stage a repository file. +- When the repository input already exists, call \`upload_file\` directly; do not click "Upload file(s)" or the drop zone first. +- A filename chip, generated commit summary, and enabled "Commit changes" button mean the file is staged only. Click "Commit changes", wait, and verify the file under "Files and versions" before reporting upload success. +- For model-card media, commit the asset first, then edit README Markdown to reference it, commit README, and verify the rendered model card.`, }, { name: 'mozilla-addons-developer', diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index ca8e3e348..f4ce45aa8 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -367,6 +367,7 @@ export class Agent extends LoopDetector { // Default off; user opts in via Settings → "Strict secret handling". this.strictSecretMode = false; this._lastAxScopes = new Map(); // tabId -> { documentToken, pageUrl }, captured by the latest AX read + this._uploadSelectorRecoveryRequired = new Map(); // tabId -> prior ambiguous match count; cleared only by inspection/navigation/cleanup // Productive browsing often mixes reads and scrolling, so exact-call loop // detection cannot tell when the agent already has enough evidence to // answer. Track long observation-only streaks and remind it to deliver a @@ -1696,6 +1697,7 @@ export class Agent extends LoopDetector { */ _clearPageLoopState(tabId) { super._clearPageLoopState(tabId); + this._uploadSelectorRecoveryRequired.delete(tabId); this.deliveryObservationStreaks.delete(tabId); this.bulkApiMutationClicks.delete(tabId); this.bulkApiMutationHints.delete(tabId); @@ -1707,6 +1709,12 @@ export class Agent extends LoopDetector { } } + _clearUploadSelectorRecoveryAfterInspection(tabId, name, response) { + if (name !== 'get_interactive_elements' || !Array.isArray(response)) return false; + this._uploadSelectorRecoveryRequired.delete(tabId); + return true; + } + _rememberAxScope(tabId, documentToken, pageUrl = '') { const next = { documentToken: String(documentToken || ''), @@ -1747,6 +1755,7 @@ export class Agent extends LoopDetector { && this._isSuccessfulExecutionEvidence(result) && result?.noProgress !== true && result?.verified !== false + && result?.remoteStateVerified !== false && result?.inconclusive !== true; } @@ -3207,7 +3216,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d ) { return 'action_failed'; } - if (toolResult?.inconclusive || toolResult?.verified === false) { + if ( + toolResult?.inconclusive + || toolResult?.verified === false + || toolResult?.remoteStateVerified === false + ) { return 'action_unverified'; } if ( @@ -11888,7 +11901,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } case 'upload_file': { if (parsed.success === false) return `upload failed: ${this._truncate(parsed.error || '', 110)}`; - if (parsed.attached) return `uploaded ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`; + if (parsed.remoteStateVerified === false) { + const localState = parsed.attachmentState === 'page_consumed' + ? 'page consumed attachment' + : 'file attached to input'; + return `${localState} (remote submission unverified)`; + } + if (parsed.attached) return `attached ${this._truncate(parsed.attached.name || '', 60)} (${parsed.attached.size} bytes)`; return parsed.verified === false ? `upload sent (unverified)` : `uploaded ${this._truncate(parsed.file || '', 70)}`; } case 'new_tab': { @@ -13485,6 +13504,17 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'upload_file') { const UPLOAD_MAX_BYTES = 25 * 1024 * 1024; try { + if (this._uploadSelectorRecoveryRequired.has(tabId)) { + return { + success: false, + dispatched: false, + noDispatch: true, + ambiguous: true, + matchCount: Number(this._uploadSelectorRecoveryRequired.get(tabId)) || 0, + recoveryRequired: 'get_interactive_elements', + error: 'A previous upload selector matched multiple file inputs. Call get_interactive_elements now and use the exact selector returned on the intended file-input record before retrying upload_file; do not guess another selector variant.', + }; + } if (args.filePath) { return { success: false, @@ -13696,7 +13726,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d dispatched = true; el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true })); - return { success: true, dispatched: true, file: ${JSON.stringify(filename)}, size: len }; + const attachmentState = el.files && el.files.length + ? 'input_attached' + : 'page_consumed'; + return { success: true, dispatched: true, file: ${JSON.stringify(filename)}, size: len, attachmentState }; } catch (e) { return { success: false, dispatched, error: e.message || String(e) }; } @@ -13714,20 +13747,72 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } const res = results && results[0]; if (!res || !res.success) { + if (res?.ambiguous) this._uploadSelectorRecoveryRequired.set(tabId, Number(res.matchCount) || 0); return { success: false, dispatched: res?.dispatched === true, ...(res?.ambiguous ? { ambiguous: true, matchCount: Number(res.matchCount) || 0, + noDispatch: true, + recoveryRequired: 'get_interactive_elements', } : {}), - error: res ? res.error : 'Failed to attach file to input element', + error: res?.ambiguous + ? `${res.error} Call get_interactive_elements and use the exact, unique selector returned on the intended file-input record; do not guess another selector variant.` + : (res ? res.error : 'Failed to attach file to input element'), }; } + let attachmentState = res.attachmentState === 'page_consumed' + ? 'page_consumed' + : 'input_attached'; + if (attachmentState === 'input_attached') { + // Let change handlers queued through a microtask/timer consume or + // replace the input before we classify its stable local state. + await new Promise(resolve => setTimeout(resolve, 25)); + const settleProbeCode = ` + (function() { + // WebBrain file attachment settle probe. + const selector = ${JSON.stringify(args.selector)}; + const matches = []; + const collectDeepMatches = (root) => { + matches.push(...root.querySelectorAll(selector)); + for (const element of root.querySelectorAll('*')) { + if (element.shadowRoot) collectDeepMatches(element.shadowRoot); + } + }; + try { + collectDeepMatches(document); + } catch { + return null; + } + if (matches.length !== 1) return { attachmentState: 'page_consumed' }; + const input = matches[0]; + if (!(input instanceof HTMLInputElement) || input.type !== 'file') { + return { attachmentState: 'page_consumed' }; + } + const expectedName = ${JSON.stringify(filename)}; + const expectedSize = ${Number(res.size) || 0}; + const attached = Array.from(input.files || []).some(file => ( + file.name === expectedName && file.size === expectedSize + )); + return { attachmentState: attached ? 'input_attached' : 'page_consumed' }; + })(); + `; + try { + const settledResults = await browser.tabs.executeScript(tabId, { code: settleProbeCode }); + const settledState = settledResults?.[0]?.attachmentState; + if (settledState === 'input_attached' || settledState === 'page_consumed') { + attachmentState = settledState; + } + } catch { /* Navigation/detachment leaves the initial state unverified. */ } + } return { success: true, attached: { name: filename, size: res.size }, file: filename, + verified: false, + attachmentState, + remoteStateVerified: false, }; } catch (e) { return { success: false, error: e.message || String(e) }; @@ -14698,6 +14783,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'read_page') { response = applyReadPageWindow(response, args); } + this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); return response; } catch (e) { // Content script might not be injected — try injecting it @@ -14727,6 +14813,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (name === 'read_page') { response = applyReadPageWindow(response, args); } + this._clearUploadSelectorRecoveryAfterInspection(tabId, name, response); return response; } catch (e2) { let pageUrl = ''; diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js index 7e26f1753..0441620c6 100644 --- a/src/firefox/src/agent/tools.js +++ b/src/firefox/src/agent/tools.js @@ -769,7 +769,7 @@ export const AGENT_TOOLS = [ type: 'function', function: { name: 'upload_file', - description: 'Attach a file directly to an existing without clicking the page upload control. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide downloadId (preferred — re-fetches the file from its original URL without an OS dialog), or omit it only when the user must pick a local file through WebBrain\'s own picker. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. NOTE: Firefox cannot set arbitrary local file paths (no CDP); only downloadId and user-picker flows are supported.', + description: 'Attach a file directly to an existing without clicking the page upload control. This only proves that the page input received or consumed the file; it does NOT prove a remote upload, form submission, or repository commit. Do NOT click "Choose file", "Select a file", an upload drop zone, or the input first when the input already exists. Provide downloadId (preferred — re-fetches the file from its original URL without an OS dialog), or omit it only when the user must pick a local file through WebBrain\'s own picker. If the selector is ambiguous, call get_interactive_elements and use the exact selector on the intended file-input record before retrying. If no file input exists because the widget creates it lazily, one guarded click on its add-files control may initialize the widget; then retry upload_file with the exact selector returned or discovered. NOTE: Firefox cannot set arbitrary local file paths (no CDP); only downloadId and user-picker flows are supported.', parameters: { type: 'object', properties: { @@ -1444,7 +1444,7 @@ CLICKING — read this: - For buttons and links you can SEE, click by visible text: \`click({text: "Publish release"})\`. Default matching is EXACT (case-insensitive). If exact fails (no match), the system automatically tries prefix then substring matching — but if multiple elements match at any level, it returns an ambiguity error instead of guessing. - If you get an ambiguity error, use a more specific text string, switch to \`click({index: N})\` from \`get_interactive_elements\`, or use a selector. - You can explicitly control matching with \`textMatch\`: \`"exact"\` (default), \`"prefix"\`, or \`"contains"\`. -- FILE UPLOADS: when the page already has an \`\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. If the file is already downloaded, find the exact selector and call \`upload_file({selector, downloadId})\` directly; omit downloadId only when the user must choose a local file through WebBrain's own picker. Exception: if \`upload_file\` reports that no input exists because the widget creates it lazily, make one guarded click on the widget's add-files control to initialize it. A blocked-picker result may return the new exact selector; retry \`upload_file\` with that selector. Never substitute a generic \`input[type="file"]\` selector when multiple file inputs exist. +- FILE UPLOADS: when the page already has an \`\`, do not click "Choose file", "Select a file", "Browse", the upload drop zone, or the input first. Call \`get_interactive_elements\` when needed and use the exact \`selector\` returned on the intended file-input record. If the file is downloaded, call \`upload_file({selector, downloadId})\`; omit downloadId only for WebBrain's user picker. \`attachmentState\` proves only local input attachment/page consumption; it does NOT prove a remote upload or submit. Verify the filename/status in the page, then activate and verify the required Submit/Commit control. If the selector is ambiguous, a fresh \`get_interactive_elements\` call is required before retrying. Exception: if no input exists because the widget creates it lazily, make one guarded click on its add-files control to initialize it. - Order of preference: 1. \`click({text: "..."})\` — visible text. Most reliable. 2. \`click({index: N})\` — index from get_interactive_elements MADE THIS SAME TURN. diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index 00c3ff30d..c66e9f84b 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -842,6 +842,69 @@ return queryInteractiveFull().map(c => c.el); } + function _cssString(value) { + return String(value || '') + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"') + .replace(/\0/g, '\uFFFD') + .replace(/[\n\r\f]/g, ch => `\\${ch.codePointAt(0).toString(16)} `); + } + + function _deepSelectorMatches(selector) { + const matches = []; + const visit = (root) => { + root.querySelectorAll(selector).forEach(el => matches.push(el)); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) visit(host.shadowRoot); + }); + }; + try { visit(document); } catch { return []; } + return matches; + } + + function _fileInputPath(el) { + const parts = []; + for (let node = el; node && node.nodeType === 1 && parts.length < 10; node = node.parentElement) { + let part = String(node.tagName || '').toLowerCase(); + if (!part) break; + if (node.id) { + try { part += `#${CSS.escape(node.id)}`; } catch {} + parts.unshift(part); + break; + } + const parent = node.parentElement; + if (parent) { + const sameTag = Array.from(parent.children).filter(child => child.tagName === node.tagName); + if (sameTag.length > 1) part += `:nth-of-type(${sameTag.indexOf(node) + 1})`; + } + parts.unshift(part); + } + return parts.join(' > '); + } + + function _uniqueFileInputSelector(el) { + if (!(el instanceof HTMLInputElement) || el.type !== 'file') return ''; + const candidates = []; + if (el.id) { + try { candidates.push(`#${CSS.escape(el.id)}`); } catch {} + } + if (el.name) candidates.push(`input[type="file"][name="${_cssString(el.name)}"]`); + const accept = el.getAttribute('accept'); + const acceptPart = accept == null + ? ':not([accept])' + : `[accept="${_cssString(accept)}"]`; + const multiplePart = el.hasAttribute('multiple') ? '[multiple]' : ':not([multiple])'; + candidates.push(`input[type="file"]${acceptPart}${multiplePart}`); + candidates.push(`input[type="file"]${acceptPart}`); + const path = _fileInputPath(el); + if (path) candidates.push(path); + for (const selector of candidates) { + const matches = _deepSelectorMatches(selector); + if (matches.length === 1 && matches[0] === el) return selector; + } + return ''; + } + window.__wb_resolve_click_target_for_submit_probe = function resolveClickTargetForSubmitProbe(params = {}) { if (params?.index == null) return null; const index = Number(params.index); @@ -852,7 +915,7 @@ function getInteractiveElementsFull() { return queryInteractiveFull().map((c, i) => { const el = c.el; - return { + const result = { index: i, tag: el.tagName.toLowerCase(), type: el.type || '', @@ -864,6 +927,13 @@ rect: { x: Math.round(c.rect.x), y: Math.round(c.rect.y), w: Math.round(c.rect.width), h: Math.round(c.rect.height) }, inShadowDOM: c.inShadow, }; + if (el instanceof HTMLInputElement && el.type === 'file') { + const selector = _uniqueFileInputSelector(el); + result.accept = el.getAttribute('accept'); + result.multiple = el.hasAttribute('multiple'); + if (selector) result.selector = selector; + } + return result; }); } diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs index 3eb8a1977..8d417c053 100644 --- a/test/fixtures/run.mjs +++ b/test/fixtures/run.mjs @@ -1052,6 +1052,49 @@ test('CDP upload selector bridge resolves hidden and open-shadow file inputs', a }); for (const browserKind of ['chrome', 'firefox']) { + test(`get_interactive_elements (${browserKind}): identifies the Hugging Face repository file input`, async (page) => { + const setupHtml = browserKind === 'firefox' ? setupFirefoxHtml : setupChromeHtml; + await setupHtml(page, ` + + +
+ +
`); + + const elements = await call(page, 'get_interactive_elements_cdp', {}); + const fileInputs = elements.filter((element) => element.type === 'file'); + if (fileInputs.length !== 2) { + throw new Error(`expected both Hugging Face-shaped file inputs, got ${JSON.stringify(fileInputs)}`); + } + + const repositoryInput = fileInputs.find((element) => element.accept === null); + const editorMediaInput = fileInputs.find((element) => element.accept === 'image/png,image/jpeg'); + if (!repositoryInput || repositoryInput.multiple !== true) { + throw new Error(`file metadata mismatch: ${JSON.stringify(repositoryInput)}`); + } + if (!editorMediaInput || editorMediaInput.multiple !== true) { + throw new Error(`raw editor-media accept metadata mismatch: ${JSON.stringify(editorMediaInput)}`); + } + if (!repositoryInput.selector || !repositoryInput.selector.includes(':not([accept])')) { + throw new Error(`expected a no-accept repository selector, got ${JSON.stringify(repositoryInput)}`); + } + + const selectorCheck = await page.evaluate((selector) => { + const matches = document.querySelectorAll(selector); + return { + count: matches.length, + repository: matches[0] === document.querySelector('[data-repository-drop-zone]'), + }; + }, repositoryInput.selector); + if (selectorCheck.count !== 1 || !selectorCheck.repository) { + throw new Error(`selector did not uniquely target the repository drop zone: ${JSON.stringify(selectorCheck)}`); + } + }); + test(`file picker guard (${browserKind}): blocks the native chooser and returns the exact input`, async (page) => { await setupContentHtml(page, ` @@ -1409,7 +1452,14 @@ test('Firefox upload_file resolves one open-shadow input and rejects ambiguous p selector: '#shadow-upload', downloadId: 9001, }); - if (!uploaded?.success || uploaded.attached?.name !== 'shadow-upload.txt' || uploaded.attached?.size !== 2) { + if ( + !uploaded?.success + || uploaded.attached?.name !== 'shadow-upload.txt' + || uploaded.attached?.size !== 2 + || uploaded.attachmentState !== 'input_attached' + || uploaded.verified !== false + || uploaded.remoteStateVerified !== false + ) { throw new Error(`open-shadow upload failed: ${JSON.stringify(uploaded)}`); } const state = await page.evaluate(() => { @@ -1431,6 +1481,29 @@ test('Firefox upload_file resolves one open-shadow input and rejects ambiguous p throw new Error(`open-shadow upload state mismatch: ${JSON.stringify(state)}`); } + await page.evaluate(() => { + const input = document.querySelector('#host-a').shadowRoot.querySelector('#shadow-upload'); + input.addEventListener('change', () => { + queueMicrotask(() => { input.value = ''; }); + }, { once: true }); + }); + const consumed = await agent.executeTool(77, 'upload_file', { + selector: '#shadow-upload', + downloadId: 9001, + }); + const consumedCount = await page.evaluate(() => ( + document.querySelector('#host-a').shadowRoot.querySelector('#shadow-upload').files.length + )); + if ( + consumed?.success !== true + || consumed.attachmentState !== 'page_consumed' + || consumed.verified !== false + || consumed.remoteStateVerified !== false + || consumedCount !== 0 + ) { + throw new Error(`queued file consumption was misclassified: ${JSON.stringify({ consumed, consumedCount })}`); + } + const ambiguous = await agent.executeTool(77, 'upload_file', { selector: 'input[type="file"]', downloadId: 9001, @@ -1440,10 +1513,36 @@ test('Firefox upload_file resolves one open-shadow input and rejects ambiguous p || ambiguous.dispatched !== false || ambiguous.ambiguous !== true || ambiguous.matchCount !== 2 + || ambiguous.recoveryRequired !== 'get_interactive_elements' || !/exact, unique selector/.test(ambiguous.error || '') ) { throw new Error(`ambiguous pierced selector did not fail closed: ${JSON.stringify(ambiguous)}`); } + + const blockedRetry = await agent.executeTool(77, 'upload_file', { + selector: '#shadow-upload', + downloadId: 9001, + }); + if ( + blockedRetry?.success !== false + || blockedRetry.noDispatch !== true + || blockedRetry.matchCount !== 2 + || blockedRetry.recoveryRequired !== 'get_interactive_elements' + ) { + throw new Error(`upload retry was not blocked pending inspection: ${JSON.stringify(blockedRetry)}`); + } + if (!agent._clearUploadSelectorRecoveryAfterInspection( + 77, + 'get_interactive_elements', + [{ selector: '#shadow-upload' }], + )) { + throw new Error('fresh interactive-element inspection did not clear upload recovery'); + } + agent._uploadSelectorRecoveryRequired.set(77, 2); + agent._clearRunLoopState(77); + if (agent._uploadSelectorRecoveryRequired.has(77)) { + throw new Error('Firefox run cleanup did not clear upload recovery'); + } } finally { if (originalBrowser === undefined) delete globalThis.browser; else globalThis.browser = originalBrowser; diff --git a/test/run.js b/test/run.js index 9a0cc6cc0..641f20e6b 100644 --- a/test/run.js +++ b/test/run.js @@ -2309,6 +2309,28 @@ test('matches www.github.com', () => { assert.equal(a?.name, 'github'); }); +test('matches Hugging Face upload pages with staged-upload guidance and rejects spoofed hosts', () => { + const url = 'https://huggingface.co/acme/example-model/upload/main'; + const chromeAdapter = getActiveAdapter(url); + const firefoxAdapter = getActiveAdapterFx(url); + + assert.equal(chromeAdapter?.name, 'huggingface'); + assert.equal(firefoxAdapter?.name, 'huggingface'); + assert.equal(firefoxAdapter?.notes, chromeAdapter?.notes); + assert.match(chromeAdapter?.notes || '', /input\[type="file"\]:not\(\[accept\]\)/); + assert.match(chromeAdapter?.notes || '', /accept\*="image".*extended-description editor/i); + assert.match(chromeAdapter?.notes || '', /filename chip.*commit summary.*enabled "Commit changes".*staged only/is); + assert.match(chromeAdapter?.notes || '', /Click "Commit changes".*verify the file under "Files and versions"/is); + + for (const resolveAdapter of [getActiveAdapter, getActiveAdapterFx]) { + assert.notEqual( + resolveAdapter('https://huggingface.co.evil.example/acme/example-model/upload/main')?.name, + 'huggingface', + 'lookalike hosts must not activate Hugging Face upload guidance', + ); + } +}); + test('matches Mozilla Add-ons Developer Hub and guides version submission', () => { const sourceUrl = 'https://addons.mozilla.org/en-US/developers/addon/example-addon/versions/submit/6358210/source'; const chromeAdapter = getActiveAdapter(sourceUrl); @@ -43774,6 +43796,67 @@ test('browser batches keep leading reads, then require fresh evidence after unsa } }); +test('remote-unverified file attachment blocks a queued commit action in every prompt tier', async () => { + for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { + for (const tier of ['compact', 'mid', 'full']) { + const agent = new AgentClass({ + getActive: () => ({ promptTier: tier, supportsVision: false }), + getVisionProvider: async () => null, + }); + const executed = []; + const messages = []; + agent._ensureGateSetting = async () => {}; + agent._skipPermissionGate = true; + agent._currentUrl = async () => 'https://huggingface.co/acme/model/upload/main'; + agent._rememberMastodonObservation = async () => null; + agent._recordProgressObservation = async () => null; + agent._autoRecordProgressAction = () => null; + agent._persist = () => {}; + agent.executeTool = async (_tabId, name) => { + executed.push(name); + if (name === 'upload_file') { + return { + success: true, + attachmentState: 'input_attached', + remoteStateVerified: false, + }; + } + return { success: true, verified: true }; + }; + + const result = await agent._executeToolBatch( + label === 'chrome' ? 815 : 816, + [ + { + id: 'attach', + function: { + name: 'upload_file', + arguments: JSON.stringify({ selector: 'input[type="file"]:not([accept])', downloadId: 1 }), + }, + }, + { + id: 'commit', + function: { name: 'click_ax', arguments: JSON.stringify({ ref_id: 'ref_commit' }) }, + }, + ], + messages, + () => {}, + { supportsVision: false }, + null, + new Set(['upload_file', 'click_ax']), + 1, + ); + + assert.equal(result.action, 'continue', `${label}/${tier}: unverified attachment should start a fresh turn`); + assert.deepEqual(executed, ['upload_file'], `${label}/${tier}: queued Commit ran before attachment observation`); + const skipped = JSON.parse(messages.find(message => message.tool_call_id === 'commit').content); + assert.equal(skipped.skipped, true, `${label}/${tier}: queued Commit was not skipped`); + assert.equal(skipped.triggeringTool, 'upload_file', `${label}/${tier}: wrong action triggered the boundary`); + assert.equal(skipped.reason, 'action_unverified', `${label}/${tier}: wrong fresh-turn reason`); + } + } +}); + test('fresh-turn batch interruptions preserve configured auto-screenshots', async () => { for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { for (const autoScreenshot of ['state_change', 'every_step']) { @@ -50043,6 +50126,7 @@ test('upload_file schema accepts downloadId and no longer hard-requires filePath assert.deepEqual(up.function.parameters.required, ['selector'], 'filePath should no longer be required'); assert.match(up.function.description, /without opening the page or OS file-picker dialog/i); assert.match(up.function.description, /Do NOT click "Choose file", "Select a file"/); + assert.match(up.function.description, /does NOT prove a remote upload, form submission, or repository commit/i); }); test('Chrome click paths suppress native file choosers and redirect to upload_file', async () => { @@ -50389,6 +50473,9 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i assert.equal(result.success, true); assert.equal(result.file, realPath); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(result.verified, false); + assert.equal(result.remoteStateVerified, false); assert.equal(args.filePath, realPath); assert.deepEqual(uploaded, [[realPath]]); assert.deepEqual(releasedGroups, ['upload-query-1'], 'successful uploads must release selector handles'); @@ -50398,22 +50485,71 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i const fallback = await agent.executeTool(42, 'upload_file', fallbackArgs); assert.equal(fallback.success, true); assert.equal(fallback.file, exactPath); + assert.equal(fallback.attachmentState, 'input_attached'); + assert.equal(fallback.verified, false); + assert.equal(fallback.remoteStateVerified, false); assert.equal(fallbackArgs.filePath, exactPath, 'an unresolved downloadId must not replace a supplied absolute path'); assert.deepEqual(uploaded, [[realPath], [exactPath]]); expectedPath = realPath; + cdpClientCh.getFileInputFiles = async () => []; + const consumed = await agent.executeTool(42, 'upload_file', { + selector: 'input[type=file]', + downloadId: 9123, + }); + assert.equal(consumed.success, true); + assert.equal(consumed.attachmentState, 'page_consumed'); + assert.equal(consumed.verified, false); + assert.equal(consumed.remoteStateVerified, false); + assert.match(consumed.note, /does not prove a remote upload or form submission/i); + cdpClientCh.getFileInputFiles = async () => [{ name: expectedPath.split('/').pop(), size: 123, readable: true }]; + selectorMatches = ['input-501', 'input-502']; const ambiguous = await agent.executeTool(42, 'upload_file', { selector: 'input[type=file]', downloadId: 9123, }); assert.equal(ambiguous.success, false); + assert.equal(ambiguous.dispatched, false); + assert.equal(ambiguous.noDispatch, true); + assert.equal(ambiguous.ambiguous, true); + assert.equal(ambiguous.matchCount, 2); + assert.equal(ambiguous.recoveryRequired, 'get_interactive_elements'); assert.match(ambiguous.error, /matched 2 elements/); assert.match(ambiguous.error, /exact, unique selector/); - assert.deepEqual(uploaded, [[realPath], [exactPath]], 'ambiguous selectors must fail before attaching the file'); + assert.deepEqual(uploaded, [[realPath], [exactPath], [realPath]], 'ambiguous selectors must fail before attaching the file'); + + const queryCountBeforeBlockedRetry = queryCount; + const blockedRetry = await agent.executeTool(42, 'upload_file', { + selector: 'input[type=file]:not([accept])', + downloadId: 9123, + }); + assert.equal(blockedRetry.success, false); + assert.equal(blockedRetry.noDispatch, true); + assert.equal(blockedRetry.matchCount, 2); + assert.equal(blockedRetry.recoveryRequired, 'get_interactive_elements'); + assert.equal(queryCount, queryCountBeforeBlockedRetry, 'retry must not query the DOM before a fresh inspection'); + + assert.equal( + agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_accessibility_tree', {}), + false, + 'an unrelated observation must not clear upload recovery', + ); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), true); + assert.equal( + agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_interactive_elements', [{ selector: 'input[type=file]:not([accept])' }]), + true, + ); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false); + agent._uploadSelectorRecoveryRequired.set(42, 2); + agent._clearRunLoopState(42); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'run cleanup must clear upload recovery'); + agent._uploadSelectorRecoveryRequired.set(42, 2); + agent._clearPageLoopState(42); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false, 'navigation cleanup must clear upload recovery'); assert.deepEqual( releasedGroups, - ['upload-query-1', 'upload-query-2', 'upload-query-3'], + ['upload-query-1', 'upload-query-2', 'upload-query-3', 'upload-query-4'], 'early upload failures must release selector handles', ); } finally { @@ -50431,6 +50567,31 @@ test('upload_file schema accepts downloadId and no longer hard-requires filePath assert.deepEqual(up.function.parameters.required, ['selector'], 'filePath should no longer be required'); assert.match(up.function.description, /without clicking the page upload control/i); assert.match(up.function.description, /Do NOT click "Choose file", "Select a file"/); + assert.match(up.function.description, /does NOT prove a remote upload, form submission, or repository commit/i); +}); + +test('upload_file digests preserve local attachment and remote-unverified semantics', () => { + for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { + const agent = new AgentClass({}); + const attached = agent._digestToolResult('upload_file', JSON.stringify({ + success: true, + attached: { name: 'asset.bin', size: 12 }, + verified: false, + attachmentState: 'input_attached', + remoteStateVerified: false, + })); + assert.match(attached, /file attached to input.*remote submission unverified/i, `${label}: attached digest lost local-only state`); + assert.doesNotMatch(attached, /\buploaded\b/i, `${label}: attached digest claimed remote upload`); + + const consumed = agent._digestToolResult('upload_file', JSON.stringify({ + success: true, + verified: false, + attachmentState: 'page_consumed', + remoteStateVerified: false, + })); + assert.match(consumed, /page consumed attachment.*remote submission unverified/i, `${label}: consumed digest lost local-only state`); + assert.doesNotMatch(consumed, /\buploaded\b/i, `${label}: consumed digest claimed remote upload`); + } }); test('upload_file (firefox) rejects non-complete downloads and missing picker base64', async () => { @@ -50487,6 +50648,7 @@ test('upload_file (firefox) re-fetches downloadId with manual redirect handling const originalFetch = globalThis.fetch; const executedScripts = []; const fetchCalls = []; + let injectedAttachmentState = 'input_attached'; try { globalThis.browser = { downloads: { @@ -50501,7 +50663,10 @@ test('upload_file (firefox) re-fetches downloadId with manual redirect handling }, async executeScript(tabId, details) { executedScripts.push(details.code); - return [{ success: true, file: 'test.zip', size: 4 }]; + if (details.code.includes('WebBrain file attachment settle probe')) { + return [{ attachmentState: injectedAttachmentState }]; + } + return [{ success: true, file: 'test.zip', size: 4, attachmentState: 'input_attached' }]; }, }, }; @@ -50546,13 +50711,16 @@ test('upload_file (firefox) re-fetches downloadId with manual redirect handling assert.equal(result.success, true); assert.equal(result.file, 'test.zip'); + assert.equal(result.attachmentState, 'input_attached'); + assert.equal(result.verified, false); + assert.equal(result.remoteStateVerified, false); assert.equal(fetchCalls.length, 2); assert.equal(fetchCalls[0].opts.redirect, 'manual'); assert.equal(fetchCalls[0].opts.credentials, 'include'); assert.equal(fetchCalls[1].opts.redirect, 'manual'); assert.equal(fetchCalls[1].opts.credentials, 'omit'); - assert.equal(executedScripts.length, 1); + assert.equal(executedScripts.length, 2); assert.ok(executedScripts[0].includes('new DataTransfer()'), 'Script should use DataTransfer'); assert.ok(executedScripts[0].includes('dt.items.add(file)'), 'Script should add file to DataTransfer'); assert.ok(executedScripts[0].includes('el.files = dt.files'), 'Script should assign DataTransfer files to input'); @@ -50560,6 +50728,14 @@ test('upload_file (firefox) re-fetches downloadId with manual redirect handling assert.ok(executedScripts[0].includes('collectDeepMatches(element.shadowRoot)'), 'Script should search open shadow roots'); assert.ok(executedScripts[0].includes('matches.length > 1'), 'Script should reject ambiguous selectors'); assert.ok(executedScripts[0].includes('exact, unique selector'), 'Script should return actionable ambiguity guidance'); + assert.ok(executedScripts[1].includes('WebBrain file attachment settle probe'), 'Script should re-check after queued change handlers'); + + injectedAttachmentState = 'page_consumed'; + const consumed = await agent.executeTool(42, 'upload_file', args); + assert.equal(consumed.success, true); + assert.equal(consumed.attachmentState, 'page_consumed'); + assert.equal(consumed.verified, false); + assert.equal(consumed.remoteStateVerified, false); } finally { if (originalBrowser === undefined) delete globalThis.browser; else globalThis.browser = originalBrowser; From 47336ab6627b7f2357a973f6552f2a86cf5e90c1 Mon Sep 17 00:00:00 2001 From: Barack Sokullu Date: Tue, 4 Aug 2026 15:49:46 +0300 Subject: [PATCH 2/2] Keep upload recovery gated after inspection --- src/chrome/src/agent/agent.js | 7 +++++++ src/chrome/src/content/content.js | 21 +++++++++++++++++++++ src/firefox/src/agent/agent.js | 7 +++++++ src/firefox/src/content/content.js | 21 +++++++++++++++++++++ test/fixtures/run.mjs | 23 ++++++++++++++++++++--- test/run.js | 17 ++++++++++++++++- 6 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index eca1587d8..135cea152 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -1679,6 +1679,13 @@ export class Agent extends LoopDetector { _clearUploadSelectorRecoveryAfterInspection(tabId, name, response) { if (name !== 'get_interactive_elements' || !Array.isArray(response)) return false; + const hasVerifiedFileInputSelector = response.some(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + && typeof element.selector === 'string' + && element.selector.trim().length > 0 + )); + if (!hasVerifiedFileInputSelector) return false; this._uploadSelectorRecoveryRequired.delete(tabId); return true; } diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js index caa18d08e..5ab829a32 100644 --- a/src/chrome/src/content/content.js +++ b/src/chrome/src/content/content.js @@ -2842,6 +2842,27 @@ return a.rect.left - b.rect.left; }); + // Upload controls are commonly CSS-hidden behind a styled drop zone. Keep + // visible action indices stable by appending any omitted file inputs only + // after the visual sort; their records expose selectors for upload_file. + const appendOmittedFileInputs = (root) => { + try { + root.querySelectorAll('input[type="file"]').forEach(el => { + if (seen.has(el)) return; + seen.add(el); + collected.push({ + el, + rect: el.getBoundingClientRect(), + inShadow: root !== document, + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) appendOmittedFileInputs(host.shadowRoot); + }); + } catch (e) {} + }; + appendOmittedFileInputs(document); + return collected; } diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index f4ce45aa8..47469d8c4 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -1711,6 +1711,13 @@ export class Agent extends LoopDetector { _clearUploadSelectorRecoveryAfterInspection(tabId, name, response) { if (name !== 'get_interactive_elements' || !Array.isArray(response)) return false; + const hasVerifiedFileInputSelector = response.some(element => ( + element?.tag === 'input' + && String(element.type || '').toLowerCase() === 'file' + && typeof element.selector === 'string' + && element.selector.trim().length > 0 + )); + if (!hasVerifiedFileInputSelector) return false; this._uploadSelectorRecoveryRequired.delete(tabId); return true; } diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js index c66e9f84b..2c5f70053 100644 --- a/src/firefox/src/content/content.js +++ b/src/firefox/src/content/content.js @@ -831,6 +831,27 @@ return a.rect.left - b.rect.left; }); + // Upload controls are commonly CSS-hidden behind a styled drop zone. Keep + // visible action indices stable by appending any omitted file inputs only + // after the visual sort; their records expose selectors for upload_file. + const appendOmittedFileInputs = (root) => { + try { + root.querySelectorAll('input[type="file"]').forEach(el => { + if (seen.has(el)) return; + seen.add(el); + collected.push({ + el, + rect: el.getBoundingClientRect(), + inShadow: root !== document, + }); + }); + root.querySelectorAll('*').forEach(host => { + if (host.shadowRoot) appendOmittedFileInputs(host.shadowRoot); + }); + } catch (e) {} + }; + appendOmittedFileInputs(document); + return collected; } diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjs index 8d417c053..33f75c60e 100644 --- a/test/fixtures/run.mjs +++ b/test/fixtures/run.mjs @@ -1056,13 +1056,13 @@ for (const browserKind of ['chrome', 'firefox']) { const setupHtml = browserKind === 'firefox' ? setupFirefoxHtml : setupChromeHtml; await setupHtml(page, `
- +
`); const elements = await call(page, 'get_interactive_elements_cdp', {}); @@ -1531,10 +1531,27 @@ test('Firefox upload_file resolves one open-shadow input and rejects ambiguous p ) { throw new Error(`upload retry was not blocked pending inspection: ${JSON.stringify(blockedRetry)}`); } + if (agent._clearUploadSelectorRecoveryAfterInspection( + 77, + 'get_interactive_elements', + [], + )) { + throw new Error('empty inspection cleared upload recovery'); + } + if (!agent._uploadSelectorRecoveryRequired.has(77)) { + throw new Error('empty inspection removed upload recovery state'); + } + if (agent._clearUploadSelectorRecoveryAfterInspection( + 77, + 'get_interactive_elements', + [{ tag: 'input', type: 'file' }], + )) { + throw new Error('selector-less file-input inspection cleared upload recovery'); + } if (!agent._clearUploadSelectorRecoveryAfterInspection( 77, 'get_interactive_elements', - [{ selector: '#shadow-upload' }], + [{ tag: 'input', type: 'file', selector: '#shadow-upload' }], )) { throw new Error('fresh interactive-element inspection did not clear upload recovery'); } diff --git a/test/run.js b/test/run.js index 641f20e6b..640339b3b 100644 --- a/test/run.js +++ b/test/run.js @@ -50537,7 +50537,22 @@ test('upload_file prefers a valid downloadId and falls back to filePath for an i ); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), true); assert.equal( - agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_interactive_elements', [{ selector: 'input[type=file]:not([accept])' }]), + agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_interactive_elements', []), + false, + 'an inspection without file inputs must not clear upload recovery', + ); + assert.equal( + agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_interactive_elements', [{ tag: 'input', type: 'file' }]), + false, + 'a file-input record without a verified selector must not clear upload recovery', + ); + assert.equal(agent._uploadSelectorRecoveryRequired.has(42), true); + assert.equal( + agent._clearUploadSelectorRecoveryAfterInspection(42, 'get_interactive_elements', [{ + tag: 'input', + type: 'file', + selector: 'input[type=file]:not([accept])', + }]), true, ); assert.equal(agent._uploadSelectorRecoveryRequired.has(42), false);