From 73c353ecfb9cca4ae9628eda6d7783319df63348 Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Fri, 17 Jul 2026 17:49:59 +0000 Subject: [PATCH] fix(browser): handle absolute https Wikipedia URLs in throughput benchmark The CSS selector :not([href*=':']) rejected article body links rendered as absolute https:// URLs (e.g. https://en.wikipedia.org/wiki/Foo) because 'https:' contains a colon. Wikipedia now renders some body links in this format, causing ~54% of iterations to fail across all providers on the 2026-07-17 run. Three fixes: - Replace CSS :not() filter with JS-based isArticleLink() that only checks for namespace colons after /wiki/, handling relative, protocol-relative, and absolute URL formats - Skip goBack and dependent actions when click fails to avoid 30s timeouts on stub articles with no body links - Add stub-filtering retry logic to workflow URL resolution - Fix hardcoded expectedActions=50 in run.ts to use ACTIONS_PER_SESSION (10) --- .../browser-throughput-benchmarks.yml | 28 +++++++++-- src/browser/throughput-benchmark.ts | 47 ++++++++++++++++--- src/run.ts | 4 +- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/.github/workflows/browser-throughput-benchmarks.yml b/.github/workflows/browser-throughput-benchmarks.yml index 35a50453..5e5d3d36 100644 --- a/.github/workflows/browser-throughput-benchmarks.yml +++ b/.github/workflows/browser-throughput-benchmarks.yml @@ -42,11 +42,29 @@ jobs: urls=() for i in $(seq 1 "$count"); do # Follow Special:Random's redirect once to a concrete article URL. - url=$(curl -sL -A 'ComputeSDK-Benchmark' -o /dev/null -w '%{url_effective}' \ - 'https://en.wikipedia.org/wiki/Special:Random') - if [ -z "$url" ] || [[ "$url" == *"Special:Random"* ]]; then - url='https://en.wikipedia.org/wiki/Special:Random' - fi + # Retry up to 5 times until we find an article that has at least + # one article-body link (href containing "/wiki/" without a colon, + # which excludes namespace pages like Help:, File:, Special:). + # Stub articles without body links cause the click action to time + # out and skew throughput results. + for attempt in 1 2 3 4 5; do + url=$(curl -sL -A 'ComputeSDK-Benchmark' -o /dev/null -w '%{url_effective}' \ + 'https://en.wikipedia.org/wiki/Special:Random') + if [ -z "$url" ] || [[ "$url" == *"Special:Random"* ]]; then + url='https://en.wikipedia.org/wiki/Special:Random' + break + fi + # Fetch the article HTML and check for article-body links. + # The benchmark selects #mw-content-text a[href*="/wiki/"] and + # filters in JS for links without a namespace colon after /wiki/. + # Body links appear as protocol-relative (//en.wikipedia.org/wiki/) + # or absolute (https://en.wikipedia.org/wiki/) URLs. + html=$(curl -sL -A 'ComputeSDK-Benchmark' "$url") + if echo "$html" | grep -oP 'href="[^"]*"' \ + | grep -qP '/wiki/[^:]*$'; then + break + fi + done urls+=("$url") done # Emit as a newline-delimited multiline output. Pure bash — avoids a diff --git a/src/browser/throughput-benchmark.ts b/src/browser/throughput-benchmark.ts index 26164880..a965a567 100644 --- a/src/browser/throughput-benchmark.ts +++ b/src/browser/throughput-benchmark.ts @@ -48,10 +48,21 @@ export function navUrlForIteration(i: number): string { if (NAV_URLS.length > 0) return NAV_URLS[i % NAV_URLS.length]; return RANDOM_URL; } -// Match article-body links across both classic MediaWiki HTML (relative "/wiki/Foo") -// and Parsoid read-HTML (protocol-relative absolute "//en.wikipedia.org/wiki/Foo"). -// :not([href*=":"]) still excludes namespace pages (Help:, File:) and external http(s) links. -const ARTICLE_LINK = '#mw-content-text a[href*="/wiki/"]:not([href*=":"])'; +// Match article-body links across classic MediaWiki HTML (relative "/wiki/Foo"), +// Parsoid read-HTML (protocol-relative "//en.wikipedia.org/wiki/Foo"), and the +// newer absolute form ("https://en.wikipedia.org/wiki/Foo"). +// We can't use :not([href*=":"]) because that also rejects https: URLs. Instead +// we select all /wiki/ links in the content area and filter in JS by checking +// that the path segment after /wiki/ has no namespace colon (Help:, File:, etc.). +const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; + +/** Returns true if an href points to a real article (not a namespace page). */ +function isArticleLink(href: string | null): boolean { + if (!href) return false; + const match = href.match(/\/wiki\/([^#]*)/); + if (!match) return false; + return !match[1].includes(':'); +} const ACTION_TIMEOUT_MS = 30_000; @@ -131,14 +142,38 @@ async function runActionLoop(page: Page, results: ActionResult[], navigateUrl: s } // 5. Click first article link (filter out meta pages like Help:, File:, etc.) + let clickSucceeded = false; { const r = await timeAction(async () => { - const link = await page.waitForSelector(ARTICLE_LINK, { timeout: 10_000 }); - await link.click(); + // Wait for any /wiki/ link to appear, then filter in JS for article + // links (no namespace colon after /wiki/). This handles relative, + // protocol-relative, and absolute https:// URL formats. + await page.waitForSelector(ARTICLE_LINK_SELECTOR, { timeout: 10_000 }); + const links = await page.$$(ARTICLE_LINK_SELECTOR); + for (const link of links) { + const href = await link.getAttribute('href'); + if (isArticleLink(href)) { + await link.click(); + return; + } + } + throw new Error('No article body link found on page'); }); + clickSucceeded = r.success; results.push({ index: baseIdx + 5, type: 'click', durationMs: r.durationMs, success: r.success, error: r.error }); } + // If the click failed (e.g. stub article with no body links), skip the + // remaining actions that depend on having navigated to a new page. + // Without this, goBack navigates to a blank page and the final + // waitForSelector times out for 30s, inflating task time. + if (!clickSucceeded) { + for (const idx of [6, 7, 8, 9, 10]) { + results.push({ index: baseIdx + idx, type: idx <= 8 ? (idx === 6 || idx === 10 ? 'waitForSelector' : idx === 7 ? 'screenshot' : 'textContent') : 'goBack', durationMs: 0, success: false, error: 'skipped: click failed' }); + } + continue; + } + // 6. Wait for #firstHeading on the new page { const r = await timeAction(() => page.waitForSelector(FIRST_HEADING)); diff --git a/src/run.ts b/src/run.ts index 431e62a7..227cb1f2 100644 --- a/src/run.ts +++ b/src/run.ts @@ -37,6 +37,7 @@ import type { StorageBenchmarkResult } from './storage/types.js'; import type { SnapshotForkBenchmarkResult } from './storage/snapshot-fork-types.js'; import type { DatasetPreset } from './storage/snapshot-fork-types.js'; import type { BrowserBenchmarkResult } from './browser/types.js'; +import { ACTIONS_PER_SESSION } from './browser/throughput-types.js'; import type { ThroughputBenchmarkResult, ThroughputTimingResult } from './browser/throughput-types.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -409,8 +410,7 @@ async function runBrowserThroughput(toRun: typeof throughputProviders): Promise< console.log(`${r.provider}: SKIPPED (${r.skipReason})`); continue; } - const expectedActions = 50; - const fullSuccess = r.iterations.filter(i => !i.error && i.actionsCompleted === expectedActions).length; + const fullSuccess = r.iterations.filter(i => !i.error && i.actionsCompleted === ACTIONS_PER_SESSION).length; const total = r.iterations.length; const aps = r.summary.actionsPerSecond.median; const taskMed = r.summary.taskMs.median;