From b710276754bc539b7565b280baf915b9c09dfa20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 7 Jul 2026 14:03:26 +0200 Subject: [PATCH] test: remove slow-test ratchet pins --- scripts/vitest-slow-test-reporter.ts | 56 +-------- scripts/write-xcuitest-cache-metadata.mjs | 108 +++++++++++------- .../vitest-slow-test-reporter.test.ts | 5 +- .../android/__tests__/devices.test.ts | 86 +++++++++++++- .../__tests__/snapshot-helper-session.test.ts | 12 +- src/platforms/android/devices.ts | 8 +- .../android/snapshot-helper-session.ts | 15 ++- src/platforms/android/snapshot-helper.ts | 1 + .../apple/core/__tests__/index.test.ts | 74 +++++++----- .../core/__tests__/runner-xctestrun.test.ts | 62 ++++------ 10 files changed, 249 insertions(+), 178 deletions(-) diff --git a/scripts/vitest-slow-test-reporter.ts b/scripts/vitest-slow-test-reporter.ts index 64e9c38a5..34aa6bbc7 100644 --- a/scripts/vitest-slow-test-reporter.ts +++ b/scripts/vitest-slow-test-reporter.ts @@ -6,10 +6,7 @@ import type { Reporter, TestCase, TestModule } from 'vitest/node'; * Unit tests must not wait real time: measured 2026-07-04, the unit suite's * wall clock was bounded by files whose tests slept through production * timeouts (a 10.8s test proving "times out" by waiting the full 10s budget). - * This reporter fails the run when a unit test exceeds the budget UNLESS it - * is pinned below. The pin is a ratchet: it may only shrink, or grow in the - * same PR that adds the entry here with a justification — the same - * only-changes-in-reviewed-diffs rule as the guarantee-matrix gap list. + * This reporter fails the run when a unit test exceeds the enforced budget. * * Budgets are per suite family: integration scenarios drive a real daemon * request path and get more room; unit tests get 2.5s, which is already @@ -23,49 +20,6 @@ const INTEGRATION_BUDGET_MS = 15_000; // gate reports without failing. const ENFORCE_FACTOR = 2; -// Ratchet pin: known offenders at gate introduction (tracking issue #1098). -// Every entry is a test that waits real time (production timeout constants, -// 1Hz poll loops, real retry backoff) and shrinks as those are converted to -// budget-wiring assertions or budget-derived poll intervals. -const PINNED_SLOW_UNIT_TESTS = new Set([ - "src/__tests__/daemon-entrypoint.test.ts :: daemon runtime starts HTTP transport in-process and shuts down cleanly", - "src/daemon/__tests__/artifact-materialization.test.ts :: materializeArtifact extracts iOS app bundle tar archives and returns the installable .app path", - "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp preserves write failures after a successful run-as probe", - "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp writes React Native Android dev prefs", - "src/daemon/__tests__/runtime-hints.test.ts :: applyRuntimeHintsToApp writes iOS simulator React Native defaults", - "src/daemon/__tests__/runtime-hints.test.ts :: clearRuntimeHintsFromApp removes managed Android runtime prefs but preserves unrelated entries", - "src/daemon/handlers/__tests__/session-replay-vars.test.ts :: runReplayScriptFile skips Maestro runFlow.when.visible commands when absent", - "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact cleans URL materialization when IPA payload resolution fails", - "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing nested app tar", - "src/platforms/__tests__/install-source.test.ts :: prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing one IPA", - "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted falls back to ANDROID_SDK_ROOT when PATH is incomplete", - "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted launches emulator in headless mode when requested", - "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted launches emulator with GUI by default", - "src/platforms/android/__tests__/devices.test.ts :: ensureAndroidEmulatorBooted reuses running emulator for headless requests", - "src/platforms/android/__tests__/devices.test.ts :: listAndroidDevices falls back to model when emulator avd name is unavailable", - "src/platforms/android/__tests__/input-actions.test.ts :: fillAndroid uses chunk-safe shell input and retries when verification still fails", - "src/platforms/android/__tests__/app-lifecycle-install.test.ts :: installAndroidApp .aab reports missing bundletool tooling", - "src/platforms/android/__tests__/app-lifecycle-install.test.ts :: installAndroidApp installs .aab via bundletool build-apks + install-apks", - "src/platforms/android/__tests__/app-lifecycle-install.test.ts :: installAndroidApp installs .apk via adb install -r", - "src/platforms/android/__tests__/app-lifecycle-install.test.ts :: installAndroidApp resolves packageName and launchTarget from nested archive artifacts", - "src/platforms/android/__tests__/perf.test.ts :: stopAndroidSimpleperfProfile fails before pull when remote artifact never stabilizes", - "src/platforms/android/__tests__/snapshot-helper-session.test.ts :: allows a persistent session snapshot to use the helper command budget", - "src/platforms/apple/core/__tests__/index.test.ts :: openIosSimulatorApp times out instead of hanging indefinitely", - "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot restores prior visible overrides", - "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot skips known redundant status bar commands", - "src/platforms/apple/core/__tests__/index.test.ts :: prepareSimulatorStatusBarForScreenshot still normalizes when snapshotting current overrides fails", - "src/platforms/apple/core/__tests__/index.test.ts :: screenshotIos retries simulator capture timeouts and eventually succeeds", - "src/platforms/apple/core/__tests__/runner-adoption.test.ts :: adoption is skipped on artifact fingerprint mismatch", - "src/platforms/apple/core/__tests__/runner-adoption.test.ts :: adoption is skipped when the probe fails", - "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun falls back to scan when cache manifest is stale", - "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds after cached macOS runner repair failure", - "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds cached runner when Swift build flags mismatch", - "src/platforms/apple/core/__tests__/runner-client.test.ts :: ensureXctestrun rebuilds foreign artifacts when metadata does not match", - "src/platforms/apple/core/__tests__/runner-xctestrun.test.ts :: setup metadata script matches expected iOS simulator cache metadata", - "src/recording/__tests__/recording-scripts.test.ts :: recording overlay Swift script typechecks", - "src/utils/__tests__/daemon-client.test.ts :: cleanupFailedDaemonStartupMetadata retains live startup daemon on timeout", -]); - type Offender = { key: string; durationMs: number; budgetMs: number; enforce: boolean }; function budgetForPath(relativePath: string): number { @@ -90,8 +44,6 @@ export function classifySlowTest(params: { const budgetMs = budgetForPath(relative); if (params.durationMs <= budgetMs) return null; const fullKey = `${relative} :: ${params.fullName.split(' > ').join(' ')}`; - if (PINNED_SLOW_UNIT_TESTS.has(`${relative} :: ${params.name}`)) return null; - if (PINNED_SLOW_UNIT_TESTS.has(fullKey)) return null; return { key: fullKey, durationMs: params.durationMs, @@ -119,10 +71,10 @@ export function reportSlowTests( } if (failing.length === 0) return false; write( - `\nSlow-test gate: ${failing.length} test(s) exceeded ${ENFORCE_FACTOR}x the wall-clock budget.\n` + + `\nSlow-test gate: ${failing.length} test(s) exceeded ${ENFORCE_FACTOR}x the wall-clock budget.\n` + `Tests must not wait real time — inject the timeout/poll budget or assert the budget is\n` + - `wired instead of waiting it out (docs/agents/testing.md). If the wait is genuinely\n` + - `irreducible, pin it in scripts/vitest-slow-test-reporter.ts in this PR with a reason.\n` + + `wired instead of waiting it out (docs/agents/testing.md). If the runtime cost is genuinely\n` + + `irreducible, document the reason in the owning test or move it out of the unit lane.\n` + failing.map(line).join('\n'), ); return true; diff --git a/scripts/write-xcuitest-cache-metadata.mjs b/scripts/write-xcuitest-cache-metadata.mjs index e764c66d8..d2a7291fc 100644 --- a/scripts/write-xcuitest-cache-metadata.mjs +++ b/scripts/write-xcuitest-cache-metadata.mjs @@ -3,19 +3,16 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; -const args = process.argv.slice(2); -const [platform, derivedPath, destination] = args; +let platform = ''; +let derivedPath = ''; +let destination = ''; +let projectRoot = ''; +let metadataPath = ''; -if (!platform || !derivedPath || !destination) { - console.error( - 'Usage: write-xcuitest-cache-metadata.mjs ', - ); - process.exit(1); -} - -const projectRoot = process.cwd(); -const metadataPath = path.join(derivedPath, '.agent-device-runner-cache.json'); +const USAGE = + 'Usage: write-xcuitest-cache-metadata.mjs '; const DEFAULT_IOS_RUNNER_APP_BUNDLE_ID = 'com.callstack.agentdevice.runner'; @@ -215,39 +212,62 @@ function resolveSandboxBuildArgs() { ]; } -const appBundleId = resolveRunnerAppBundleId(); -const testBundleId = resolveRunnerTestBundleId(); -const metadata = { - schemaVersion: 2, - packageVersion: readPackageVersion(), - runnerSourceFingerprint: computeRunnerSourceFingerprint(), - ...resolveRunnerToolchainFingerprint(), - platformName: resolvePlatformName(), - deviceKind: resolveDeviceKind(), - target: resolveTarget(), - buildDestinationFamily: resolveBuildDestinationFamily(), - runnerBundleBuildSettings: [ - `AGENT_DEVICE_IOS_RUNNER_APP_BUNDLE_ID=${appBundleId}`, - `AGENT_DEVICE_IOS_RUNNER_TEST_BUNDLE_ID=${testBundleId}`, - ], - runnerSigningBuildSettings: resolveSigningBuildSettings(), - runnerPerformanceBuildSettings: [ - 'COMPILER_INDEX_STORE_ENABLE=NO', - 'ENABLE_CODE_COVERAGE=NO', - 'ONLY_ACTIVE_ARCH=YES', - 'ENABLE_PREVIEWS=NO', - 'ENABLE_DEBUG_DYLIB=NO', - ], - runnerSandboxBuildArgs: resolveSandboxBuildArgs(), -}; - -const artifacts = resolveRunnerCacheArtifacts(); -if (artifacts) { - metadata.artifacts = artifacts; -} - -fs.mkdirSync(path.dirname(metadataPath), { recursive: true }); -fs.writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`); +export function writeXcuitestCacheMetadata(args = process.argv.slice(2), cwd = process.cwd()) { + const [nextPlatform, nextDerivedPath, nextDestination] = args; + if (!nextPlatform || !nextDerivedPath || !nextDestination) { + throw new Error(USAGE); + } + + platform = nextPlatform; + derivedPath = nextDerivedPath; + destination = nextDestination; + projectRoot = cwd; + metadataPath = path.join(derivedPath, '.agent-device-runner-cache.json'); + + const appBundleId = resolveRunnerAppBundleId(); + const testBundleId = resolveRunnerTestBundleId(); + const metadata = { + schemaVersion: 2, + packageVersion: readPackageVersion(), + runnerSourceFingerprint: computeRunnerSourceFingerprint(), + ...resolveRunnerToolchainFingerprint(), + platformName: resolvePlatformName(), + deviceKind: resolveDeviceKind(), + target: resolveTarget(), + buildDestinationFamily: resolveBuildDestinationFamily(), + runnerBundleBuildSettings: [ + `AGENT_DEVICE_IOS_RUNNER_APP_BUNDLE_ID=${appBundleId}`, + `AGENT_DEVICE_IOS_RUNNER_TEST_BUNDLE_ID=${testBundleId}`, + ], + runnerSigningBuildSettings: resolveSigningBuildSettings(), + runnerPerformanceBuildSettings: [ + 'COMPILER_INDEX_STORE_ENABLE=NO', + 'ENABLE_CODE_COVERAGE=NO', + 'ONLY_ACTIVE_ARCH=YES', + 'ENABLE_PREVIEWS=NO', + 'ENABLE_DEBUG_DYLIB=NO', + ], + runnerSandboxBuildArgs: resolveSandboxBuildArgs(), + }; + + const artifacts = resolveRunnerCacheArtifacts(); + if (artifacts) { + metadata.artifacts = artifacts; + } + + fs.mkdirSync(path.dirname(metadataPath), { recursive: true }); + fs.writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`); + return metadata; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + writeXcuitestCacheMetadata(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} function resolveRunnerCacheArtifacts() { const xctestrunPath = findXctestrun(derivedPath); diff --git a/src/__tests__/vitest-slow-test-reporter.test.ts b/src/__tests__/vitest-slow-test-reporter.test.ts index fc36a9b52..08981949e 100644 --- a/src/__tests__/vitest-slow-test-reporter.test.ts +++ b/src/__tests__/vitest-slow-test-reporter.test.ts @@ -32,7 +32,7 @@ test('integration paths get the larger budget', () => { assert.equal(offender, null); }); -test('pinned tests never become offenders', () => { +test('known slow tests are reported when they exceed the budget', () => { const offender = classifySlowTest({ root: '/repo', moduleId: '/repo/src/platforms/android/__tests__/app-lifecycle-install.test.ts', @@ -40,7 +40,8 @@ test('pinned tests never become offenders', () => { fullName: 'installAndroidApp installs .apk via adb install -r', durationMs: 9_000, }); - assert.equal(offender, null); + assert.ok(offender); + assert.equal(offender.enforce, true); }); test('reportSlowTests fails only on enforced offenders and prints both bands', () => { diff --git a/src/platforms/android/__tests__/devices.test.ts b/src/platforms/android/__tests__/devices.test.ts index fd05cd6ee..635cc086a 100644 --- a/src/platforms/android/__tests__/devices.test.ts +++ b/src/platforms/android/__tests__/devices.test.ts @@ -1,6 +1,6 @@ import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; -import { appendFileSync, promises as fs, writeFileSync } from 'node:fs'; +import { appendFileSync, existsSync, promises as fs, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; vi.mock('../../../utils/exec.ts', async (importOriginal) => { @@ -12,7 +12,8 @@ const execActual = await vi.importActual('../../../utils/exec.ts'); import { AppError } from '../../../kernel/errors.ts'; -import { runCmdDetached } from '../../../utils/exec.ts'; +import { runCmdDetached, withCommandExecutorOverride } from '../../../utils/exec.ts'; +import type { CommandExecutorOverride, ExecResult } from '../../../utils/exec.ts'; import { ensureAndroidEmulatorBooted, listAndroidDevices, @@ -199,7 +200,14 @@ async function withMockedAndroidTools( ANDROID_SDK_ROOT: undefined, ANDROID_HOME: undefined, }, - async () => await run({ emulatorLogPath, emulatorBootedPath }), + async () => + await withCommandExecutorOverride( + createMockedAndroidToolsCommandOverride({ + emulatorBootedPath, + avdNameMode: options.avdNameMode ?? 'success', + }), + async () => await run({ emulatorLogPath, emulatorBootedPath }), + ), ); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); @@ -238,13 +246,83 @@ async function withMockedAndroidSdkRoot( ANDROID_SDK_ROOT: sdkRoot, ANDROID_HOME: undefined, }, - async () => await run({ emulatorLogPath, emulatorBootedPath, sdkRoot }), + async () => + await withCommandExecutorOverride( + createMockedAndroidToolsCommandOverride({ emulatorBootedPath }), + async () => await run({ emulatorLogPath, emulatorBootedPath, sdkRoot }), + ), ); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); } } +function createMockedAndroidToolsCommandOverride(params: { + emulatorBootedPath: string; + avdNameMode?: 'success' | 'missing'; +}): CommandExecutorOverride { + return async (cmd, args) => { + if (cmd === 'emulator') return handleMockedEmulatorCommand(args); + if (cmd === 'adb') return handleMockedAdbCommand(args, params); + throw new Error(`Unexpected command: ${cmd} ${args.join(' ')}`); + }; +} + +function handleMockedEmulatorCommand(args: string[]): ExecResult { + const emulatorKey = args.join('\0'); + if (emulatorKey === '-list-avds') return execResult('Pixel_9_Pro_XL\n'); + throw new Error(`Unexpected emulator args: ${args.join(' ')}`); +} + +function handleMockedAdbCommand( + args: string[], + params: { emulatorBootedPath: string; avdNameMode?: 'success' | 'missing' }, +): ExecResult { + const adbArgs = stripAdbSerial(args); + const adbKey = adbArgs.join('\0'); + if (adbKey === 'devices\0-l') return mockedAdbDevicesResult(params.emulatorBootedPath); + if (isAdbAvdNameProbe(adbKey)) { + return execResult(params.avdNameMode === 'missing' ? '' : 'Pixel_9_Pro_XL\n'); + } + if (adbKey === 'shell\0getprop\0sys.boot_completed') { + return execResult(existsSync(params.emulatorBootedPath) ? '1\n' : '0\n'); + } + if (adbKey === 'shell\0getprop\0ro.build.characteristics') return execResult('phone\n'); + if (isAdbPackageProbe(adbArgs)) return execResult('false\n'); + if (adbKey === 'shell\0pm\0list\0features') return execResult('\n'); + throw new Error(`Unexpected adb args: ${args.join(' ')}`); +} + +function mockedAdbDevicesResult(emulatorBootedPath: string): ExecResult { + const bootedDevice = existsSync(emulatorBootedPath) + ? [ + 'emulator-5554 device product:sdk_gphone64 model:Pixel_9_Pro_XL device:emu64a transport_id:2', + ] + : []; + return execResult(['List of devices attached', ...bootedDevice, ''].join('\n')); +} + +function isAdbAvdNameProbe(adbKey: string): boolean { + return ( + adbKey === 'emu\0avd\0name' || + adbKey === 'shell\0getprop\0ro.boot.qemu.avd_name' || + adbKey === 'shell\0getprop\0persist.sys.avd_name' + ); +} + +function isAdbPackageProbe(adbArgs: string[]): boolean { + return adbArgs[0] === 'shell' && adbArgs[1] === 'cmd' && adbArgs[2] === 'package'; +} + +function stripAdbSerial(args: string[]): string[] { + if (args[0] === '-s') return args.slice(2); + return args; +} + +function execResult(stdout = '', stderr = '', exitCode = 0): ExecResult { + return { stdout, stderr, exitCode }; +} + test('resolveAndroidEmulatorAvdName ignores probe timeouts and keeps probing', async () => { const calls: string[][] = []; const results = [ diff --git a/src/platforms/android/__tests__/snapshot-helper-session.test.ts b/src/platforms/android/__tests__/snapshot-helper-session.test.ts index c5a898846..636c1d013 100644 --- a/src/platforms/android/__tests__/snapshot-helper-session.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper-session.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, test } from 'vitest'; import { captureAndroidSnapshotWithHelperSession, resetAndroidSnapshotHelperSessions, + resolveAndroidSnapshotHelperSessionRequestTimeoutMs, } from '../snapshot-helper.ts'; import type { AndroidAdbExecutor, AndroidAdbProcess, AndroidAdbProvider } from '../adb-executor.ts'; @@ -115,8 +116,15 @@ test('starts and reuses a persistent Android snapshot helper session', async () test('allows a persistent session snapshot to use the helper command budget', async () => { const calls: string[][] = []; - // The delay must stay above the previous 3s session cap to guard the regression. - const provider = createSessionProvider({ calls, responseDelayMs: 3_200 }); + const provider = createSessionProvider({ calls, responseDelayMs: 25 }); + + assert.equal( + resolveAndroidSnapshotHelperSessionRequestTimeoutMs({ + timeoutMs: 10, + commandTimeoutMs: 4_000, + }), + 4_000, + ); const output = await captureAndroidSnapshotWithHelperSession({ adb: provider.exec, diff --git a/src/platforms/android/devices.ts b/src/platforms/android/devices.ts index 8a2805a7c..f66c98d0a 100644 --- a/src/platforms/android/devices.ts +++ b/src/platforms/android/devices.ts @@ -506,7 +506,9 @@ export async function ensureAndroidEmulatorBooted(params: { export async function waitForAndroidBoot(serial: string, timeoutMs = 60000): Promise { const timeoutBudget = timeoutMs; const deadline = Deadline.fromTimeoutMs(timeoutBudget); - const maxAttempts = Math.max(1, Math.ceil(timeoutBudget / ANDROID_BOOT_POLL_MS)); + // Aim for at least 20 polls without busy-looping, capped at the production cadence. + const pollMs = Math.min(ANDROID_BOOT_POLL_MS, Math.max(50, Math.floor(timeoutBudget / 20))); + const maxAttempts = Math.max(1, Math.ceil(timeoutBudget / pollMs)); let lastBootResult: ExecResult | undefined; let timedOut = false; try { @@ -539,8 +541,8 @@ export async function waitForAndroidBoot(serial: string, timeoutMs = 60000): Pro }, { maxAttempts, - baseDelayMs: ANDROID_BOOT_POLL_MS, - maxDelayMs: ANDROID_BOOT_POLL_MS, + baseDelayMs: pollMs, + maxDelayMs: pollMs, jitter: 0, shouldRetry: (error) => { const reason = classifyBootFailure({ diff --git a/src/platforms/android/snapshot-helper-session.ts b/src/platforms/android/snapshot-helper-session.ts index 8bb434d29..72551899b 100644 --- a/src/platforms/android/snapshot-helper-session.ts +++ b/src/platforms/android/snapshot-helper-session.ts @@ -137,6 +137,16 @@ export function getAndroidSnapshotHelperSessionDeviceKey( return `${device.platform}:${device.id}`; } +export function resolveAndroidSnapshotHelperSessionRequestTimeoutMs(params: { + timeoutMs: number; + commandTimeoutMs: number; +}): number { + return Math.min( + params.commandTimeoutMs, + Math.max(params.timeoutMs + SESSION_REQUEST_OVERHEAD_MS, 3_000), + ); +} + export async function resetAndroidSnapshotHelperSessions(): Promise { await Promise.all( [...sessions.keys()].map((deviceKey) => stopAndroidSnapshotHelperSession(deviceKey)), @@ -264,10 +274,7 @@ async function requestSessionSnapshot( const requestId = `snapshot-${Date.now()}-${Math.random().toString(16).slice(2)}`; // Keep the session request generous enough for slow UIAutomator captures, but never // beyond the command budget the caller already assigned to this snapshot. - const timeoutMs = Math.min( - resolved.commandTimeoutMs, - Math.max(resolved.timeoutMs + SESSION_REQUEST_OVERHEAD_MS, 3_000), - ); + const timeoutMs = resolveAndroidSnapshotHelperSessionRequestTimeoutMs(resolved); const response = await sendSessionCommand(session, `snapshot ${requestId}`, timeoutMs); return parseSessionSnapshotResponse(response, requestId); } diff --git a/src/platforms/android/snapshot-helper.ts b/src/platforms/android/snapshot-helper.ts index c6ab6ae72..c3a82b4ef 100644 --- a/src/platforms/android/snapshot-helper.ts +++ b/src/platforms/android/snapshot-helper.ts @@ -12,6 +12,7 @@ export { captureAndroidSnapshotWithHelperSession, getAndroidSnapshotHelperSessionDeviceKey, resetAndroidSnapshotHelperSessions, + resolveAndroidSnapshotHelperSessionRequestTimeoutMs, stopAndroidSnapshotHelperSession, stopAndroidSnapshotHelperSessionForDevice, } from './snapshot-helper-session.ts'; diff --git a/src/platforms/apple/core/__tests__/index.test.ts b/src/platforms/apple/core/__tests__/index.test.ts index 24d94fdfc..56bd410d1 100644 --- a/src/platforms/apple/core/__tests__/index.test.ts +++ b/src/platforms/apple/core/__tests__/index.test.ts @@ -77,7 +77,11 @@ import { } from '../screenshot-status-bar.ts'; import { runAppleRunnerCommand } from '../runner/runner-client.ts'; import { iosRunnerOverrides } from '../../interactions.ts'; -import { IOS_DEVICE_INSTALL_TIMEOUT_MS, IOS_SIMULATOR_TERMINATE_TIMEOUT_MS } from '../config.ts'; +import { + IOS_DEVICE_INSTALL_TIMEOUT_MS, + IOS_SIMULATOR_FOCUS_TIMEOUT_MS, + IOS_SIMULATOR_TERMINATE_TIMEOUT_MS, +} from '../config.ts'; import type { DeviceInfo } from '../../../../kernel/device.ts'; import { withDiagnosticsScope } from '../../../../utils/diagnostics.ts'; import { AppError } from '../../../../kernel/errors.ts'; @@ -1035,34 +1039,26 @@ test('captureSimulatorScreenshotWithFallback uses simulator runner fallback by d } }); -test( - 'openIosSimulatorApp times out instead of hanging indefinitely', - { timeout: 15_000 }, - async () => { - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-ios-focus-timeout-test-')); - const openPath = path.join(tmpDir, 'open'); - await fs.writeFile(openPath, '#!/bin/sh\nsleep 10\n', 'utf8'); - await fs.chmod(openPath, 0o755); - - const previousPath = process.env.PATH; - process.env.PATH = `${tmpDir}${path.delimiter}${previousPath ?? ''}`; +test('openIosSimulatorApp times out instead of hanging indefinitely', async () => { + mockRunCmd.mockImplementation(async (cmd, args, options) => { + assert.equal(cmd, 'open'); + assert.deepEqual(args, ['-a', 'Simulator']); + assert.equal(options?.timeoutMs, IOS_SIMULATOR_FOCUS_TIMEOUT_MS); + throw new AppError('COMMAND_FAILED', 'open timed out after 10000ms', { + timeoutMs: options?.timeoutMs, + }); + }); - try { - await assert.rejects( - () => openIosSimulatorApp(), - (error: unknown) => { - assert.equal(error instanceof AppError, true); - assert.equal((error as AppError).code, 'COMMAND_FAILED'); - assert.match((error as AppError).message, /open timed out after 10000ms/); - return true; - }, - ); - } finally { - process.env.PATH = previousPath; - await fs.rm(tmpDir, { recursive: true, force: true }); - } - }, -); + await assert.rejects( + () => openIosSimulatorApp(), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'COMMAND_FAILED'); + assert.match((error as AppError).message, /open timed out after 10000ms/); + return true; + }, + ); +}); test('prepareSimulatorStatusBarForScreenshot restores prior visible overrides', async () => { await withMockedXcrun( @@ -1263,6 +1259,28 @@ test('screenshotIos retries simulator capture timeouts and eventually succeeds', process.env.PATH = `${tmpDir}${path.delimiter}${previousPath ?? ''}`; process.env.AGENT_DEVICE_TEST_COMMAND_LOG = commandLogPath; process.env.AGENT_DEVICE_TEST_SCREENSHOT_COUNT_FILE = screenshotCountPath; + mockRetryWithPolicy.mockImplementation(async (fn, policy, options) => { + assert.ok(policy); + assert.ok(options); + assert.equal(options.phase, 'ios_simulator_screenshot'); + assert.equal(policy.maxAttempts, 5); + assert.equal(policy.baseDelayMs, 1_000); + assert.equal(policy.maxDelayMs, 5_000); + let lastError: unknown; + for (let attempt = 1; attempt <= policy.maxAttempts; attempt += 1) { + try { + return await fn({ + attempt, + maxAttempts: policy.maxAttempts, + deadline: options.deadline, + }); + } catch (error) { + lastError = error; + if (!policy.shouldRetry?.(error, attempt)) throw error; + } + } + throw lastError; + }); try { await screenshotIos(IOS_TEST_SIMULATOR, outPath); diff --git a/src/platforms/apple/core/__tests__/runner-xctestrun.test.ts b/src/platforms/apple/core/__tests__/runner-xctestrun.test.ts index 946a73b87..d57deaaaa 100644 --- a/src/platforms/apple/core/__tests__/runner-xctestrun.test.ts +++ b/src/platforms/apple/core/__tests__/runner-xctestrun.test.ts @@ -1,9 +1,9 @@ import { test, vi } from 'vitest'; import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; import type { DeviceInfo } from '../../../../kernel/device.ts'; import { withCommandExecutorOverride } from '../../../../utils/exec.ts'; @@ -209,8 +209,21 @@ test('scoreXctestrunCandidate penalizes macos and env xctestrun files for simula test('setup metadata script matches expected iOS simulator cache metadata', async () => { await withTempDir('runner-cache-metadata-', async (root) => { + const repoRoot = process.cwd(); + const scriptPath = path.join(repoRoot, 'scripts', 'write-xcuitest-cache-metadata.mjs'); + const projectRoot = path.join(root, 'project'); + const derivedRoot = path.join(root, 'derived'); const binDir = path.join(root, 'bin'); - fs.mkdirSync(binDir); + fs.mkdirSync(binDir, { recursive: true }); + fs.mkdirSync(derivedRoot, { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'apple-runner', 'AgentDeviceRunner'), { + recursive: true, + }); + fs.writeFileSync(path.join(projectRoot, 'package.json'), '{"version":"0.19.0"}\n'); + fs.writeFileSync( + path.join(projectRoot, 'apple-runner', 'AgentDeviceRunner', 'Runner.swift'), + 'final class Runner {}\n', + ); writeExecutable( path.join(binDir, 'xcodebuild'), ['#!/bin/sh', 'printf "Xcode 26.2\\nBuild version 17C52\\n"'].join('\n'), @@ -226,53 +239,24 @@ test('setup metadata script matches expected iOS simulator cache metadata', asyn 'esac', ].join('\n'), ); - writeExecutable( - path.join(binDir, 'plutil'), - [ - '#!/bin/sh', - "cat <<'JSON'", - '{"TestConfigurations":[{"TestTargets":[{"ProductPaths":["__TESTROOT__/Debug-iphonesimulator/AgentDeviceRunner.app","__TESTROOT__/Debug-iphonesimulator/AgentDeviceRunnerUITests-Runner.app"]}]}]}', - 'JSON', - ].join('\n'), - ); - fs.writeFileSync( - path.join( - root, - 'AgentDeviceRunner_AgentDeviceRunnerUITests_iphonesimulator26.2-arm64.xctestrun', - ), - '{}', - ); - fs.mkdirSync(path.join(root, 'Debug-iphonesimulator', 'AgentDeviceRunner.app'), { - recursive: true, - }); - fs.mkdirSync(path.join(root, 'Debug-iphonesimulator', 'AgentDeviceRunnerUITests-Runner.app'), { - recursive: true, - }); const previousPath = process.env.PATH; process.env.PATH = `${binDir}${path.delimiter}${previousPath ?? ''}`; try { - execFileSync( - process.execPath, - [ - 'scripts/write-xcuitest-cache-metadata.mjs', - 'ios', - root, - 'generic/platform=iOS Simulator', - ], - { - cwd: process.cwd(), - env: { ...process.env, PATH: process.env.PATH }, - stdio: ['ignore', 'ignore', 'inherit'], - }, + const { writeXcuitestCacheMetadata } = await import( + `${pathToFileURL(scriptPath).href}?case=${Date.now()}` + ); + writeXcuitestCacheMetadata( + ['ios', derivedRoot, 'generic/platform=iOS Simulator'], + projectRoot, ); const actual = JSON.parse( - fs.readFileSync(path.join(root, '.agent-device-runner-cache.json'), 'utf8'), + fs.readFileSync(path.join(derivedRoot, '.agent-device-runner-cache.json'), 'utf8'), ); const { artifacts: _actualArtifacts, ...actualComparable } = actual; const { artifacts: _expectedArtifacts, ...expectedComparable } = - resolveExpectedRunnerCacheMetadata(iosSimulator); + resolveExpectedRunnerCacheMetadata(iosSimulator, projectRoot); assert.deepEqual(actualComparable, expectedComparable); } finally {