From cdfc2fc0ec81e4a31d113fccfdcceb0d859f0165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 7 Jul 2026 17:29:10 +0200 Subject: [PATCH 1/2] fix: keep iOS synthesized drags off AX --- .../RunnerTests+CommandExecution.swift | 225 ++++++++++++++++-- .../RunnerTests+Interaction.swift | 167 ++++++++++++- .../RunnerTests+SequenceExecution.swift | 32 ++- .../apple/core/__tests__/index.test.ts | 24 ++ 4 files changed, 426 insertions(+), 22 deletions(-) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 0771c52f3..293017ef7 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -182,6 +182,74 @@ extension RunnerTests { XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) } + func testSkipAppActivationPreflightIncludesCoordinateOnlySynthesizedGestures() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20,"synthesized":true}"# + ) + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true} + ]} + """ + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) + XCTAssertTrue(shouldSkipAppActivationPreflight(drag)) + XCTAssertTrue(shouldSkipAppActivationPreflight(scroll)) + XCTAssertTrue(shouldSkipAppActivationPreflight(sequence)) + } + + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let selectorTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# + ) + let standardDrag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let mixedSequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) + } + + func testSkipAppActivationPreflightRequiresCachedTarget() throws { + currentApp = nil + currentBundleId = nil + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + } + func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws { let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) abandonedMainThreadWorkCount = 1 @@ -732,9 +800,9 @@ extension RunnerTests { message: "dragged" ) case .scroll: - // Fused frame-resolve + drag scroll for non-tvOS. Resolves the interaction frame via - // resolvedTouchReferenceFrame, computes drag endpoints with the Swift port of - // buildScrollGesturePlan, then runs the same non-synthesized drag path scroll's drag used. + // Fused frame-resolve + drag scroll for non-tvOS. On iOS this intentionally stays on the + // AX-free synthesized coordinate lane so scroll keeps working when XCTest cannot serialize + // the accessibility tree. guard let direction = command.direction, direction == "up" || direction == "down" || direction == "left" || direction == "right" else { @@ -746,7 +814,7 @@ extension RunnerTests { ) ) } - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) + let frame = scrollReferenceFrame(app: activeApp) guard frame.width > 0, frame.height > 0 else { return Response( ok: false, @@ -786,7 +854,7 @@ extension RunnerTests { x2: frame.minX + plan.x2, y2: frame.minY + plan.y2, durationMs: command.durationMs, - synthesized: command.durationMs != nil, + synthesized: shouldUseSynthesizedScrollPath(), message: "scrolled" ) case .desktopScroll: @@ -1139,10 +1207,9 @@ extension RunnerTests { } } - /// Shared drag execution for `.drag` and the fused `.scroll`. Mirrors the original `.drag` body - /// exactly: keyboardAvoidingDragPoints -> resolvedDragVisualizationFrame -> synthesized branch - /// (16-10000ms clamp) or non-synthesized dragAt with coordinateDragHoldDuration -> - /// gestureResponse(.drag). `.scroll` uses the synthesized path only when a duration is requested. + /// Shared drag execution for `.drag` and the fused `.scroll`. The iOS synthesized lane avoids + /// keyboard/window lookup and `XCUICoordinate` fallback so coordinate gestures remain usable on + /// screens whose XCTest accessibility tree is unhealthy. private func executeDragGesture( activeApp: XCUIApplication, x: Double, @@ -1153,6 +1220,24 @@ extension RunnerTests { synthesized: Bool, message: String ) -> Response { + let commandName = dragCommandName(message: message) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return Response( + ok: false, + error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") + ) + } + if synthesized, let synthesizedResponse = executeSynthesizedDragGesture( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message + ) { + return synthesizedResponse + } let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) let dragFrame = resolvedDragVisualizationFrame( app: activeApp, @@ -1203,6 +1288,73 @@ extension RunnerTests { ) } + private func executeSynthesizedDragGesture( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double?, + message: String + ) -> Response? { +#if os(iOS) + guard let plan = axFreeSynthesizedDragPlan(app: activeApp, x: x, y: y, x2: x2, y2: y2) + else { + return Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "\(dragCommandName(message: message)) could not resolve a finite synthesized coordinate frame" + ) + ) + } + let durationMs = min(max(durationMs ?? 250, 16), 10000) + let dragFrame = axFreeDragVisualizationFrame( + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + referenceFrame: plan.referenceFrame + ) + let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + synthesizedDragAt( + app: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs + ) + } + if case .performed = outcome { + return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) + } + return unsupportedResponse(for: outcome) +#else + return nil +#endif + } + + private func scrollReferenceFrame(app: XCUIApplication) -> CGRect { +#if os(iOS) + return synthesizedGestureReferenceFrame(app: app) ?? CGRect(x: 0, y: 0, width: 0, height: 0) +#else + return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) +#endif + } + + private func shouldUseSynthesizedScrollPath() -> Bool { +#if os(iOS) + return true +#else + return false +#endif + } + + private func dragCommandName(message: String) -> String { + return message == "scrolled" ? "scroll" : "drag" + } + private func currentXCTestFailureCount() -> Int { return testRun?.failureCount ?? 0 } @@ -1235,20 +1387,57 @@ extension RunnerTests { private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { #if os(iOS) - // Coordinate-only synthesized taps can run after an AX-fatal screen because they do not need - // app activation, window lookup, keyboard lookup, or element resolution. Selector/text taps - // intentionally stay on the normal AX path because they need an element query. - return command.command == .tap - && command.synthesized == true - && command.x != nil - && command.y != nil - && command.text == nil - && command.selectorKey == nil + // Coordinate-only synthesized gestures can run after an AX-fatal screen because they do not + // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text + // interactions intentionally stay on the normal AX path because they need an element query. + guard command.text == nil, command.selectorKey == nil else { return false } + guard hasCachedTargetForActivationSkip(command: command) else { return false } + switch command.command { + case .tap: + return command.synthesized == true + && command.x != nil + && command.y != nil + case .drag: + return command.synthesized == true + && command.x != nil + && command.y != nil + && command.x2 != nil + && command.y2 != nil + case .scroll: + return true + case .sequence: + guard let steps = command.steps, !steps.isEmpty else { return false } + return steps.allSatisfy { step in + if step.kind == "tap" { + return step.synthesized == true && step.x != nil && step.y != nil + } + if step.kind == "drag" { + return step.synthesized == true + && step.x != nil + && step.y != nil + && step.x2 != nil + && step.y2 != nil + } + return false + } + default: + return false + } #else return false #endif } + private func hasCachedTargetForActivationSkip(command: Command) -> Bool { + guard currentApp != nil else { return false } + guard let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return true + } + return currentBundleId == bundleId + } + private func resolveAppWithoutActivation(command: Command) -> XCUIApplication { guard let bundleId = command.appBundleId? .trimmingCharacters(in: .whitespacesAndNewlines), diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 7f015113d..2305426f6 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -36,6 +36,11 @@ extension RunnerTests { let y2: Double } + struct SynthesizedDragPlan { + let points: DragPoints + let referenceFrame: CGRect + } + struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool @@ -790,8 +795,19 @@ extension RunnerTests { durationMs: Double ) -> RunnerInteractionOutcome { #if os(iOS) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return .unsupported( + message: "synthesized coordinate drag requires finite coordinates", + hint: "Retry with finite x, y, x2, and y2 values." + ) + } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let frame = app.frame + guard let frame = synthesizedGestureReferenceFrame(app: app) else { + return .unsupported( + message: "synthesized coordinate drag could not resolve a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeSwipe( @@ -804,7 +820,7 @@ extension RunnerTests { ) { return .unsupported( message: message, - hint: "Falling back to XCTest coordinate drag may be slower; update Xcode if this persists." + hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." ) } return .performed @@ -823,8 +839,20 @@ extension RunnerTests { func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { #if os(iOS) + guard x.isFinite, y.isFinite else { + return .unsupported( + message: "synthesized coordinate tap requires finite coordinates", + hint: "Retry with finite x and y values." + ) + } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation) + guard let frame = synthesizedGestureReferenceFrame(app: app) else { + return .unsupported( + message: "synthesized coordinate tap could not resolve a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, x: Double(point.x), @@ -994,6 +1022,90 @@ extension RunnerTests { #endif } + func axFreeSynthesizedDragPlan( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double + ) -> SynthesizedDragPlan? { +#if os(iOS) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, + let frame = synthesizedGestureReferenceFrame(app: app) + else { + return nil + } + return SynthesizedDragPlan( + points: DragPoints(x: x, y: y, x2: x2, y2: y2), + referenceFrame: frame + ) +#else + return nil +#endif + } + + func axFreeDragVisualizationFrame( + x: Double, + y: Double, + x2: Double, + y2: Double, + referenceFrame: CGRect + ) -> DragVisualizationFrame { + return DragVisualizationFrame( + x: x, + y: y, + x2: x2, + y2: y2, + referenceWidth: Double(referenceFrame.width), + referenceHeight: Double(referenceFrame.height) + ) + } + + func synthesizedGestureReferenceFrame(app _: XCUIApplication) -> CGRect? { +#if os(iOS) + return finiteSynthesizedReferenceFrame( + appFrame: .zero, + fallbackBounds: .zero, + fallbackScreenshotSize: { XCUIScreen.main.screenshot().image.size } + ) +#else + return nil +#endif + } + + func finiteSynthesizedReferenceFrame( + appFrame: CGRect, + fallbackBounds: CGRect, + fallbackScreenshotSize: () -> CGSize + ) -> CGRect? { + if isUsableReferenceFrame(appFrame) { + return appFrame + } + if isUsableReferenceFrame(fallbackBounds) { + return CGRect(x: 0, y: 0, width: fallbackBounds.width, height: fallbackBounds.height) + } + let screenshotSize = fallbackScreenshotSize() + guard screenshotSize.width.isFinite, screenshotSize.height.isFinite, + screenshotSize.width > 0, + screenshotSize.height > 0 + else { + return nil + } + return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height) + } + + private func isUsableReferenceFrame(_ frame: CGRect) -> Bool { + return !frame.isNull + && !frame.isEmpty + && !frame.isInfinite + && frame.minX.isFinite + && frame.minY.isFinite + && frame.width.isFinite + && frame.height.isFinite + && frame.width > 0 + && frame.height > 0 + } + func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { @@ -1296,6 +1408,55 @@ extension RunnerTests { } } + func testFiniteSynthesizedReferenceFramePrefersValidAppFrame() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: CGRect(x: 4, y: 8, width: 390, height: 844), + fallbackBounds: CGRect(x: 0, y: 0, width: 1, height: 1), + fallbackScreenshotSize: { CGSize(width: 2, height: 2) } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 4, y: 8, width: 390, height: 844)) + } + + func testFiniteSynthesizedReferenceFrameFallsBackWithoutUsingInvalidAppFrame() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: CGRect(x: 20, y: 30, width: 430, height: 932), + fallbackScreenshotSize: { + XCTFail("screenshot fallback should not be used when screen bounds are finite") + return CGSize(width: 1, height: 1) + } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 0, y: 0, width: 430, height: 932)) + } + + func testFiniteSynthesizedReferenceFrameFallsBackToScreenshotSize() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: .zero, + fallbackScreenshotSize: { CGSize(width: 430, height: 932) } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 0, y: 0, width: 430, height: 932)) + } + + func testFiniteSynthesizedReferenceFrameRejectsInvalidSources() { + XCTAssertNil( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: .zero, + fallbackScreenshotSize: { CGSize(width: .infinity, height: 932) } + ) + ) + } + func testDesktopScrollWheelDeltasMapDirections() throws { XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "up", pixels: 120)).vertical, 120) XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "down", pixels: 120)).vertical, -120) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 12427228b..d2c8b640a 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -165,8 +165,30 @@ extension RunnerTests { // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } if step.kind == "drag", step.synthesized == true { - let dragPoints = keyboardAvoidingDragPoints( + let dragPoints: DragPoints +#if os(iOS) + guard let dragPlan = axFreeSynthesizedDragPlan( + app: activeApp, + x: x, + y: y, + x2: step.x2 ?? x, + y2: step.y2 ?? y + ) else { + let nowMs = ProcessInfo.processInfo.systemUptime * 1000 + return SequenceStepOutcome( + outcome: .unsupported( + message: "synthesized coordinate drag could not resolve a finite coordinate frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ), + gestureStartUptimeMs: nowMs, + gestureEndUptimeMs: nowMs + ) + } + dragPoints = dragPlan.points +#else + dragPoints = keyboardAvoidingDragPoints( app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) +#endif let durationMs = min(max(step.durationMs ?? 250, 16), 10000) let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { synthesizedDragAt( @@ -188,6 +210,13 @@ extension RunnerTests { gestureEndUptimeMs: timing.gestureEndUptimeMs ) } +#if os(iOS) + return SequenceStepOutcome( + outcome: outcome, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs + ) +#else let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250) let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) { dragAt( @@ -207,6 +236,7 @@ extension RunnerTests { gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs, gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs ) +#endif } let (timing, outcome) = performGesture(activeApp) { switch step.kind { diff --git a/src/platforms/apple/core/__tests__/index.test.ts b/src/platforms/apple/core/__tests__/index.test.ts index 56bd410d1..24146138b 100644 --- a/src/platforms/apple/core/__tests__/index.test.ts +++ b/src/platforms/apple/core/__tests__/index.test.ts @@ -390,6 +390,30 @@ test('iosRunnerOverrides maps iOS scroll to a single fused scroll command', asyn }); }); +test('iosRunnerOverrides maps iOS scroll without duration to a fused runner scroll', async () => { + mockRunAppleRunnerCommand.mockResolvedValueOnce({ + x: 200, + y: 640, + x2: 200, + y2: 240, + referenceWidth: 400, + referenceHeight: 800, + }); + + const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, { + appBundleId: 'com.example.App', + }); + + await overrides.scroll('down', { pixels: 400 }); + + assert.deepEqual(mockRunAppleRunnerCommand.mock.calls[0]?.[1], { + command: 'scroll', + direction: 'down', + pixels: 400, + appBundleId: 'com.example.App', + }); +}); + test('iosRunnerOverrides maps tvOS scroll duration to remote press hold duration', async () => { mockRunAppleRunnerCommand.mockResolvedValueOnce({ ok: true, From 47f006372b15a1456c444fd83d2ccd43c598b286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 7 Jul 2026 18:31:45 +0200 Subject: [PATCH 2/2] fix: address iOS synthesized drag review --- .../RunnerTests+CommandExecution.swift | 182 ++++++++++++------ .../RunnerTests+Interaction.swift | 58 +++++- .../RunnerTests+SequenceExecution.swift | 9 +- .../RunnerTests+Snapshot.swift | 1 + .../RunnerTests+SnapshotCapturePlan.swift | 1 + .../RunnerTests.swift | 1 + 6 files changed, 184 insertions(+), 68 deletions(-) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 293017ef7..66494c8b5 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -182,7 +182,7 @@ extension RunnerTests { XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) } - func testSkipAppActivationPreflightIncludesCoordinateOnlySynthesizedGestures() throws { + func testSkipAppActivationPreflightOnlyIncludesCoordinateOnlySynthesizedTaps() throws { currentApp = app currentBundleId = nil defer { @@ -192,25 +192,8 @@ extension RunnerTests { let tap = try runnerCommandFixture( #"{"command":"tap","commandId":"tap-1","x":10,"y":20,"synthesized":true}"# ) - let drag = try runnerCommandFixture( - #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"# - ) - let scroll = try runnerCommandFixture( - #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# - ) - let sequence = try runnerCommandFixture( - """ - {"command":"sequence","commandId":"seq-1","steps":[ - {"kind":"tap","x":10,"y":20,"synthesized":true}, - {"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true} - ]} - """ - ) XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) - XCTAssertTrue(shouldSkipAppActivationPreflight(drag)) - XCTAssertTrue(shouldSkipAppActivationPreflight(scroll)) - XCTAssertTrue(shouldSkipAppActivationPreflight(sequence)) } func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { @@ -250,6 +233,33 @@ extension RunnerTests { XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) } + func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) + } + func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws { let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) abandonedMainThreadWorkCount = 1 @@ -855,7 +865,9 @@ extension RunnerTests { y2: frame.minY + plan.y2, durationMs: command.durationMs, synthesized: shouldUseSynthesizedScrollPath(), - message: "scrolled" + message: "scrolled", + synthesizedReferenceFrame: frame, + allowSynthesizedCoordinateFallback: false ) case .desktopScroll: guard let direction = command.direction, @@ -1218,7 +1230,9 @@ extension RunnerTests { y2: Double, durationMs: Double?, synthesized: Bool, - message: String + message: String, + synthesizedReferenceFrame: CGRect? = nil, + allowSynthesizedCoordinateFallback: Bool = true ) -> Response { let commandName = dragCommandName(message: message) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { @@ -1234,7 +1248,9 @@ extension RunnerTests { x2: x2, y2: y2, durationMs: durationMs, - message: message + message: message, + referenceFrame: synthesizedReferenceFrame, + allowCoordinateFallback: allowSynthesizedCoordinateFallback ) { return synthesizedResponse } @@ -1295,11 +1311,33 @@ extension RunnerTests { x2: Double, y2: Double, durationMs: Double?, - message: String + message: String, + referenceFrame: CGRect?, + allowCoordinateFallback: Bool ) -> Response? { #if os(iOS) - guard let plan = axFreeSynthesizedDragPlan(app: activeApp, x: x, y: y, x2: x2, y2: y2) + guard let plan = axFreeSynthesizedDragPlan( + app: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + referenceFrame: referenceFrame, + avoidKeyboardWhenSafe: true + ) else { + if allowCoordinateFallback { + return executeCoordinateDragFallback( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message, + fallback: nil + ) + } return Response( ok: false, error: ErrorPayload( @@ -1323,21 +1361,77 @@ extension RunnerTests { y: plan.points.y, x2: plan.points.x2, y2: plan.points.y2, - durationMs: durationMs + durationMs: durationMs, + referenceFrame: plan.referenceFrame ) } if case .performed = outcome { return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) } + if allowCoordinateFallback { + return executeCoordinateDragFallback( + activeApp: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs, + message: message, + fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) + ) + } return unsupportedResponse(for: outcome) #else return nil #endif } + private func executeCoordinateDragFallback( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double?, + message: String, + fallback: GestureFallback? + ) -> Response { + let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = resolvedDragVisualizationFrame( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2 + ) + let holdDuration = synthesizedSwipeFallbackHoldDuration(durationMs: durationMs ?? 250) + let (timing, outcome) = performGesture(activeApp) { + dragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + holdDuration: holdDuration + ) + } + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse( + message: message, + timing: timing, + frame: .drag(dragFrame), + fallback: fallback + ) + } + private func scrollReferenceFrame(app: XCUIApplication) -> CGRect { #if os(iOS) - return synthesizedGestureReferenceFrame(app: app) ?? CGRect(x: 0, y: 0, width: 0, height: 0) + guard let frame = synthesizedGestureReferenceFrame(app: app) else { + return CGRect(x: 0, y: 0, width: 0, height: 0) + } + return synthesizedFrameAvoidingKeyboardWhenSafe(app: app, frame: frame) #else return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) #endif @@ -1387,42 +1481,16 @@ extension RunnerTests { private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { #if os(iOS) - // Coordinate-only synthesized gestures can run after an AX-fatal screen because they do not + // Coordinate-only synthesized taps can run after an AX-fatal screen because they do not // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text // interactions intentionally stay on the normal AX path because they need an element query. + // Scroll/drag/sequence keep the normal foreground guard and stabilization path. guard command.text == nil, command.selectorKey == nil else { return false } guard hasCachedTargetForActivationSkip(command: command) else { return false } - switch command.command { - case .tap: - return command.synthesized == true - && command.x != nil - && command.y != nil - case .drag: - return command.synthesized == true - && command.x != nil - && command.y != nil - && command.x2 != nil - && command.y2 != nil - case .scroll: - return true - case .sequence: - guard let steps = command.steps, !steps.isEmpty else { return false } - return steps.allSatisfy { step in - if step.kind == "tap" { - return step.synthesized == true && step.x != nil && step.y != nil - } - if step.kind == "drag" { - return step.synthesized == true - && step.x != nil - && step.y != nil - && step.x2 != nil - && step.y2 != nil - } - return false - } - default: - return false - } + return command.command == .tap + && command.synthesized == true + && command.x != nil + && command.y != nil #else return false #endif diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 2305426f6..627b6654f 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -792,7 +792,8 @@ extension RunnerTests { y: Double, x2: Double, y2: Double, - durationMs: Double + durationMs: Double, + referenceFrame: CGRect? = nil ) -> RunnerInteractionOutcome { #if os(iOS) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { @@ -802,7 +803,7 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let frame = synthesizedGestureReferenceFrame(app: app) else { + guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else { return .unsupported( message: "synthesized coordinate drag could not resolve a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." @@ -837,7 +838,7 @@ extension RunnerTests { #endif } - func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double, referenceFrame: CGRect? = nil) -> RunnerInteractionOutcome { #if os(iOS) guard x.isFinite, y.isFinite else { return .unsupported( @@ -846,7 +847,7 @@ extension RunnerTests { ) } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - guard let frame = synthesizedGestureReferenceFrame(app: app) else { + guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else { return .unsupported( message: "synthesized coordinate tap could not resolve a finite screen frame", hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." @@ -1027,16 +1028,21 @@ extension RunnerTests { x: Double, y: Double, x2: Double, - y2: Double + y2: Double, + referenceFrame: CGRect? = nil, + avoidKeyboardWhenSafe: Bool = false ) -> SynthesizedDragPlan? { #if os(iOS) guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, - let frame = synthesizedGestureReferenceFrame(app: app) + let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else { return nil } + let points = avoidKeyboardWhenSafe + ? keyboardAvoidingSynthesizedDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) + : DragPoints(x: x, y: y, x2: x2, y2: y2) return SynthesizedDragPlan( - points: DragPoints(x: x, y: y, x2: x2, y2: y2), + points: points, referenceFrame: frame ) #else @@ -1061,7 +1067,7 @@ extension RunnerTests { ) } - func synthesizedGestureReferenceFrame(app _: XCUIApplication) -> CGRect? { + func synthesizedGestureReferenceFrame(app: XCUIApplication) -> CGRect? { #if os(iOS) return finiteSynthesizedReferenceFrame( appFrame: .zero, @@ -1094,6 +1100,40 @@ extension RunnerTests { return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height) } + func synthesizedFrameAvoidingKeyboardWhenSafe(app: XCUIApplication, frame: CGRect) -> CGRect { +#if os(iOS) + guard shouldProbeKeyboardForSynthesizedGesture() else { return frame } + return frameAvoidingKeyboard(app: app, frame: frame) +#else + return frame +#endif + } + + func keyboardAvoidingSynthesizedDragPoints( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double + ) -> DragPoints { +#if os(iOS) + guard shouldProbeKeyboardForSynthesizedGesture() else { + return DragPoints(x: x, y: y, x2: x2, y2: y2) + } + return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) +#else + return DragPoints(x: x, y: y, x2: x2, y2: y2) +#endif + } + + private func shouldProbeKeyboardForSynthesizedGesture() -> Bool { +#if os(iOS) + return !lastSnapshotHadAccessibilityUnavailable +#else + return false +#endif + } + private func isUsableReferenceFrame(_ frame: CGRect) -> Bool { return !frame.isNull && !frame.isEmpty @@ -1452,7 +1492,7 @@ extension RunnerTests { finiteSynthesizedReferenceFrame( appFrame: .infinite, fallbackBounds: .zero, - fallbackScreenshotSize: { CGSize(width: .infinity, height: 932) } + fallbackScreenshotSize: { CGSize(width: CGFloat.infinity, height: 932) } ) ) } diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index d2c8b640a..20b766ef0 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -166,13 +166,15 @@ extension RunnerTests { } if step.kind == "drag", step.synthesized == true { let dragPoints: DragPoints + let referenceFrame: CGRect? #if os(iOS) guard let dragPlan = axFreeSynthesizedDragPlan( app: activeApp, x: x, y: y, x2: step.x2 ?? x, - y2: step.y2 ?? y + y2: step.y2 ?? y, + avoidKeyboardWhenSafe: true ) else { let nowMs = ProcessInfo.processInfo.systemUptime * 1000 return SequenceStepOutcome( @@ -185,9 +187,11 @@ extension RunnerTests { ) } dragPoints = dragPlan.points + referenceFrame = dragPlan.referenceFrame #else dragPoints = keyboardAvoidingDragPoints( app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) + referenceFrame = nil #endif let durationMs = min(max(step.durationMs ?? 250, 16), 10000) let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { @@ -197,7 +201,8 @@ extension RunnerTests { y: dragPoints.y, x2: dragPoints.x2, y2: dragPoints.y2, - durationMs: durationMs + durationMs: durationMs, + referenceFrame: referenceFrame ) } if case .performed = outcome { diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 838ad1815..24cd6dba7 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -387,6 +387,7 @@ extension RunnerTests { func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) + lastSnapshotHadAccessibilityUnavailable = true invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) // This is a planned terminal result, so it carries the structured verdict like every other // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 373f7c4bc..2f4e5315f 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -349,6 +349,7 @@ extension RunnerTests { state: String, reason: (reason: String, code: String)? ) -> DataPayload { + lastSnapshotHadAccessibilityUnavailable = reason?.code == "ax-rejected" let payload = capture.payload let quality = SnapshotQuality( state: state, diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b5dc7cb36..cde46dd34 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -52,6 +52,7 @@ final class RunnerTests: XCTestCase { let maxRecordingFps = 120 var needsPostSnapshotInteractionDelay = false var needsFirstInteractionDelay = false + var lastSnapshotHadAccessibilityUnavailable = false var activeRecording: ScreenRecorder? let commandJournal = RunnerCommandJournal() // Coalesces duplicate transport sends of the same commandId onto the single in-flight