diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 0771c52f3..66494c8b5 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -182,6 +182,84 @@ extension RunnerTests { XCTAssertNil(xctestRecordedFailureResponse(command: tapCommand, response: runnerFatalResponse)) } + func testSkipAppActivationPreflightOnlyIncludesCoordinateOnlySynthesizedTaps() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let tap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","x":10,"y":20,"synthesized":true}"# + ) + + XCTAssertTrue(shouldSkipAppActivationPreflight(tap)) + } + + func testSkipAppActivationPreflightRejectsSelectorAndMixedSequenceGestures() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let selectorTap = try runnerCommandFixture( + #"{"command":"tap","commandId":"tap-1","selectorKey":"label","selectorValue":"Search","synthesized":true}"# + ) + let standardDrag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40}"# + ) + let mixedSequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"doubleTap","x":30,"y":40} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(selectorTap)) + XCTAssertFalse(shouldSkipAppActivationPreflight(standardDrag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(mixedSequence)) + } + + func testSkipAppActivationPreflightRequiresCachedTarget() throws { + currentApp = nil + currentBundleId = nil + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + } + + func testSkipAppActivationPreflightKeepsDragScrollAndSequenceOnForegroundGuard() throws { + currentApp = app + currentBundleId = nil + defer { + currentApp = nil + currentBundleId = nil + } + let drag = try runnerCommandFixture( + #"{"command":"drag","commandId":"drag-1","x":10,"y":20,"x2":30,"y2":40,"synthesized":true}"# + ) + let scroll = try runnerCommandFixture( + #"{"command":"scroll","commandId":"scroll-1","direction":"down","pixels":400}"# + ) + let sequence = try runnerCommandFixture( + """ + {"command":"sequence","commandId":"seq-1","steps":[ + {"kind":"tap","x":10,"y":20,"synthesized":true}, + {"kind":"drag","x":10,"y":200,"x2":10,"y2":100,"synthesized":true} + ]} + """ + ) + + XCTAssertFalse(shouldSkipAppActivationPreflight(drag)) + XCTAssertFalse(shouldSkipAppActivationPreflight(scroll)) + XCTAssertFalse(shouldSkipAppActivationPreflight(sequence)) + } + func testExecuteDispatchedReturnsBusyBeforeMainThreadFastPath() throws { let command = try runnerCommandFixture(#"{"command":"snapshot","commandId":"snapshot-busy"}"#) abandonedMainThreadWorkCount = 1 @@ -732,9 +810,9 @@ extension RunnerTests { message: "dragged" ) case .scroll: - // Fused frame-resolve + drag scroll for non-tvOS. Resolves the interaction frame via - // resolvedTouchReferenceFrame, computes drag endpoints with the Swift port of - // buildScrollGesturePlan, then runs the same non-synthesized drag path scroll's drag used. + // Fused frame-resolve + drag scroll for non-tvOS. On iOS this intentionally stays on the + // AX-free synthesized coordinate lane so scroll keeps working when XCTest cannot serialize + // the accessibility tree. guard let direction = command.direction, direction == "up" || direction == "down" || direction == "left" || direction == "right" else { @@ -746,7 +824,7 @@ extension RunnerTests { ) ) } - let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame) + let frame = scrollReferenceFrame(app: activeApp) guard frame.width > 0, frame.height > 0 else { return Response( ok: false, @@ -786,8 +864,10 @@ extension RunnerTests { x2: frame.minX + plan.x2, y2: frame.minY + plan.y2, durationMs: command.durationMs, - synthesized: command.durationMs != nil, - message: "scrolled" + synthesized: shouldUseSynthesizedScrollPath(), + message: "scrolled", + synthesizedReferenceFrame: frame, + allowSynthesizedCoordinateFallback: false ) case .desktopScroll: guard let direction = command.direction, @@ -1139,10 +1219,9 @@ extension RunnerTests { } } - /// Shared drag execution for `.drag` and the fused `.scroll`. Mirrors the original `.drag` body - /// exactly: keyboardAvoidingDragPoints -> resolvedDragVisualizationFrame -> synthesized branch - /// (16-10000ms clamp) or non-synthesized dragAt with coordinateDragHoldDuration -> - /// gestureResponse(.drag). `.scroll` uses the synthesized path only when a duration is requested. + /// Shared drag execution for `.drag` and the fused `.scroll`. The iOS synthesized lane avoids + /// keyboard/window lookup and `XCUICoordinate` fallback so coordinate gestures remain usable on + /// screens whose XCTest accessibility tree is unhealthy. private func executeDragGesture( activeApp: XCUIApplication, x: Double, @@ -1151,8 +1230,30 @@ extension RunnerTests { y2: Double, durationMs: Double?, synthesized: Bool, - message: String + message: String, + synthesizedReferenceFrame: CGRect? = nil, + allowSynthesizedCoordinateFallback: Bool = true ) -> Response { + let commandName = dragCommandName(message: message) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return Response( + ok: false, + error: ErrorPayload(code: "INVALID_ARGS", message: "\(commandName) requires finite coordinates") + ) + } + if synthesized, let synthesizedResponse = executeSynthesizedDragGesture( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message, + referenceFrame: synthesizedReferenceFrame, + allowCoordinateFallback: allowSynthesizedCoordinateFallback + ) { + return synthesizedResponse + } let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) let dragFrame = resolvedDragVisualizationFrame( app: activeApp, @@ -1203,6 +1304,151 @@ extension RunnerTests { ) } + private func executeSynthesizedDragGesture( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double?, + message: String, + referenceFrame: CGRect?, + allowCoordinateFallback: Bool + ) -> Response? { +#if os(iOS) + guard let plan = axFreeSynthesizedDragPlan( + app: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + referenceFrame: referenceFrame, + avoidKeyboardWhenSafe: true + ) + else { + if allowCoordinateFallback { + return executeCoordinateDragFallback( + activeApp: activeApp, + x: x, + y: y, + x2: x2, + y2: y2, + durationMs: durationMs, + message: message, + fallback: nil + ) + } + return Response( + ok: false, + error: ErrorPayload( + code: "INVALID_ARGS", + message: "\(dragCommandName(message: message)) could not resolve a finite synthesized coordinate frame" + ) + ) + } + let durationMs = min(max(durationMs ?? 250, 16), 10000) + let dragFrame = axFreeDragVisualizationFrame( + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + referenceFrame: plan.referenceFrame + ) + let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { + synthesizedDragAt( + app: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs, + referenceFrame: plan.referenceFrame + ) + } + if case .performed = outcome { + return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame)) + } + if allowCoordinateFallback { + return executeCoordinateDragFallback( + activeApp: activeApp, + x: plan.points.x, + y: plan.points.y, + x2: plan.points.x2, + y2: plan.points.y2, + durationMs: durationMs, + message: message, + fallback: gestureFallback(strategy: "xctest-coordinate-drag", from: outcome) + ) + } + return unsupportedResponse(for: outcome) +#else + return nil +#endif + } + + private func executeCoordinateDragFallback( + activeApp: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + durationMs: Double?, + message: String, + fallback: GestureFallback? + ) -> Response { + let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2) + let dragFrame = resolvedDragVisualizationFrame( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2 + ) + let holdDuration = synthesizedSwipeFallbackHoldDuration(durationMs: durationMs ?? 250) + let (timing, outcome) = performGesture(activeApp) { + dragAt( + app: activeApp, + x: dragPoints.x, + y: dragPoints.y, + x2: dragPoints.x2, + y2: dragPoints.y2, + holdDuration: holdDuration + ) + } + if let response = unsupportedResponse(for: outcome) { + return response + } + return gestureResponse( + message: message, + timing: timing, + frame: .drag(dragFrame), + fallback: fallback + ) + } + + private func scrollReferenceFrame(app: XCUIApplication) -> CGRect { +#if os(iOS) + guard let frame = synthesizedGestureReferenceFrame(app: app) else { + return CGRect(x: 0, y: 0, width: 0, height: 0) + } + return synthesizedFrameAvoidingKeyboardWhenSafe(app: app, frame: frame) +#else + return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) +#endif + } + + private func shouldUseSynthesizedScrollPath() -> Bool { +#if os(iOS) + return true +#else + return false +#endif + } + + private func dragCommandName(message: String) -> String { + return message == "scrolled" ? "scroll" : "drag" + } + private func currentXCTestFailureCount() -> Int { return testRun?.failureCount ?? 0 } @@ -1235,20 +1481,31 @@ extension RunnerTests { private func shouldSkipAppActivationPreflight(_ command: Command) -> Bool { #if os(iOS) - // Coordinate-only synthesized taps can run after an AX-fatal screen because they do not need - // app activation, window lookup, keyboard lookup, or element resolution. Selector/text taps - // intentionally stay on the normal AX path because they need an element query. + // Coordinate-only synthesized taps can run after an AX-fatal screen because they do not + // need app activation, window lookup, keyboard lookup, or element resolution. Selector/text + // interactions intentionally stay on the normal AX path because they need an element query. + // Scroll/drag/sequence keep the normal foreground guard and stabilization path. + guard command.text == nil, command.selectorKey == nil else { return false } + guard hasCachedTargetForActivationSkip(command: command) else { return false } return command.command == .tap && command.synthesized == true && command.x != nil && command.y != nil - && command.text == nil - && command.selectorKey == nil #else return false #endif } + private func hasCachedTargetForActivationSkip(command: Command) -> Bool { + guard currentApp != nil else { return false } + guard let bundleId = command.appBundleId?.trimmingCharacters(in: .whitespacesAndNewlines), + !bundleId.isEmpty + else { + return true + } + return currentBundleId == bundleId + } + private func resolveAppWithoutActivation(command: Command) -> XCUIApplication { guard let bundleId = command.appBundleId? .trimmingCharacters(in: .whitespacesAndNewlines), diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 7f015113d..627b6654f 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -36,6 +36,11 @@ extension RunnerTests { let y2: Double } + struct SynthesizedDragPlan { + let points: DragPoints + let referenceFrame: CGRect + } + struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool @@ -787,11 +792,23 @@ extension RunnerTests { y: Double, x2: Double, y2: Double, - durationMs: Double + durationMs: Double, + referenceFrame: CGRect? = nil ) -> RunnerInteractionOutcome { #if os(iOS) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite else { + return .unsupported( + message: "synthesized coordinate drag requires finite coordinates", + hint: "Retry with finite x, y, x2, and y2 values." + ) + } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let frame = app.frame + guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else { + return .unsupported( + message: "synthesized coordinate drag could not resolve a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeSwipe( @@ -804,7 +821,7 @@ extension RunnerTests { ) { return .unsupported( message: message, - hint: "Falling back to XCTest coordinate drag may be slower; update Xcode if this persists." + hint: "Private XCTest event synthesis is required for AX-free coordinate drag on iOS; update Xcode if this persists." ) } return .performed @@ -821,10 +838,22 @@ extension RunnerTests { #endif } - func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome { + func synthesizedTapAt(app: XCUIApplication, x: Double, y: Double, referenceFrame: CGRect? = nil) -> RunnerInteractionOutcome { #if os(iOS) + guard x.isFinite, y.isFinite else { + return .unsupported( + message: "synthesized coordinate tap requires finite coordinates", + hint: "Retry with finite x and y values." + ) + } let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app)) - let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation) + guard let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) else { + return .unsupported( + message: "synthesized coordinate tap could not resolve a finite screen frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ) + } + let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation) if let message = RunnerSynthesizedGesture.synthesizeTap( withApplication: app, x: Double(point.x), @@ -994,6 +1023,129 @@ extension RunnerTests { #endif } + func axFreeSynthesizedDragPlan( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double, + referenceFrame: CGRect? = nil, + avoidKeyboardWhenSafe: Bool = false + ) -> SynthesizedDragPlan? { +#if os(iOS) + guard x.isFinite, y.isFinite, x2.isFinite, y2.isFinite, + let frame = referenceFrame ?? synthesizedGestureReferenceFrame(app: app) + else { + return nil + } + let points = avoidKeyboardWhenSafe + ? keyboardAvoidingSynthesizedDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) + : DragPoints(x: x, y: y, x2: x2, y2: y2) + return SynthesizedDragPlan( + points: points, + referenceFrame: frame + ) +#else + return nil +#endif + } + + func axFreeDragVisualizationFrame( + x: Double, + y: Double, + x2: Double, + y2: Double, + referenceFrame: CGRect + ) -> DragVisualizationFrame { + return DragVisualizationFrame( + x: x, + y: y, + x2: x2, + y2: y2, + referenceWidth: Double(referenceFrame.width), + referenceHeight: Double(referenceFrame.height) + ) + } + + func synthesizedGestureReferenceFrame(app: XCUIApplication) -> CGRect? { +#if os(iOS) + return finiteSynthesizedReferenceFrame( + appFrame: .zero, + fallbackBounds: .zero, + fallbackScreenshotSize: { XCUIScreen.main.screenshot().image.size } + ) +#else + return nil +#endif + } + + func finiteSynthesizedReferenceFrame( + appFrame: CGRect, + fallbackBounds: CGRect, + fallbackScreenshotSize: () -> CGSize + ) -> CGRect? { + if isUsableReferenceFrame(appFrame) { + return appFrame + } + if isUsableReferenceFrame(fallbackBounds) { + return CGRect(x: 0, y: 0, width: fallbackBounds.width, height: fallbackBounds.height) + } + let screenshotSize = fallbackScreenshotSize() + guard screenshotSize.width.isFinite, screenshotSize.height.isFinite, + screenshotSize.width > 0, + screenshotSize.height > 0 + else { + return nil + } + return CGRect(x: 0, y: 0, width: screenshotSize.width, height: screenshotSize.height) + } + + func synthesizedFrameAvoidingKeyboardWhenSafe(app: XCUIApplication, frame: CGRect) -> CGRect { +#if os(iOS) + guard shouldProbeKeyboardForSynthesizedGesture() else { return frame } + return frameAvoidingKeyboard(app: app, frame: frame) +#else + return frame +#endif + } + + func keyboardAvoidingSynthesizedDragPoints( + app: XCUIApplication, + x: Double, + y: Double, + x2: Double, + y2: Double + ) -> DragPoints { +#if os(iOS) + guard shouldProbeKeyboardForSynthesizedGesture() else { + return DragPoints(x: x, y: y, x2: x2, y2: y2) + } + return keyboardAvoidingDragPoints(app: app, x: x, y: y, x2: x2, y2: y2) +#else + return DragPoints(x: x, y: y, x2: x2, y2: y2) +#endif + } + + private func shouldProbeKeyboardForSynthesizedGesture() -> Bool { +#if os(iOS) + return !lastSnapshotHadAccessibilityUnavailable +#else + return false +#endif + } + + private func isUsableReferenceFrame(_ frame: CGRect) -> Bool { + return !frame.isNull + && !frame.isEmpty + && !frame.isInfinite + && frame.minX.isFinite + && frame.minY.isFinite + && frame.width.isFinite + && frame.height.isFinite + && frame.width > 0 + && frame.height > 0 + } + func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? { if performTvRemoteSwipeIfAvailable(direction: direction) { @@ -1296,6 +1448,55 @@ extension RunnerTests { } } + func testFiniteSynthesizedReferenceFramePrefersValidAppFrame() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: CGRect(x: 4, y: 8, width: 390, height: 844), + fallbackBounds: CGRect(x: 0, y: 0, width: 1, height: 1), + fallbackScreenshotSize: { CGSize(width: 2, height: 2) } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 4, y: 8, width: 390, height: 844)) + } + + func testFiniteSynthesizedReferenceFrameFallsBackWithoutUsingInvalidAppFrame() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: CGRect(x: 20, y: 30, width: 430, height: 932), + fallbackScreenshotSize: { + XCTFail("screenshot fallback should not be used when screen bounds are finite") + return CGSize(width: 1, height: 1) + } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 0, y: 0, width: 430, height: 932)) + } + + func testFiniteSynthesizedReferenceFrameFallsBackToScreenshotSize() throws { + let frame = try XCTUnwrap( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: .zero, + fallbackScreenshotSize: { CGSize(width: 430, height: 932) } + ) + ) + + XCTAssertEqual(frame, CGRect(x: 0, y: 0, width: 430, height: 932)) + } + + func testFiniteSynthesizedReferenceFrameRejectsInvalidSources() { + XCTAssertNil( + finiteSynthesizedReferenceFrame( + appFrame: .infinite, + fallbackBounds: .zero, + fallbackScreenshotSize: { CGSize(width: CGFloat.infinity, height: 932) } + ) + ) + } + func testDesktopScrollWheelDeltasMapDirections() throws { XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "up", pixels: 120)).vertical, 120) XCTAssertEqual(try XCTUnwrap(desktopScrollWheelDeltas(direction: "down", pixels: 120)).vertical, -120) diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift index 12427228b..20b766ef0 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift @@ -165,8 +165,34 @@ extension RunnerTests { // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below. } if step.kind == "drag", step.synthesized == true { - let dragPoints = keyboardAvoidingDragPoints( + let dragPoints: DragPoints + let referenceFrame: CGRect? +#if os(iOS) + guard let dragPlan = axFreeSynthesizedDragPlan( + app: activeApp, + x: x, + y: y, + x2: step.x2 ?? x, + y2: step.y2 ?? y, + avoidKeyboardWhenSafe: true + ) else { + let nowMs = ProcessInfo.processInfo.systemUptime * 1000 + return SequenceStepOutcome( + outcome: .unsupported( + message: "synthesized coordinate drag could not resolve a finite coordinate frame", + hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates." + ), + gestureStartUptimeMs: nowMs, + gestureEndUptimeMs: nowMs + ) + } + dragPoints = dragPlan.points + referenceFrame = dragPlan.referenceFrame +#else + dragPoints = keyboardAvoidingDragPoints( app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y) + referenceFrame = nil +#endif let durationMs = min(max(step.durationMs ?? 250, 16), 10000) let (timing, outcome) = performGesture(activeApp, idleTimeout: false) { synthesizedDragAt( @@ -175,7 +201,8 @@ extension RunnerTests { y: dragPoints.y, x2: dragPoints.x2, y2: dragPoints.y2, - durationMs: durationMs + durationMs: durationMs, + referenceFrame: referenceFrame ) } if case .performed = outcome { @@ -188,6 +215,13 @@ extension RunnerTests { gestureEndUptimeMs: timing.gestureEndUptimeMs ) } +#if os(iOS) + return SequenceStepOutcome( + outcome: outcome, + gestureStartUptimeMs: timing.gestureStartUptimeMs, + gestureEndUptimeMs: timing.gestureEndUptimeMs + ) +#else let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250) let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) { dragAt( @@ -207,6 +241,7 @@ extension RunnerTests { gestureStartUptimeMs: fallbackTiming.gestureStartUptimeMs, gestureEndUptimeMs: fallbackTiming.gestureEndUptimeMs ) +#endif } let (timing, outcome) = performGesture(activeApp) { switch step.kind { diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 838ad1815..24cd6dba7 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -387,6 +387,7 @@ extension RunnerTests { func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload { NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message) + lastSnapshotHadAccessibilityUnavailable = true invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason) // This is a planned terminal result, so it carries the structured verdict like every other // planned snapshot — downstream sparse handling keys off the verdict, not node shapes. diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 373f7c4bc..2f4e5315f 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -349,6 +349,7 @@ extension RunnerTests { state: String, reason: (reason: String, code: String)? ) -> DataPayload { + lastSnapshotHadAccessibilityUnavailable = reason?.code == "ax-rejected" let payload = capture.payload let quality = SnapshotQuality( state: state, diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index b5dc7cb36..cde46dd34 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -52,6 +52,7 @@ final class RunnerTests: XCTestCase { let maxRecordingFps = 120 var needsPostSnapshotInteractionDelay = false var needsFirstInteractionDelay = false + var lastSnapshotHadAccessibilityUnavailable = false var activeRecording: ScreenRecorder? let commandJournal = RunnerCommandJournal() // Coalesces duplicate transport sends of the same commandId onto the single in-flight diff --git a/src/platforms/apple/core/__tests__/index.test.ts b/src/platforms/apple/core/__tests__/index.test.ts index 56bd410d1..24146138b 100644 --- a/src/platforms/apple/core/__tests__/index.test.ts +++ b/src/platforms/apple/core/__tests__/index.test.ts @@ -390,6 +390,30 @@ test('iosRunnerOverrides maps iOS scroll to a single fused scroll command', asyn }); }); +test('iosRunnerOverrides maps iOS scroll without duration to a fused runner scroll', async () => { + mockRunAppleRunnerCommand.mockResolvedValueOnce({ + x: 200, + y: 640, + x2: 200, + y2: 240, + referenceWidth: 400, + referenceHeight: 800, + }); + + const { overrides } = iosRunnerOverrides(IOS_TEST_SIMULATOR, { + appBundleId: 'com.example.App', + }); + + await overrides.scroll('down', { pixels: 400 }); + + assert.deepEqual(mockRunAppleRunnerCommand.mock.calls[0]?.[1], { + command: 'scroll', + direction: 'down', + pixels: 400, + appBundleId: 'com.example.App', + }); +}); + test('iosRunnerOverrides maps tvOS scroll duration to remote press hold duration', async () => { mockRunAppleRunnerCommand.mockResolvedValueOnce({ ok: true,