From 4784bdb6eebc691cfd6f94378593e0f65733cb75 Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Wed, 19 Aug 2026 08:39:59 +0200 Subject: [PATCH 1/3] feat(drive): Add the `jpdrive` accessibility driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Debugging the macOS app from a conversation needs a way to read what is actually on screen and act on it. Screenshots answer "what does it look like" but not "what is the button called, is it enabled, does the menu item exist" — and none of that is reachable from the app's own test bundle, which runs inside the process and cannot see menu enablement, the pasteboard, or a relaunch. `jpdrive` reads and drives another application through its accessibility tree, and writes one JSON document to stdout either way: a result, or an error with a non-zero exit status. It is what the `debug_app_*` tools shell out to. A standalone SwiftPM package rather than a target in the app's Xcode project, so the binary lands at a predictable path and nothing has to search derived data for it. The logic lives in a `DriveKit` library with a one-line executable on top, because SwiftPM cannot cleanly test an executable target and the tree traversal is where the bugs are. The tests run against a fake accessibility tree, so they need no running app and no accessibility grant. The package mirrors the app's strictness: Swift 6 language mode, `ExistentialAny`, and warnings as errors. SwiftPM 6.0 has no first-class setting for the last of those, so it goes through `unsafeFlags`, which is rejected only for a package consumed as a dependency — this one never is. Accessibility is gated by TCC, and a grant given to a terminal does not obviously reach a tool that terminal started. `just drive-doctor` reports whether the calling process may read another app's tree, so that question is answered by running it under each host rather than guessed at. Signed-off-by: Jean Mertz --- apps/macos/Tools/jpdrive/Package.swift | 34 + apps/macos/Tools/jpdrive/README.md | 298 ++++ .../jpdrive/Sources/DriveKit/AXElement.swift | 359 +++++ .../Sources/DriveKit/AXErrorName.swift | 24 + .../Tools/jpdrive/Sources/DriveKit/Act.swift | 1218 +++++++++++++++++ .../jpdrive/Sources/DriveKit/Ambient.swift | 80 ++ .../jpdrive/Sources/DriveKit/Arguments.swift | 425 ++++++ .../jpdrive/Sources/DriveKit/Doctor.swift | 95 ++ .../jpdrive/Sources/DriveKit/DriveError.swift | 81 ++ .../jpdrive/Sources/DriveKit/Driver.swift | 68 + .../Tools/jpdrive/Sources/DriveKit/Dump.swift | 119 ++ .../jpdrive/Sources/DriveKit/Duration.swift | 15 + .../jpdrive/Sources/DriveKit/Element.swift | 186 +++ .../Tools/jpdrive/Sources/DriveKit/Menu.swift | 52 + .../jpdrive/Sources/DriveKit/Output.swift | 44 + .../jpdrive/Sources/DriveKit/Pixels.swift | 252 ++++ .../Sources/DriveKit/ProcessTable.swift | 58 + .../Tools/jpdrive/Sources/DriveKit/Tree.swift | 183 +++ .../jpdrive/Sources/DriveKit/WindowIDs.swift | 133 ++ .../jpdrive/Sources/DriveKit/Windows.swift | 74 + .../Tools/jpdrive/Sources/jpdrive/main.swift | 3 + .../Tests/DriveKitTests/ActTests.swift | 249 ++++ .../Tests/DriveKitTests/ArgumentsTests.swift | 201 +++ .../Tests/DriveKitTests/ClickTests.swift | 115 ++ .../Tests/DriveKitTests/DragTests.swift | 229 ++++ .../Tests/DriveKitTests/FakeElement.swift | 173 +++ .../Tests/DriveKitTests/FakePoster.swift | 35 + .../Tests/DriveKitTests/MenuTests.swift | 272 ++++ .../Tests/DriveKitTests/PixelsTests.swift | 231 ++++ .../Tests/DriveKitTests/ResizeTests.swift | 103 ++ .../Tests/DriveKitTests/TreeTests.swift | 164 +++ .../Tests/DriveKitTests/TypeTests.swift | 186 +++ .../Tests/DriveKitTests/WaitForTests.swift | 137 ++ .../Tests/DriveKitTests/WindowIDsTests.swift | 129 ++ .../Tests/DriveKitTests/WindowsTests.swift | 61 + justfile | 46 + 36 files changed, 6132 insertions(+) create mode 100644 apps/macos/Tools/jpdrive/Package.swift create mode 100644 apps/macos/Tools/jpdrive/README.md create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift create mode 100644 apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift diff --git a/apps/macos/Tools/jpdrive/Package.swift b/apps/macos/Tools/jpdrive/Package.swift new file mode 100644 index 000000000..5e3cee840 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Package.swift @@ -0,0 +1,34 @@ +// swift-tools-version: 6.0 + +import PackageDescription + +// Mirrors the app's `project.yml`: an existential is spelled `any P`, and a +// warning that never fails a build is a warning nobody fixes. +// +// SwiftPM 6.0 has no first-class setting for warnings-as-errors, and +// `unsafeFlags` is rejected only for a package consumed as a dependency, which +// this one never is. +let strict: [SwiftSetting] = [ + .swiftLanguageMode(.v6), + .enableUpcomingFeature("ExistentialAny"), + .unsafeFlags(["-warnings-as-errors"]), +] + +let package = Package( + name: "jpdrive", + platforms: [.macOS(.v15)], + products: [ + .executable(name: "jpdrive", targets: ["jpdrive"]) + ], + targets: [ + // The driver's logic, in a library so it can be tested. SwiftPM cannot + // cleanly test an executable target, and the traversal is where the bugs + // are. + .target(name: "DriveKit", swiftSettings: strict), + + // One line, calling into the library. + .executableTarget(name: "jpdrive", dependencies: ["DriveKit"], swiftSettings: strict), + + .testTarget(name: "DriveKitTests", dependencies: ["DriveKit"], swiftSettings: strict), + ] +) diff --git a/apps/macos/Tools/jpdrive/README.md b/apps/macos/Tools/jpdrive/README.md new file mode 100644 index 000000000..58206ee13 --- /dev/null +++ b/apps/macos/Tools/jpdrive/README.md @@ -0,0 +1,298 @@ +# jpdrive + +Reads and acts on a running macOS app's accessibility tree, speaking JSON. The +`debug_app_*` tools shell out to it; the Rust side stays the presenter, parsing +the JSON and rendering markdown. + +Swift rather than Rust because `AXUIElement` is CoreFoundation-shaped: ordinary +code here, unsafe bindings or a 784-download crate there. + +External rather than an in-app automation socket, deliberately. Driving through +`AXUIElement` means a broken accessibility tree breaks the tooling, which is the +pressure that keeps the app's accessibility honest. + +## Build + +```sh +just build-drive +``` + +The binary lands at `.build/release/jpdrive` under this directory. + +## The TCC question + +Everything downstream depends on one unknown: does a binary launched as a child +of `just serve-tools` inherit the Accessibility grant given to the terminal? + +macOS attributes TCC to the *responsible process*, which for a command-line tool +is normally the terminal rather than the tool. That is the same mechanism behind +the `sample(1)` note in `.config/jp/tools/src/debug_jp/profile_sampling.rs` about +granting Terminal *Developer Tools*. Apple documents neither the algorithm nor +its stability, so the answer has to be measured. + +`jpdrive doctor` measures it. Run it three ways, with Accessibility granted to +the terminal application and the app running: + +Check the target first. An empty `pgrep` means the app is not running, and a +run without a target reports the trust flag alone, which is the half of the +answer that can be wrong: + +```sh +pgrep -f JP.app # must print exactly one pid +``` + +```sh +# 1. Directly from the terminal. +.build/release/jpdrive doctor --pid $(pgrep -f JP.app) + +# 2. Through just, which adds the process layer the tools will run under. +just drive-doctor $(pgrep -f JP.app) +``` + +The third case, a child of `jp-tools` under `just serve-tools`, needs a tool that +shells out to the driver. Reaching it means writing the first `debug_app_*` tool, +which is why cases 1 and 2 come first: if the grant already fails at case 2, +nothing is learned by going further. + +Compare `trusted` and `probe.axError` across the runs. `trusted: true` with a +window count means the grant inherits. `trusted: false`, or `api_disabled` / +`cannot_complete` from the probe, means it does not, and the driver needs its own +signed bundle or its own grant. + +The report lists the ancestor chain, so a `false` says which processes were +candidates for holding the grant. + +The check never prompts. `AXIsProcessTrustedWithOptions` with +`kAXTrustedCheckOptionPrompt` would raise the system dialog and change the state +being measured. + +### Result + +**The grant inherits.** With Accessibility granted to Ghostty, case 2 reports +`trusted: true` and a window count from a chain of six: + +``` +ghostty → login → fish → just → sh → jpdrive +``` + +So `tree`, `windows`, `menu`, and `act` need no signed bundle and no grant of +their own. They can assume the terminal's. Case 3, a child of `jp-tools` under +`just serve-tools`, adds one more process of the same kind and is still +unmeasured. + +Observations across the runs, on macOS with Ghostty as the terminal: + +- Process depth is not the variable. Run directly from the shell (chain of four, + up to `ghostty`) and through `just` (chain of six, adding `sh` and `just`), the + report is identical. Whatever governs the grant, it is not the number of + processes between the terminal and the driver. +- The trust flag and the probe agree. `trusted: false` came with + `ax_error: api_disabled` from a real read against a running app, which is what + the accessibility API returns to an untrusted caller; `trusted: true` came with + a window count. No case has been seen where the two disagree. +- Untested: a terminal instance started *before* the grant. The `false` runs and + the `true` run may differ by the grant alone, by a relaunch, or by both, so + "the grant is not visible to this terminal instance" is not yet ruled out as a + separate failure mode. + +## Screen Recording is a second grant + +`windowid` answers the window server rather than the accessibility API, and the +two are governed by different TCC grants. Enumerating windows needs neither, so +the command works with nothing granted at all; reading a window's *title*, and +capturing its content with `screencapture -l`, need Screen Recording. + +That is why the report pairs the list with a `screen_recording` flag rather than +refusing outright. Missing the grant, a capture succeeds and returns the desktop +where the window should be, so the caller has to know before it writes a file. +An untitled window in the list is the same fact seen from the other side. + +The pane is System Settings ▸ Privacy & Security ▸ Screen & System Audio +Recording, and as with Accessibility it is the terminal application that needs +it, not the driver. + +### Result + +**The grant inherits.** Measured with Ghostty as the terminal, the driver run as +a child of `jp-tools` under `just serve-tools`: before the grant, +`screen_recording` came back `false` and `debug_app_screenshot` refused; after +granting Screen Recording to Ghostty and restarting it, the same call captured +the window. + +So the flag is worth trusting, and this grant reaches a driver six processes +deep from the terminal, same as Accessibility does. + +Untested: whether the restart was necessary. The grant and the restart happened +together, so nothing here separates them. + +## What the sidebar looks like through accessibility + +SwiftUI's `.accessibilityIdentifier` does not land on the element that owns +behaviour. For a `List` row it lands two levels below it: + +``` +AXOutline AXIdentifier: sidebar.list AXRows: 1065, AXVisibleRows: 9 + AXRow AXSelected settable: true + AXCell AXSelected settable: false, AXScrollToVisible settable: true + AXUnknown AXIdentifier: sidebar.row. + AXAttributedDescription: ", <n> events" + no actions, no children +``` + +So addressing an element and acting on it are two different steps. The identified +element has no actions at all: no `AXPress`, nothing. Selecting a row means +walking up to the `AXRow` and writing `AXSelected`. + +That write is preferable to a synthesized click for a reason beyond determinism. +Every row exists as an accessibility element, but only nine are on screen: the +outline's frame is 41658pt tall against a 398pt viewport. A click at +`AXActivationPoint` would miss an off-screen row, or land on whichever row +occupies those coordinates instead. An `AXSelected` write is independent of +scroll position. + +`AXScrollToVisible` appears as a settable *attribute* on a sidebar cell and as an +*action* on a transcript event, so scrolling has to try both forms. + +The sidebar materialises every row; the transcript does not. Only one +`transcript.event.*` element exists at a time, so an identifier that names an +unrendered event cannot be waited for, only scrolled to. + +Writing `AXSelected` on a row 690 places down a thousand-row list selects it and +brings it into view, so selecting a row needs no scrolling step of its own. The +transcript still does. + +### The identified element cannot be walked upwards + +The `AXUnknown` carrying the identifier reports no `AXParent`, and no +`AXTopLevelUIElement` either, unlike the cell and row above it. Climbing from it +arrives nowhere. + +So resolving an identifier means keeping the chain the search descended through, +not finding the element and navigating from it afterwards. Anything that acts on +an ancestor of an identified element depends on this. + +### Cost + +An accessibility round-trip to this app costs roughly 3ms, and that number sets +every other budget: + +- Reading the first few rows under `sidebar.` takes 250ms. +- Finding one row 690 places down takes 5.8s, because the search reads about two + thousand elements to get there and cannot prune on the way: every identifier in + the sidebar sits on a leaf. + +Hence the batched reads and the match budget. Anything that polls should resolve +an element once and re-read that reference, rather than searching each time. + +## Acting on an element + +Each step names exactly one mechanism, because the mechanism depends on what the +element is and guessing hides regressions: + +| step | addressed by | mechanism | +| --- | --- | --- | +| `select` | identifier | write `AXSelected` on the nearest ancestor accepting it | +| `press` | identifier | `AXPress` on the element itself | +| `type` | identifier | write `AXValue`, then `AXConfirm` | +| `perform` | identifier | a named action, for the verbs with no step of their own | +| `menu` | titled path | `AXPress` on the item the path resolves to | +| `click` | identifier | synthesized mouse event at `AXActivationPoint` | + +`press` and `menu` end in the same call and are not redundant: they differ in what +they address by, and that is what a test pins. `closeAll:` is an `AppKit` selector +name that survives the item moving to another menu, so a script keyed on it cannot +notice the menu bar being rearranged. `["File", "Close All"]` names the structure +the user sees, and a path that stops resolving reports how far it got and what that +level holds instead — which is the assertion failure a layout test wants to read. + +A step that names the wrong mechanism fails and says which actions the element +does accept. There is no fallback chain: if a sidebar row stopped accepting +`AXSelected`, a driver that quietly fell back to a synthesized click would keep +every script green while the app's accessibility rotted, which is the failure this +tool exists to prevent. + +`select` and `type` read the attribute back afterwards, because a write can be +accepted and discarded. `press` cannot: nothing observable says a button did +anything, so its result reports no confirmation rather than claiming one. + +### A menu step has to bring the app forward + +`menu` writes `AXFrontmost` on the application and waits for it to take, which +makes it the one step that takes focus from whatever had it. + +Without it almost nothing in the menu bar can be pressed. AppKit disables every +item that acts on the front window or the responder chain while the application +is in the background, and against a driven instance that is most of the bar: +`Close`, `Copy`, `Select All`, `Show Sidebar`, and every `SwiftUI` command +reading a `@FocusedValue` all report `AXEnabled: 0`. `New Window` and `Close +All` do not, which is what makes the difference easy to miss — a first menu step +against an app-level item works, and the next one silently does nothing. + +So the item's enabled state is checked before it is pressed, rather than +trusting `AXPress` to report a refusal. A disabled item accepts the press and +answers success. + +An element that reports no `AXEnabled` at all is not disabled. Plenty carry no +such attribute, and reading its absence as a refusal would reject them all. + +### Typing writes the value, and then has to commit it + +`type` writes `AXValue` and performs `AXConfirm`. Both are needed, and the second +one is the part that was not obvious. + +Writing `AXValue` on a `SwiftUI` `TextField` changes the text the field displays +and leaves the binding behind it untouched. Measured against the conversation +filter: after the write the field read back `"accessibility"` and the list still +showed all 1,066 rows. Deleting one character by hand then filtered on +`"accessibilit"` — the keystroke made the binding resync from whatever the field +held by then. So a `type` that only wrote the value would report success while the +application carried on as though nothing had been typed. + +`AXConfirm` commits through the path the binding observes. A field advertising no +confirm action is not a failure — some publish every change as it happens — so the +result reports `committed` separately from `confirmed`: the text being in the field +and the application having seen it are different facts. + +Synthesizing key events was rejected on three counts: the events go wherever focus +is, so a window activating mid-sequence types into it instead; posting them fast +enough to be useful means pauses between characters, which makes the step flaky +rather than deterministic; and event posting is global process state, so it could +not sit behind the element abstraction the rest of the driver is tested through. + +The remaining cost is that per-character behaviour never runs. A field that +validates each keystroke, or completes as you type, sees one change rather than a +dozen. + +### Clicking is the last resort + +`click` raises the element's window and posts a mouse event at its +`AXActivationPoint`. It is the only step whose effect is not addressed to an +element: the event goes to whatever occupies that screen coordinate, which is why +the window is raised first and why an occluding window from another application +will still swallow it. + +An element reporting no activation point is refused rather than clicked at the +origin. A sidebar row is exactly that case, and it wants `select`. + +Posting is behind an `EventPoster`, so where the driver aimed can be asserted in a +test even though where the event lands cannot. + +### Apple Events are a separate pathway, and that one does not inherit + +Reading the same tree through AppleScript fails from the terminal the driver +succeeds from: + +``` +System Events got an error: osascript is not allowed assistive access. (-1719) +``` + +Two different checks. `AXIsProcessTrusted`, which the driver calls, resolves to +the responsible process and finds the terminal. `System Events` requires the +calling binary itself to be listed, and the calling binary is `/usr/bin/osascript` +— shared by everything on the machine, so granting it grants far more than the +driver needs. + +This is the second reason the driver is a binary of its own rather than a shell +script over `osascript`, alongside the one at the top of this file. It also means +AppleScript is not a fallback when the driver is missing a verb: the verb has to +be added here. diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift new file mode 100644 index 000000000..4ee15b376 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift @@ -0,0 +1,359 @@ +import ApplicationServices +import Foundation + +/// An element of a running application's accessibility tree. +/// +/// Every method here is one or more synchronous round-trips to the target's main +/// thread. That cost dominates everything the driver does, so callers batch reads +/// with ``values(_:)`` rather than reading attributes one at a time, and hold onto +/// an element they will read again instead of walking to it twice. +/// +/// A reference stays valid while the underlying element lives. Once it is gone, +/// reads answer `invalid_ui_element` rather than crashing. +struct AXElement { + let element: AXUIElement + + /// The root element of the application owning `pid`. + /// + /// Succeeds whether or not the process exists; the first read is what fails. + static func application(pid: pid_t) -> AXElement { + return AXElement(element: AXUIElementCreateApplication(pid)) + } + + /// `AXRole`, or `nil` when the element does not report one. + var role: String? { + return read(kAXRoleAttribute).flatMap { $0 as? String } + } + + /// `AXIdentifier`, or `nil` when the element carries none. + /// + /// SwiftUI's `.accessibilityIdentifier` surfaces here, but it also composites + /// with identifiers the framework generates itself, so a value like + /// `"workspace-AppWindow-1, SidebarNavigationSplitView"` is possible. + var identifier: String? { + return read(kAXIdentifierAttribute).flatMap { $0 as? String } + } + + /// The element's human-readable label. + /// + /// Tries `AXAttributedDescription`, then `AXDescription`, then `AXTitle`. + /// SwiftUI populates the first of those for list rows and leaves the others + /// empty, while AppKit controls tend to do the reverse. + var label: String? { + if let attributed = read(Self.attributedDescription) as? NSAttributedString { + return attributed.string + } + + for name in [kAXDescriptionAttribute, kAXTitleAttribute] { + guard let text = read(name).flatMap({ $0 as? String }), !text.isEmpty else { + continue + } + return text + } + + return nil + } + + /// `AXAttributedDescription`, which has no constant in the SDK headers. + static let attributedDescription = "AXAttributedDescription" + + /// `AXActivationPoint`, which has no constant in the SDK headers. + /// + /// Where the element says a click on it belongs, in screen coordinates. Not + /// always the middle of its frame. + static let activationPoint = "AXActivationPoint" + + /// The element's children, or an empty array when it has none. + /// + /// A round-trip of its own. A walk should take children from ``read(_:)``, + /// which fetches them alongside everything else it needs. + var children: [AXElement] { + guard let value = read(kAXChildrenAttribute), let raw = value as? [AXUIElement] else { + return [] + } + return raw.map { AXElement(element: $0) } + } + + /// Actions the element accepts, such as `AXPress`. + var actions: [String] { + var names: CFArray? + guard AXUIElementCopyActionNames(element, &names) == .success, + let names = names as? [String] + else { + return [] + } + return names + } + + /// Every attribute name the element advertises. + func names() -> [String] { + var names: CFArray? + guard AXUIElementCopyAttributeNames(element, &names) == .success, + let names = names as? [String] + else { + return [] + } + return names + } + + /// Read one attribute, or `nil` when the read fails or the value is absent. + /// + /// Use ``values(_:)`` when reading more than one: this costs a round-trip per + /// call, which is what makes a naive tree walk take seconds. + func read(_ name: String) -> CFTypeRef? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success + else { + return nil + } + guard let value, CFGetTypeID(value) != CFNullGetTypeID() else { return nil } + return value + } + + /// Read several attributes in one round-trip. + /// + /// Results are positional and the same count as `names`. An attribute that + /// could not be read arrives as CoreFoundation's null or as an `AXValue` + /// boxing the error, both of which ``text(_:)`` reports rather than discards. + func values(_ names: [String]) -> [CFTypeRef?] { + guard !names.isEmpty else { return [] } + + var raw: CFArray? + let status = AXUIElementCopyMultipleAttributeValues( + element, + names as CFArray, + AXCopyMultipleAttributeOptions(), + &raw + ) + + guard status == .success, + let values = raw as? [CFTypeRef], + values.count == names.count + else { + return Array(repeating: nil, count: names.count) + } + + return values + } + + /// Whether `name` can be written on this element. + /// + /// A failed query reports as not settable: the accessibility API answers this + /// for every attribute it advertises, so a failure means the element is gone + /// or the attribute is not really there. + func isSettable(_ name: String) -> Bool { + var settable = DarwinBoolean(false) + guard AXUIElementIsAttributeSettable(element, name as CFString, &settable) == .success + else { + return false + } + return settable.boolValue + } + + /// Perform `action`, returning the API's own status. + func perform(_ action: String) -> AXError { + return AXUIElementPerformAction(element, action as CFString) + } +} + +extension AXElement: Element { + /// Read the named attributes and the element's children in one round-trip. + /// + /// Children come back in the same batch as everything else: asking for them + /// separately would add a hop per element, and every walk asks for them. + func read(_ names: [String]) -> Reading<AXElement> { + let values = self.values(names + [kAXChildrenAttribute]) + + let children = (values.last.flatMap { $0 } as? [AXUIElement] ?? []) + .map { AXElement(element: $0) } + + return Reading( + text: values.dropLast().map(Self.optionalText), + children: children + ) + } + + /// Read a boolean attribute. + /// + /// `CFBoolean` bridges to `NSNumber` rather than to `Bool`, so a direct cast + /// answers `nil` for a perfectly good `0` or `1`. + func flag(_ name: String) -> Bool? { + guard let value = read(name) as? NSNumber else { return nil } + return value.boolValue + } + + /// Write a boolean attribute, answering the API's own status. + func setFlag(_ name: String, _ value: Bool) -> AXError { + return AXUIElementSetAttributeValue( + element, + name as CFString, + value ? kCFBooleanTrue : kCFBooleanFalse + ) + } + + /// The point held in an attribute, in screen coordinates. + func point(_ name: String) -> CGPoint? { + guard let value = read(name), CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + + let boxed = unsafeDowncast(value, to: AXValue.self) + guard AXValueGetType(boxed) == .cgPoint else { return nil } + + var point = CGPoint.zero + guard AXValueGetValue(boxed, .cgPoint, &point) else { return nil } + + return point + } + + /// The size held in an attribute, in points. + func size(_ name: String) -> CGSize? { + guard let value = read(name), CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + + let boxed = unsafeDowncast(value, to: AXValue.self) + guard AXValueGetType(boxed) == .cgSize else { return nil } + + var size = CGSize.zero + guard AXValueGetValue(boxed, .cgSize, &size) else { return nil } + + return size + } + + /// Write a size attribute, answering the API's own status. + /// + /// The value has to be boxed in an `AXValue`: the API takes `CFTypeRef` and a + /// bare `CGSize` is not one, so passing it any other way fails the write with + /// no indication of why. + func setSize(_ name: String, _ value: CGSize) -> AXError { + var size = value + guard let boxed = AXValueCreate(.cgSize, &size) else { + return .failure + } + + return AXUIElementSetAttributeValue(element, name as CFString, boxed) + } + + /// Write a string attribute, answering the API's own status. + func setText(_ name: String, _ value: String) -> AXError { + return AXUIElementSetAttributeValue(element, name as CFString, value as CFString) + } + + /// The elements held in an attribute. + func elements(_ name: String) -> [AXElement] { + guard let value = read(name) else { return [] } + + if let raw = value as? [AXUIElement] { + return raw.map { AXElement(element: $0) } + } + + guard CFGetTypeID(value) == AXUIElementGetTypeID() else { return [] } + return [AXElement(element: unsafeDowncast(value, to: AXUIElement.self))] + } +} + +extension AXElement { + /// Render an attribute value as text, or `nil` when there is no value. + /// + /// A batched read answers an absent attribute with CoreFoundation's null and an + /// unreadable one with a boxed error. Both are facts a dump wants to see and a + /// caller reading one attribute wants as nothing at all. + static func optionalText(_ value: CFTypeRef?) -> String? { + guard let value, CFGetTypeID(value) != CFNullGetTypeID() else { return nil } + + if CFGetTypeID(value) == AXValueGetTypeID(), + AXValueGetType(unsafeDowncast(value, to: AXValue.self)) == .axError + { + return nil + } + + return text(value) + } + + /// Render an attribute value as text. + /// + /// Values arrive as CoreFoundation types, including geometry boxed in + /// `AXValue` and references to other elements. Everything becomes a string so + /// that a reader can see which attributes exist and which carry identifiers + /// without this growing a case per boxed type. + static func text(_ value: CFTypeRef) -> String { + // An attribute the element advertises but cannot answer for, such as + // `AXSubrole` on an element that has none. + if CFGetTypeID(value) == CFNullGetTypeID() { + return "<null>" + } + if let text = value as? String { + return text + } + // Labels arrive as attributed strings more often than plain ones, and the + // attributes carry nothing the driver acts on. + if let attributed = value as? NSAttributedString { + return attributed.string + } + if let number = value as? NSNumber { + return number.stringValue + } + if let elements = value as? [AXUIElement] { + return "<\(elements.count) AXUIElement>" + } + if let array = value as? [Any] { + return "<array of \(array.count)>" + } + + let typeID = CFGetTypeID(value) + if typeID == AXUIElementGetTypeID() { + return "<AXUIElement>" + } + if typeID == AXValueGetTypeID() { + // The conditional form is rejected here: every CoreFoundation type is + // bridged as a class, so the compiler sees a cast that cannot fail. + // The type ID check above is the real test. + return text(unsafeDowncast(value, to: AXValue.self)) + } + return "<CFTypeID \(typeID)>" + } + + /// Render the geometry boxed in an `AXValue`. + /// + /// `AXActivationPoint` and `AXFrame` decide where a synthesized click lands, + /// so these arrive as numbers a reader can check against the screen rather + /// than as an opaque marker. + static func text(_ value: AXValue) -> String { + let type = AXValueGetType(value) + + switch type { + // A batched read reports a per-attribute failure by boxing the error + // rather than by failing the whole call. + case .axError: + var status = AXError.success + guard AXValueGetValue(value, .axError, &status) else { break } + return "<\(status.name)>" + + case .cgPoint: + var point = CGPoint.zero + guard AXValueGetValue(value, .cgPoint, &point) else { break } + return "\(point.x),\(point.y)" + + case .cgSize: + var size = CGSize.zero + guard AXValueGetValue(value, .cgSize, &size) else { break } + return "\(size.width)x\(size.height)" + + case .cgRect: + var rect = CGRect.zero + guard AXValueGetValue(value, .cgRect, &rect) else { break } + return "\(rect.origin.x),\(rect.origin.y) \(rect.size.width)x\(rect.size.height)" + + case .cfRange: + var range = CFRange() + guard AXValueGetValue(value, .cfRange, &range) else { break } + return "\(range.location)+\(range.length)" + + default: + break + } + + return "<AXValue \(type.rawValue)>" + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift new file mode 100644 index 000000000..68add5ffe --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift @@ -0,0 +1,24 @@ +import ApplicationServices + +extension AXError { + /// A stable snake_case name for this error. + /// + /// Only the codes a read or an action can realistically produce are named. + /// Anything else keeps its numeric code rather than being flattened into + /// "unknown", so an unexpected failure stays traceable to a header. + var name: String { + switch self { + case .success: return "success" + case .apiDisabled: return "api_disabled" + case .cannotComplete: return "cannot_complete" + case .invalidUIElement: return "invalid_ui_element" + case .notImplemented: return "not_implemented" + case .attributeUnsupported: return "attribute_unsupported" + case .actionUnsupported: return "action_unsupported" + case .noValue: return "no_value" + case .illegalArgument: return "illegal_argument" + case .failure: return "failure" + default: return "ax_error_\(rawValue)" + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift new file mode 100644 index 000000000..2faf78dc7 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift @@ -0,0 +1,1218 @@ +import ApplicationServices +import Foundation + +/// One thing to do to one element. +/// +/// Each case names its own mechanism. There is no step that picks a mechanism +/// based on what the element supports: a script that says `select` against +/// something unselectable fails and says so, which is how a change in the app's +/// accessibility becomes visible instead of being absorbed by a fallback. +/// +/// Decoded from a single-key object, so a step reads as what it does: +/// +/// ```json +/// {"select": {"identifier": "sidebar.row.17855681129"}} +/// ``` +enum Step: Decodable { + /// Write `AXSelected` on the nearest ancestor that accepts it. + /// + /// The mechanism for list and outline rows, where the identified element is + /// below the one that owns selection. Independent of scroll position, so it + /// reaches a row that is not on screen. + case select(Target) + + /// Perform `AXPress` on the identified element itself. + /// + /// The mechanism for buttons and menu items, which advertise the action. + case press(Target) + + /// Synthesize a mouse click at the element's activation point. + /// + /// The last resort, and the only step that depends on the world outside the + /// accessibility tree: the window has to be frontmost and the element on + /// screen, or the click lands somewhere else entirely. + case click(Target) + + /// Perform a named accessibility action on the identified element. + /// + /// The long tail. `press` is this with `AXPress` and a better error message, + /// and is worth keeping because it is the overwhelmingly common case; anything + /// else an element advertises — `AXConfirm`, `AXShowMenu`, `AXScrollToVisible`, + /// `AXCancel` — is reached through here rather than by growing a step per verb. + case perform(ActionTarget) + + /// Put text into a text field. + case type(TypeTarget) + + /// Set an element's size, which for a window resizes it. + /// + /// The one step that changes the shape of what is on screen rather than what + /// is in it, and the only way to observe what a resize costs: a drag of a + /// window's edge cannot be synthesized against a background application, and + /// resizing is where a view that re-measures its contents shows up. + case resize(SizeTarget) + + /// Drag the pointer across an element, with the button held. + /// + /// The gesture no other step can stand in for. `resize` sets a window's size + /// in one write, which is not a drag: nothing enters live resize, and a view + /// that behaves differently *during* a gesture than after it looks correct to + /// every other step here. + /// + /// Not only for window edges. Any two points on any element — a split + /// divider's handle, a stretch of text to select, a row to drag out — is the + /// same gesture with different endpoints. + /// + /// Depends on the world outside the tree in the same way `click` does: the + /// events go to whatever occupies those coordinates, so the window is raised + /// first and has to be on screen. + case drag(DragTarget) + + /// What to drag across, and along what path. + struct DragTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// Names the coordinate space, not necessarily the thing that reacts. A + /// window's own frame is how its resize corner is addressed, and the + /// window is what reacts. + let identifier: String + + /// Where the button goes down, as a fraction of the element's frame. + let from: Offset + + /// Where it comes up. + let to: Offset + + /// How many moves to post between the two, not counting the press. + /// + /// Defaults to 24. The number is the point of the step: a drag posted as + /// one jump exercises a single frame, and the behaviour usually under + /// question is what happens across many. + let steps: Int? + + /// How long to pause between moves, in milliseconds. Defaults to 8. + let pauseMs: Int? + + private enum CodingKeys: String, CodingKey { + case identifier + case from + case to + case steps + case pauseMs = "pause_ms" + } + } + + /// A point on an element, as a fraction of its frame. + /// + /// Fractions rather than points, so a script says "the right edge, halfway + /// down" and keeps meaning it after the window is resized. + /// + /// `1.0` is the far edge exactly, and is what a window resize wants. The + /// region that resizes a window is a few points wide and straddles the frame + /// boundary, so aiming even five points inside lands in the content instead: + /// the gesture runs, the pointer moves, and whatever is under it gets dragged + /// rather than the window resized. Measured on a running window — `0.995` of + /// a 1070-point window grabs text, `1.0` grabs the edge. + struct Offset: Decodable { + let dx: Double + let dy: Double + } + + /// What to resize, and to what. + struct SizeTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// A window carries one, so it is addressed the same way as anything else + /// rather than through a step that means "the frontmost window". + let identifier: String + + /// The width to ask for, in points. + let width: Double + + /// The height to ask for, in points. + let height: Double + } + + /// What action to perform, and on what. + struct ActionTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + let identifier: String + + /// The action's own name, spelled as the accessibility API spells it. + /// + /// Not translated from a friendlier vocabulary: a step that says + /// `AXConfirm` can be checked against what `jpdrive dump` reported for the + /// element, and a friendlier name could not. + let action: String + } + + /// Press a menu item, addressed by the titles leading to it. + case menu(MenuTarget) + + /// What to type, and where. + struct TypeTarget: Decodable { + /// The field's `AXIdentifier`, matched exactly. + let identifier: String + + /// The text to put in the field, replacing what is there. + /// + /// Written as a value and then confirmed, rather than typed a character at + /// a time. Two calls, neither of which can be derailed by focus moving to + /// another application halfway through, which a synthesized keystroke can. + /// + /// The confirm is not optional dressing. Writing `AXValue` on a `SwiftUI` + /// text field changes the text the field displays without the binding + /// behind it noticing, so the application carries on as though nothing was + /// typed. Confirming commits the edit through the path the binding does + /// observe. + /// + /// The cost is that per-character behaviour never runs. A field that + /// validates each keystroke, or completes as you type, sees one change + /// rather than a dozen. Assert the consequence — the list that narrowed, + /// the button that enabled — rather than assuming the field's own handlers + /// fired for every character. + let text: String + } + + /// Wait until an element with the given identifier exists. + case waitFor(WaitTarget) + + /// A path through a menu. + struct MenuTarget: Decodable { + /// Titles from the top of the menu downwards, such as `["File", "Close"]`. + /// + /// Titles rather than identifiers, because the structure is the thing worth + /// asserting. An identifier like `closeAll:` is an `AppKit` selector name: + /// it survives the item moving to a different menu, so a script keyed on it + /// cannot notice the menu bar being rearranged. A path cannot miss that. + /// + /// A path that does not resolve reports how far it got and what that level + /// holds, which is the assertion failure a layout test wants to read. + let path: [String] + + /// The element whose shown menu the path starts from. + /// + /// Absent, the path starts at the menu bar. Present, it starts at the menu + /// that element is currently displaying, which `AXShowMenu` puts up. + /// + /// A title is the only way to name a context menu item: `SwiftUI` does not + /// carry an accessibility identifier onto the `NSMenuItem` it bridges a + /// menu button to, so every item in one reports the same selector name. + let under: String? + + /// Spelled out so `under` can be left off, both here and on the wire. + init(path: [String], under: String? = nil) { + self.path = path + self.under = under + } + } + + /// What a wait addresses, and for how long. + struct WaitTarget: Decodable { + /// The `AXIdentifier` to wait for, matched exactly. + let identifier: String + + /// Identifier of a container to search inside, resolved once before + /// polling begins. + /// + /// Strongly worth setting. A search for something absent has no early exit + /// and reads every element in the application, which against a thousand-row + /// sidebar takes longer than a typical timeout allows for a single attempt. + /// Scoping to the container the element will appear in makes each poll + /// cheap. + /// + /// A container that does not exist fails immediately, rather than being + /// waited for. + let under: String? + + /// How long to keep trying. Defaults to 5000. + let timeoutMs: Int? + + /// How long to pause between attempts. Defaults to 100. + /// + /// Not the kind of sleep the driver avoids. Waiting a fixed duration and + /// assuming the work finished is a guess; pausing between two observations + /// of a condition is how polling stays off a busy loop that would flood the + /// target with accessibility traffic. + let intervalMs: Int? + + /// Spelled out, because the decoder converts no cases of its own: without + /// these two, a step naming `timeout_ms` decodes as though it had named + /// nothing and silently waits the default. + private enum CodingKeys: String, CodingKey { + case identifier + case under + case timeoutMs = "timeout_ms" + case intervalMs = "interval_ms" + } + } + + /// What a step addresses. + struct Target: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// Exact rather than by prefix: `sidebar.row.1785` is a prefix of many + /// rows, and acting on whichever one happened to be found first is not a + /// thing a script can mean. + let identifier: String + } + + private enum CodingKeys: String, CodingKey { + case select + case press + case click + case perform + case type + case menu + case waitFor = "wait_for" + case resize + case drag + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + if let target = try container.decodeIfPresent(Target.self, forKey: .select) { + self = .select(target) + return + } + if let target = try container.decodeIfPresent(Target.self, forKey: .press) { + self = .press(target) + return + } + if let target = try container.decodeIfPresent(Target.self, forKey: .click) { + self = .click(target) + return + } + if let target = try container.decodeIfPresent(ActionTarget.self, forKey: .perform) { + self = .perform(target) + return + } + if let target = try container.decodeIfPresent(TypeTarget.self, forKey: .type) { + self = .type(target) + return + } + if let target = try container.decodeIfPresent(MenuTarget.self, forKey: .menu) { + self = .menu(target) + return + } + if let target = try container.decodeIfPresent(WaitTarget.self, forKey: .waitFor) { + self = .waitFor(target) + return + } + if let target = try container.decodeIfPresent(SizeTarget.self, forKey: .resize) { + self = .resize(target) + return + } + if let target = try container.decodeIfPresent(DragTarget.self, forKey: .drag) { + self = .drag(target) + return + } + + throw DecodingError.dataCorrupted( + .init( + codingPath: container.codingPath, + debugDescription: + "expected one of select, press, click, perform, type, menu, wait_for, " + + "resize, drag" + ) + ) + } +} + +/// What a step did. +struct StepResult: Encodable, Equatable { + /// The step that ran, named as it was written. + let step: String + + let identifier: String + + /// The role of the element the step acted on. + /// + /// Not always the identified element: `select` climbs to the ancestor that + /// owns selection, and reporting the role it reached is how a restructuring of + /// the view surfaces as a changed role rather than as a puzzling failure. + let role: String + + /// Whether the intended change was observed after the step ran. + /// + /// A write can succeed and change nothing, so this is read back from the + /// element rather than inferred from the API's status. + /// + /// Absent for a step with nothing to read back. Pressing a button runs + /// arbitrary code in the target and has no attribute that says it worked, so + /// reporting `true` there would be claiming more than was checked. + let confirmed: Bool? + + /// Where a click was aimed, in screen coordinates. + /// + /// Only `click` reports this. A click is the one step whose outcome depends on + /// a number the caller cannot otherwise see, and "it clicked the wrong thing" + /// is unanswerable without knowing where it clicked. + let point: String? + + /// Whether an edit was committed through the element's confirm action. + /// + /// Only `type` reports this. `false` means the field took the text but + /// advertises no `AXConfirm`, so whether the application noticed depends on it + /// watching the value directly — worth knowing, because the text being in the + /// field and the application having seen it are different facts. + let committed: Bool? + + /// The size the element ended at, as `WIDTHxHEIGHT` in points. + /// + /// Only `resize` reports this. A window clamps a size to its own limits, so + /// what was asked for and what happened are different facts and the second is + /// the one worth reading. + let size: String? + + /// How many moves a drag posted between pressing and releasing. + /// + /// Only `drag` reports this. It is what separates a gesture from a jump, and + /// a caller asking why a view did not react during one wants to know how many + /// chances it had. + let moves: Int? + + init( + step: String, + identifier: String, + role: String, + confirmed: Bool? = nil, + committed: Bool? = nil, + point: String? = nil, + size: String? = nil, + moves: Int? = nil + ) { + self.step = step + self.identifier = identifier + self.role = role + self.confirmed = confirmed + self.committed = committed + self.point = point + self.size = size + self.moves = moves + } +} + +/// Runs a single step against a running application. +enum Act { + /// How far `select` looks above the identified element for one that accepts + /// selection. + /// + /// The known chain is two levels, from the identified element through the cell + /// to the row. The cap is above that so an extra wrapper does not break the + /// step, and low enough that a miss fails rather than selecting the window. + private static let maxAncestors = 4 + + /// Resolve the step's target and act on it. + static func run(_ step: Step, pid: pid_t) throws(DriveError) -> StepResult { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return try run(step, in: AXElement.application(pid: pid), poster: SystemEventPoster()) + } + + /// Run a step against an already-resolved root. + /// + /// Split from ``run(_:pid:)`` so the part with the logic in it can be exercised + /// against a tree that is not a running application. `poster` is separate for + /// the same reason: a click is aimed using the tree but delivered outside it. + /// `activation` is how long a menu step waits for the application to come + /// forward and for the item to enable, and is a parameter so a test of either + /// wait does not have to sit through the real one. + static func run<E: Element>( + _ step: Step, + in root: E, + poster: any EventPoster = SystemEventPoster(), + activation: Duration = activationTimeout + ) throws(DriveError) -> StepResult { + switch step { + case .select(let target): + return try select(target, in: root) + + case .waitFor(let target): + return try waitFor(target, in: root) + + case .press(let target): + return try press(target, in: root) + + case .perform(let target): + return try perform(target.action, on: target.identifier, in: root, step: "perform") + + case .type(let target): + return try type(target, in: root) + + case .menu(let target): + return try menu(target, in: root, within: activation) + + case .click(let target): + return try click(target, in: root, poster: poster) + + case .resize(let target): + return try resize(target, in: root) + + case .drag(let target): + return try drag(target, in: root, poster: poster, activation: activation) + } + } + + /// How many moves a drag posts when it does not say. + private static let defaultDragSteps = 24 + + /// How long a drag pauses between moves when it does not say. + private static let defaultDragPause = Duration.milliseconds(8) + + /// Drag the pointer from one point on an element to another. + private static func drag<E: Element>( + _ target: Step.DragTarget, + in root: E, + poster: any EventPoster, + activation: Duration + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard + let origin = element.point(kAXPositionAttribute), + let size = element.size(kAXSizeAttribute) + else { + throw DriveError( + kind: .notClickable, + message: "\(target.identifier) reports no frame to drag across", + hint: "an element with no position or size cannot be aimed at" + ) + } + + let steps = max(target.steps ?? defaultDragSteps, 1) + let pause = target.pauseMs.map { Duration.milliseconds($0) } ?? defaultDragPause + + let start = point(target.from, in: origin, size) + let end = point(target.to, in: origin, size) + let route = (0...steps).map { step in + let progress = Double(step) / Double(steps) + return CGPoint( + x: start.x + (end.x - start.x) * progress, + y: start.y + (end.y - start.y) * progress + ) + } + + // Activated, and then raised, and both are needed. + // + // `AXRaise` orders a window forward *within its own application*. Global + // ordering between applications follows activation, so raising a + // background app's window leaves it under the active app's windows: the + // gesture lands on whatever is on top at those coordinates, which is + // whatever the person at the keyboard is using. Measured, not assumed — a + // drag posted without this was received by the frontmost terminal. + // + // The cost is that a gesture takes focus. Nothing here can give it back: + // this process handles one step and exits, so the restore belongs to + // whatever drives the whole list. + activate(root, within: activation) + raiseWindow(in: path) + + guard poster.drag(through: route, pausing: pause) else { + throw DriveError( + kind: .actionFailed, + message: + "could not post a drag from \(start.x),\(start.y) to \(end.x),\(end.y)", + hint: nil + ) + } + + return StepResult( + step: "drag", + identifier: target.identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + point: "\(start.x),\(start.y) -> \(end.x),\(end.y)", + moves: route.count - 1 + ) + } + + /// One fractional offset as a screen coordinate inside a frame. + private static func point( + _ offset: Step.Offset, in origin: CGPoint, _ size: CGSize + ) + -> CGPoint + { + CGPoint( + x: origin.x + size.width * offset.dx, + y: origin.y + size.height * offset.dy + ) + } + + /// Ask the identified element to take a new size. + /// + /// The size it ends at is read back and reported rather than assumed: a window + /// clamps to its own minimum and maximum, so asking for something outside those + /// succeeds and lands somewhere else. `confirmed` says whether it landed on + /// what was asked for. + private static func resize<E: Element>( + _ target: Step.SizeTarget, in root: E + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: "list what is addressable with: jpdrive tree --identifier <prefix>" + ) + } + + let role = element.read([kAXRoleAttribute]).text[0] ?? "" + + guard element.isSettable(kAXSizeAttribute) else { + throw DriveError( + kind: .notEditable, + message: "\(target.identifier) does not accept a write to AXSize", + hint: "a window does; most elements inside one do not" + ) + } + + let wanted = CGSize(width: target.width, height: target.height) + let status = element.setSize(kAXSizeAttribute, wanted) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXSize on \(target.identifier) failed: \(status.name)", + hint: nil + ) + } + + let reached = element.size(kAXSizeAttribute) + + return StepResult( + step: "resize", + identifier: target.identifier, + role: role, + confirmed: reached == wanted, + size: reached.map { "\(Int($0.width))x\(Int($0.height))" } + ) + } + + /// Click where the identified element says a click belongs. + /// + /// The last resort among the steps, and the only one whose effect is not + /// addressed to the element: the event goes to whatever occupies that screen + /// coordinate. Prefer `select` for rows and `press` for controls, both of which + /// reach their target regardless of what is on top of it or whether it is + /// scrolled into view. + private static func click<E: Element>( + _ target: Step.Target, + in root: E, + poster: any EventPoster + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard let point = element.point(AXElement.activationPoint) else { + throw DriveError( + kind: .notClickable, + message: "\(target.identifier) reports no AXActivationPoint", + hint: + "an element with no place to be clicked is usually one that wants `select` " + + "or `press` instead" + ) + } + + // Raised first, because the click lands on whatever is at that coordinate + // rather than on the element that named it. A window behind another one + // would otherwise have its click swallowed by the window in front. + raiseWindow(in: path) + + guard poster.click(at: point) else { + throw DriveError( + kind: .actionFailed, + message: "could not post a click at \(point.x),\(point.y)", + hint: nil + ) + } + + return StepResult( + step: "click", + identifier: target.identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + point: "\(point.x),\(point.y)" + ) + } + + /// Bring the application forward, ignoring a refusal. + /// + /// Best effort, unlike ``front(_:within:)``, which fails a menu step that + /// cannot activate: there the activation *is* the step, because AppKit + /// disables every item acting on the front window until the application is + /// frontmost. A pointer gesture only needs to be on top of the z-order, and a + /// tree that is not a running application — a test's — has nothing to + /// activate and a gesture against it is still worth posting. + private static func activate<E: Element>(_ root: E, within timeout: Duration) { + guard root.flag(kAXFrontmostAttribute) != true else { return } + guard root.setFlag(kAXFrontmostAttribute, true) == .success else { return } + + _ = poll(untilTrue: { root.flag(kAXFrontmostAttribute) == true }, within: timeout) + } + + /// Bring the window holding the addressed element to the front, if it has one. + /// + /// Found along the path the search descended, for the same reason the selection + /// owner is: the identified element does not report a parent to climb from. + private static func raiseWindow<E: Element>(in path: [E]) { + for element in path where element.read([kAXRoleAttribute]).text[0] == kAXWindowRole { + _ = element.perform(kAXRaiseAction) + return + } + } + + /// Press the identified element. + private static func press<E: Element>( + _ target: Step.Target, in root: E + ) throws(DriveError) + -> StepResult + { + return try perform(kAXPressAction, on: target.identifier, in: root, step: "press") + } + + /// Perform `action` on the element with `identifier`. + /// + /// `step` names the result, so `press` reports itself rather than the general + /// mechanism it is a shorthand for. + private static func perform<E: Element>( + _ action: String, + on identifier: String, + in root: E, + step: String + ) throws(DriveError) -> StepResult { + let path = try find(identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: nil + ) + } + + // Checked before performing, so the error can name what the element does + // accept. Performing an unsupported action answers `action_unsupported` + // with nothing to act on. + let actions = element.actions + guard actions.contains(action) else { + throw DriveError( + kind: .actionUnsupported, + message: "\(identifier) does not accept \(action)", + hint: actions.isEmpty + ? "it advertises no actions at all; a list row is activated with `select`" + : "it accepts: \(actions.joined(separator: ", "))" + ) + } + + let status = element.perform(action) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "performing \(action) on \(identifier) answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: step, + identifier: identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: nil + ) + } + + /// Put text into the identified field. + private static func type<E: Element>( + _ target: Step.TypeTarget, in root: E + ) throws(DriveError) + -> StepResult + { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard element.isSettable(kAXValueAttribute) else { + throw DriveError( + kind: .notEditable, + message: "\(target.identifier) does not accept a write to AXValue", + hint: "a static label and a disabled field both look like this; check with " + + "`jpdrive dump --settable`" + ) + } + + let status = element.setText(kAXValueAttribute, target.text) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXValue to \(target.identifier) answered \(status.name)", + hint: nil + ) + } + + let committed = try confirm(element, identifier: target.identifier) + + // `confirmed` says the field holds the text, and nothing more. Whether the + // application reacted is the caller's assertion to make, against whatever + // the typing was supposed to change. + let after = element.read([kAXValueAttribute, kAXRoleAttribute]) + + return StepResult( + step: "type", + identifier: target.identifier, + role: after.text[1] ?? "<none>", + confirmed: after.text[0] == target.text, + committed: committed + ) + } + + /// Commit an edit, if the element offers a way to. + /// + /// Answers whether it did. An element with no confirm action is not a failure: + /// some fields publish every change as it happens and need nothing further. + private static func confirm<E: Element>( + _ element: E, identifier: String + ) throws(DriveError) -> Bool { + guard element.actions.contains(kAXConfirmAction) else { return false } + + let status = element.perform(kAXConfirmAction) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "confirming \(identifier) answered \(status.name)", + hint: + "the text was written but not committed, so the application has not seen it" + ) + } + + return true + } + + /// Attributes read while walking a menu path. + private static let menuBatch = [ + kAXRoleAttribute, + AXElement.attributedDescription, + kAXDescriptionAttribute, + kAXTitleAttribute, + ] + + /// How long to wait for the application to come forward, and for the item + /// addressed through it to be enabled. + static let activationTimeout = Duration.milliseconds(2000) + + /// Press the menu item at the end of a titled path. + private static func menu<E: Element>( + _ target: Step.MenuTarget, in root: E, within timeout: Duration + ) throws(DriveError) + -> StepResult + { + guard !target.path.isEmpty else { + throw DriveError( + kind: .badUsage, + message: "a menu step needs a path, such as [\"File\", \"Close\"]", + hint: nil + ) + } + + let start: E + let origin: String + + if let owner = target.under { + // A menu already on screen. No activation: showing it required the + // application to be active, and asking again would be a no-op at best. + start = try shownMenu(of: owner, in: root) + origin = "'\(owner)' is showing a menu that" + } else { + // The one step that takes focus from whatever had it. AppKit disables + // every menu item that acts on the front window or on the responder + // chain while the application is in the background, which is most of + // the menu bar: without this, a path resolves to an item that cannot + // be pressed. + try front(root, within: timeout) + + guard let bar = root.elements(kAXMenuBarAttribute).first else { + throw DriveError( + kind: .notFound, + message: "the application reports no menu bar", + hint: "an agent or accessory application has none" + ) + } + start = bar + origin = "the menu bar" + } + + var current = start + var reached: [String] = [] + + for title in target.path { + guard let next = child(titled: title, of: current) else { + throw DriveError( + kind: .notFound, + message: reached.isEmpty + ? "\(origin) holds no item titled '\(title)'" + : "'\(reached.joined(separator: " > "))' holds no item titled '\(title)'", + hint: "it holds: \(titles(of: current).joined(separator: ", "))" + ) + } + current = next + reached.append(title) + } + + let path = target.path.joined(separator: " > ") + try waitUntilEnabled(current, named: path, within: timeout) + + let actions = current.actions + guard actions.contains(kAXPressAction) else { + throw DriveError( + kind: .actionUnsupported, + message: "'\(path)' does not accept AXPress", + hint: actions.isEmpty + ? "the path names a submenu rather than an item; name the item inside it" + : "it accepts: \(actions.joined(separator: ", "))" + ) + } + + let status = current.perform(kAXPressAction) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "pressing '\(path)' answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: "menu", + identifier: path, + role: current.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: nil + ) + } + + /// The menu an element is currently displaying. + /// + /// A shown menu hangs off the element that opened it, after that element's + /// own children, which is why a capped or filtered read passes straight over + /// it. + private static func shownMenu<E: Element>( + of identifier: String, in root: E + ) throws(DriveError) -> E { + let path = try find(identifier, from: root) + guard let owner = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: nil + ) + } + + let children = owner.read([kAXRoleAttribute]).children + for child in children where child.read([kAXRoleAttribute]).text[0] == kAXMenuRole { + return child + } + + throw DriveError( + kind: .notFound, + message: "\(identifier) is not showing a menu", + hint: """ + open one first, in an earlier step: \ + {"perform": {"identifier": "\(identifier)", "action": "AXShowMenu"}} + """ + ) + } + + /// Bring the application forward, and wait until it reports that it is. + /// + /// Writing `AXFrontmost` is a request. The window server grants it a moment + /// later, and the menu validation that depends on it later still. + private static func front<E: Element>( + _ root: E, within timeout: Duration + ) throws(DriveError) { + guard root.flag(kAXFrontmostAttribute) != true else { return } + + let status = root.setFlag(kAXFrontmostAttribute, true) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "bringing the application forward answered \(status.name)", + hint: "a menu item that acts on the front window is disabled until it is" + ) + } + + guard poll(untilTrue: { root.flag(kAXFrontmostAttribute) == true }, within: timeout) + else { + throw DriveError( + kind: .timeout, + message: + "the application did not come forward within \(timeout.milliseconds)ms", + hint: "another application may be holding focus with a modal panel" + ) + } + } + + /// Wait for an item to stop reporting itself disabled. + /// + /// An element that reports no `AXEnabled` at all is not disabled: plenty + /// carry no such attribute, and treating its absence as a refusal would + /// reject every one of them. + private static func waitUntilEnabled<E: Element>( + _ item: E, named path: String, within timeout: Duration + ) throws(DriveError) { + if poll(untilTrue: { item.flag(kAXEnabledAttribute) != false }, within: timeout) { + return + } + + throw DriveError( + kind: .disabled, + message: "'\(path)' is disabled", + hint: + "an item acting on a selection is disabled while nothing is selected, and one " + + "acting on the front window while no window has focus" + ) + } + + /// Poll `condition` until it holds, or `timeout` elapses. + private static func poll(untilTrue condition: () -> Bool, within timeout: Duration) -> Bool + { + let clock = ContinuousClock() + let started = clock.now + + while true { + if condition() { return true } + guard clock.now - started < timeout else { return false } + Thread.sleep(forTimeInterval: defaultInterval.seconds) + } + } + + /// The child of `parent` whose title is `title`. + /// + /// Descends through `AXMenu`, which carries no title of its own: a bar item's + /// items live inside one, so a path names `["File", "Close"]` rather than + /// spelling out the container between them. + private static func child<E: Element>(titled title: String, of parent: E) -> E? { + for child in parent.read([]).children { + let text = child.read(menuBatch).text + + if text[1] ?? text[2] ?? text[3] == title { + return child + } + + guard text[0] == "AXMenu", let found = self.child(titled: title, of: child) else { + continue + } + return found + } + + return nil + } + + /// The titles a level offers, for saying what a path could have named instead. + private static func titles<E: Element>(of parent: E) -> [String] { + var found: [String] = [] + + for child in parent.read([]).children { + let text = child.read(menuBatch).text + + if let title = text[1] ?? text[2] ?? text[3], !title.isEmpty { + found.append(title) + continue + } + + // An untitled `AXMenu` is the container a path skips, so what it holds + // is what this level effectively offers. + guard text[0] == "AXMenu" else { continue } + found.append(contentsOf: titles(of: child)) + } + + return found + } + + /// Select the row that owns the identified element. + private static func select<E: Element>( + _ target: Step.Target, in root: E + ) throws(DriveError) + -> StepResult + { + let path = try find(target.identifier, from: root) + + guard let owner = selectionOwner(in: path) else { + throw DriveError( + kind: .notSelectable, + message: + "neither \(target.identifier) nor its \(maxAncestors) nearest ancestors accept " + + "a write to AXSelected", + hint: "check what the element reports with: jpdrive dump --settable" + ) + } + + let status = owner.setFlag(kAXSelectedAttribute, true) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXSelected to \(target.identifier) answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: "select", + identifier: target.identifier, + role: owner.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: owner.flag(kAXSelectedAttribute) ?? false + ) + } + + /// Default time to keep polling for an element to appear. + private static let defaultTimeout = Duration.milliseconds(5000) + + /// Default pause between polling attempts. + private static let defaultInterval = Duration.milliseconds(100) + + /// Wait until an element with the target's identifier exists. + /// + /// Returns as soon as it is found, including on the first attempt when it was + /// already there. + private static func waitFor<E: Element>( + _ target: Step.WaitTarget, in root: E + ) throws(DriveError) + -> StepResult + { + // Resolved once, before the loop. This is the expensive search, and paying + // it on every attempt is what makes an unscoped wait useless. + let scope: E + if let under = target.under { + guard let container = try find(under, from: root).last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(under) to wait inside", + hint: "`under` names a container that must already exist" + ) + } + scope = container + } else { + scope = root + } + + let timeout = target.timeoutMs.map { Duration.milliseconds($0) } ?? defaultTimeout + let interval = target.intervalMs.map { Duration.milliseconds($0) } ?? defaultInterval + + let clock = ContinuousClock() + let started = clock.now + var attempts = 0 + + while true { + attempts += 1 + + if let path = try? find(target.identifier, from: scope), let found = path.last { + return StepResult( + step: "wait_for", + identifier: target.identifier, + role: found.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: true + ) + } + + guard clock.now - started < timeout else { break } + Thread.sleep(forTimeInterval: interval.seconds) + } + + let elapsed = clock.now - started + throw DriveError( + kind: .timeout, + message: + "\(target.identifier) did not appear within \(timeout.milliseconds)ms " + + "(\(attempts) attempts over \(elapsed.milliseconds)ms)", + hint: attempts == 1 + ? "one attempt exhausted the timeout; scope the search with `under`" + : nil + ) + } + + /// The nearest element at or above the end of `path` that accepts a write to + /// `AXSelected`. + /// + /// Walks the chain the search descended rather than reading `AXParent`. The + /// identified element is a SwiftUI leaf that does not report a parent, so + /// climbing from it arrives nowhere, while the chain that reached it is known + /// for free and is not subject to that. + private static func selectionOwner<E: Element>(in path: [E]) -> E? { + for element in path.suffix(maxAncestors + 1).reversed() + where element.isSettable(kAXSelectedAttribute) { + return element + } + return nil + } + + /// What a search reads at each element. + /// + /// Children arrive alongside, so the identifier is all the search asks for. + private static let searchBatch = [kAXIdentifierAttribute] + + /// Find the element whose identifier is exactly `identifier`, and the chain of + /// elements that reached it. + /// + /// The path comes back rather than the element alone because acting on an + /// element often means acting on one of its ancestors, and this tree cannot + /// reliably be walked upwards. + /// + /// Depth-first with an early exit, reading only what the search needs. Reading + /// every attribute of each element on the way past would make a step against + /// this app's few thousand elements cost seconds. + private static func find<E: Element>( + _ identifier: String, from root: E + ) throws(DriveError) + -> [E] + { + var stack = [[root]] + + while let path = stack.popLast() { + guard let element = path.last else { continue } + let reading = element.read(searchBatch) + + if reading.text[0] == identifier { + return path + } + + // Reversed, so a depth-first walk visits siblings in the order the + // application reports them. + for child in reading.children.reversed() { + stack.append(path + [child]) + } + } + + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: "list what is addressable with: jpdrive tree --identifier <prefix>" + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift new file mode 100644 index 000000000..0ad381f26 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift @@ -0,0 +1,80 @@ +import AppKit +import Foundation + +/// The state a driven run borrows from whoever is at the keyboard. +/// +/// Which application is in front, and where the pointer is. Neither belongs to +/// the app under test: a synthesized gesture has to take both — mouse events go +/// to whatever is on top at a coordinate, and the ordering between applications +/// follows activation — and a run that takes them owes them back. +/// +/// Deliberately not window geometry. A step that resizes a window did the thing +/// it was asked to do, and putting the window back would undo the effect under +/// test. What a run borrows is restored; what it was told to change is not. +/// +/// Read and written separately rather than as one capture-and-restore pair, so a +/// caller composes what it needs and decides for itself when a restore is owed. +enum Ambient { + /// The bundle identifier of the frontmost application. + /// + /// `nil` when there is none, or when it has no identifier — a process + /// launched without a bundle has neither. + static func frontmost() -> FrontmostReport { + FrontmostReport(bundleID: NSWorkspace.shared.frontmostApplication?.bundleIdentifier) + } + + /// Bring the application with `bundleID` back to the front. + /// + /// Through `NSWorkspace`, which asks the application to activate itself, so + /// this needs no permission beyond launching one. Answers whether an + /// application with that identifier was found to ask. + static func activate(bundleID: String) -> FrontmostReport { + guard + let app = NSRunningApplication.runningApplications(withBundleIdentifier: bundleID) + .first + else { + return FrontmostReport(bundleID: nil) + } + + app.activate() + return FrontmostReport(bundleID: bundleID) + } + + /// Where the pointer is, in the coordinates a synthesized event uses. + /// + /// `NSEvent.mouseLocation` is bottom-left origin and screen coordinates are + /// top-left, so the y is flipped here rather than at each call site. The + /// height flipped against is the *main* screen's, which is what the window + /// server measures global coordinates from. + static func pointer() -> PointerReport { + let location = NSEvent.mouseLocation + let height = NSScreen.screens.first?.frame.height ?? 0 + + return PointerReport(x: location.x, y: height - location.y) + } + + /// Put the pointer back at `point`. + /// + /// Warped rather than moved: `CGWarpMouseCursorPosition` relocates the cursor + /// without synthesizing motion, so nothing under it takes a hover, and no + /// application sees a gesture it has to interpret. + static func movePointer(to point: CGPoint) -> PointerReport { + CGWarpMouseCursorPosition(point) + return PointerReport(x: point.x, y: point.y) + } +} + +/// Which application is in front. +struct FrontmostReport: Encodable, Equatable { + let bundleID: String? + + private enum CodingKeys: String, CodingKey { + case bundleID = "bundle_id" + } +} + +/// Where the pointer is, in top-left-origin screen coordinates. +struct PointerReport: Encodable, Equatable { + let x: CGFloat + let y: CGFloat +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift new file mode 100644 index 000000000..64fdbcfb0 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift @@ -0,0 +1,425 @@ +import Foundation + +/// A parsed command line. +enum Command { + /// Report whether this process may read another app's accessibility tree. + case doctor(pid: pid_t?) + + /// Print the elements and attributes under an application. + case dump(DumpOptions) + + /// Report the elements under an application, identified and described. + case tree(TreeOptions) + + /// List the application's windows. + case windows(pid: pid_t) + + /// Report the window-server identifiers of the application's windows. + case windowid(pid: pid_t) + + /// Report the application's menu bar. + case menu(pid: pid_t, options: TreeOptions) + + /// Do one thing to one element. + case act(step: Step, pid: pid_t) + + /// Report the colours along one row or column of a screenshot. + case pixels(PixelOptions) + + /// Report which application is in front, or put one there. + case frontmost(set: String?) + + /// Report where the pointer is, or put it somewhere. + case pointer(set: CGPoint?) +} + +/// What to walk, and how much of it. +struct DumpOptions { + let pid: pid_t + + /// How deep to recurse before reporting a node's children as elided. + let maxDepth: Int + + /// How many children to walk at each level, or `0` for all of them. + let maxSiblings: Int + + /// Whether to ask, per attribute, if it can be written. + let settable: Bool +} + +/// The command line the driver accepts. +/// +/// Hand-rolled rather than pulled from `swift-argument-parser`: the package has +/// no other dependency, and keeping it that way means the build needs no network +/// and no resolved manifest. +enum Arguments { + /// Usage text, embedded in every bad-usage error. + static let usage = """ + usage: jpdrive doctor [--pid <pid>] + jpdrive tree --pid <pid> [--identifier <prefix>] [--max-matches <n>] + [--frames] [--depth <n>] [--max-siblings <n>] + jpdrive windows --pid <pid> + jpdrive windowid --pid <pid> + jpdrive menu --pid <pid> [--depth <n>] [--max-siblings <n>] + jpdrive dump --pid <pid> [--depth <n>] [--max-siblings <n>] [--settable] + jpdrive act --pid <pid> --json '<step>' + a step is a single-key object, e.g. + {"resize":{"identifier":"w","width":1400,"height":900}} + jpdrive frontmost [--set <bundle-id>] + jpdrive pointer [--set <x>,<y>] + jpdrive pixels --image <path> --scan row|column --at <n> + [--from <n>] [--to <n>] + """ + + /// Depth cap for `dump` when `--depth` is not given. + /// + /// A SwiftUI window nests deeply: the wrapper groups between a `List` and its + /// rows are several levels on their own, so a cap low enough to be tidy hides + /// the elements worth seeing. + static let defaultDepth = 20 + + /// Sibling cap for `dump` when `--max-siblings` is not given. + /// + /// A thousand sidebar rows are a thousand copies of one shape, and walking + /// them all costs a round-trip per attribute per element. Five is enough to + /// see the shape and to tell a homogeneous list from a mixed one. + static let defaultSiblings = 5 + + /// Match budget for a filtered `tree` when `--max-matches` is not given. + /// + /// Identifiers sit on leaves, so a prefix search cannot prune on the way down + /// and an unbounded one reads every element in the application. Five answers + /// what a list looks like; looking up one known identifier wants `1`. + static let defaultMatches = 5 + + /// Parse `arguments`, which excludes the executable path. + static func parse(_ arguments: [String]) throws(DriveError) -> Command { + guard let subcommand = arguments.first else { + throw DriveError(kind: .badUsage, message: usage, hint: nil) + } + + let options = try options(arguments.dropFirst()) + + switch subcommand { + case "doctor": + return .doctor(pid: options.pid) + + case "dump": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, + message: "dump needs --pid <pid>", + hint: usage + ) + } + return .dump( + DumpOptions( + pid: pid, + maxDepth: options.depth ?? defaultDepth, + maxSiblings: options.siblings ?? defaultSiblings, + settable: options.settable + ) + ) + + case "tree": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "tree needs --pid <pid>", hint: usage) + } + return .tree( + TreeOptions( + pid: pid, + identifierPrefix: options.identifier, + maxMatches: options.matches ?? defaultMatches, + maxDepth: options.depth ?? defaultDepth, + maxSiblings: options.siblings ?? defaultSiblings, + frames: options.frames + ) + ) + + case "frontmost": + return .frontmost(set: options.set) + + case "pointer": + guard let raw = options.set else { + return .pointer(set: nil) + } + + let parts = raw.split(separator: ",") + guard + parts.count == 2, + let x = Double(parts[0].trimmingCharacters(in: .whitespaces)), + let y = Double(parts[1].trimmingCharacters(in: .whitespaces)) + else { + throw DriveError( + kind: .badUsage, + message: "pointer --set takes <x>,<y>", + hint: usage + ) + } + return .pointer(set: CGPoint(x: x, y: y)) + + case "windows": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "windows needs --pid <pid>", hint: usage) + } + return .windows(pid: pid) + + case "windowid": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "windowid needs --pid <pid>", hint: usage) + } + return .windowid(pid: pid) + + case "menu": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "menu needs --pid <pid>", hint: usage) + } + return .menu( + pid: pid, + options: TreeOptions( + pid: pid, + identifierPrefix: options.identifier, + maxMatches: options.matches ?? defaultMatches, + maxDepth: options.depth ?? defaultDepth, + // A menu bar is a couple of hundred elements and every one of + // them is a thing you might press, so the default that keeps a + // thousand-row list readable would only hide half the verbs. + maxSiblings: options.siblings ?? 0, + frames: options.frames + ) + ) + + case "act": + guard let pid = options.pid else { + throw DriveError(kind: .badUsage, message: "act needs --pid <pid>", hint: usage) + } + guard let json = options.json else { + throw DriveError( + kind: .badUsage, message: "act needs --json '<step>'", hint: usage) + } + + let step: Step + do { + step = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + } catch { + throw DriveError( + kind: .badUsage, + message: "could not read the step: \(error)", + hint: #"a step is a single-key object, e.g. {"select":{"identifier":"…"}}"# + ) + } + + return .act(step: step, pid: pid) + + case "pixels": + guard let image = options.image else { + throw DriveError( + kind: .badUsage, message: "pixels needs --image <path>", hint: usage) + } + guard let scan = options.scan else { + throw DriveError( + kind: .badUsage, + message: "pixels needs --scan row or --scan column", + hint: usage + ) + } + guard let at = options.at else { + throw DriveError( + kind: .badUsage, message: "pixels needs --at <n>", hint: usage) + } + + return .pixels( + PixelOptions( + image: image, + axis: scan, + at: at, + from: options.from, + to: options.to + ) + ) + + default: + throw DriveError( + kind: .badUsage, + message: "unknown subcommand '\(subcommand)'", + hint: usage + ) + } + } + + /// Flags accepted by any subcommand, whether or not that subcommand reads + /// them. Keeping one parser means `--pid` behaves identically everywhere. + private struct Options { + var pid: pid_t? + var depth: Int? + var siblings: Int? + var matches: Int? + var settable = false + var identifier: String? + var json: String? + var frames = false + var image: String? + var scan: PixelOptions.Axis? + var at: Int? + var from: Int? + var to: Int? + var set: String? + } + + private static func options(_ arguments: ArraySlice<String>) throws(DriveError) -> Options { + var options = Options() + var rest = arguments.makeIterator() + + while let argument = rest.next() { + switch argument { + case "--pid": + guard let value = rest.next(), + let raw = Int(value), + let pid = pid_t(exactly: raw) + else { + throw DriveError( + kind: .badUsage, + message: "--pid takes an integer process id", + hint: """ + \(usage). `--pid $(pgrep -f JP.app)` expands to nothing \ + when the app is not running, which lands here rather \ + than reporting app_not_running + """ + ) + } + options.pid = pid + + case "--depth": + guard let value = rest.next(), let depth = Int(value), depth > 0 else { + throw DriveError( + kind: .badUsage, + message: "--depth takes a positive integer", + hint: usage + ) + } + options.depth = depth + + case "--max-siblings": + guard let value = rest.next(), let siblings = Int(value), siblings >= 0 else { + throw DriveError( + kind: .badUsage, + message: + "--max-siblings takes a non-negative integer, where 0 means all", + hint: usage + ) + } + options.siblings = siblings + + case "--settable": + options.settable = true + + case "--max-matches": + guard let value = rest.next(), let matches = Int(value), matches > 0 else { + throw DriveError( + kind: .badUsage, + message: "--max-matches takes a positive integer", + hint: usage + ) + } + options.matches = matches + + case "--frames": + options.frames = true + + case "--identifier": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--identifier takes a value", + hint: usage + ) + } + options.identifier = value + + case "--set": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--set takes a value", + hint: usage + ) + } + options.set = value + + case "--json": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--json takes a value", + hint: usage + ) + } + options.json = value + + case "--image": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--image takes a path", + hint: usage + ) + } + options.image = value + + case "--scan": + guard let value = rest.next(), let axis = PixelOptions.Axis(rawValue: value) + else { + throw DriveError( + kind: .badUsage, + message: "--scan takes `row` or `column`", + hint: usage + ) + } + options.scan = axis + + case "--at": + guard let value = rest.next(), let at = Int(value), at >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--at takes a non-negative integer", + hint: usage + ) + } + options.at = at + + case "--from": + guard let value = rest.next(), let from = Int(value), from >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--from takes a non-negative integer", + hint: usage + ) + } + options.from = from + + case "--to": + guard let value = rest.next(), let to = Int(value), to >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--to takes a non-negative integer", + hint: usage + ) + } + options.to = to + + default: + throw DriveError( + kind: .badUsage, + message: "unknown argument '\(argument)'", + hint: usage + ) + } + } + + return options + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift new file mode 100644 index 000000000..b6a474a2b --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift @@ -0,0 +1,95 @@ +import ApplicationServices +import Foundation + +/// What `jpdrive doctor` observed. +struct DoctorReport: Encodable { + /// `AXIsProcessTrusted()` for this process. + let trusted: Bool + + /// This process and its ancestors, nearest first. One of these holds the + /// Accessibility grant when `trusted` is true. + let processes: [ProcessLink] + + /// A real read against a target app, present when `--pid` was given. + let probe: WindowProbe? +} + +/// The outcome of reading a target application's window list. +struct WindowProbe: Encodable { + let pid: pid_t + + /// The target's short command name. + let command: String + + /// How many windows were read, when the read succeeded. + let windowCount: Int? + + /// The accessibility error, when it did not. + let axError: String? +} + +/// Answers whether this process may read another application's accessibility +/// tree, and records the evidence for why. +/// +/// `AXIsProcessTrusted()` alone is not enough: it reports what TCC believes +/// about the responsible process, which is not always the process making the +/// call. So the report pairs the flag with a real `AXUIElementCopyAttributeValue` +/// against a live app, and with the ancestor chain the grant might be attributed +/// to. Apple documents neither the attribution algorithm nor its stability, so +/// this records observations rather than asserting a rule. +enum Doctor { + /// Run every probe and collect the results. + /// + /// Throws only when `pid` names a process that is not running. A refused + /// accessibility read is an observation the report carries, not a failure of + /// the diagnostic. + static func run(targetPid pid: pid_t?) throws(DriveError) -> DoctorReport { + let probe: WindowProbe? + if let pid { + guard let record = ProcessTable.record(for: pid) else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + probe = windowProbe(pid: pid, command: ProcessTable.name(of: record)) + } else { + probe = nil + } + + return DoctorReport( + trusted: AXIsProcessTrusted(), + processes: ProcessTable.ancestry(from: getpid()), + probe: probe + ) + } + + /// Read the target's window list, reporting the accessibility error instead + /// of the count when the read is refused. + /// + /// Uses the non-prompting trust path throughout: a spike that raises the + /// system's "grant access" dialog changes the state it is measuring. + private static func windowProbe(pid: pid_t, command: String) -> WindowProbe { + let app = AXUIElementCreateApplication(pid) + var value: CFTypeRef? + let status = AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) + + guard status == .success else { + return WindowProbe( + pid: pid, + command: command, + windowCount: nil, + axError: status.name + ) + } + + let windows = value as? [AXUIElement] + return WindowProbe( + pid: pid, + command: command, + windowCount: windows?.count ?? 0, + axError: nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift new file mode 100644 index 000000000..c08b6ebaf --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift @@ -0,0 +1,81 @@ +import Foundation + +/// A failure reported as JSON on stdout, alongside a non-zero exit status. +/// +/// Every exit path produces either a result document or one of these, so a +/// caller never has to scrape prose off stderr to find out what happened. +struct DriveError: Error, Encodable { + /// Machine-readable discriminator. Callers switch on this; the message is + /// for humans and may be reworded freely. + enum Kind: String, Encodable { + /// The command line named an unknown subcommand or was missing a value. + case badUsage = "bad_usage" + + /// No process is running under the given pid. + case appNotRunning = "app_not_running" + + /// The accessibility API refused the request for want of a TCC grant. + case notPermitted = "not_permitted" + + /// No element carries the identifier the step addressed. + case identifierNotFound = "identifier_not_found" + + /// The addressed element and its nearest ancestors do not accept a write + /// to `AXSelected`. + case notSelectable = "not_selectable" + + /// An attribute write was refused by the accessibility API. + case writeFailed = "write_failed" + + /// The addressed element does not accept a write to its value. + case notEditable = "not_editable" + + /// The addressed element reports nowhere on screen to click. + case notClickable = "not_clickable" + + /// The addressed element does not accept the action the step performs. + case actionUnsupported = "action_unsupported" + + /// The addressed element is present but refuses to act while disabled. + case disabled = "disabled" + + /// An action was refused by the accessibility API. + case actionFailed = "action_failed" + + /// An element waited for did not appear in time. + case timeout = "timeout" + + /// The application reports no element of the requested kind. + case notFound = "not_found" + + /// The result could not be encoded as JSON. + case encodingFailed = "encoding_failed" + } + + let kind: Kind + + /// One sentence saying what went wrong. + let message: String + + /// What the operator can do about it, when there is something to do. + var hint: String? +} + +extension DriveError { + /// Names the System Settings pane that grants Accessibility. + /// + /// macOS attributes the grant to the responsible process, which for a + /// command-line tool is normally the terminal rather than the tool, so this + /// points at the terminal and not at `jpdrive`. + static let accessibilityHint = """ + grant Accessibility to the terminal application running this command, \ + under System Settings > Privacy & Security > Accessibility, then start \ + a new terminal session + """ +} + +/// Envelope that makes an error document distinguishable from a result document +/// by its top-level key alone. +struct ErrorDocument: Encodable { + let error: DriveError +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift new file mode 100644 index 000000000..d3e222961 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift @@ -0,0 +1,68 @@ +import Foundation + +/// The driver's entry point. +/// +/// Everything below this is internal to the library, so the tests reach it with +/// `@testable import` and the executable target stays a single line. +public enum Driver { + /// Parse the process arguments, run the command, and exit. + /// + /// Writes one JSON document to stdout either way: a result, or an error with a + /// non-zero exit status. + public static func run() -> Never { + do throws(DriveError) { + try dispatch(Array(CommandLine.arguments.dropFirst())) + } catch { + Output.writeError(error) + exit(1) + } + + exit(0) + } + + /// Run one command and write its result. + static func dispatch(_ arguments: [String]) throws(DriveError) { + switch try Arguments.parse(arguments) { + case .doctor(let pid): + try Output.write(try Doctor.run(targetPid: pid)) + + case .dump(let options): + try Output.write(try Dump.walk(options)) + + case .tree(let options): + guard let tree = try Tree.read(options) else { + throw DriveError( + kind: .identifierNotFound, + message: + "no element's identifier begins with \(options.identifierPrefix ?? "")", + hint: "drop --identifier to see what the application reports" + ) + } + try Output.write(tree) + + case .windows(let pid): + try Output.write(try Windows.read(pid: pid)) + + case .windowid(let pid): + try Output.write(try WindowIDs.read(pid: pid)) + + case .menu(let pid, let options): + try Output.write(try Menu.read(pid: pid, options: options)) + + case .act(let step, let pid): + try Output.write(try Act.run(step, pid: pid)) + + case .pixels(let options): + try Output.write(try Pixels.read(options)) + + case .frontmost(let set): + let report = + if let set { Ambient.activate(bundleID: set) } else { Ambient.frontmost() } + try Output.write(report) + + case .pointer(let set): + let report = if let set { Ambient.movePointer(to: set) } else { Ambient.pointer() } + try Output.write(report) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift new file mode 100644 index 000000000..9b9ae62dd --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift @@ -0,0 +1,119 @@ +import ApplicationServices +import Foundation + +/// One accessibility attribute, as reported name and rendered value. +/// +/// A list of pairs rather than a dictionary, so attribute names reach the JSON +/// exactly as the accessibility API spells them. `JSONEncoder`'s snake-case key +/// strategy rewrites dictionary keys, which would turn `AXIdentifier` into +/// `ax_identifier` and make the dump a poor record of what the app reports. +struct DumpAttribute: Encodable { + let name: String + let value: String + + /// Whether the accessibility API reports this attribute as writable, when + /// settability was asked for. + /// + /// This decides how the driver changes state. Writing `AXSelected` on a row is + /// deterministic; synthesizing a click at a screen coordinate depends on the + /// window being frontmost and unobscured. + /// + /// Absent unless requested: answering it costs one round-trip per attribute, + /// which doubles the cost of a walk. + let settable: Bool? +} + +/// One element of an application's accessibility tree, with everything it reports. +struct DumpNode: Encodable { + /// `AXRole`, lifted out of the attributes because it is what a reader scans + /// for. + let role: String + + /// Every attribute the element reports, minus the two that only lead back into + /// the tree, sorted by name. + let attributes: [DumpAttribute] + + /// Actions the element accepts, such as `AXPress`. + let actions: [String] + + let children: [DumpNode] + + /// How many children were dropped to keep the walk bounded. + /// + /// Absent when every child was walked. A sidebar of a thousand conversations + /// repeats one row shape a thousand times, so the count is the useful part and + /// the repetition is not. + let elidedChildren: Int? +} + +/// Walks an application's accessibility tree and reports everything it finds. +/// +/// This is a design instrument. SwiftUI's mapping onto accessibility elements is +/// undocumented and not one-to-one, so decisions about how to address and act on +/// an element are made by reading a real dump rather than by predicting where a +/// `.accessibilityIdentifier` lands. +/// +/// Unfiltered by intent: every attribute of every element it visits, so nothing +/// that turns out to matter has been quietly dropped. [`Tree`](Tree) is the +/// filtered counterpart for everyday use. +enum Dump { + /// Walk the tree rooted at the application owning `options.pid`. + static func walk(_ options: DumpOptions) throws(DriveError) -> DumpNode { + guard ProcessTable.record(for: options.pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(options.pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + // Checked up front rather than reported per element: without the grant + // every read fails, and a tree of identical refusals says less than one + // error naming the pane that fixes it. + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return node(AXElement.application(pid: options.pid), depth: 0, options: options) + } + + /// Attributes that only lead back into the tree, and so are not recorded. + /// + /// `AXChildren` is what the walk recurses into, and `AXParent` points at the + /// element that just reported this one. + private static let structuralAttributes: Set<String> = [ + kAXChildrenAttribute, + kAXParentAttribute, + ] + + private static func node(_ element: AXElement, depth: Int, options: DumpOptions) -> DumpNode + { + let names = + element.names() + .filter { !structuralAttributes.contains($0) } + .sorted() + + let attributes = zip(names, element.values(names)).map { name, value in + DumpAttribute( + name: name, + value: value.map(AXElement.text) ?? "<null>", + settable: options.settable ? element.isSettable(name) : nil + ) + } + + let all = depth < options.maxDepth ? element.children : [] + let walked = options.maxSiblings > 0 ? Array(all.prefix(options.maxSiblings)) : all + + return DumpNode( + role: attributes.first { $0.name == kAXRoleAttribute }?.value ?? "<none>", + attributes: attributes, + actions: element.actions, + children: walked.map { node($0, depth: depth + 1, options: options) }, + elidedChildren: all.count > walked.count ? all.count - walked.count : nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift new file mode 100644 index 000000000..09ef2dbce --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift @@ -0,0 +1,15 @@ +import Foundation + +extension Duration { + /// The duration in whole milliseconds, for reporting. + var milliseconds: Int { + let (seconds, attoseconds) = components + return Int(seconds) * 1000 + Int(attoseconds / 1_000_000_000_000_000) + } + + /// The duration in seconds, for the APIs that take a `TimeInterval`. + var seconds: TimeInterval { + let (seconds, attoseconds) = components + return TimeInterval(seconds) + TimeInterval(attoseconds) / 1e18 + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift new file mode 100644 index 000000000..dd2ef2a30 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift @@ -0,0 +1,186 @@ +import ApplicationServices + +extension Optional where Wrapped == String { + /// The value read as a boolean. + /// + /// The accessibility API renders `AXEnabled`, `AXMain` and their like as `"0"` + /// or `"1"`. Anything else, including an absent attribute, is neither true nor + /// false. + var axFlag: Bool? { + switch self { + case "0": return false + case "1": return true + default: return nil + } + } +} + +/// Posts synthesized input to the window server. +/// +/// Behind a protocol because posting is the one thing the driver does that is not +/// addressed to an element. A click goes to whatever occupies a screen +/// coordinate, which is global state and cannot be exercised against a fake tree +/// the way every other step can. +protocol EventPoster { + /// Click once at `point`, in screen coordinates. + /// + /// Answers whether the events could be built and posted, which is not whether + /// anything received them. + func click(at point: CGPoint) -> Bool + + /// Press at the first point of `path`, move through the rest, release at the + /// last. + /// + /// `pause` separates one move from the next. Without it the moves are posted + /// faster than the target can consume them and the window server delivers a + /// coalesced few, which is the opposite of what a drag is usually being + /// synthesized to exercise: what a view does *during* the gesture, frame by + /// frame. + /// + /// Answers whether every event could be built and posted. + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool +} + +/// Posts through `CoreGraphics`. +struct SystemEventPoster: EventPoster { + func click(at point: CGPoint) -> Bool { + guard + let down = event(.leftMouseDown, at: point), + let up = event(.leftMouseUp, at: point) + else { + return false + } + + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + return true + } + + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool { + guard let first = path.first, let last = path.last else { return false } + guard let down = event(.leftMouseDown, at: first) else { return false } + + down.post(tap: .cghidEventTap) + + for point in path.dropFirst() { + guard let moved = event(.leftMouseDragged, at: point) else { + // Released wherever it got to rather than returned from. A drag + // abandoned with the button still down leaves the whole machine + // holding a mouse button nobody is pressing, which outlives this + // process and is not something a failed test should do to the + // person running it. + release(at: point) + return false + } + + moved.post(tap: .cghidEventTap) + Thread.sleep(forTimeInterval: pause.seconds) + } + + guard let up = event(.leftMouseUp, at: last) else { + release(at: last) + return false + } + + up.post(tap: .cghidEventTap) + return true + } + + /// Let the button go, on a path that could not be finished. + private func release(at point: CGPoint) { + event(.leftMouseUp, at: point)?.post(tap: .cghidEventTap) + } + + private func event(_ type: CGEventType, at point: CGPoint) -> CGEvent? { + CGEvent( + mouseEventSource: nil, + mouseType: type, + mouseCursorPosition: point, + mouseButton: .left + ) + } +} + +/// Attribute text and children, read together. +/// +/// The pair exists because reading them separately costs an extra round-trip per +/// element, and walking to a child is the most common read the driver makes. +struct Reading<E> { + /// One entry per requested name, positionally, `nil` where the element has no + /// value for that attribute. + let text: [String?] + + let children: [E] +} + +/// One element of an accessibility tree, as the driver's traversal needs it. +/// +/// The traversal is where the driver's logic lives: pruning a filtered walk, +/// spending a match budget, finding which ancestor of an identified element owns +/// selection. None of that is about the accessibility API, and all of it has been +/// wrong at least once. Behind this protocol it can be tested against a fake tree +/// instead of against a running application. +/// +/// Deliberately narrow. Everything here is something a walk actually does, so a +/// fake stays small enough to read at a glance and cannot drift far from the real +/// implementation. +protocol Element { + /// Read the named attributes and the element's children. + /// + /// Implementations batch: this is one round-trip in the real one. + func read(_ names: [String]) -> Reading<Self> + + /// Actions the element accepts, such as `AXPress`. + /// + /// Separate from ``read(_:)`` because it costs its own round-trip and most + /// elements a filtered walk passes through are discarded unread. + var actions: [String] { get } + + /// Whether `name` can be written on this element. + func isSettable(_ name: String) -> Bool + + /// Read a boolean attribute, `nil` when it is absent or not a boolean. + func flag(_ name: String) -> Bool? + + /// Write a boolean attribute, answering the accessibility API's own status. + /// + /// A successful write is not a successful change: the target can accept the + /// value and do nothing with it. Read it back to find out. + func setFlag(_ name: String, _ value: Bool) -> AXError + + /// Write a string attribute, answering the accessibility API's own status. + func setText(_ name: String, _ value: String) -> AXError + + /// Perform an action, answering the accessibility API's own status. + /// + /// What the action did is not observable from here. Pressing a button runs + /// arbitrary code in the target, and success means the press was delivered, + /// not that anything came of it. + func perform(_ action: String) -> AXError + + /// The point held in an attribute, in screen coordinates. + /// + /// `nil` when the attribute is absent or holds something else. Separate from + /// ``read(_:)`` because a caller aiming a click needs the numbers, not the + /// text they render as. + func point(_ name: String) -> CGPoint? + + /// The size held in an attribute, in points. + /// + /// `nil` when the attribute is absent or holds something else. + func size(_ name: String) -> CGSize? + + /// Write a size attribute, answering the accessibility API's own status. + /// + /// As with every other write here, success is not change: a window clamps a + /// size to its own minimum and maximum, so what it ends up at has to be read + /// back. + func setSize(_ name: String, _ value: CGSize) -> AXError + + /// The elements held in an attribute, such as `AXWindows` or `AXMenuBar`. + /// + /// Answers a single element as a one-element array, since the accessibility + /// API spells "the menu bar" and "the windows" the same way apart from the + /// plural. + func elements(_ name: String) -> [Self] +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift new file mode 100644 index 000000000..a1c595b9d --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift @@ -0,0 +1,52 @@ +import ApplicationServices +import Foundation + +/// Reads an application's menu bar. +/// +/// Reported as a tree, because a menu is one: bar, then bar items, then menus, +/// then items. What makes it worth its own subcommand is the root — reaching the +/// menu bar from the application element takes an attribute that holds it, not a +/// walk through the window hierarchy. +/// +/// Menu items are pressed with `act press`; they advertise `AXPress` where a list +/// row does not. +enum Menu { + /// Read the menu bar of the application owning `pid`. + static func read(pid: pid_t, options: TreeOptions) throws(DriveError) -> TreeNode { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + let app = AXElement.application(pid: pid) + + guard let bar = app.elements(kAXMenuBarAttribute).first else { + throw DriveError( + kind: .notFound, + message: "the application reports no menu bar", + hint: "an agent or accessory application has none" + ) + } + + guard let tree = Tree.walk(from: bar, options: options) else { + throw DriveError( + kind: .notFound, + message: "the menu bar held nothing matching", + hint: nil + ) + } + + return tree + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift new file mode 100644 index 000000000..47ca031d5 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift @@ -0,0 +1,44 @@ +import Foundation + +/// Writes the driver's JSON documents. +/// +/// Both results and errors go to stdout, so a caller reads one stream and +/// distinguishes the two by the top-level `error` key or by the exit status. +enum Output { + /// Encode `value` as pretty JSON on stdout, with a trailing newline. + static func write(_ value: some Encodable) throws(DriveError) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + + // Snake case on the wire, matching every other JSON payload JP produces. + encoder.keyEncodingStrategy = .convertToSnakeCase + + let data: Data + do { + data = try encoder.encode(value) + } catch { + throw DriveError( + kind: .encodingFailed, + message: "could not encode the result as JSON: \(error)", + hint: nil + ) + } + + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data("\n".utf8)) + } + + /// Write an error document. + /// + /// Falls back to hand-built JSON, so a caller still gets something parseable + /// in the case where even the error will not encode. + static func writeError(_ error: DriveError) { + do throws(DriveError) { + try write(ErrorDocument(error: error)) + } catch { + let message = error.message.replacingOccurrences(of: "\"", with: "'") + let json = #"{"error":{"kind":"encoding_failed","message":"\#(message)"}}"# + "\n" + FileHandle.standardOutput.write(Data(json.utf8)) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift new file mode 100644 index 000000000..d9a0c8d8d --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift @@ -0,0 +1,252 @@ +import CoreGraphics +import Foundation +import ImageIO + +/// One stretch of identical pixels along a scanline. +struct PixelRun: Encodable, Equatable { + /// Where the run begins, in pixels along the scan. + let start: Int + + /// How many pixels it covers. + let count: Int + + /// The colour, `#RRGGBB` when opaque and `#RRGGBBAA` when it is not. + let color: String +} + +/// What one scan across an image found. +struct PixelReport: Encodable, Equatable { + /// The image's width in pixels, which on a retina display is twice its width + /// in points. + let width: Int + + /// The image's height in pixels. + let height: Int + + /// The colour space the values are reported in. + /// + /// Always sRGB. Stated anyway, because the numbers mean nothing without it: + /// the same screenshot read in the display's own profile and in sRGB gives two + /// different sets of values for the same pixels, and a light grey moves by + /// several steps between them. + /// + /// sRGB because that is the space colours are *written* in — a palette + /// constant, a value from a colour picker, a hex in a design note — so a + /// reading can be compared against the thing it was supposed to be. + let colorSpace: String + + /// Which way the scan ran: `row` or `column`. + let scan: String + + /// The row or column that was read, in pixels. + let at: Int + + /// The runs along it, in order, covering the scanned range without gaps. + let runs: [PixelRun] +} + +/// What to scan, and where. +struct PixelOptions { + /// Which way a scan runs. + enum Axis: String { + /// Left to right, across one row. + case row + + /// Top to bottom, down one column. + case column + } + + /// The PNG to read. + let image: String + + let axis: Axis + + /// The row or column to read, in pixels. + let at: Int + + /// Where along the scan to start, in pixels. The near edge when absent. + let from: Int? + + /// Where along the scan to stop, inclusive, in pixels. The far edge when + /// absent. + let to: Int? +} + +/// Reads the pixels of a screenshot. +/// +/// Answers the questions the accessibility tree cannot: what colour something is, +/// and how wide a drawn thing is. A hairline, a selection fill, a divider and a +/// row separator are all invisible to the tree, and all obvious in a scanline. +/// +/// Reads a file rather than capturing one. Capture already has a home +/// (`screencapture`, driven by `debug_app_screenshot`), and the only ways to +/// capture from inside this process are deprecated. It also makes this testable +/// against an image built by hand, with no window server and no grants. +enum Pixels { + /// Scan `options.image` and report the runs along the requested line. + static func read(_ options: PixelOptions) throws(DriveError) -> PixelReport { + let bitmap = try Bitmap(path: options.image) + let extent = options.axis == .row ? bitmap.width : bitmap.height + let across = options.axis == .row ? bitmap.height : bitmap.width + + guard options.at >= 0, options.at < across else { + throw DriveError( + kind: .notFound, + message: + "\(options.axis.rawValue) \(options.at) is outside the image, which is " + + "\(bitmap.width)x\(bitmap.height) pixels", + hint: "a row is indexed down from the top and a column across from the left" + ) + } + + let from = max(options.from ?? 0, 0) + let to = min(options.to ?? extent - 1, extent - 1) + + guard from <= to else { + throw DriveError( + kind: .badUsage, + message: "--from \(from) is past --to \(to)", + hint: "both are pixel offsets along the scan, and --to is inclusive" + ) + } + + let line = (from...to).map { along in + options.axis == .row + ? bitmap.pixel(x: along, y: options.at) + : bitmap.pixel(x: options.at, y: along) + } + + return PixelReport( + width: bitmap.width, + height: bitmap.height, + colorSpace: bitmap.colorSpace, + scan: options.axis.rawValue, + at: options.at, + runs: runs(of: line, startingAt: from) + ) + } + + /// Collapse `line` into runs of one colour, the first starting at `start`. + /// + /// The whole point of the output shape: a scan across a window is thousands of + /// pixels and a handful of colours, and the edges between them are the + /// measurements a reader is after. + static func runs(of line: [Pixel], startingAt start: Int) -> [PixelRun] { + var runs: [PixelRun] = [] + + for (offset, pixel) in line.enumerated() { + if let last = runs.last, last.color == pixel.hex { + runs[runs.count - 1] = PixelRun( + start: last.start, count: last.count + 1, color: last.color) + continue + } + + runs.append(PixelRun(start: start + offset, count: 1, color: pixel.hex)) + } + + return runs + } +} + +/// One pixel, as read out of an image. +struct Pixel: Equatable { + let red: UInt8 + let green: UInt8 + let blue: UInt8 + let alpha: UInt8 + + /// `#RRGGBB` when opaque, `#RRGGBBAA` when not. + /// + /// Alpha is left off the common case so the values read the way a colour + /// picker reports them, and included when it is not 255 because a translucent + /// pixel that printed as opaque would be a lie about what is on screen. + var hex: String { + let rgb = String(format: "#%02X%02X%02X", red, green, blue) + return alpha == 255 ? rgb : rgb + String(format: "%02X", alpha) + } +} + +/// An image's pixels, in the image's own colour space. +private struct Bitmap { + let width: Int + let height: Int + let colorSpace: String + + /// RGBA, row-major, four bytes per pixel and no row padding. + private let bytes: [UInt8] + + /// Decode the PNG at `path`. + init(path: String) throws(DriveError) { + guard + let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw DriveError( + kind: .notFound, + message: "could not read an image at \(path)", + hint: "debug_app_screenshot writes one, and reports where it put it" + ) + } + + width = image.width + height = image.height + + // Converted rather than read raw. `screencapture` writes in the display's + // profile, which is often unnamed and never the space a palette was + // written in: a `#DBDBDB` divider comes back as `#D6D6D6` read that way, + // which looks like a bug in the app rather than a difference of space. + let target = CGColorSpace(name: CGColorSpace.sRGB) + + guard + let target, + let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: width * 4, + space: target, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + else { + throw DriveError( + kind: .notFound, + message: "could not open \(path) as an 8-bit RGBA image", + hint: nil + ) + } + + colorSpace = "sRGB" + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + + guard let data = context.data else { + throw DriveError( + kind: .notFound, + message: "the drawing context for \(path) reported no pixels", + hint: nil + ) + } + + bytes = [UInt8]( + UnsafeBufferPointer( + start: data.assumingMemoryBound(to: UInt8.self), + count: width * height * 4 + )) + } + + /// The pixel at `x`, `y`, counted from the top-left corner. + /// + /// The buffer runs in the same direction: a bitmap context's first row is the + /// top of what was drawn into it, so a screenshot's rows and this buffer's rows + /// are the same rows in the same order. + func pixel(x: Int, y: Int) -> Pixel { + let offset = (y * width + x) * 4 + + return Pixel( + red: bytes[offset], + green: bytes[offset + 1], + blue: bytes[offset + 2], + alpha: bytes[offset + 3] + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift new file mode 100644 index 000000000..9aed316e4 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift @@ -0,0 +1,58 @@ +import Darwin +import Foundation + +/// One process in the chain from the driver up towards `launchd`. +struct ProcessLink: Encodable { + let pid: pid_t + + /// Short command name from the kernel process table. The kernel truncates it + /// to 16 bytes, so `Terminal` and `iTerm2` arrive whole but a long binary + /// name does not. + let command: String +} + +/// Process identity read from the kernel through `sysctl(KERN_PROC_PID)`. +/// +/// The spike needs the ancestor chain because TCC attributes a grant to the +/// responsible process, and the chain is the list of candidates for that role. +enum ProcessTable { + /// Depth limit for the ancestor walk. A shell-to-`launchd` chain is a + /// handful of processes; the limit only guards against a process table that + /// changes underneath the walk. + private static let maxDepth = 32 + + /// `pid` and its ancestors, nearest first, stopping below `launchd`. + static func ancestry(from pid: pid_t) -> [ProcessLink] { + var links: [ProcessLink] = [] + var current = pid + + while current > 1, links.count < maxDepth { + guard let record = record(for: current) else { break } + links.append(ProcessLink(pid: current, command: name(of: record))) + current = record.kp_eproc.e_ppid + } + + return links + } + + /// The kernel's record for `pid`, or `nil` when no such process is running. + static func record(for pid: pid_t) -> kinfo_proc? { + var selector: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, pid] + var record = kinfo_proc() + var size = MemoryLayout<kinfo_proc>.stride + + let result = sysctl(&selector, u_int(selector.count), &record, &size, nil, 0) + + // Querying a pid that no longer exists succeeds and writes nothing, so + // the written size is what separates a dead pid from a live one. + guard result == 0, size > 0 else { return nil } + return record + } + + /// The short command name held in a kernel record. + static func name(of record: kinfo_proc) -> String { + return withUnsafeBytes(of: record.kp_proc.p_comm) { bytes in + return String(decoding: bytes.prefix { $0 != 0 }, as: UTF8.self) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift new file mode 100644 index 000000000..c9a6f4ad5 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift @@ -0,0 +1,183 @@ +import ApplicationServices +import Foundation + +/// One element, as the driver reports it. +/// +/// An absent field means the element does not report that attribute. An +/// `AXUnknown` row carries a label and no value; a text field carries both. +struct TreeNode: Encodable, Equatable { + let role: String + let identifier: String? + let label: String? + let value: String? + let enabled: Bool? + let focused: Bool? + + /// The element's frame in screen coordinates, only when frames were asked for. + /// + /// Left out by default: coordinates change whenever a window moves or a list + /// scrolls, so including them turns every diff between two snapshots into + /// noise. + let frame: String? + + let actions: [String] + let children: [TreeNode] + + /// How many of this element's children are missing from ``children``. + /// + /// Every reason a child goes missing is counted the same, because they answer + /// one question: is there more here than I am looking at? The depth limit, the + /// per-level sibling cap, the match budget running out, and a filter discarding + /// a branch that held no match all leave the reader in the same position, and + /// the last two are the easiest to mistake for an element having no children at + /// all. + let elidedChildren: Int? +} + +/// What to walk, and what to keep. +struct TreeOptions { + let pid: pid_t + + /// Keep only elements whose identifier begins with this, along with the + /// ancestors that lead to them. `nil` keeps everything. + /// + /// A prefix rather than an exact match, because the useful question to ask of a + /// tree is "what is under `sidebar.`". Acting on an element is the opposite + /// case and matches exactly. + let identifierPrefix: String? + + /// How many matches to find before stopping. + /// + /// This is the bound that matters. Every identifier in this app's sidebar sits + /// on a leaf, so a prefix search cannot prune on the way down and an unbounded + /// one visits every element in the application. Stopping at a handful of + /// matches answers "what does the sidebar look like" for the cost of the first + /// handful rather than of all thousand. + /// + /// Set this to `1` when looking up one identifier already known, or the walk + /// continues past it looking for a second. + let maxMatches: Int + + let maxDepth: Int + let maxSiblings: Int + let frames: Bool +} + +/// Reads an application's accessibility tree into something a person can scan. +/// +/// Where [`Dump`](Dump) reports every attribute of every element for design work, +/// this reports the handful that identify and describe an element, and prunes +/// branches holding nothing that matched. +enum Tree { + /// Attributes read for every node, in one batch. Order matters, since values + /// come back positionally. + static let batch = [ + kAXRoleAttribute, + kAXIdentifierAttribute, + AXElement.attributedDescription, + kAXDescriptionAttribute, + kAXTitleAttribute, + kAXValueAttribute, + kAXEnabledAttribute, + kAXFocusedAttribute, + "AXFrame", + ] + + /// Walk the tree of the application owning `options.pid`. + /// + /// Returns `nil` when a prefix was given and nothing matched it. + static func read(_ options: TreeOptions) throws(DriveError) -> TreeNode? { + guard ProcessTable.record(for: options.pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(options.pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return walk(from: AXElement.application(pid: options.pid), options: options) + } + + /// Walk from `root`, spending a fresh match budget. + static func walk<E: Element>(from root: E, options: TreeOptions) -> TreeNode? { + // An unfiltered walk has no matches to count, so the budget only bounds a + // filtered one. + var remaining = options.identifierPrefix == nil ? Int.max : options.maxMatches + return node(root, depth: 0, options: options, remaining: &remaining) + } + + private static func node<E: Element>( + _ element: E, + depth: Int, + options: TreeOptions, + remaining: inout Int + ) -> TreeNode? { + let reading = element.read(batch) + let text = reading.text + + let identifier = text[1] + let matches = + options.identifierPrefix.map { identifier?.hasPrefix($0) ?? false } ?? false + if matches { + remaining -= 1 + } + + // Every child the element has, whether or not this walk descends into it. + // The count is what tells a reader there is more here; without it a node + // stopped at the depth limit is indistinguishable from a leaf. + let available = reading.children + let all = depth < options.maxDepth ? available : [] + + // The sibling cap is for reading an unfiltered tree, where every level is + // worth seeing but a thousand copies of one row are not. Under a filter the + // match budget does the bounding instead: a cap here would hide the eight + // hundredth row from a search that named it. + let capped = options.identifierPrefix == nil && options.maxSiblings > 0 + + var children: [TreeNode] = [] + var visited = 0 + + for child in all { + guard remaining > 0 else { break } + guard !capped || visited < options.maxSiblings else { break } + visited += 1 + + guard + let node = node( + child, depth: depth + 1, options: options, remaining: &remaining) + else { continue } + children.append(node) + } + + // A branch is kept when it matches, or when something under it does. The + // ancestors are what make a match locatable rather than a bare hit. + guard matches || !children.isEmpty || options.identifierPrefix == nil else { + return nil + } + + return TreeNode( + role: text[0] ?? "<none>", + identifier: identifier, + label: text[2] ?? text[3] ?? text[4], + value: text[5], + enabled: text[6].axFlag, + focused: text[7].axFlag, + frame: options.frames ? text[8] : nil, + // Read only for a node being kept. Actions cost their own round-trip and + // most elements a filtered walk passes through are discarded. + actions: element.actions, + children: children, + elidedChildren: available.count > children.count + ? available.count - children.count + : nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift new file mode 100644 index 000000000..063735580 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift @@ -0,0 +1,133 @@ +import CoreGraphics +import Foundation + +/// A window the window server can be told to capture. +struct CaptureWindow: Encodable, Equatable { + /// The window server's identifier, in the form `screencapture -l` takes. + let id: CGWindowID + + /// The window's title, or `nil` when this process holds no Screen Recording + /// grant: the window server withholds other applications' titles until it + /// does. + let title: String? + + let width: Int + let height: Int +} + +/// What `jpdrive windowid` observed. +struct WindowIDReport: Encodable, Equatable { + /// Whether this process may read other applications' screen content. + /// + /// Enumerating windows needs no grant, so a report can list windows that + /// cannot be captured. A caller that acts on the list without reading this + /// gets a picture of the desktop where it expected a window. + let screenRecording: Bool + + /// The application's capturable windows, front to back. + let windows: [CaptureWindow] + + /// Windows the application has that are not on the active Space. + /// + /// Reported separately because the two look identical from the outside and + /// mean opposite things. A window on another desktop is absent from every + /// on-screen enumeration and from the accessibility tree, so an app that has + /// one and nothing else is indistinguishable from an app with no window at + /// all — except by asking for windows on every Space, which is this list. + let otherSpaces: [CaptureWindow] +} + +/// Resolves an application's window-server identifiers. +/// +/// Separate from `Windows`, which reads the accessibility tree: the two answer +/// different questions and neither identifier converts into the other. An +/// accessibility window has a title and a frame but no number the capture tools +/// accept, and a window-server window has that number but nothing structural. +enum WindowIDs { + /// The layer ordinary application windows sit on. + /// + /// Everything else the window server reports for an application is chrome — + /// tooltips, drag images, the shadow behind a menu — and capturing one of + /// those instead of the window is a silent wrong answer rather than a + /// failure. + static let normalLayer = 0 + + /// The capturable windows of the application owning `pid`. + static func read(pid: pid_t) throws(DriveError) -> WindowIDReport { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + let onScreen = + CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] ?? [] + + // Every Space, not just the active one. The difference between the two + // lists is what says a window exists somewhere the screen cannot show it. + let everywhere = + CGWindowListCopyWindowInfo( + [.excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] ?? [] + + let here = capturable(from: onScreen, pid: pid) + let all = capturable(from: everywhere, pid: pid) + let shown = Set(here.map(\.id)) + + // The preflight variant, never the requesting one: raising the system's + // permission dialog from a background tool leaves a prompt nobody is + // watching, in front of the app being measured. + return WindowIDReport( + screenRecording: CGPreflightScreenCaptureAccess(), + windows: here, + otherSpaces: all.filter { !shown.contains($0.id) } + ) + } + + /// The windows in `listed` that belong to `pid` and can be captured, + /// in the order the window server reported them, which is front to back. + /// + /// A window with no area is dropped: `AppKit` keeps zero-sized windows + /// around for panels that have never been shown, and capturing one produces + /// an empty file. + static func capturable(from listed: [[String: Any]], pid: pid_t) -> [CaptureWindow] { + return listed.compactMap { window -> CaptureWindow? in + guard integer(window[kCGWindowOwnerPID as String]) == Int(pid), + integer(window[kCGWindowLayer as String]) == normalLayer, + let number = integer(window[kCGWindowNumber as String]), + let id = CGWindowID(exactly: number), + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let width = integer(bounds["Width"]), + let height = integer(bounds["Height"]), + width > 0, height > 0 + else { + return nil + } + + return CaptureWindow( + id: id, + title: window[kCGWindowName as String] as? String, + width: width, + height: height + ) + } + } + + /// One of the window server's numbers, whichever numeric type it arrives as. + /// + /// The list holds `CFNumber`s in untyped dictionaries. Bridged, those cast to + /// `Int` while the value is whole and only to `Double` otherwise, which is a + /// distinction window bounds can cross: a window on a scaled display sits at + /// fractional points. + private static func integer(_ value: Any?) -> Int? { + if let int = value as? Int { return int } + if let double = value as? Double { return Int(double) } + return nil + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift new file mode 100644 index 000000000..a2a5ab144 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift @@ -0,0 +1,74 @@ +import ApplicationServices +import Foundation + +/// One of an application's windows. +struct WindowSummary: Encodable, Equatable { + let identifier: String? + let title: String? + + /// Whether this is the application's main window. + let main: Bool? + + let minimized: Bool? + + /// Position and size in screen coordinates. + /// + /// Included here, unlike in a tree, because a window's frame is what the + /// listing is for: which window is where, and how big. + let frame: String? +} + +/// Lists an application's windows. +/// +/// Separate from a tree walk because the useful facts about a window are its own — +/// which one is main, which is minimized, where it sits — rather than what it +/// contains. +enum Windows { + /// Attributes read for every window, in one batch. + static let batch = [ + kAXIdentifierAttribute, + kAXTitleAttribute, + kAXMainAttribute, + kAXMinimizedAttribute, + "AXFrame", + ] + + /// List the windows of the application owning `pid`. + static func read(pid: pid_t) throws(DriveError) -> [WindowSummary] { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return list(of: AXElement.application(pid: pid)) + } + + /// The windows an application element reports. + /// + /// An application with no windows answers an empty list, which is a state a + /// running app can legitimately be in. + static func list<E: Element>(of app: E) -> [WindowSummary] { + return app.elements(kAXWindowsAttribute).map { window in + let text = window.read(batch).text + + return WindowSummary( + identifier: text[0], + title: text[1], + main: text[2].axFlag, + minimized: text[3].axFlag, + frame: text[4] + ) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift b/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift new file mode 100644 index 000000000..7e1373e35 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift @@ -0,0 +1,3 @@ +import DriveKit + +Driver.run() diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift new file mode 100644 index 000000000..b5f8c9ba0 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift @@ -0,0 +1,249 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act") +struct ActTests { + /// The case the driver exists for, and the one that was broken: the identifier + /// is on a leaf two levels below the element that owns selection. + @Test("select writes AXSelected on the row, not on the identified element") + func selectsTheOwningRow() throws { + let root = FakeElement.sidebar(rowCount: 3) + let step = Step.select(.init(identifier: "sidebar.row.1")) + + let result = try Act.run(step, in: root) + + #expect( + result + == StepResult( + step: "select", + identifier: "sidebar.row.1", + role: "AXRow", + confirmed: true + ) + ) + #expect(result.confirmed == true) + + let rows = try #require(root.children.first?.children) + #expect(rows[1].attributes[kAXSelectedAttribute] == "1") + + // The leaf that carried the identifier must not have been written to. It + // reports no AXSelected at all, and a driver that wrote there would report + // success while selecting nothing. + let leaf = try #require(rows[1].children.first?.children.first) + #expect(leaf.attributes[kAXSelectedAttribute] == nil) + } + + /// Selection reaches a row regardless of where it sits, which is what makes the + /// attribute write preferable to a synthesized click. + @Test("select reaches a row far down a long list") + func selectsADeepRow() throws { + let root = FakeElement.sidebar(rowCount: 1000) + + let result = try Act.run(.select(.init(identifier: "sidebar.row.987")), in: root) + + #expect(result.confirmed == true) + let rows = try #require(root.children.first?.children) + #expect(rows[987].attributes[kAXSelectedAttribute] == "1") + } + + @Test("select reports the identifier it could not find") + func reportsAMissingIdentifier() { + let root = FakeElement.sidebar(rowCount: 3) + + #expect(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.nope")), in: root) + } + } + + /// An element nothing in its chain can select is a failure, not a fallback onto + /// some other mechanism. + @Test("select fails when no ancestor accepts the write") + func failsWhenNothingIsSelectable() throws { + let leaf = FakeElement(role: "AXUnknown", identifier: "lonely") + let root = FakeElement(role: "AXApplication", children: [leaf]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "lonely")), in: root) + } + + #expect(error.kind == .notSelectable) + } + + /// A write the accessibility API refuses is reported, not silently treated as + /// an unconfirmed success. + @Test("select reports a refused write") + func reportsARefusedWrite() throws { + let root = FakeElement.sidebar(rowCount: 2) + let rows = try #require(root.children.first?.children) + rows[0].writeStatus = .cannotComplete + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .writeFailed) + #expect(error.message.contains("cannot_complete")) + } + + /// A write can be accepted and do nothing. The step reports that as an + /// unconfirmed success rather than as a failure, because the distinction is + /// what tells a caller the mechanism stopped working. + @Test("select reports an accepted write that changed nothing") + func reportsAnIneffectiveWrite() throws { + let leaf = FakeElement(role: "AXUnknown", identifier: "row") + let row = FakeElement(role: "AXRow", settable: [kAXSelectedAttribute], children: [leaf]) + let root = FakeElement(role: "AXApplication", children: [row]) + row.ignoresWrites = true + + let result = try Act.run(.select(.init(identifier: "row")), in: root) + + #expect(result.role == "AXRow") + #expect( + result.confirmed == false, + "a write the target discarded must not report as confirmed" + ) + } + + /// A sidebar row is the case that makes `click` the wrong tool: the identified + /// element has no activation point, and `select` reaches it whether or not it + /// is on screen. + @Test("click fails on a row, which wants select instead") + func clickFailsOnARow() throws { + let root = FakeElement.sidebar(rowCount: 1) + + let error = try #require(throws: DriveError.self) { + try Act.run( + .click(.init(identifier: "sidebar.row.0")), in: root, poster: FakePoster()) + } + + #expect(error.kind == .notClickable) + } + + @Test("press performs AXPress on the identified element") + func pressesTheElement() throws { + let item = FakeElement( + role: "AXMenuItem", + identifier: "terminate:", + actions: ["AXCancel", "AXPress", "AXPick"] + ) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "terminate:")), in: root) + + #expect(item.performed == ["AXPress"]) + #expect(result.step == "press") + #expect(result.role == "AXMenuItem") + } + + /// Nothing readable says a press worked, so the step must not claim it did. + /// Reporting `true` here would be the one dishonest field in the output. + @Test("press reports no confirmation") + func pressDoesNotClaimConfirmation() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "go")), in: root) + + #expect(result.confirmed == nil) + } + + /// A sidebar row is the case this catches: it advertises no actions at all, so + /// the error points at the step that does work on it. + @Test("press fails on an element that does not accept it") + func pressFailsWithoutTheAction() throws { + let root = FakeElement.sidebar(rowCount: 1) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint?.contains("select") == true) + // The press must not have been attempted anyway. + let leaf = root.children.first?.children.first?.children.first?.children.first + #expect(leaf?.performed.isEmpty == true) + } + + /// An element with other actions gets told what it does accept, which is how a + /// script author finds the right verb without dumping the tree. + @Test("press names the actions an element does accept") + func pressNamesAvailableActions() throws { + let item = FakeElement(role: "AXRow", identifier: "row", actions: ["AXShowMenu"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "row")), in: root) + } + + #expect(error.hint?.contains("AXShowMenu") == true) + } + + /// `press` is a shorthand for `perform` with `AXPress`, and must keep saying so + /// in its result rather than reporting the mechanism underneath. + @Test("press names itself, not the general mechanism") + func pressNamesItself() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "go")), in: root) + + #expect(result.step == "press") + } + + /// The escape hatch for the actions with no step of their own. A text field + /// offers `AXConfirm` and no `AXPress`, so this is the only way to reach it. + @Test("perform runs any action the element advertises") + func performsANamedAction() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "sidebar.filter", + actions: ["AXShowMenu", "AXConfirm"] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .perform(.init(identifier: "sidebar.filter", action: "AXConfirm")), + in: root + ) + + #expect(field.performed == ["AXConfirm"]) + #expect(result.step == "perform") + } + + @Test("perform fails on an action the element does not advertise") + func performRejectsAnUnknownAction() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "sidebar.filter", + actions: ["AXConfirm"] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run( + .perform(.init(identifier: "sidebar.filter", action: "AXPress")), + in: root + ) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint == "it accepts: AXConfirm") + #expect(field.performed.isEmpty) + } + + @Test("press reports a refused action") + func pressReportsARefusal() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + item.performStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [item]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "go")), in: root) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("cannot_complete")) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift new file mode 100644 index 000000000..41534c061 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift @@ -0,0 +1,201 @@ +import Testing + +@testable import DriveKit + +@Suite("Arguments") +struct ArgumentsTests { + @Test("doctor takes an optional pid") + func doctorPid() throws { + guard case .doctor(let pid) = try Arguments.parse(["doctor", "--pid", "42"]) else { + Issue.record("expected a doctor command") + return + } + #expect(pid == 42) + + guard case .doctor(let none) = try Arguments.parse(["doctor"]) else { + Issue.record("expected a doctor command") + return + } + #expect(none == nil) + } + + @Test("tree defaults its bounds") + func treeDefaults() throws { + guard case .tree(let options) = try Arguments.parse(["tree", "--pid", "42"]) else { + Issue.record("expected a tree command") + return + } + + #expect(options.pid == 42) + #expect(options.identifierPrefix == nil) + #expect(options.maxMatches == Arguments.defaultMatches) + #expect(options.maxDepth == Arguments.defaultDepth) + #expect(options.maxSiblings == Arguments.defaultSiblings) + #expect(!options.frames) + } + + @Test("tree takes every bound") + func treeFlags() throws { + let parsed = try Arguments.parse([ + "tree", "--pid", "42", "--identifier", "sidebar.", "--max-matches", "1", + "--depth", "3", "--max-siblings", "0", "--frames", + ]) + + guard case .tree(let options) = parsed else { + Issue.record("expected a tree command") + return + } + + #expect(options.identifierPrefix == "sidebar.") + #expect(options.maxMatches == 1) + #expect(options.maxDepth == 3) + #expect(options.maxSiblings == 0) + #expect(options.frames) + } + + /// Zero means "every sibling", which is a different thing from the cap being + /// unset, so it has to survive parsing rather than be rejected as non-positive. + @Test("max-siblings accepts zero for no cap") + func zeroSiblingsIsAllowed() throws { + guard + case .dump(let options) = try Arguments.parse([ + "dump", "--pid", "1", "--max-siblings", "0", + ]) + else { + Issue.record("expected a dump command") + return + } + + #expect(options.maxSiblings == 0) + } + + @Test("windows takes only a pid") + func windowsPid() throws { + guard case .windows(let pid) = try Arguments.parse(["windows", "--pid", "42"]) else { + Issue.record("expected a windows command") + return + } + #expect(pid == 42) + } + + @Test("windowid takes only a pid") + func windowidPid() throws { + guard case .windowid(let pid) = try Arguments.parse(["windowid", "--pid", "42"]) else { + Issue.record("expected a windowid command") + return + } + #expect(pid == 42) + } + + /// A menu bar is small and every item in it is a thing to press, so the sibling + /// cap that keeps a thousand-row list readable would only hide verbs here. + @Test("menu walks every sibling by default") + func menuHasNoSiblingCap() throws { + guard case .menu(let pid, let options) = try Arguments.parse(["menu", "--pid", "42"]) + else { + Issue.record("expected a menu command") + return + } + + #expect(pid == 42) + #expect(options.maxSiblings == 0) + } + + @Test("act decodes a step") + func actStep() throws { + let parsed = try Arguments.parse([ + "act", "--pid", "42", "--json", #"{"select":{"identifier":"sidebar.row.7"}}"#, + ]) + + guard case .act(let step, let pid) = parsed else { + Issue.record("expected an act command") + return + } + + #expect(pid == 42) + guard case .select(let target) = step else { + Issue.record("expected a select step") + return + } + #expect(target.identifier == "sidebar.row.7") + } + + /// Every field of a step is spelled the way the tool definition documents it, + /// and a mismatch is silent: an unrecognised key decodes as absent, so a wait + /// given a short timeout would wait the default instead and the run would look + /// merely slow. + @Test("act decodes every field of a wait") + func actWaitFields() throws { + let parsed = try Arguments.parse([ + "act", "--pid", "42", "--json", + #"{"wait_for":{"identifier":"transcript.scroll","under":"sidebar.list","timeout_ms":1500,"interval_ms":25}}"#, + ]) + + guard case .act(let step, _) = parsed, case .waitFor(let target) = step else { + Issue.record("expected a wait_for step") + return + } + + #expect(target.identifier == "transcript.scroll") + #expect(target.under == "sidebar.list") + #expect(target.timeoutMs == 1500) + #expect(target.intervalMs == 25) + } + + @Test( + "a step naming no known verb is rejected", + arguments: [ + #"{"nope":{"identifier":"x"}}"#, + #"{"wait":{"identifier":"x"}}"#, + #"{}"#, + #"not json"#, + #"{"select":{}}"#, + ] + ) + func rejectsAMalformedStep(json: String) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(["act", "--pid", "1", "--json", json]) + } + + #expect(error.kind == .badUsage) + } + + @Test( + "a command missing its pid is rejected", + arguments: [ + ["tree"], ["dump"], ["windows"], ["windowid"], ["menu"], ["act", "--json", "{}"], + ] + ) + func requiresAPid(arguments: [String]) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(arguments) + } + + #expect(error.kind == .badUsage) + } + + /// An empty command substitution is the shape this most often takes: + /// `--pid $(pgrep -f JP.app)` expands to nothing when the app is not running, + /// leaving the flag with no value. + @Test("a pid flag with no value is rejected with a usable hint") + func rejectsAMissingPidValue() throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(["doctor", "--pid"]) + } + + #expect(error.kind == .badUsage) + #expect(error.hint?.contains("pgrep") == true) + } + + @Test( + "unknown input is rejected", + arguments: [["fly", "--pid", "1"], ["tree", "--pid", "1", "--nope"], []] + ) + func rejectsUnknownInput(arguments: [String]) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(arguments) + } + + #expect(error.kind == .badUsage) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift new file mode 100644 index 000000000..d9986c121 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift @@ -0,0 +1,115 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.click") +struct ClickTests { + /// A button inside a window, which is the shape a click needs: something with a + /// point, under something that can be raised. + private func app(activationPoint: CGPoint? = CGPoint(x: 120, y: 340)) -> FakeElement { + let button = FakeElement( + role: "AXButton", identifier: "toolbar.open", actions: ["AXPress"]) + if let activationPoint { + button.points[AXElement.activationPoint] = activationPoint + } + + let window = FakeElement( + role: kAXWindowRole, + identifier: "workspace-AppWindow-1", + actions: [kAXRaiseAction], + children: [button] + ) + + return FakeElement(role: "AXApplication", children: [window]) + } + + @Test("clicks where the element says a click belongs") + func clicksTheActivationPoint() throws { + let root = app() + let poster = FakePoster() + + let result = try Act.run( + .click(.init(identifier: "toolbar.open")), + in: root, + poster: poster + ) + + #expect(poster.clicks == [CGPoint(x: 120, y: 340)]) + #expect(result.step == "click") + #expect(result.role == "AXButton") + #expect(result.point == "120.0,340.0") + } + + /// The event goes to whatever occupies the coordinate, so a window behind + /// another one would have its click swallowed. Raising is what makes the + /// coordinate mean the element that named it. + @Test("raises the window before clicking") + func raisesTheWindowFirst() throws { + let root = app() + let window = try #require(root.children.first) + + _ = try Act.run( + .click(.init(identifier: "toolbar.open")), in: root, poster: FakePoster()) + + #expect(window.performed == [kAXRaiseAction]) + } + + /// A sidebar row has no activation point of its own, and pointing at `select` + /// is more use than clicking at the origin would be. + @Test("fails on an element with nowhere to click") + func failsWithoutAnActivationPoint() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run( + .click(.init(identifier: "toolbar.open")), + in: app(activationPoint: nil), + poster: poster + ) + } + + #expect(error.kind == .notClickable) + #expect(error.hint?.contains("select") == true) + #expect(poster.clicks.isEmpty, "nothing may be clicked when there is no point to click") + } + + @Test("reports a click that could not be posted") + func reportsAFailedPost() throws { + let poster = FakePoster() + poster.succeeds = false + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "toolbar.open")), in: app(), poster: poster) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("120.0,340.0")) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "nope")), in: app(), poster: poster) + } + + #expect(error.kind == .identifierNotFound) + #expect(poster.clicks.isEmpty) + } + + /// An element outside any window still has a point, and clicking it is better + /// than refusing because there was nothing to raise. + @Test("clicks without a window to raise") + func clicksWithoutAWindow() throws { + let element = FakeElement(role: "AXButton", identifier: "loose") + element.points[AXElement.activationPoint] = CGPoint(x: 1, y: 2) + let root = FakeElement(role: "AXApplication", children: [element]) + let poster = FakePoster() + + _ = try Act.run(.click(.init(identifier: "loose")), in: root, poster: poster) + + #expect(poster.clicks == [CGPoint(x: 1, y: 2)]) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift new file mode 100644 index 000000000..93e471d96 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift @@ -0,0 +1,229 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.drag") +struct DragTests { + /// A window with a known frame, which is all a drag needs: somewhere to + /// measure fractions against, and something to raise. + private func app( + origin: CGPoint? = CGPoint(x: 100, y: 200), + size: CGSize? = CGSize(width: 800, height: 600) + ) -> FakeElement { + let window = FakeElement( + role: kAXWindowRole, + identifier: "workspace-AppWindow-1", + actions: [kAXRaiseAction] + ) + if let origin { + window.points[kAXPositionAttribute] = origin + } + if let size { + window.sizes[kAXSizeAttribute] = size + } + + return FakeElement(role: "AXApplication", children: [window]) + } + + private func step( + from: (Double, Double), + to: (Double, Double), + steps: Int? = nil, + pauseMs: Int? = nil + ) -> Step { + .drag( + .init( + identifier: "workspace-AppWindow-1", + from: .init(dx: from.0, dy: from.1), + to: .init(dx: to.0, dy: to.1), + steps: steps, + pauseMs: pauseMs + ) + ) + } + + /// Fractions are resolved against the element's own frame, so a script says + /// "the right edge, halfway down" rather than a screen coordinate that stops + /// being right the moment the window moves. + @Test("resolves fractional offsets against the element's frame") + func resolvesOffsets() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (1.0, 0.5), to: (0.5, 0.5), steps: 2), in: app(), poster: poster) + + // Right edge, halfway down: 100 + 800, 200 + 300. Halfway across: 100 + 400. + #expect( + poster.drags == [ + [ + CGPoint(x: 900, y: 500), + CGPoint(x: 700, y: 500), + CGPoint(x: 500, y: 500), + ] + ] + ) + } + + /// The step exists to produce many frames rather than one jump, so the count + /// is asserted rather than assumed: a drag delivered as a single move cannot + /// show what a view does *during* a gesture, which is the whole reason for it. + @Test("posts one move per step, plus the press") + func postsOneMovePerStep() throws { + let poster = FakePoster() + + let result = try Act.run( + step(from: (0, 0), to: (1, 0), steps: 12), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 13) + #expect(result.moves == 12) + #expect(result.step == "drag") + #expect(result.role == kAXWindowRole) + } + + @Test("defaults to enough moves to be a gesture") + func defaultsToAGesture() throws { + let poster = FakePoster() + + _ = try Act.run(step(from: (0, 0), to: (1, 1)), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 25) + #expect(poster.pauses == [.milliseconds(8)]) + } + + @Test("honours a stated pause between moves") + func honoursThePause() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 3, pauseMs: 40), in: app(), poster: poster) + + #expect(poster.pauses == [.milliseconds(40)]) + } + + /// A drag of zero steps is a click with extra words. Clamped rather than + /// rejected, so a caller computing the count from a distance cannot produce a + /// path with nothing in it. + @Test("clamps a step count below one") + func clampsZeroSteps() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 0), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 2) + } + + /// The events land on whatever occupies the coordinates, so a window behind + /// another would have the gesture swallowed. + @Test("raises the window before dragging") + func raisesTheWindowFirst() throws { + let root = app() + let window = try #require(root.children.first) + + _ = try Act.run(step(from: (1, 0.5), to: (0.5, 0.5)), in: root, poster: FakePoster()) + + #expect(window.performed == [kAXRaiseAction]) + } + + /// Raising alone is not enough, and this is the assertion that says so. + /// + /// `AXRaise` orders a window forward within its own application; the ordering + /// *between* applications follows activation. A drag posted at a background + /// window's coordinates without this was received by the frontmost terminal + /// instead — measured, and the reason the step takes focus. + @Test("brings the application forward before dragging") + func activatesTheApplication() throws { + let root = app() + + _ = try Act.run(step(from: (1, 0.5), to: (0.5, 0.5)), in: root, poster: FakePoster()) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// A tree that is not a running application has nothing to activate, and a + /// gesture against one is still worth posting: every other assertion in this + /// file depends on that. + @Test("drags even when the application cannot be brought forward") + func dragsWithoutActivating() throws { + let root = app() + root.writeStatus = .cannotComplete + let poster = FakePoster() + + _ = try Act.run(step(from: (0, 0), to: (1, 1), steps: 2), in: root, poster: poster) + + #expect(poster.drags.first?.count == 3) + } + + @Test("fails on an element with no frame to measure") + func failsWithoutAFrame() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1)), in: app(size: nil), poster: poster) + } + + #expect(error.kind == .notClickable) + #expect(poster.drags.isEmpty, "nothing may be dragged across a frame that is not known") + } + + @Test("reports a drag that could not be posted") + func reportsAFailedPost() throws { + let poster = FakePoster() + poster.succeeds = false + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1)), in: app(), poster: poster) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("100.0,200.0")) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run( + .drag( + .init( + identifier: "nope", + from: .init(dx: 0, dy: 0), + to: .init(dx: 1, dy: 1), + steps: nil, + pauseMs: nil + ) + ), + in: app(), + poster: poster + ) + } + + #expect(error.kind == .identifierNotFound) + #expect(poster.drags.isEmpty) + } + + /// Decoded from the wire, because the snake-cased key is spelled by hand and a + /// mismatch there reads as the default silently applying. + @Test("decodes a step from its written form") + func decodesFromJSON() throws { + let json = """ + {"drag": {"identifier": "transcript.text", "from": {"dx": 0.1, "dy": 0.2}, + "to": {"dx": 0.8, "dy": 0.6}, "steps": 6, "pause_ms": 15}} + """ + + let decoded = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + + guard case .drag(let target) = decoded else { + Issue.record("expected a drag step, got \(decoded)") + return + } + + #expect(target.identifier == "transcript.text") + #expect(target.from.dx == 0.1) + #expect(target.to.dy == 0.6) + #expect(target.steps == 6) + #expect(target.pauseMs == 15) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift new file mode 100644 index 000000000..aa8deef81 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift @@ -0,0 +1,173 @@ +import ApplicationServices + +@testable import DriveKit + +/// An accessibility element that is not one. +/// +/// Reference semantics on purpose: a step writes to an element and then reads it +/// back, and a test asserts on what was written. With a value type the write would +/// land on a copy and every such assertion would pass vacuously. +final class FakeElement: Element { + /// Attribute values by name. A name that is absent here reads as `nil`, which + /// is what the real implementation answers for an attribute the element does + /// not report. + var attributes: [String: String] + + /// Attribute names this element accepts writes for. + let settable: Set<String> + + var actions: [String] + var children: [FakeElement] + + /// Counts every call to ``read(_:)``, so a test can pin how much of a tree a + /// walk touched rather than only what it returned. + private(set) var reads = 0 + + /// What `setFlag` should answer, for exercising a refused write. + var writeStatus: AXError = .success + + /// Accept writes and discard them. + /// + /// The accessibility API lets a target answer `success` and then do nothing, + /// which is why a step reads back rather than trusting the status. Without this + /// there is no way to tell a driver that reads back from one that pretends to. + var ignoresWrites = false + + /// Called during every ``read(_:)``, after ``reads`` is incremented and before + /// children are handed back. + /// + /// This is how a test makes an element appear partway through a wait. A fixture + /// that has the element from the start cannot tell polling from a single lucky + /// look. + var onRead: ((FakeElement) -> Void)? + + /// Element-valued attributes, such as `AXWindows` and `AXMenuBar`. + var related: [String: [FakeElement]] = [:] + + /// Point-valued attributes, such as `AXActivationPoint`. + var points: [String: CGPoint] = [:] + + /// Size-valued attributes, such as `AXSize`. + var sizes: [String: CGSize] = [:] + + /// Actions performed on this element, in order. + private(set) var performed: [String] = [] + + /// What `perform` should answer, for exercising a refused action. + var performStatus: AXError = .success + + init( + role: String, + identifier: String? = nil, + label: String? = nil, + settable: Set<String> = [], + actions: [String] = [], + children: [FakeElement] = [] + ) { + self.attributes = [kAXRoleAttribute: role] + self.attributes[kAXIdentifierAttribute] = identifier + self.attributes[AXElement.attributedDescription] = label + self.settable = settable + self.actions = actions + self.children = children + } + + func read(_ names: [String]) -> Reading<FakeElement> { + reads += 1 + onRead?(self) + return Reading(text: names.map { attributes[$0] }, children: children) + } + + func isSettable(_ name: String) -> Bool { + return settable.contains(name) + } + + func flag(_ name: String) -> Bool? { + switch attributes[name] { + case "1": return true + case "0": return false + default: return nil + } + } + + func setFlag(_ name: String, _ value: Bool) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + attributes[name] = value ? "1" : "0" + return .success + } + + func setText(_ name: String, _ value: String) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + attributes[name] = value + return .success + } + + func perform(_ action: String) -> AXError { + guard performStatus == .success else { return performStatus } + performed.append(action) + return .success + } + + func point(_ name: String) -> CGPoint? { + return points[name] + } + + func size(_ name: String) -> CGSize? { + return sizes[name] + } + + func setSize(_ name: String, _ value: CGSize) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + sizes[name] = value + return .success + } + + func elements(_ name: String) -> [FakeElement] { + return related[name] ?? [] + } +} + +extension FakeElement { + /// The sidebar shape this app actually produces, at whatever size a test needs. + /// + /// Three elements per conversation, with the identifier on the leaf and + /// `AXSelected` writable only on the row. Reproducing that here is the point: + /// the driver has to address one element and act on another. + static func sidebar(rowCount: Int) -> FakeElement { + let rows = (0..<rowCount).map { index in + FakeElement( + role: "AXRow", + settable: [kAXSelectedAttribute], + actions: ["AXShowDefaultUI"], + children: [ + FakeElement( + role: "AXCell", + children: [ + FakeElement( + role: "AXUnknown", + identifier: "sidebar.row.\(index)", + label: "Conversation \(index), 4 events" + ) + ] + ) + ] + ) + } + + return FakeElement( + role: "AXApplication", + children: [ + FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + label: "Conversations", + actions: ["AXShowMenu"], + children: rows + ) + ] + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift new file mode 100644 index 000000000..808d79006 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift @@ -0,0 +1,35 @@ +import ApplicationServices + +@testable import DriveKit + +/// Records clicks and drags instead of posting them. +/// +/// A real click goes to the window server and lands on whatever occupies the +/// coordinate, so the only part a test can hold still is where the driver aimed. +final class FakePoster: EventPoster { + /// Every point clicked, in order. + private(set) var clicks: [CGPoint] = [] + + /// Every drag's path, in order. + private(set) var drags: [[CGPoint]] = [] + + /// The pause each drag was asked to wait between moves. + private(set) var pauses: [Duration] = [] + + /// What `click` and `drag` should answer, for exercising a post that could + /// not be built. + var succeeds = true + + func click(at point: CGPoint) -> Bool { + guard succeeds else { return false } + clicks.append(point) + return true + } + + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool { + guard succeeds else { return false } + drags.append(path) + pauses.append(pause) + return true + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift new file mode 100644 index 000000000..002559049 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift @@ -0,0 +1,272 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +/// The menu bar's own walk is [`Tree`](Tree)'s, already covered by `TreeTests`. +/// What is specific here is the root: the menu bar hangs off an attribute of the +/// application rather than sitting in its children, and every level of it should be +/// reported rather than capped. +@Suite("Menu") +struct MenuTests { + /// The whole menu bar, with no cap, because every item in it is something a + /// script might press. + private func options() -> TreeOptions { + return TreeOptions( + pid: 0, + identifierPrefix: nil, + maxMatches: 100, + maxDepth: 20, + maxSiblings: 0, + frames: false + ) + } + + /// An application whose menu bar hangs off the attribute the real one uses. + private func app() -> FakeElement { + let app = FakeElement(role: "AXApplication") + app.related[kAXMenuBarAttribute] = [menuBar()] + return app + } + + private func menuBar() -> FakeElement { + return FakeElement( + role: "AXMenuBar", + children: [ + FakeElement( + role: "AXMenuBarItem", + label: "File", + children: [ + FakeElement( + role: "AXMenu", + children: [ + FakeElement( + role: "AXMenuItem", + identifier: "performClose:", + label: "Close", + actions: ["AXCancel", "AXPress", "AXPick"] + ), + FakeElement( + role: "AXMenuItem", + identifier: "closeAll:", + label: "Close All", + actions: ["AXCancel", "AXPress", "AXPick"] + ), + ] + ) + ] + ) + ] + ) + } + + @Test("every menu item is reported, with the action that activates it") + func reportsEveryItem() throws { + let tree = try #require(Tree.walk(from: menuBar(), options: options())) + + #expect(tree.role == "AXMenuBar") + let items = try #require(tree.children.first?.children.first?.children) + #expect(items.count == 2) + #expect(items.map(\.identifier) == ["performClose:", "closeAll:"]) + #expect(items[0].actions.contains("AXPress")) + #expect(tree.children.first?.children.first?.elidedChildren == nil) + } + + /// Menu items advertise `AXPress` where a list row advertises nothing, so they + /// are the case `press` was built for. + @Test("a menu item can be pressed by identifier") + func pressesAMenuItem() throws { + let bar = menuBar() + + let result = try Act.run(.press(.init(identifier: "closeAll:")), in: bar) + + #expect(result.role == "AXMenuItem") + let items = try #require(bar.children.first?.children.first?.children) + #expect(items[1].performed == ["AXPress"]) + #expect(items[0].performed.isEmpty, "only the addressed item may be pressed") + } + + /// The path names the two titled levels a user sees and skips the `AXMenu` + /// between them, because that container has no title to name. + @Test("a titled path resolves through the intervening menu") + func resolvesATitledPath() throws { + let root = app() + + let result = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(result.step == "menu") + #expect(result.identifier == "File > Close All") + #expect(result.role == "AXMenuItem") + + let bar = try #require(root.elements(kAXMenuBarAttribute).first) + let items = try #require(bar.children.first?.children.first?.children) + #expect(items[1].performed == ["AXPress"]) + #expect(items[0].performed.isEmpty) + } + + /// The point of addressing by title: an item that moved to another menu keeps + /// its identifier, so only a path notices. The failure has to say what the + /// level does hold, or the test that catches the move cannot say what changed. + @Test("a path that does not resolve names what the level holds") + func reportsWhatTheLevelHolds() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File", "Quit"])), in: app()) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("'File' holds no item titled 'Quit'")) + #expect(error.hint == "it holds: Close, Close All") + } + + /// A context menu's items cannot be addressed any other way: `SwiftUI` gives + /// every one of them the same selector name, so a title is all there is. + @Test("a path can start at the menu an element is showing") + func resolvesUnderAShownMenu() throws { + let item = FakeElement(role: "AXMenuItem", label: "Copy Link", actions: ["AXPress"]) + let owner = FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [ + FakeElement(role: "AXRow", identifier: "sidebar.row.0"), + FakeElement(role: "AXMenu", children: [item]), + ] + ) + let root = FakeElement(role: "AXApplication", children: [owner]) + + let result = try Act.run( + .menu(.init(path: ["Copy Link"], under: "sidebar.list")), in: root) + + #expect(result.step == "menu") + #expect(result.identifier == "Copy Link") + #expect(item.performed == ["AXPress"]) + } + + /// The menu closes as soon as the application deactivates, so "press an item + /// in it" fails far more often than "open it" does. The error has to name the + /// step that was missed. + @Test("a path under an element that shows no menu says how to open one") + func reportsAnUnshownMenu() throws { + let owner = FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [FakeElement(role: "AXRow", identifier: "sidebar.row.0")] + ) + let root = FakeElement(role: "AXApplication", children: [owner]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["Copy Link"], under: "sidebar.list")), in: root) + } + + #expect(error.kind == .notFound) + #expect(error.message == "sidebar.list is not showing a menu") + #expect(error.hint?.contains("AXShowMenu") == true) + } + + @Test("a missing top-level menu is reported against the bar") + func reportsAMissingTopLevelMenu() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["Edit", "Copy"])), in: app()) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("the menu bar holds no item titled 'Edit'")) + #expect(error.hint == "it holds: File") + } + + /// Stopping at a bar item addresses the menu, not an item in it, and pressing a + /// menu is not what the script meant. + @Test("a path stopping at a submenu is rejected") + func rejectsAPathToASubmenu() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File"])), in: app()) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint?.contains("name the item inside it") == true) + } + + @Test("an empty path is rejected") + func rejectsAnEmptyPath() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: [])), in: app()) + } + + #expect(error.kind == .badUsage) + } + + /// The reason every menu test would otherwise pass while the real thing did + /// nothing: AppKit disables every item acting on the front window or the + /// responder chain while the application is in the background, which a driven + /// app always is. + @Test("a menu step brings the application forward first") + func bringsTheApplicationForward() throws { + let root = app() + + _ = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// An application already in front must not be written to: the write is what + /// steals focus, and a run of several menu steps would take it repeatedly. + @Test("an application already in front is left alone") + func leavesAFrontApplicationAlone() throws { + let root = app() + root.attributes[kAXFrontmostAttribute] = "1" + // Any write from here on fails, so a needless one fails the step. + root.writeStatus = .failure + + let result = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(result.identifier == "File > Close All") + } + + /// A disabled item swallows `AXPress` and answers success, so a step that + /// pressed it anyway would report having done something it did not do. + @Test("a disabled item is refused rather than pressed") + func refusesADisabledItem() throws { + let root = app() + let bar = try #require(root.elements(kAXMenuBarAttribute).first) + let items = try #require(bar.children.first?.children.first?.children) + items[1].attributes[kAXEnabledAttribute] = "0" + + let error = try #require(throws: DriveError.self) { + try Act.run( + .menu(.init(path: ["File", "Close All"])), + in: root, + activation: .milliseconds(1) + ) + } + + #expect(error.kind == .disabled) + #expect(error.message == "'File > Close All' is disabled") + #expect(items[1].performed.isEmpty, "a disabled item must not be pressed") + } + + /// Most elements report no `AXEnabled` at all, and reading its absence as a + /// refusal would reject every one of them. + @Test("an item reporting no enabled state is pressed") + func pressesAnItemWithNoEnabledState() throws { + let root = app() + + let result = try Act.run( + .menu(.init(path: ["File", "Close All"])), + in: root, + activation: .milliseconds(1) + ) + + #expect(result.identifier == "File > Close All") + } + + /// The menu bar comes from the application's attribute. An app without one is a + /// real case, and it must not be reported as a missing menu item. + @Test("an application with no menu bar says so") + func reportsNoMenuBar() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File"])), in: FakeElement(role: "AXApplication")) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("no menu bar")) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift new file mode 100644 index 000000000..28bef284e --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift @@ -0,0 +1,231 @@ +import CoreGraphics +import Foundation +import ImageIO +import Testing +import UniformTypeIdentifiers + +@testable import DriveKit + +/// Tests for reading a screenshot's pixels. +/// +/// Everything here works against a PNG written by the test, so none of it needs a +/// window server, a running app or a Screen Recording grant. +@Suite("Pixels") +struct PixelsTests { + /// A four-by-two image, written to a temporary file and removed afterwards. + /// + /// Rows top to bottom, each row left to right, as `#RRGGBB` strings. Written + /// through `CGImageDestination` so the file is a real PNG decoded by the same + /// path a screenshot takes. + private func withImage( + rows: [[String]], + _ body: (String) throws -> Void + ) throws { + let height = rows.count + let width = try #require(rows.first?.count) + + var bytes: [UInt8] = [] + for row in rows { + for hex in row { + let value = try #require(UInt32(hex.dropFirst(), radix: 16)) + bytes.append(UInt8((value >> 16) & 0xFF)) + bytes.append(UInt8((value >> 8) & 0xFF)) + bytes.append(UInt8(value & 0xFF)) + bytes.append(255) + } + } + + let space = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: width, + height: height, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: width * 4, + space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let path = NSTemporaryDirectory() + "/jpdrive-pixels-\(UUID().uuidString).png" + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + defer { try? FileManager.default.removeItem(atPath: path) } + try body(path) + } + + /// Two colours across a row, which is the shape every real question takes: a + /// wide background, a narrow line, and the offset where one becomes the other. + @Test("collapses a row into runs of one colour") + func scansARow() throws { + try withImage(rows: [ + ["#FFFFFF", "#FFFFFF", "#DBDBDB", "#FFFFFF"], + ["#000000", "#000000", "#000000", "#000000"], + ]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect(report.width == 4) + #expect(report.height == 2) + #expect(report.colorSpace == "sRGB") + #expect( + report.runs == [ + PixelRun(start: 0, count: 2, color: "#FFFFFF"), + PixelRun(start: 2, count: 1, color: "#DBDBDB"), + PixelRun(start: 3, count: 1, color: "#FFFFFF"), + ] + ) + } + } + + /// Rows are indexed down from the top, the way a screenshot is read, not up + /// from the bottom the way CoreGraphics draws. + @Test("counts rows down from the top") + func rowsCountFromTheTop() throws { + try withImage(rows: [ + ["#FF0000", "#FF0000"], + ["#00FF00", "#00FF00"], + ["#0000FF", "#0000FF"], + ]) { path in + let top = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + let bottom = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 2, from: nil, to: nil)) + + #expect(top.runs == [PixelRun(start: 0, count: 2, color: "#FF0000")]) + #expect(bottom.runs == [PixelRun(start: 0, count: 2, color: "#0000FF")]) + } + } + + @Test("collapses a column into runs of one colour") + func scansAColumn() throws { + try withImage(rows: [ + ["#FFFFFF", "#111111"], + ["#FFFFFF", "#111111"], + ["#222222", "#111111"], + ]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .column, at: 0, from: nil, to: nil)) + + #expect(report.scan == "column") + #expect( + report.runs == [ + PixelRun(start: 0, count: 2, color: "#FFFFFF"), + PixelRun(start: 2, count: 1, color: "#222222"), + ] + ) + } + } + + /// A window is nine hundred points wide and the interesting part is a few of + /// them, so a scan can be bounded. The offsets stay absolute, because they are + /// what gets compared against a frame from the accessibility tree. + @Test("bounds a scan and keeps the offsets absolute") + func boundsAScan() throws { + try withImage(rows: [["#FFFFFF", "#AAAAAA", "#BBBBBB", "#FFFFFF"]]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: 1, to: 2)) + + #expect( + report.runs == [ + PixelRun(start: 1, count: 1, color: "#AAAAAA"), + PixelRun(start: 2, count: 1, color: "#BBBBBB"), + ] + ) + } + } + + /// Reading past the edge is a mistake worth reporting rather than clamping: a + /// silently moved scan answers a question nobody asked. + @Test("refuses a line outside the image") + func refusesALineOutside() throws { + try withImage(rows: [["#FFFFFF"]]) { path in + #expect(throws: DriveError.self) { + try Pixels.read( + PixelOptions(image: path, axis: .row, at: 7, from: nil, to: nil)) + } + } + } + + @Test("reports an image it cannot read") + func reportsAMissingImage() { + #expect(throws: DriveError.self) { + try Pixels.read( + PixelOptions( + image: "/no/such/screenshot.png", axis: .row, at: 0, from: nil, to: nil)) + } + } + + /// A translucent pixel carries its alpha, so it cannot be mistaken for an + /// opaque one of the same colour. + @Test("spells an opaque colour without alpha and a translucent one with it") + func spellsAlphaOnlyWhenItMatters() { + #expect(Pixel(red: 0xDB, green: 0xDB, blue: 0xDB, alpha: 255).hex == "#DBDBDB") + #expect(Pixel(red: 0xDB, green: 0xDB, blue: 0xDB, alpha: 128).hex == "#DBDBDB80") + } + + @Test("collapses an empty line into no runs") + func emptyLine() { + #expect(Pixels.runs(of: [], startingAt: 0).isEmpty) + } + + /// A screenshot is written in the display's profile, which is not the space a + /// palette constant was written in. Read raw, a `#DBDBDB` divider comes back + /// as something several steps off and looks like a bug in the app. + /// + /// The image here is tagged Display P3 and holds the P3 encoding of sRGB + /// `#DBDBDB`, so a reader that converts reports the value the palette names + /// and a reader that does not reports `#DBDBDB` itself — which is the wrong + /// answer, arrived at by leaving the numbers alone. + @Test("reports colours in sRGB whatever space the image is tagged with") + func convertsToSRGB() throws { + let p3 = try #require(CGColorSpace(name: CGColorSpace.displayP3)) + let sRGB = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let level = CGFloat(0xDB) / 255 + let grey = try #require( + CGColor(colorSpace: sRGB, components: [level, level, level, 1])) + let converted = try #require( + grey.converted(to: p3, intent: CGColorRenderingIntent.defaultIntent, options: nil)) + let parts = try #require(converted.components) + + let path = NSTemporaryDirectory() + "/jpdrive-p3-\(UUID().uuidString).png" + defer { try? FileManager.default.removeItem(atPath: path) } + + let bytes = parts.prefix(3).map { UInt8(($0 * 255).rounded()) } + [255] + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: 1, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 4, + space: p3, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect(report.runs.first?.color == "#DBDBDB") + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift new file mode 100644 index 000000000..6240dcf08 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift @@ -0,0 +1,103 @@ +import ApplicationServices +import Foundation +import Testing + +@testable import DriveKit + +/// Tests for the `resize` step. +/// +/// A window is the only thing that accepts a write to `AXSize`, and resizing is +/// the one interaction a driver cannot reach any other way: a drag of a window's +/// edge has to be synthesized, and a synthesized drag needs the window frontmost. +@Suite("Resize") +struct ResizeTests { + /// A window that accepts a size, starting at `size`. + private func window(_ size: CGSize, settable: Bool = true) -> FakeElement { + let window = FakeElement( + role: "AXWindow", + identifier: "the-window", + settable: settable ? [kAXSizeAttribute] : [] + ) + window.sizes[kAXSizeAttribute] = size + return window + } + + private func step(width: Double, height: Double) -> Step { + .resize(Step.SizeTarget(identifier: "the-window", width: width, height: height)) + } + + @Test("writes the size it was asked for") + func writesTheSize() throws { + let window = self.window(CGSize(width: 900, height: 450)) + + let result = try Act.run(step(width: 1400, height: 900), in: window) + + #expect(window.sizes[kAXSizeAttribute] == CGSize(width: 1400, height: 900)) + #expect(result.step == "resize") + #expect(result.role == "AXWindow") + #expect(result.confirmed == true) + #expect(result.size == "1400x900") + } + + /// A window clamps to its own minimum and maximum, so the write succeeds and + /// the window lands somewhere else. Reporting what it reached is the whole + /// reason the step reads the size back instead of echoing the request. + @Test("reports the size it reached when the window clamps the request") + func reportsAClampedSize() throws { + let window = self.window(CGSize(width: 900, height: 450)) + window.ignoresWrites = true + + let result = try Act.run(step(width: 200, height: 100), in: window) + + #expect(result.confirmed == false) + #expect(result.size == "900x450") + } + + /// Most elements inside a window do not accept a size, and a step that asked + /// anyway would report success having changed nothing. + @Test("refuses an element that does not accept a size") + func refusesAnUnsizableElement() { + let element = window(CGSize(width: 900, height: 450), settable: false) + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: element) + } + } + + @Test("reports an identifier that is not in the tree") + func reportsAMissingIdentifier() { + let other = FakeElement(role: "AXWindow", identifier: "something-else") + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: other) + } + } + + @Test("surfaces a write the accessibility API refused") + func surfacesARefusedWrite() { + let window = self.window(CGSize(width: 900, height: 450)) + window.writeStatus = .cannotComplete + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: window) + } + } + + /// The step arrives as JSON from the driver's caller, so the spelling of its + /// keys is part of the contract. + @Test("decodes the step a caller writes") + func decodesTheStep() throws { + let json = #"{"resize":{"identifier":"w","width":1400,"height":900}}"# + + let decoded = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + + guard case .resize(let target) = decoded else { + Issue.record("expected a resize step, got \(decoded)") + return + } + + #expect(target.identifier == "w") + #expect(target.width == 1400) + #expect(target.height == 900) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift new file mode 100644 index 000000000..fb3677812 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift @@ -0,0 +1,164 @@ +import Testing + +@testable import DriveKit + +@Suite("Tree") +struct TreeTests { + /// Options with the bounds wide open, so a test names only what it is about. + private func options( + prefix: String? = nil, + maxMatches: Int = 100, + maxDepth: Int = 20, + maxSiblings: Int = 0, + frames: Bool = false + ) -> TreeOptions { + return TreeOptions( + pid: 0, + identifierPrefix: prefix, + maxMatches: maxMatches, + maxDepth: maxDepth, + maxSiblings: maxSiblings, + frames: frames + ) + } + + @Test("an unfiltered walk keeps every element") + func keepsEverythingUnfiltered() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.role == "AXApplication") + let outline = try #require(tree.children.first) + #expect(outline.identifier == "sidebar.list") + #expect(outline.children.count == 2) + } + + /// The bug this replaced: with a sibling cap in force, a search for a row past + /// the cap found nothing, because the cap dropped it before the filter saw it. + @Test("a filtered walk finds a match past the sibling cap") + func filterOutrunsTheSiblingCap() throws { + let root = FakeElement.sidebar(rowCount: 50) + + let tree = try #require( + Tree.walk(from: root, options: options(prefix: "sidebar.row.42", maxSiblings: 5)) + ) + + let leaf = tree.children.first?.children.first?.children.first?.children.first + #expect(leaf?.identifier == "sidebar.row.42") + } + + /// Ancestors are kept so a match can be located, but only the ones leading to it. + @Test("a filtered walk drops branches holding no match") + func prunesUnmatchedBranches() throws { + let root = FakeElement( + role: "AXApplication", + children: [ + FakeElement(role: "AXWindow", identifier: "other.window"), + FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [FakeElement(role: "AXRow", identifier: "sidebar.row.0")] + ), + ] + ) + + let tree = try #require(Tree.walk(from: root, options: options(prefix: "sidebar."))) + + #expect(tree.children.count == 1) + #expect(tree.children.first?.identifier == "sidebar.list") + + // The dropped window is still counted. A filtered read that silently + // showed one child of two would have the reader believe the application + // has one. + #expect(tree.elidedChildren == 1) + } + + @Test("a filtered walk with no match returns nothing") + func returnsNothingWhenNothingMatches() { + let root = FakeElement.sidebar(rowCount: 3) + + #expect(Tree.walk(from: root, options: options(prefix: "nope.")) == nil) + } + + /// The budget is the bound that keeps a prefix search off the whole tree, so it + /// has to actually stop the walk rather than only trim the output. + @Test("the match budget stops the walk") + func budgetStopsTheWalk() throws { + let root = FakeElement.sidebar(rowCount: 500) + + let tree = try #require( + Tree.walk(from: root, options: options(prefix: "sidebar.", maxMatches: 3)) + ) + + // One match is the outline itself, leaving two rows. + let outline = try #require(tree.children.first) + #expect(outline.children.count == 2) + + // Reads, not results: a budget that trimmed the output while still visiting + // every element would pass an assertion on the tree alone. + let rows = try #require(root.children.first?.children) + #expect(rows.dropFirst(3).allSatisfy { $0.reads == 0 }) + } + + @Test("the sibling cap reports what it skipped") + func capReportsElidedChildren() throws { + let root = FakeElement.sidebar(rowCount: 10) + + let tree = try #require(Tree.walk(from: root, options: options(maxSiblings: 4))) + + let outline = try #require(tree.children.first) + #expect(outline.children.count == 4) + #expect(outline.elidedChildren == 6) + } + + @Test("a complete level reports no elision") + func noElisionWhenComplete() throws { + let root = FakeElement.sidebar(rowCount: 3) + + let tree = try #require(Tree.walk(from: root, options: options(maxSiblings: 10))) + + #expect(tree.children.first?.elidedChildren == nil) + } + + /// The count is what separates a node stopped at the depth limit from a leaf. + /// Without it the two render identically and a reader concludes the element + /// has no children. + @Test("the depth limit stops the descent and reports what it did not reach") + func depthLimitStopsDescent() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let tree = try #require(Tree.walk(from: root, options: options(maxDepth: 1))) + + #expect(tree.children.first?.children.isEmpty == true) + #expect(tree.children.first?.elidedChildren == 2) + } + + /// Frames move whenever a window moves or a list scrolls, so they stay out + /// unless asked for. + @Test("frames are omitted by default") + func framesAreOptIn() throws { + let root = FakeElement(role: "AXWindow") + root.attributes["AXFrame"] = "0.0,0.0 100.0x100.0" + + let without = try #require(Tree.walk(from: root, options: options())) + #expect(without.frame == nil) + + let with = try #require(Tree.walk(from: root, options: options(frames: true))) + #expect(with.frame == "0.0,0.0 100.0x100.0") + } + + /// An attribute the element does not report must arrive as absent, not as the + /// text of whatever error the accessibility API answered with. + @Test("absent attributes are absent, not error text") + func absentAttributesAreNull() throws { + let root = FakeElement(role: "AXCell") + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.identifier == nil) + #expect(tree.label == nil) + #expect(tree.value == nil) + #expect(tree.enabled == nil) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift new file mode 100644 index 000000000..436ce5ab2 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift @@ -0,0 +1,186 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.type") +struct TypeTests { + /// A text field shaped like the one SwiftUI produces: `AXValue` writable, + /// `AXPress` absent. + private func field(identifier: String = "sidebar.filter") -> FakeElement { + return FakeElement( + role: "AXTextField", + identifier: identifier, + settable: [kAXValueAttribute, kAXFocusedAttribute], + actions: ["AXShowMenu", "AXConfirm"] + ) + } + + @Test("writes the text into the field") + func writesTheText() throws { + let field = field() + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect(field.attributes[kAXValueAttribute] == "driving") + #expect(result.step == "type") + #expect(result.role == "AXTextField") + #expect(result.confirmed == true) + } + + /// Writing the value alone changes the text a `SwiftUI` field shows without the + /// binding behind it noticing, so the application carries on as though nothing + /// was typed. The confirm is what the application actually observes, and a + /// `type` that skipped it would report success having done nothing. + @Test("commits the edit through the field's confirm action") + func commitsTheEdit() throws { + let field = field() + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect(field.performed == ["AXConfirm"]) + #expect(result.committed == true) + } + + /// A field that publishes every change as it happens needs nothing committing, + /// so this is reported rather than treated as a failure. + @Test("reports a field with no confirm action as uncommitted") + func reportsAnUncommittedWrite() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "live", + settable: [kAXValueAttribute], + actions: [] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run(.type(.init(identifier: "live", text: "x")), in: root) + + #expect(result.confirmed == true) + #expect(result.committed == false) + #expect(field.performed.isEmpty) + } + + /// Text in the field that the application never saw is the worst outcome to + /// report as success, so a refused confirm fails the step. + @Test("fails when the edit cannot be committed") + func failsOnARefusedConfirm() throws { + let field = field() + field.performStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "sidebar.filter", text: "x")), in: root) + } + + #expect(error.kind == .actionFailed) + #expect(error.hint?.contains("not committed") == true) + } + + /// Typing replaces rather than appends, so a script does not have to clear the + /// field first and a second step cannot silently concatenate. + @Test("replaces what the field already held") + func replacesExistingText() throws { + let field = field() + field.attributes[kAXValueAttribute] = "old" + let root = FakeElement(role: "AXApplication", children: [field]) + + _ = try Act.run(.type(.init(identifier: "sidebar.filter", text: "new")), in: root) + + #expect(field.attributes[kAXValueAttribute] == "new") + } + + /// Clearing is typing nothing, not a step of its own. + @Test("an empty string clears the field") + func clearsTheField() throws { + let field = field() + field.attributes[kAXValueAttribute] = "something" + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run(.type(.init(identifier: "sidebar.filter", text: "")), in: root) + + #expect(field.attributes[kAXValueAttribute] == "") + #expect(result.confirmed == true) + } + + /// A static label and a disabled field both resolve by identifier and both + /// refuse the write. Failing here beats reporting a write that went nowhere. + @Test("fails on an element whose value is not writable") + func failsOnAReadOnlyElement() throws { + let label = FakeElement(role: "AXStaticText", identifier: "subtitle") + let root = FakeElement(role: "AXApplication", children: [label]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "subtitle", text: "x")), in: root) + } + + #expect(error.kind == .notEditable) + #expect(label.attributes[kAXValueAttribute] == nil) + } + + @Test("reports a refused write") + func reportsARefusedWrite() throws { + let field = field() + field.writeStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "sidebar.filter", text: "x")), in: root) + } + + #expect(error.kind == .writeFailed) + #expect(error.message.contains("cannot_complete")) + } + + /// The accessibility API lets a target accept a write and discard it, which is + /// the whole reason the step reads back instead of trusting the status. + @Test("reports an accepted write that did not take") + func reportsAnIneffectiveWrite() throws { + let field = field() + field.ignoresWrites = true + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect( + result.confirmed == false, + "a field that discarded the text must not report as confirmed" + ) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let root = FakeElement(role: "AXApplication", children: [field()]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "nope", text: "x")), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// Only the addressed field is written to, so a step cannot quietly clobber a + /// second field that happens to sit nearby. + @Test("leaves other fields alone") + func leavesOtherFieldsAlone() throws { + let first = field(identifier: "one") + let second = field(identifier: "two") + let root = FakeElement(role: "AXApplication", children: [first, second]) + + _ = try Act.run(.type(.init(identifier: "two", text: "x")), in: root) + + #expect(first.attributes[kAXValueAttribute] == nil) + #expect(second.attributes[kAXValueAttribute] == "x") + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift new file mode 100644 index 000000000..7510632c2 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift @@ -0,0 +1,137 @@ +import Testing + +@testable import DriveKit + +@Suite("Act.waitFor") +struct WaitForTests { + /// A container that produces the awaited element on its third read, and not + /// before. + /// + /// The delay is what makes a wait test mean anything: against a tree that + /// already holds the element, polling and not polling look identical. + private func appearsOnThirdRead() -> FakeElement { + let container = FakeElement(role: "AXScrollArea", identifier: "transcript.scroll") + container.onRead = { element in + guard element.reads == 3 else { return } + element.children = [ + FakeElement(role: "AXGroup", identifier: "transcript.event.1") + ] + } + return container + } + + private func step( + _ identifier: String, + under: String? = nil, + timeoutMs: Int? = nil, + intervalMs: Int? = 1 + ) -> Step { + return .waitFor( + .init( + identifier: identifier, under: under, timeoutMs: timeoutMs, + intervalMs: intervalMs) + ) + } + + @Test("an element already present is returned on the first attempt") + func returnsImmediately() throws { + let root = FakeElement.sidebar(rowCount: 2) + + // A zero timeout permits exactly one attempt, so a pass here cannot have + // come from a retry. + let result = try Act.run(step("sidebar.row.1", timeoutMs: 0), in: root) + + #expect(result.step == "wait_for") + #expect(result.role == "AXUnknown") + #expect(result.confirmed == true) + } + + /// Half of a pair. This one proves the fixture genuinely withholds the element, + /// so that the passing case below is evidence of retrying rather than of the + /// element having been there all along. + @Test("one attempt is not enough for an element that appears later") + func oneAttemptIsNotEnough() throws { + let container = appearsOnThirdRead() + let root = FakeElement(role: "AXApplication", children: [container]) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("transcript.event.1", timeoutMs: 0), in: root) + } + + #expect(error.kind == .timeout) + } + + @Test("polling finds an element that appears later") + func findsAnElementThatAppearsLater() throws { + let container = appearsOnThirdRead() + let root = FakeElement(role: "AXApplication", children: [container]) + + let result = try Act.run(step("transcript.event.1", timeoutMs: 2000), in: root) + + #expect(result.confirmed == true) + #expect(result.role == "AXGroup") + #expect( + container.reads >= 3, "the element cannot have been found before its third read") + } + + @Test("an element that never appears times out") + func timesOut() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("never.appears", timeoutMs: 20), in: root) + } + + #expect(error.kind == .timeout) + #expect(error.message.contains("never.appears")) + } + + /// A single attempt eating the whole timeout is the failure mode that makes an + /// unscoped wait useless, so the error says what to do about it. + @Test("a timeout after one attempt suggests scoping") + func suggestsScopingAfterOneAttempt() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("never.appears", timeoutMs: 0), in: root) + } + + #expect(error.hint?.contains("under") == true) + } + + /// Waiting inside something that does not exist is a mistake in the script, not + /// a condition that might come true. + @Test("a missing container fails at once rather than being waited for") + func missingContainerFailsFast() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("anything", under: "no.such.container", timeoutMs: 5000), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// The reason `under` exists. Without it every attempt re-reads the whole + /// application, and against this app's sidebar one attempt outlasts a typical + /// timeout. + @Test("scoping keeps polling off the rest of the tree") + func scopingBoundsThePolling() throws { + let sidebar = FakeElement.sidebar(rowCount: 20) + let outline = try #require(sidebar.children.first) + let container = FakeElement(role: "AXScrollArea", identifier: "transcript.scroll") + let root = FakeElement(role: "AXApplication", children: [outline, container]) + + let error = try #require(throws: DriveError.self) { + try Act.run( + step("transcript.event.1", under: "transcript.scroll", timeoutMs: 30), + in: root + ) + } + #expect(error.kind == .timeout) + + // Read once while resolving the container, and never again. Repeated reads + // here would mean each poll was walking the sidebar. + #expect(outline.children.allSatisfy { $0.reads == 1 }) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift new file mode 100644 index 000000000..701110dc4 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift @@ -0,0 +1,129 @@ +import CoreGraphics +import Foundation +import Testing + +@testable import DriveKit + +@Suite("WindowIDs") +struct WindowIDsTests { + /// One entry shaped the way the window server reports it: every number a + /// `CFNumber`, with the bounds arriving as doubles. + private func entry( + id: Int, + pid: Int, + layer: Int = 0, + title: String? = "JP", + width: Double = 1200, + height: Double = 800 + ) -> [String: Any] { + var window: [String: Any] = [ + kCGWindowNumber as String: id, + kCGWindowOwnerPID as String: pid, + kCGWindowLayer as String: layer, + kCGWindowBounds as String: ["X": 0.0, "Y": 0.0, "Width": width, "Height": height], + ] + window[kCGWindowName as String] = title + return window + } + + @Test("reports the window server's number and the window's size") + func reportsIdentifiers() { + let listed = [entry(id: 7412, pid: 4321)] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: "JP", width: 1200, height: 800) + ] + ) + } + + /// Every application on the desktop is in the list, so a capture that took the + /// first entry would photograph whatever happened to be frontmost. + @Test("keeps only the windows the pid owns") + func filtersByOwner() { + let listed = [ + entry(id: 1, pid: 999, title: "Terminal"), + entry(id: 2, pid: 4321), + entry(id: 3, pid: 111, title: "Finder"), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// Tooltips, drag images and menu shadows are the app's too, and capturing one + /// in place of the window is a wrong answer rather than a failure. + @Test("keeps only windows on the normal layer") + func dropsChrome() { + let listed = [ + entry(id: 1, pid: 4321, layer: 25, title: "tooltip"), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// A panel that has never been shown sits in the list at zero size, and + /// capturing it produces an empty file. + @Test("drops windows with no area") + func dropsEmptyWindows() { + let listed = [ + entry(id: 1, pid: 4321, width: 0, height: 0), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// The window server withholds other applications' titles until the Screen + /// Recording grant is given, which is the state a first run is in. + @Test("a window with no readable title is still reported") + func toleratesAMissingTitle() { + let listed = [entry(id: 7412, pid: 4321, title: nil)] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: nil, width: 1200, height: 800) + ] + ) + } + + /// What actually arrives is a `CFArray` of `CFDictionary`, so every number in it + /// is an `NSNumber` once bridged, and a reader that only understood Swift's own + /// numeric types would report an application with no windows at all. + @Test("reads the numbers as the bridged types the window server hands over") + func readsBridgedNumbers() { + let listed: [[String: Any]] = [ + [ + kCGWindowNumber as String: NSNumber(value: 7412), + kCGWindowOwnerPID as String: NSNumber(value: 4321), + kCGWindowLayer as String: NSNumber(value: 0), + kCGWindowName as String: "JP", + kCGWindowBounds as String: [ + "X": NSNumber(value: 0.0), + "Y": NSNumber(value: 0.0), + "Width": NSNumber(value: 1200.0), + "Height": NSNumber(value: 800.0), + ], + ] + ] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: "JP", width: 1200, height: 800) + ] + ) + } + + /// Front-to-back is the window server's own order, and it is the only thing + /// telling a caller which of two windows to capture. + @Test("preserves the order the window server reported") + func preservesOrder() { + let listed = [ + entry(id: 3, pid: 4321), + entry(id: 1, pid: 4321), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [3, 1, 2]) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift new file mode 100644 index 000000000..437fa98e0 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift @@ -0,0 +1,61 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Windows") +struct WindowsTests { + @Test("reports each window's own facts") + func reportsWindowFacts() { + let main = FakeElement(role: "AXWindow", identifier: "workspace-AppWindow-1") + main.attributes[kAXTitleAttribute] = "JP" + main.attributes[kAXMainAttribute] = "1" + main.attributes[kAXMinimizedAttribute] = "0" + main.attributes["AXFrame"] = "0.0,0.0 1200.0x800.0" + + let other = FakeElement(role: "AXWindow", identifier: "workspace-AppWindow-2") + other.attributes[kAXMainAttribute] = "0" + other.attributes[kAXMinimizedAttribute] = "1" + + let app = FakeElement(role: "AXApplication") + app.related[kAXWindowsAttribute] = [main, other] + + #expect( + Windows.list(of: app) == [ + WindowSummary( + identifier: "workspace-AppWindow-1", + title: "JP", + main: true, + minimized: false, + frame: "0.0,0.0 1200.0x800.0" + ), + WindowSummary( + identifier: "workspace-AppWindow-2", + title: nil, + main: false, + minimized: true, + frame: nil + ), + ] + ) + } + + /// A running application with every window closed is a normal state, not an + /// error. + @Test("an application with no windows reports an empty list") + func noWindows() { + #expect(Windows.list(of: FakeElement(role: "AXApplication")).isEmpty) + } + + /// Windows are read from the application's own attribute, not found by walking + /// into the hierarchy. A listing that descended would pick up sheets and popups + /// as if they were windows. + @Test("windows are read from the attribute, not from the children") + func doesNotWalkChildren() { + let child = FakeElement(role: "AXWindow", identifier: "not.a.window") + let app = FakeElement(role: "AXApplication", children: [child]) + + #expect(Windows.list(of: app).isEmpty) + #expect(child.reads == 0) + } +} diff --git a/justfile b/justfile index 01ec2a8d4..9981f9d3f 100644 --- a/justfile +++ b/justfile @@ -194,6 +194,52 @@ build-ffi PROFILE="debug": (_install "cbindgen@" + cbindgen_version) echo "library: $out/libjp_ffi.a" >&2 echo "header: $out/include/jp_ffi.h" >&2 +# Build the `jpdrive` accessibility driver that the `debug_app_*` tools shell out +# to. +# +# A standalone SwiftPM package rather than a target in the app's Xcode project, +# so the binary lands at a predictable path with no derived-data lookup. +[group('build')] +[macos] +build-drive CONFIG="release": + #!/usr/bin/env sh + set -eu + + swift build --package-path apps/macos/Tools/jpdrive -c {{CONFIG}} + + bin=$(swift build --package-path apps/macos/Tools/jpdrive -c {{CONFIG}} --show-bin-path) + echo "binary: $bin/jpdrive" >&2 + +# Run the `jpdrive` test suite. +# +# Covers the driver's traversal against a fake accessibility tree, so it needs no +# running app and no accessibility grant. +[group('test')] +[macos] +test-drive *ARGS: + swift test --package-path apps/macos/Tools/jpdrive {{ARGS}} + +# Report whether this process may read another app's accessibility tree. +# +# Run under the terminal, under `just`, and under `serve-tools` to find out +# whether a TCC grant given to the terminal reaches a tool it started. See +# `apps/macos/Tools/jpdrive/README.md`. +# +# PID is the target application's process id, e.g. `$(pgrep -f JP.app)`. +[group('debug')] +[macos] +drive-doctor PID="": build-drive + #!/usr/bin/env sh + set -eu + + bin=$(swift build --package-path apps/macos/Tools/jpdrive -c release --show-bin-path) + + if [ -n "{{PID}}" ]; then + "$bin/jpdrive" doctor --pid "{{PID}}" + else + "$bin/jpdrive" doctor + fi + # Generate the macOS app's Xcode project from `apps/macos/project.yml`. # # The project file is generated rather than committed, so `project.yml` stays the From c8b0cf5bac22377f396a75f6905833eafff33abb Mon Sep 17 00:00:00 2001 From: Jean Mertz <git@jeanmertz.com> Date: Fri, 21 Aug 2026 07:07:49 +0200 Subject: [PATCH 2/3] review feedback Signed-off-by: Jean Mertz <git@jeanmertz.com> --- apps/macos/Tests/ClipboardPolicyTests.swift | 71 ++++ apps/macos/Tools/jpdrive/README.md | 328 ++++++++++-------- .../jpdrive/Sources/DriveKit/AXElement.swift | 37 +- .../Tools/jpdrive/Sources/DriveKit/Act.swift | 96 +++-- .../jpdrive/Sources/DriveKit/DriveError.swift | 4 + .../Tools/jpdrive/Sources/DriveKit/Dump.swift | 69 ++-- .../jpdrive/Sources/DriveKit/Element.swift | 47 +++ .../jpdrive/Sources/DriveKit/Pixels.swift | 42 ++- .../Tools/jpdrive/Sources/DriveKit/Tree.swift | 11 +- .../jpdrive/Sources/DriveKit/WindowIDs.swift | 25 +- .../Tests/DriveKitTests/ActTests.swift | 38 +- .../Tests/DriveKitTests/ClickTests.swift | 30 ++ .../Tests/DriveKitTests/DragTests.swift | 65 +++- .../Tests/DriveKitTests/DumpTests.swift | 121 +++++++ .../Tests/DriveKitTests/FakeElement.swift | 29 ++ .../Tests/DriveKitTests/PixelsTests.swift | 68 ++++ .../Tests/DriveKitTests/TreeTests.swift | 19 + 17 files changed, 859 insertions(+), 241 deletions(-) create mode 100644 apps/macos/Tests/ClipboardPolicyTests.swift create mode 100644 apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift diff --git a/apps/macos/Tests/ClipboardPolicyTests.swift b/apps/macos/Tests/ClipboardPolicyTests.swift new file mode 100644 index 000000000..5fd405a9e --- /dev/null +++ b/apps/macos/Tests/ClipboardPolicyTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing + +/// The UI suite must never touch the *system* pasteboard. +/// +/// There is one of those and it belongs to whoever is at the keyboard. A test +/// that triggers a copy into it destroys what they had, and saving and +/// restoring around the test is not a fix: a pasteboard item can be a promise +/// its owner fulfils lazily, so a restore puts back a degraded copy and an +/// early exit puts back nothing at all. +/// +/// A *named* pasteboard has none of that problem, so the UI tests use one: a +/// debug build reads `JP_DEBUG_PASTEBOARD` and copies there instead (see +/// ``DebugState/pasteboard``), and `WorkspaceFixture.copiedText()` reads it +/// back. Copy Link is covered end to end without a clipboard being lost. +/// +/// What this forbids is therefore narrow and exact: the spellings that mean +/// "the one everybody shares". It is a source scan rather than a rule in a +/// document because a rule in a document is not enforced by anything. +@Suite("ClipboardPolicy") +struct ClipboardPolicyTests { + /// The spellings that reach the system pasteboard. + /// + /// `NSPasteboard(name: .general)` is the same object as + /// `NSPasteboard.general`, so naming it counts too. + static let forbidden = [ + "NSPasteboard.general", + "UIPasteboard.general", + "Name.general", + "name: .general", + ] + + @Test("no UI test reaches for the system pasteboard") + func uiTestsDoNotTouchThePasteboard() throws { + let sources = try Self.uiTestSources() + + // A scan over nothing passes for the wrong reason, and would keep + // passing if the directory were renamed. + #expect(sources.count >= 3, "expected to find the UI test sources to scan") + + for source in sources { + let text = try String(contentsOf: source, encoding: .utf8) + for symbol in Self.forbidden where text.contains(symbol) { + Issue.record( + """ + \(source.lastPathComponent) reaches the system pasteboard through \ + `\(symbol)`. Copy through the fixture's own pasteboard instead: the app \ + writes to the one `JP_DEBUG_PASTEBOARD` names, and \ + `WorkspaceFixture.copiedText()` reads it back. + """ + ) + } + } + } + + /// Every Swift file in `apps/macos/UITests`. + /// + /// Located from this file's compile-time path. The app is not sandboxed and + /// these tests are hosted by it, so the checkout is readable from here. + static func uiTestSources() throws -> [URL] { + // .../apps/macos/Tests/ClipboardPolicyTests.swift + let directory = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("UITests") + + return try FileManager.default + .contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .filter { $0.pathExtension == "swift" } + } +} diff --git a/apps/macos/Tools/jpdrive/README.md b/apps/macos/Tools/jpdrive/README.md index 58206ee13..98124d4f7 100644 --- a/apps/macos/Tools/jpdrive/README.md +++ b/apps/macos/Tools/jpdrive/README.md @@ -1,15 +1,15 @@ # jpdrive -Reads and acts on a running macOS app's accessibility tree, speaking JSON. The -`debug_app_*` tools shell out to it; the Rust side stays the presenter, parsing -the JSON and rendering markdown. +Reads and acts on a running macOS app's accessibility tree, speaking JSON. +The `debug_app_*` tools shell out to it; the Rust side stays the presenter, +parsing the JSON and rendering markdown. Swift rather than Rust because `AXUIElement` is CoreFoundation-shaped: ordinary code here, unsafe bindings or a 784-download crate there. -External rather than an in-app automation socket, deliberately. Driving through -`AXUIElement` means a broken accessibility tree breaks the tooling, which is the -pressure that keeps the app's accessibility honest. +External rather than an in-app automation socket, deliberately. +Driving through `AXUIElement` means a broken accessibility tree breaks the +tooling, which is the pressure that keeps the app's accessibility honest. ## Build @@ -25,17 +25,20 @@ Everything downstream depends on one unknown: does a binary launched as a child of `just serve-tools` inherit the Accessibility grant given to the terminal? macOS attributes TCC to the *responsible process*, which for a command-line tool -is normally the terminal rather than the tool. That is the same mechanism behind -the `sample(1)` note in `.config/jp/tools/src/debug_jp/profile_sampling.rs` about -granting Terminal *Developer Tools*. Apple documents neither the algorithm nor -its stability, so the answer has to be measured. +is normally the terminal rather than the tool. +That is the same mechanism behind the `sample(1)` note in +`.config/jp/tools/src/debug_jp/profile_sampling.rs` about granting Terminal +*Developer Tools*. +Apple documents neither the algorithm nor its stability, so the answer has to be +measured. -`jpdrive doctor` measures it. Run it three ways, with Accessibility granted to -the terminal application and the app running: +`jpdrive doctor` measures it. +Run it three ways, with Accessibility granted to the terminal application and +the app running: -Check the target first. An empty `pgrep` means the app is not running, and a -run without a target reports the trust flag alone, which is the half of the -answer that can be wrong: +Check the target first. +An empty `pgrep` means the app is not running, and a run without a target +reports the trust flag alone, which is the half of the answer that can be wrong: ```sh pgrep -f JP.app # must print exactly one pid @@ -49,22 +52,23 @@ pgrep -f JP.app # must print exactly one pid just drive-doctor $(pgrep -f JP.app) ``` -The third case, a child of `jp-tools` under `just serve-tools`, needs a tool that -shells out to the driver. Reaching it means writing the first `debug_app_*` tool, -which is why cases 1 and 2 come first: if the grant already fails at case 2, -nothing is learned by going further. +The third case, a child of `jp-tools` under `just serve-tools`, needs a tool +that shells out to the driver. +Reaching it means writing the first `debug_app_*` tool, which is why cases 1 and +2 come first: if the grant already fails at case 2, nothing is learned by going +further. -Compare `trusted` and `probe.axError` across the runs. `trusted: true` with a -window count means the grant inherits. `trusted: false`, or `api_disabled` / -`cannot_complete` from the probe, means it does not, and the driver needs its own -signed bundle or its own grant. +Compare `trusted` and `probe.axError` across the runs. +`trusted: true` with a window count means the grant inherits. +`trusted: false`, or `api_disabled` / `cannot_complete` from the probe, means it +does not, and the driver needs its own signed bundle or its own grant. The report lists the ancestor chain, so a `false` says which processes were candidates for holding the grant. -The check never prompts. `AXIsProcessTrustedWithOptions` with -`kAXTrustedCheckOptionPrompt` would raise the system dialog and change the state -being measured. +The check never prompts. +`AXIsProcessTrustedWithOptions` with `kAXTrustedCheckOptionPrompt` would raise +the system dialog and change the state being measured. ### Result @@ -76,35 +80,40 @@ ghostty → login → fish → just → sh → jpdrive ``` So `tree`, `windows`, `menu`, and `act` need no signed bundle and no grant of -their own. They can assume the terminal's. Case 3, a child of `jp-tools` under -`just serve-tools`, adds one more process of the same kind and is still -unmeasured. +their own. +They can assume the terminal's. +Case 3, a child of `jp-tools` under `just serve-tools`, adds one more process of +the same kind and is still unmeasured. Observations across the runs, on macOS with Ghostty as the terminal: -- Process depth is not the variable. Run directly from the shell (chain of four, - up to `ghostty`) and through `just` (chain of six, adding `sh` and `just`), the - report is identical. Whatever governs the grant, it is not the number of - processes between the terminal and the driver. -- The trust flag and the probe agree. `trusted: false` came with - `ax_error: api_disabled` from a real read against a running app, which is what - the accessibility API returns to an untrusted caller; `trusted: true` came with - a window count. No case has been seen where the two disagree. -- Untested: a terminal instance started *before* the grant. The `false` runs and - the `true` run may differ by the grant alone, by a relaunch, or by both, so - "the grant is not visible to this terminal instance" is not yet ruled out as a - separate failure mode. +- Process depth is not the variable. + Run directly from the shell (chain of four, up to `ghostty`) and through + `just` (chain of six, adding `sh` and `just`), the report is identical. + Whatever governs the grant, it is not the number of processes between the + terminal and the driver. +- The trust flag and the probe agree. + `trusted: false` came with `ax_error: api_disabled` from a real read against a + running app, which is what the accessibility API returns to an untrusted + caller; `trusted: true` came with a window count. + No case has been seen where the two disagree. +- Untested: a terminal instance started *before* the grant. + The `false` runs and the `true` run may differ by the grant alone, by a + relaunch, or by both, so "the grant is not visible to this terminal instance" + is not yet ruled out as a separate failure mode. ## Screen Recording is a second grant `windowid` answers the window server rather than the accessibility API, and the -two are governed by different TCC grants. Enumerating windows needs neither, so -the command works with nothing granted at all; reading a window's *title*, and -capturing its content with `screencapture -l`, need Screen Recording. +two are governed by different TCC grants. +Enumerating windows needs neither, so the command works with nothing granted at +all; reading a window's *title*, and capturing its content with `screencapture +-l`, need Screen Recording. That is why the report pairs the list with a `screen_recording` flag rather than -refusing outright. Missing the grant, a capture succeeds and returns the desktop -where the window should be, so the caller has to know before it writes a file. +refusing outright. +Missing the grant, a capture succeeds and returns the desktop where the window +should be, so the caller has to know before it writes a file. An untitled window in the list is the same fact seen from the other side. The pane is System Settings ▸ Privacy & Security ▸ Screen & System Audio @@ -122,13 +131,14 @@ the window. So the flag is worth trusting, and this grant reaches a driver six processes deep from the terminal, same as Accessibility does. -Untested: whether the restart was necessary. The grant and the restart happened -together, so nothing here separates them. +Untested: whether the restart was necessary. +The grant and the restart happened together, so nothing here separates them. ## What the sidebar looks like through accessibility SwiftUI's `.accessibilityIdentifier` does not land on the element that owns -behaviour. For a `List` row it lands two levels below it: +behaviour. +For a `List` row it lands two levels below it: ``` AXOutline AXIdentifier: sidebar.list AXRows: 1065, AXVisibleRows: 9 @@ -139,37 +149,37 @@ AXOutline AXIdentifier: sidebar.list AXRows: 1065, AXVisibleRows: 9 no actions, no children ``` -So addressing an element and acting on it are two different steps. The identified -element has no actions at all: no `AXPress`, nothing. Selecting a row means -walking up to the `AXRow` and writing `AXSelected`. +So addressing an element and acting on it are two different steps. +The identified element has no actions at all: no `AXPress`, nothing. +Selecting a row means walking up to the `AXRow` and writing `AXSelected`. That write is preferable to a synthesized click for a reason beyond determinism. Every row exists as an accessibility element, but only nine are on screen: the -outline's frame is 41658pt tall against a 398pt viewport. A click at -`AXActivationPoint` would miss an off-screen row, or land on whichever row -occupies those coordinates instead. An `AXSelected` write is independent of -scroll position. +outline's frame is 41658pt tall against a 398pt viewport. +A click at `AXActivationPoint` would miss an off-screen row, or land on +whichever row occupies those coordinates instead. +An `AXSelected` write is independent of scroll position. -`AXScrollToVisible` appears as a settable *attribute* on a sidebar cell and as an -*action* on a transcript event, so scrolling has to try both forms. +`AXScrollToVisible` appears as a settable *attribute* on a sidebar cell and as +an *action* on a transcript event, so scrolling has to try both forms. -The sidebar materialises every row; the transcript does not. Only one -`transcript.event.*` element exists at a time, so an identifier that names an -unrendered event cannot be waited for, only scrolled to. +The sidebar materialises every row; the transcript does not. +Only one `transcript.event.*` element exists at a time, so an identifier that +names an unrendered event cannot be waited for, only scrolled to. Writing `AXSelected` on a row 690 places down a thousand-row list selects it and -brings it into view, so selecting a row needs no scrolling step of its own. The -transcript still does. +brings it into view, so selecting a row needs no scrolling step of its own. +The transcript still does. ### The identified element cannot be walked upwards The `AXUnknown` carrying the identifier reports no `AXParent`, and no -`AXTopLevelUIElement` either, unlike the cell and row above it. Climbing from it -arrives nowhere. +`AXTopLevelUIElement` either, unlike the cell and row above it. +Climbing from it arrives nowhere. So resolving an identifier means keeping the chain the search descended through, -not finding the element and navigating from it afterwards. Anything that acts on -an ancestor of an identified element depends on this. +not finding the element and navigating from it afterwards. +Anything that acts on an ancestor of an identified element depends on this. ### Cost @@ -178,104 +188,125 @@ every other budget: - Reading the first few rows under `sidebar.` takes 250ms. - Finding one row 690 places down takes 5.8s, because the search reads about two - thousand elements to get there and cannot prune on the way: every identifier in - the sidebar sits on a leaf. + thousand elements to get there and cannot prune on the way: every identifier + in the sidebar sits on a leaf. -Hence the batched reads and the match budget. Anything that polls should resolve -an element once and re-read that reference, rather than searching each time. +Hence the batched reads and the match budget. +Anything that polls should resolve an element once and re-read that reference, +rather than searching each time. ## Acting on an element Each step names exactly one mechanism, because the mechanism depends on what the element is and guessing hides regressions: -| step | addressed by | mechanism | -| --- | --- | --- | -| `select` | identifier | write `AXSelected` on the nearest ancestor accepting it | -| `press` | identifier | `AXPress` on the element itself | -| `type` | identifier | write `AXValue`, then `AXConfirm` | -| `perform` | identifier | a named action, for the verbs with no step of their own | -| `menu` | titled path | `AXPress` on the item the path resolves to | -| `click` | identifier | synthesized mouse event at `AXActivationPoint` | - -`press` and `menu` end in the same call and are not redundant: they differ in what -they address by, and that is what a test pins. `closeAll:` is an `AppKit` selector -name that survives the item moving to another menu, so a script keyed on it cannot -notice the menu bar being rearranged. `["File", "Close All"]` names the structure -the user sees, and a path that stops resolving reports how far it got and what that -level holds instead — which is the assertion failure a layout test wants to read. +| step | addressed by | mechanism | +| --------- | ------------ | ------------------------------------------------------- | +| `select` | identifier | write `AXSelected` on the nearest ancestor accepting it | +| `press` | identifier | `AXPress` on the element itself | +| `type` | identifier | write `AXValue`, then `AXConfirm` | +| `perform` | identifier | a named action, for the verbs with no step of their own | +| `menu` | titled path | `AXPress` on the item the path resolves to | +| `click` | identifier | synthesized mouse event at `AXActivationPoint` | + +`press` and `menu` end in the same call and are not redundant: they differ in +what they address by, and that is what a test pins. +`closeAll:` is an `AppKit` selector name that survives the item moving to +another menu, so a script keyed on it cannot notice the menu bar being +rearranged. +`["File", "Close All"]` names the structure the user sees, and a path that stops +resolving reports how far it got and what that level holds instead — which is +the assertion failure a layout test wants to read. A step that names the wrong mechanism fails and says which actions the element -does accept. There is no fallback chain: if a sidebar row stopped accepting -`AXSelected`, a driver that quietly fell back to a synthesized click would keep -every script green while the app's accessibility rotted, which is the failure this -tool exists to prevent. +does accept. +There is no fallback chain: if a sidebar row stopped accepting `AXSelected`, a +driver that quietly fell back to a synthesized click would keep every script +green while the app's accessibility rotted, which is the failure this tool +exists to prevent. `select` and `type` read the attribute back afterwards, because a write can be -accepted and discarded. `press` cannot: nothing observable says a button did -anything, so its result reports no confirmation rather than claiming one. - -### A menu step has to bring the app forward - -`menu` writes `AXFrontmost` on the application and waits for it to take, which -makes it the one step that takes focus from whatever had it. - -Without it almost nothing in the menu bar can be pressed. AppKit disables every -item that acts on the front window or the responder chain while the application -is in the background, and against a driven instance that is most of the bar: -`Close`, `Copy`, `Select All`, `Show Sidebar`, and every `SwiftUI` command -reading a `@FocusedValue` all report `AXEnabled: 0`. `New Window` and `Close -All` do not, which is what makes the difference easy to miss — a first menu step -against an app-level item works, and the next one silently does nothing. +accepted and discarded. +`press` cannot: nothing observable says a button did anything, so its result +reports no confirmation rather than claiming one. + +### Some steps have to bring the app forward + +`menu`, `click` and `drag` write `AXFrontmost` on the application and wait for +it to take, which makes them the steps that take focus from whatever had it. +A step that cannot bring the application forward fails rather than acting +anyway. + +Without it almost nothing in the menu bar can be pressed. +AppKit disables every item that acts on the front window or the responder chain +while the application is in the background, and against a driven instance that +is most of the bar: `Close`, `Copy`, `Select All`, `Show Sidebar`, and every +`SwiftUI` command reading a `@FocusedValue` all report `AXEnabled: 0`. +`New Window` and `Close All` do not, which is what makes the difference easy to +miss — a first menu step against an app-level item works, and the next one +silently does nothing. So the item's enabled state is checked before it is pressed, rather than -trusting `AXPress` to report a refusal. A disabled item accepts the press and -answers success. +trusting `AXPress` to report a refusal. +A disabled item accepts the press and answers success. -An element that reports no `AXEnabled` at all is not disabled. Plenty carry no -such attribute, and reading its absence as a refusal would reject them all. +An element that reports no `AXEnabled` at all is not disabled. +Plenty carry no such attribute, and reading its absence as a refusal would +reject them all. ### Typing writes the value, and then has to commit it -`type` writes `AXValue` and performs `AXConfirm`. Both are needed, and the second -one is the part that was not obvious. +`type` writes `AXValue` and performs `AXConfirm`. +Both are needed, and the second one is the part that was not obvious. Writing `AXValue` on a `SwiftUI` `TextField` changes the text the field displays -and leaves the binding behind it untouched. Measured against the conversation -filter: after the write the field read back `"accessibility"` and the list still -showed all 1,066 rows. Deleting one character by hand then filtered on -`"accessibilit"` — the keystroke made the binding resync from whatever the field -held by then. So a `type` that only wrote the value would report success while the -application carried on as though nothing had been typed. - -`AXConfirm` commits through the path the binding observes. A field advertising no -confirm action is not a failure — some publish every change as it happens — so the -result reports `committed` separately from `confirmed`: the text being in the field -and the application having seen it are different facts. - -Synthesizing key events was rejected on three counts: the events go wherever focus -is, so a window activating mid-sequence types into it instead; posting them fast -enough to be useful means pauses between characters, which makes the step flaky -rather than deterministic; and event posting is global process state, so it could -not sit behind the element abstraction the rest of the driver is tested through. - -The remaining cost is that per-character behaviour never runs. A field that -validates each keystroke, or completes as you type, sees one change rather than a -dozen. +and leaves the binding behind it untouched. +Measured against the conversation filter: after the write the field read back +`"accessibility"` and the list still showed all 1,066 rows. +Deleting one character by hand then filtered on `"accessibilit"` — the +keystroke made the binding resync from whatever the field held by then. +So a `type` that only wrote the value would report success while the application +carried on as though nothing had been typed. + +`AXConfirm` commits through the path the binding observes. +A field advertising no confirm action is not a failure — some publish every +change as it happens — so the result reports `committed` separately from +`confirmed`: the text being in the field and the application having seen it are +different facts. + +Synthesizing key events was rejected on three counts: the events go wherever +focus is, so a window activating mid-sequence types into it instead; posting +them fast enough to be useful means pauses between characters, which makes the +step flaky rather than deterministic; and event posting is global process state, +so it could not sit behind the element abstraction the rest of the driver is +tested through. + +The remaining cost is that per-character behaviour never runs. +A field that validates each keystroke, or completes as you type, sees one change +rather than a dozen. ### Clicking is the last resort -`click` raises the element's window and posts a mouse event at its -`AXActivationPoint`. It is the only step whose effect is not addressed to an -element: the event goes to whatever occupies that screen coordinate, which is why -the window is raised first and why an occluding window from another application -will still swallow it. +`click` brings the application forward, raises the element's window, and posts a +mouse event at its `AXActivationPoint`. +It is the only step whose effect is not addressed to an element: the event goes +to whatever occupies that screen coordinate. + +Both the activation and the raise are needed, and neither substitutes for the +other. +`AXRaise` orders a window *within* its own application; ordering between +applications follows activation. +The driver is normally started from a terminal, which leaves that terminal +frontmost, so a click that only raised would land in the terminal while +reporting the element it aimed at. +An occluding window from another application will still swallow it. An element reporting no activation point is refused rather than clicked at the -origin. A sidebar row is exactly that case, and it wants `select`. +origin. +A sidebar row is exactly that case, and it wants `select`. -Posting is behind an `EventPoster`, so where the driver aimed can be asserted in a -test even though where the event lands cannot. +Posting is behind an `EventPoster`, so where the driver aimed can be asserted in +a test even though where the event lands cannot. ### Apple Events are a separate pathway, and that one does not inherit @@ -286,13 +317,14 @@ succeeds from: System Events got an error: osascript is not allowed assistive access. (-1719) ``` -Two different checks. `AXIsProcessTrusted`, which the driver calls, resolves to -the responsible process and finds the terminal. `System Events` requires the -calling binary itself to be listed, and the calling binary is `/usr/bin/osascript` -— shared by everything on the machine, so granting it grants far more than the -driver needs. +Two different checks. +`AXIsProcessTrusted`, which the driver calls, resolves to the responsible +process and finds the terminal. +`System Events` requires the calling binary itself to be listed, and the calling +binary is `/usr/bin/osascript` — shared by everything on the machine, so +granting it grants far more than the driver needs. This is the second reason the driver is a binary of its own rather than a shell -script over `osascript`, alongside the one at the top of this file. It also means -AppleScript is not a fallback when the driver is missing a verb: the verb has to -be added here. +script over `osascript`, alongside the one at the top of this file. +It also means AppleScript is not a fallback when the driver is missing a verb: +the verb has to be added here. diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift index 4ee15b376..8d379c2d3 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift @@ -110,12 +110,17 @@ struct AXElement { return value } - /// Read several attributes in one round-trip. + /// Read several attributes in one round-trip, or `nil` when the call fails. /// /// Results are positional and the same count as `names`. An attribute that /// could not be read arrives as CoreFoundation's null or as an `AXValue` /// boxing the error, both of which ``text(_:)`` reports rather than discards. - func values(_ names: [String]) -> [CFTypeRef?] { + /// + /// A `nil` return is the whole call failing, which a target that is busy or + /// exiting answers with. Reporting that as a row of absent attributes would + /// make an element nothing could be read from look exactly like one that + /// reports nothing. + func values(_ names: [String]) -> [CFTypeRef?]? { guard !names.isEmpty else { return [] } var raw: CFArray? @@ -130,7 +135,7 @@ struct AXElement { let values = raw as? [CFTypeRef], values.count == names.count else { - return Array(repeating: nil, count: names.count) + return nil } return values @@ -162,7 +167,13 @@ extension AXElement: Element { /// Children come back in the same batch as everything else: asking for them /// separately would add a hop per element, and every walk asks for them. func read(_ names: [String]) -> Reading<AXElement> { - let values = self.values(names + [kAXChildrenAttribute]) + guard let values = self.values(names + [kAXChildrenAttribute]) else { + return Reading( + text: Array(repeating: nil, count: names.count), + children: [], + failed: true + ) + } let children = (values.last.flatMap { $0 } as? [AXUIElement] ?? []) .map { AXElement(element: $0) } @@ -173,6 +184,24 @@ extension AXElement: Element { ) } + /// Enumerate the element's attributes and read them all. + /// + /// Two round-trips: one to learn the names, one to read them. Only a dump + /// pays that, which is why it is not folded into ``read(_:)``. + func reportedAttributes(settable: Bool) -> [Attribute]? { + let names = self.names().sorted() + guard !names.isEmpty else { return [] } + guard let values = values(names) else { return nil } + + return zip(names, values).map { name, value in + Attribute( + name: name, + value: value.map(Self.text) ?? "<null>", + settable: settable ? isSettable(name) : nil + ) + } + } + /// Read a boolean attribute. /// /// `CFBoolean` bridges to `NSNumber` rather than to `Bool`, so a direct cast diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift index 2faf78dc7..bca2e1d60 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift @@ -459,7 +459,7 @@ enum Act { return try menu(target, in: root, within: activation) case .click(let target): - return try click(target, in: root, poster: poster) + return try click(target, in: root, poster: poster, activation: activation) case .resize(let target): return try resize(target, in: root) @@ -475,6 +475,16 @@ enum Act { /// How long a drag pauses between moves when it does not say. private static let defaultDragPause = Duration.milliseconds(8) + /// The most moves a drag will post. + /// + /// Clamped rather than rejected, for the same reason the count is clamped + /// upwards from zero: a caller computing it from a distance should get a + /// gesture, not an error. The cap is what keeps an implausible count from + /// becoming a run that posts for minutes, or one whose route cannot be + /// allocated at all — either of which ends the process without the JSON + /// document every call promises. + private static let maxDragSteps = 1000 + /// Drag the pointer from one point on an element to another. private static func drag<E: Element>( _ target: Step.DragTarget, @@ -502,7 +512,10 @@ enum Act { ) } - let steps = max(target.steps ?? defaultDragSteps, 1) + try check(target.from, named: "from") + try check(target.to, named: "to") + + let steps = min(max(target.steps ?? defaultDragSteps, 1), maxDragSteps) let pause = target.pauseMs.map { Duration.milliseconds($0) } ?? defaultDragPause let start = point(target.from, in: origin, size) @@ -527,7 +540,7 @@ enum Act { // The cost is that a gesture takes focus. Nothing here can give it back: // this process handles one step and exits, so the restore belongs to // whatever drives the whole list. - activate(root, within: activation) + try front(root, within: activation) raiseWindow(in: path) guard poster.drag(through: route, pausing: pause) else { @@ -548,6 +561,26 @@ enum Act { ) } + /// Check that an offset names a point on the element. + /// + /// A value outside `0...1` resolves to a screen coordinate outside the + /// element's frame, and the gesture is posted into global screen space, so + /// the press or release lands on whatever occupies that point instead — + /// another application's window, or the desktop. Reading `dx` as a + /// percentage rather than a fraction is the mistake this catches, and it + /// resolves a long way outside: `100` on an 800pt window aims 80,000 points + /// to the right of it. + private static func check(_ offset: Step.Offset, named name: String) throws(DriveError) { + for (axis, value) in [("dx", offset.dx), ("dy", offset.dy)] + where !value.isFinite || value < 0 || value > 1 { + throw DriveError( + kind: .badUsage, + message: "\(name).\(axis) is \(value), which is not a fraction of the frame", + hint: "0 is the near edge of the element and 1 the far one, so 0.5 is halfway" + ) + } + } + /// One fractional offset as a screen coordinate inside a frame. private static func point( _ offset: Step.Offset, in origin: CGPoint, _ size: CGSize @@ -619,7 +652,8 @@ enum Act { private static func click<E: Element>( _ target: Step.Target, in root: E, - poster: any EventPoster + poster: any EventPoster, + activation: Duration ) throws(DriveError) -> StepResult { let path = try find(target.identifier, from: root) guard let element = path.last else { @@ -640,9 +674,13 @@ enum Act { ) } - // Raised first, because the click lands on whatever is at that coordinate - // rather than on the element that named it. A window behind another one - // would otherwise have its click swallowed by the window in front. + // Brought forward, and then raised, and both are needed — the same pair + // `drag` performs, for the same reason. `AXRaise` orders a window within + // its own application; ordering *between* applications follows + // activation. A driver invoked from a terminal leaves that terminal + // frontmost, so raising alone posts the click into the terminal while + // reporting the element it aimed at. + try front(root, within: activation) raiseWindow(in: path) guard poster.click(at: point) else { @@ -661,21 +699,6 @@ enum Act { ) } - /// Bring the application forward, ignoring a refusal. - /// - /// Best effort, unlike ``front(_:within:)``, which fails a menu step that - /// cannot activate: there the activation *is* the step, because AppKit - /// disables every item acting on the front window until the application is - /// frontmost. A pointer gesture only needs to be on top of the z-order, and a - /// tree that is not a running application — a test's — has nothing to - /// activate and a gesture against it is still worth posting. - private static func activate<E: Element>(_ root: E, within timeout: Duration) { - guard root.flag(kAXFrontmostAttribute) != true else { return } - guard root.setFlag(kAXFrontmostAttribute, true) == .success else { return } - - _ = poll(untilTrue: { root.flag(kAXFrontmostAttribute) == true }, within: timeout) - } - /// Bring the window holding the addressed element to the front, if it has one. /// /// Found along the path the search descended, for the same reason the selection @@ -953,7 +976,13 @@ enum Act { /// Bring the application forward, and wait until it reports that it is. /// /// Writing `AXFrontmost` is a request. The window server grants it a moment - /// later, and the menu validation that depends on it later still. + /// later, and whatever depends on it later still. + /// + /// Failing rather than carrying on is the point. Every caller does something + /// that only means what it says once the application is in front: a menu item + /// acting on the front window is disabled until then, and a synthesized + /// pointer event goes to whoever owns the screen coordinate, which is the + /// application the person at the keyboard is using. private static func front<E: Element>( _ root: E, within timeout: Duration ) throws(DriveError) { @@ -964,7 +993,7 @@ enum Act { throw DriveError( kind: .writeFailed, message: "bringing the application forward answered \(status.name)", - hint: "a menu item that acts on the front window is disabled until it is" + hint: "the step cannot be trusted to reach the app while it is behind another" ) } @@ -1193,11 +1222,16 @@ enum Act { -> [E] { var stack = [[root]] + var unreadable = 0 while let path = stack.popLast() { guard let element = path.last else { continue } let reading = element.read(searchBatch) + if reading.failed { + unreadable += 1 + } + if reading.text[0] == identifier { return path } @@ -1209,6 +1243,20 @@ enum Act { } } + // A miss with a gap in it is not a miss. The element may sit under one of + // the branches that could not be read, and reporting a clean + // `identifier_not_found` invites the caller to change an identifier that + // was right all along. + guard unreadable == 0 else { + throw DriveError( + kind: .readFailed, + message: + "\(identifier) was not found, and \(unreadable) element(s) could not be " + + "read", + hint: "the application may be busy or exiting; the identifier may exist" + ) + } + throw DriveError( kind: .identifierNotFound, message: "no element has the identifier \(identifier)", diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift index c08b6ebaf..2f95cd25a 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift @@ -48,6 +48,10 @@ struct DriveError: Error, Encodable { /// The application reports no element of the requested kind. case notFound = "not_found" + /// The accessibility API refused to read part of the tree, so what it + /// holds is unknown rather than known to be absent. + case readFailed = "read_failed" + /// The result could not be encoded as JSON. case encodingFailed = "encoding_failed" } diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift index 9b9ae62dd..48ab5ec25 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift @@ -1,37 +1,15 @@ import ApplicationServices import Foundation -/// One accessibility attribute, as reported name and rendered value. -/// -/// A list of pairs rather than a dictionary, so attribute names reach the JSON -/// exactly as the accessibility API spells them. `JSONEncoder`'s snake-case key -/// strategy rewrites dictionary keys, which would turn `AXIdentifier` into -/// `ax_identifier` and make the dump a poor record of what the app reports. -struct DumpAttribute: Encodable { - let name: String - let value: String - - /// Whether the accessibility API reports this attribute as writable, when - /// settability was asked for. - /// - /// This decides how the driver changes state. Writing `AXSelected` on a row is - /// deterministic; synthesizing a click at a screen coordinate depends on the - /// window being frontmost and unobscured. - /// - /// Absent unless requested: answering it costs one round-trip per attribute, - /// which doubles the cost of a walk. - let settable: Bool? -} - /// One element of an application's accessibility tree, with everything it reports. -struct DumpNode: Encodable { +struct DumpNode: Encodable, Equatable { /// `AXRole`, lifted out of the attributes because it is what a reader scans /// for. let role: String /// Every attribute the element reports, minus the two that only lead back into /// the tree, sorted by name. - let attributes: [DumpAttribute] + let attributes: [Attribute] /// Actions the element accepts, such as `AXPress`. let actions: [String] @@ -44,6 +22,14 @@ struct DumpNode: Encodable { /// repeats one row shape a thousand times, so the count is the useful part and /// the repetition is not. let elidedChildren: Int? + + /// Set when the accessibility API refused to read this element. + /// + /// Absent otherwise. Without it an element that could not be read reports no + /// attributes and no children, which is exactly how an element that has + /// neither reports — and a reader has no way to tell a gap in the walk from + /// a leaf. + let unreadable: Bool? } /// Walks an application's accessibility tree and reports everything it finds. @@ -90,30 +76,31 @@ enum Dump { kAXParentAttribute, ] - private static func node(_ element: AXElement, depth: Int, options: DumpOptions) -> DumpNode - { - let names = - element.names() - .filter { !structuralAttributes.contains($0) } - .sorted() - - let attributes = zip(names, element.values(names)).map { name, value in - DumpAttribute( - name: name, - value: value.map(AXElement.text) ?? "<null>", - settable: options.settable ? element.isSettable(name) : nil - ) - } + static func node<E: Element>(_ element: E, depth: Int, options: DumpOptions) -> DumpNode { + let reported = element.reportedAttributes(settable: options.settable) + let attributes = (reported ?? []) + .filter { !structuralAttributes.contains($0.name) } - let all = depth < options.maxDepth ? element.children : [] - let walked = options.maxSiblings > 0 ? Array(all.prefix(options.maxSiblings)) : all + // Every child the element has, whether or not this walk descends into it. + // Counted before the depth cap is applied, because a node stopped at the + // cap is otherwise indistinguishable from a leaf. + let reading = element.read([]) + let available = reading.children + let descend = depth < options.maxDepth ? available : [] + let walked = + options.maxSiblings > 0 + ? Array(descend.prefix(options.maxSiblings)) + : descend + let elided = available.count - walked.count + let unreadable = reported == nil || reading.failed return DumpNode( role: attributes.first { $0.name == kAXRoleAttribute }?.value ?? "<none>", attributes: attributes, actions: element.actions, children: walked.map { node($0, depth: depth + 1, options: options) }, - elidedChildren: all.count > walked.count ? all.count - walked.count : nil + elidedChildren: elided > 0 ? elided : nil, + unreadable: unreadable ? true : nil ) } } diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift index dd2ef2a30..ae0bf24eb 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift @@ -101,6 +101,28 @@ struct SystemEventPoster: EventPoster { } } +/// One attribute, as reported name and rendered value. +/// +/// A list of these rather than a dictionary, so attribute names reach the JSON +/// exactly as the accessibility API spells them. `JSONEncoder`'s snake-case key +/// strategy rewrites dictionary keys, which would turn `AXIdentifier` into +/// `ax_identifier` and make a dump a poor record of what the app reports. +struct Attribute: Encodable, Equatable { + let name: String + let value: String + + /// Whether the accessibility API reports this attribute as writable, when + /// settability was asked for. + /// + /// This decides how the driver changes state. Writing `AXSelected` on a row is + /// deterministic; synthesizing a click at a screen coordinate depends on the + /// window being frontmost and unobscured. + /// + /// Absent unless requested: answering it costs one round-trip per attribute, + /// which doubles the cost of a walk. + let settable: Bool? +} + /// Attribute text and children, read together. /// /// The pair exists because reading them separately costs an extra round-trip per @@ -111,6 +133,21 @@ struct Reading<E> { let text: [String?] let children: [E] + + /// Whether the call that produced this failed as a whole. + /// + /// Distinct from a `nil` in ``text``, which says the element reports no value + /// for that one attribute. This says the accessibility API refused the read, + /// so every entry is `nil` and ``children`` is empty because nothing could be + /// asked — not because the element has none. A target that is busy or exiting + /// answers that way, and the two are indistinguishable without this. + let failed: Bool + + init(text: [String?], children: [E], failed: Bool = false) { + self.text = text + self.children = children + self.failed = failed + } } /// One element of an accessibility tree, as the driver's traversal needs it. @@ -130,6 +167,16 @@ protocol Element { /// Implementations batch: this is one round-trip in the real one. func read(_ names: [String]) -> Reading<Self> + /// Every attribute the element reports, rendered and sorted by name. + /// + /// Separate from ``read(_:)``, which asks for a list of names known in + /// advance. A dump asks for whatever the element happens to have, which costs + /// an extra round-trip to enumerate and is why only a dump does it. + /// + /// `nil` when the read failed as a whole, which is not the same as an element + /// that reports no attributes. + func reportedAttributes(settable: Bool) -> [Attribute]? + /// Actions the element accepts, such as `AXPress`. /// /// Separate from ``read(_:)`` because it costs its own round-trip and most diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift index d9a0c8d8d..071752fd7 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift @@ -99,7 +99,20 @@ enum Pixels { ) } + // Checked against the image rather than against `to`, which is clamped + // below: a `--from` past the edge would otherwise be reported as being + // past a `--to` the caller never wrote. let from = max(options.from ?? 0, 0) + guard from < extent else { + throw DriveError( + kind: .badUsage, + message: + "--from \(from) is outside the image, which is " + + "\(bitmap.width)x\(bitmap.height) pixels", + hint: "a scan along a \(options.axis.rawValue) runs from 0 to \(extent - 1)" + ) + } + let to = min(options.to ?? extent - 1, extent - 1) guard from <= to else { @@ -239,14 +252,35 @@ private struct Bitmap { /// The buffer runs in the same direction: a bitmap context's first row is the /// top of what was drawn into it, so a screenshot's rows and this buffer's rows /// are the same rows in the same order. + /// + /// Colour components are divided back out by alpha. The buffer holds them + /// premultiplied, which is the only 8-bit RGBA layout a bitmap context + /// accepts, and a window capture carries real partial alpha at its rounded + /// corners and antialiased edges. Read straight out, a half-transparent red + /// reports as a dark red — a colour that was never on screen, in the same + /// spelling as one that was. func pixel(x: Int, y: Int) -> Pixel { let offset = (y * width + x) * 4 + let alpha = bytes[offset + 3] return Pixel( - red: bytes[offset], - green: bytes[offset + 1], - blue: bytes[offset + 2], - alpha: bytes[offset + 3] + red: straight(bytes[offset], alpha), + green: straight(bytes[offset + 1], alpha), + blue: straight(bytes[offset + 2], alpha), + alpha: alpha ) } + + /// One premultiplied component, divided back out by `alpha`. + /// + /// A fully transparent pixel carries no colour to recover — every component + /// is zero whatever it was before — so it reports as zero rather than as a + /// division nobody can perform. + private func straight(_ component: UInt8, _ alpha: UInt8) -> UInt8 { + guard alpha != 0, alpha != 255 else { return component } + + let recovered = (Double(component) / Double(alpha) * 255).rounded() + + return UInt8(min(recovered, 255)) + } } diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift index c9a6f4ad5..265149437 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift @@ -32,6 +32,13 @@ struct TreeNode: Encodable, Equatable { /// the last two are the easiest to mistake for an element having no children at /// all. let elidedChildren: Int? + + /// Set when the accessibility API refused to read this element. + /// + /// Absent otherwise. An element that could not be read reports no identifier, + /// no label and no children, which is how a bare container reports too — so + /// without this a gap in the walk reads as a plain element. + let unreadable: Bool? } /// What to walk, and what to keep. @@ -124,6 +131,7 @@ enum Tree { let text = reading.text let identifier = text[1] + let matches = options.identifierPrefix.map { identifier?.hasPrefix($0) ?? false } ?? false if matches { @@ -177,7 +185,8 @@ enum Tree { children: children, elidedChildren: available.count > children.count ? available.count - children.count - : nil + : nil, + unreadable: reading.failed ? true : nil ) } } diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift index 063735580..78952628e 100644 --- a/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift @@ -27,14 +27,18 @@ struct WindowIDReport: Encodable, Equatable { /// The application's capturable windows, front to back. let windows: [CaptureWindow] - /// Windows the application has that are not on the active Space. + /// Windows the application has that the screen is not currently showing. /// - /// Reported separately because the two look identical from the outside and - /// mean opposite things. A window on another desktop is absent from every - /// on-screen enumeration and from the accessibility tree, so an app that has - /// one and nothing else is indistinguishable from an app with no window at - /// all — except by asking for windows on every Space, which is this list. - let otherSpaces: [CaptureWindow] + /// Reported separately because such a window is absent from every on-screen + /// enumeration and from the accessibility tree, so an app that has one and + /// nothing else is indistinguishable from an app with no window at all — + /// except by asking for every window regardless, which is this list. + /// + /// Says nothing about *why* it is not showing. Minimized, hidden, and on + /// another Space all land here, and the window server's list carries no flag + /// separating them. `jpdrive windows` reads `AXMinimized` from the + /// accessibility tree, which distinguishes the first of the three. + let offScreen: [CaptureWindow] } /// Resolves an application's window-server identifiers. @@ -68,8 +72,9 @@ enum WindowIDs { kCGNullWindowID ) as? [[String: Any]] ?? [] - // Every Space, not just the active one. The difference between the two - // lists is what says a window exists somewhere the screen cannot show it. + // Every window, not just the ones on screen. The difference between the + // two lists is what says a window exists somewhere the screen is not + // showing it. let everywhere = CGWindowListCopyWindowInfo( [.excludeDesktopElements], @@ -86,7 +91,7 @@ enum WindowIDs { return WindowIDReport( screenRecording: CGPreflightScreenCaptureAccess(), windows: here, - otherSpaces: all.filter { !shown.contains($0.id) } + offScreen: all.filter { !shown.contains($0.id) } ) } diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift index b5f8c9ba0..4ed7ffd6c 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift @@ -49,12 +49,46 @@ struct ActTests { } @Test("select reports the identifier it could not find") - func reportsAMissingIdentifier() { + func reportsAMissingIdentifier() throws { let root = FakeElement.sidebar(rowCount: 3) - #expect(throws: DriveError.self) { + let error = try #require(throws: DriveError.self) { try Act.run(.select(.init(identifier: "sidebar.row.nope")), in: root) } + + #expect(error.kind == .identifierNotFound) + } + + /// A search that could not read part of the tree has not established that the + /// element is absent — it may sit under the branch that failed. Reporting a + /// clean miss invites the caller to change an identifier that was right, so + /// the two answers have to be different. + @Test("a miss with an unreadable branch is not reported as a clean miss") + func distinguishesAGapFromAMiss() throws { + let broken = FakeElement(role: "AXGroup") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .readFailed) + #expect(error.message.contains("could not be read")) + } + + /// The other half of the pair: a tree that read cleanly throughout still + /// reports a plain miss, so the flag above cannot be firing on everything. + @Test("a miss in a fully readable tree stays a clean miss") + func aReadableTreeReportsACleanMiss() throws { + let readable = FakeElement(role: "AXGroup") + let root = FakeElement(role: "AXApplication", children: [readable]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .identifierNotFound) } /// An element nothing in its chain can select is a failure, not a fallback onto diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift index d9986c121..ff8647b45 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift @@ -41,6 +41,36 @@ struct ClickTests { #expect(result.point == "120.0,340.0") } + /// `AXRaise` orders a window within its own application; ordering *between* + /// applications follows activation. The driver is invoked from a terminal, so + /// without this the terminal stays frontmost and the click lands there while + /// the result names the element it aimed at. + @Test("brings the application forward before clicking") + func activatesTheApplication() throws { + let root = app() + + _ = try Act.run( + .click(.init(identifier: "toolbar.open")), in: root, poster: FakePoster()) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// Posting into another application's window is worse than not clicking, so a + /// refused activation stops the step rather than aiming anyway. + @Test("refuses to click when the application cannot be brought forward") + func refusesToClickWithoutActivating() throws { + let root = app() + root.writeStatus = .cannotComplete + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "toolbar.open")), in: root, poster: poster) + } + + #expect(error.kind == .writeFailed) + #expect(poster.clicks.isEmpty, "nothing may be posted at a background application") + } + /// The event goes to whatever occupies the coordinate, so a window behind /// another one would have its click swallowed. Raising is what makes the /// coordinate mean the element that named it. diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift index 93e471d96..48f1a610c 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift @@ -141,18 +141,69 @@ struct DragTests { #expect(root.flag(kAXFrontmostAttribute) == true) } - /// A tree that is not a running application has nothing to activate, and a - /// gesture against one is still worth posting: every other assertion in this - /// file depends on that. - @Test("drags even when the application cannot be brought forward") - func dragsWithoutActivating() throws { + /// An application that will not come forward cannot be dragged in: the events + /// go to whatever owns those screen coordinates, which is the application the + /// person at the keyboard is using. Refusing is the only honest outcome, and + /// posting anyway is what this pins shut. + @Test("refuses to drag when the application cannot be brought forward") + func refusesToDragWithoutActivating() throws { let root = app() root.writeStatus = .cannotComplete let poster = FakePoster() - _ = try Act.run(step(from: (0, 0), to: (1, 1), steps: 2), in: root, poster: poster) + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1), steps: 2), in: root, poster: poster) + } + + #expect(error.kind == .writeFailed) + #expect(poster.drags.isEmpty, "nothing may be posted at a background application") + } + + /// An offset is a fraction of the element's frame. Read as a percentage — a + /// plausible misreading — `100` aims 80,000 points past an 800pt window, and + /// the gesture is posted into global screen space, so it lands on whatever is + /// there instead. + @Test( + "refuses an offset that is not a fraction of the frame", + arguments: [100.0, -0.5, 1.5, Double.nan, Double.infinity] + ) + func refusesAnOffsetOutsideTheFrame(dx: Double) throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (dx, 0.5), to: (0.5, 0.5)), in: app(), poster: poster) + } + + #expect(error.kind == .badUsage) + #expect(poster.drags.isEmpty, "nothing may be posted at a coordinate off the element") + } + + /// The far endpoint is checked too, and before anything is posted: a drag that + /// pressed on the element and released over another application would be worse + /// than one that never started. + @Test("refuses an out-of-frame destination before posting") + func refusesAnOutOfFrameDestination() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0.5, 0.5), to: (42, 0.5)), in: app(), poster: poster) + } + + #expect(error.kind == .badUsage) + #expect(poster.drags.isEmpty) + } + + /// A count nobody meant should still produce a gesture rather than an error, + /// but not one that posts for minutes or cannot allocate its route at all. + @Test("clamps a step count above the cap") + func clampsAnEnormousStepCount() throws { + let poster = FakePoster() + + let result = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 100_000_000), in: app(), poster: poster) - #expect(poster.drags.first?.count == 3) + #expect(result.moves == 1000) + #expect(poster.drags.first?.count == 1001) } @Test("fails on an element with no frame to measure") diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift new file mode 100644 index 000000000..767d86ddf --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift @@ -0,0 +1,121 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Dump") +struct DumpTests { + /// Options with the bounds wide open, so a test names only what it is about. + private func options( + maxDepth: Int = 20, + maxSiblings: Int = 0, + settable: Bool = false + ) -> DumpOptions { + return DumpOptions( + pid: 0, + maxDepth: maxDepth, + maxSiblings: maxSiblings, + settable: settable + ) + } + + @Test("an unbounded walk reports every element and elides nothing") + func walksEverything() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let node = Dump.node(root, depth: 0, options: options()) + + #expect(node.role == "AXApplication") + #expect(node.elidedChildren == nil) + + let outline = try #require(node.children.first) + #expect(outline.role == "AXOutline") + #expect(outline.children.count == 2) + #expect(outline.elidedChildren == nil) + } + + /// The bug this file was written for: the depth cap emptied the child list + /// before the count was taken, so a node stopped at the cap reported no + /// children and no elision — identical to a leaf, which is the reading + /// `elidedChildren` exists to prevent. + @Test("a node stopped at the depth limit reports what it did not walk") + func depthLimitReportsElision() throws { + let root = FakeElement.sidebar(rowCount: 3) + + let node = Dump.node(root, depth: 0, options: options(maxDepth: 1)) + + let outline = try #require(node.children.first) + #expect(outline.children.isEmpty) + #expect(outline.elidedChildren == 3) + } + + @Test("a node under the sibling cap reports what it skipped") + func siblingCapReportsElision() throws { + let root = FakeElement.sidebar(rowCount: 10) + + let node = Dump.node(root, depth: 0, options: options(maxSiblings: 4)) + + let outline = try #require(node.children.first) + #expect(outline.children.count == 4) + #expect(outline.elidedChildren == 6) + } + + /// A genuine leaf and a truncated node must not render the same way, which is + /// the whole point of the field. + @Test("a leaf reports no elision") + func leafReportsNothing() { + let node = Dump.node(FakeElement(role: "AXButton"), depth: 0, options: options()) + + #expect(node.children.isEmpty) + #expect(node.elidedChildren == nil) + } + + /// `AXChildren` is what the walk recurses into and `AXParent` points back at + /// the element that reported this one, so neither is worth recording. + @Test("the attributes that only lead back into the tree are dropped") + func dropsStructuralAttributes() { + let element = FakeElement(role: "AXRow", identifier: "row") + element.attributes[kAXChildrenAttribute] = "<2 AXUIElement>" + element.attributes[kAXParentAttribute] = "<AXUIElement>" + + let node = Dump.node(element, depth: 0, options: options()) + let names = node.attributes.map(\.name) + + #expect(!names.contains(kAXChildrenAttribute)) + #expect(!names.contains(kAXParentAttribute)) + #expect(names.contains(kAXIdentifierAttribute)) + } + + @Test("settability is answered only when it was asked for") + func settabilityIsOptIn() throws { + let element = FakeElement( + role: "AXRow", settable: [kAXSelectedAttribute]) + element.attributes[kAXSelectedAttribute] = "0" + + let without = Dump.node(element, depth: 0, options: options()) + #expect(without.attributes.allSatisfy { $0.settable == nil }) + + let with = Dump.node(element, depth: 0, options: options(settable: true)) + let selected = try #require(with.attributes.first { $0.name == kAXSelectedAttribute }) + #expect(selected.settable == true) + } + + /// An element the accessibility API refused reports no attributes and no + /// children, which is exactly how an empty element reports. Without the flag + /// a gap in the walk is indistinguishable from a leaf that is really there. + @Test("an element that could not be read is marked rather than shown empty") + func marksAnUnreadableElement() throws { + let broken = FakeElement(role: "AXGroup", identifier: "gone") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let node = Dump.node(root, depth: 0, options: options()) + + #expect(node.unreadable == nil) + + let child = try #require(node.children.first) + #expect(child.unreadable == true) + #expect(child.attributes.isEmpty) + #expect(child.role == "<none>") + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift index aa8deef81..5afda8aec 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift @@ -26,6 +26,14 @@ final class FakeElement: Element { /// What `setFlag` should answer, for exercising a refused write. var writeStatus: AXError = .success + /// Make every read of this element fail as a whole. + /// + /// What the accessibility API answers for an element whose application is + /// busy or exiting. Distinct from an element that simply reports nothing: + /// that is the confusion the `failed` flag exists to prevent, so a fake has + /// to be able to produce both. + var readFails = false + /// Accept writes and discard them. /// /// The accessibility API lets a target answer `success` and then do nothing, @@ -75,9 +83,30 @@ final class FakeElement: Element { func read(_ names: [String]) -> Reading<FakeElement> { reads += 1 onRead?(self) + + guard !readFails else { + return Reading( + text: Array(repeating: nil, count: names.count), + children: [], + failed: true + ) + } + return Reading(text: names.map { attributes[$0] }, children: children) } + func reportedAttributes(settable: Bool) -> [Attribute]? { + guard !readFails else { return nil } + + return attributes.keys.sorted().map { name in + Attribute( + name: name, + value: attributes[name] ?? "<null>", + settable: settable ? isSettable(name) : nil + ) + } + } + func isSettable(_ name: String) -> Bool { return settable.contains(name) } diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift index 28bef284e..f5a4b011c 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift @@ -228,4 +228,72 @@ struct PixelsTests { #expect(report.runs.first?.color == "#DBDBDB") } + + /// A window capture carries real partial alpha at its rounded corners and + /// antialiased edges, and the bitmap behind it is premultiplied — the only + /// 8-bit RGBA layout a context accepts. Read straight out, a half-transparent + /// red reports as a dark red: a colour that was never on screen, spelled the + /// same way as one that was. + /// + /// Round-tripped through a PNG rather than asserted on a hand-built `Pixel`, + /// because the premultiplication happens in the decode path and a test that + /// constructs the pixel itself never reaches it. + @Test("recovers the colour behind a partially transparent pixel") + func unpremultipliesPartialAlpha() throws { + // Left: `#FF0000` at half alpha, stored premultiplied as 128,0,0,128. + // Right: fully transparent, which carries no colour to recover. + let bytes: [UInt8] = [128, 0, 0, 128, 0, 0, 0, 0] + + let space = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: 2, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 8, + space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let path = NSTemporaryDirectory() + "/jpdrive-alpha-\(UUID().uuidString).png" + defer { try? FileManager.default.removeItem(atPath: path) } + + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect( + report.runs == [ + PixelRun(start: 0, count: 1, color: "#FF000080"), + PixelRun(start: 1, count: 1, color: "#00000000"), + ] + ) + } + + /// A `--from` past the edge used to be reported as being past a `--to` the + /// caller never wrote, because `to` is clamped to the image before the two + /// are compared. The message has to name the bound that was actually wrong. + @Test("refuses a start offset outside the image") + func refusesAStartOutsideTheImage() throws { + try withImage(rows: [["#FFFFFF", "#FFFFFF"]]) { path in + let error = try #require(throws: DriveError.self) { + try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: 150, to: nil)) + } + + #expect(error.kind == .badUsage) + #expect(error.message.contains("--from 150 is outside the image")) + } + } } diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift index fb3677812..350b55bff 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift @@ -148,6 +148,25 @@ struct TreeTests { #expect(with.frame == "0.0,0.0 100.0x100.0") } + /// An element the accessibility API refused reports no identifier, no label + /// and no children — which is how a bare container reports too. Without the + /// marker a gap in the walk reads as a plain element that is really there. + @Test("an element that could not be read is marked") + func marksAnUnreadableElement() throws { + let broken = FakeElement(role: "AXGroup", identifier: "gone") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.unreadable == nil) + + let child = try #require(tree.children.first) + #expect(child.unreadable == true) + #expect(child.identifier == nil) + #expect(child.role == "<none>") + } + /// An attribute the element does not report must arrive as absent, not as the /// text of whatever error the accessibility API answered with. @Test("absent attributes are absent, not error text") From 1bb82e3522709ae2829bf52845ea47d3544b8423 Mon Sep 17 00:00:00 2001 From: Jean Mertz <git@jeanmertz.com> Date: Fri, 21 Aug 2026 07:40:12 +0200 Subject: [PATCH 3/3] fixup! review feedback Signed-off-by: Jean Mertz <git@jeanmertz.com> --- .../Tools/jpdrive/Tests/DriveKitTests/DragTests.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift index 48f1a610c..064bfc71c 100644 --- a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift @@ -195,12 +195,17 @@ struct DragTests { /// A count nobody meant should still produce a gesture rather than an error, /// but not one that posts for minutes or cannot allocate its route at all. + /// + /// Any value above the cap proves it, so this one is small enough that the + /// test survives its own failure: with the clamp reverted, a count in the + /// millions allocates its whole route before the assertion runs and takes the + /// runner down instead of going red. @Test("clamps a step count above the cap") func clampsAnEnormousStepCount() throws { let poster = FakePoster() let result = try Act.run( - step(from: (0, 0), to: (1, 1), steps: 100_000_000), in: app(), poster: poster) + step(from: (0, 0), to: (1, 1), steps: 5000), in: app(), poster: poster) #expect(result.moves == 1000) #expect(poster.drags.first?.count == 1001)