diff --git a/apps/macos/Tests/ClipboardPolicyTests.swift b/apps/macos/Tests/ClipboardPolicyTests.swift new file mode 100644 index 000000000..5fd405a9e --- /dev/null +++ b/apps/macos/Tests/ClipboardPolicyTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing + +/// The UI suite must never touch the *system* pasteboard. +/// +/// There is one of those and it belongs to whoever is at the keyboard. A test +/// that triggers a copy into it destroys what they had, and saving and +/// restoring around the test is not a fix: a pasteboard item can be a promise +/// its owner fulfils lazily, so a restore puts back a degraded copy and an +/// early exit puts back nothing at all. +/// +/// A *named* pasteboard has none of that problem, so the UI tests use one: a +/// debug build reads `JP_DEBUG_PASTEBOARD` and copies there instead (see +/// ``DebugState/pasteboard``), and `WorkspaceFixture.copiedText()` reads it +/// back. Copy Link is covered end to end without a clipboard being lost. +/// +/// What this forbids is therefore narrow and exact: the spellings that mean +/// "the one everybody shares". It is a source scan rather than a rule in a +/// document because a rule in a document is not enforced by anything. +@Suite("ClipboardPolicy") +struct ClipboardPolicyTests { + /// The spellings that reach the system pasteboard. + /// + /// `NSPasteboard(name: .general)` is the same object as + /// `NSPasteboard.general`, so naming it counts too. + static let forbidden = [ + "NSPasteboard.general", + "UIPasteboard.general", + "Name.general", + "name: .general", + ] + + @Test("no UI test reaches for the system pasteboard") + func uiTestsDoNotTouchThePasteboard() throws { + let sources = try Self.uiTestSources() + + // A scan over nothing passes for the wrong reason, and would keep + // passing if the directory were renamed. + #expect(sources.count >= 3, "expected to find the UI test sources to scan") + + for source in sources { + let text = try String(contentsOf: source, encoding: .utf8) + for symbol in Self.forbidden where text.contains(symbol) { + Issue.record( + """ + \(source.lastPathComponent) reaches the system pasteboard through \ + `\(symbol)`. Copy through the fixture's own pasteboard instead: the app \ + writes to the one `JP_DEBUG_PASTEBOARD` names, and \ + `WorkspaceFixture.copiedText()` reads it back. + """ + ) + } + } + } + + /// Every Swift file in `apps/macos/UITests`. + /// + /// Located from this file's compile-time path. The app is not sandboxed and + /// these tests are hosted by it, so the checkout is readable from here. + static func uiTestSources() throws -> [URL] { + // .../apps/macos/Tests/ClipboardPolicyTests.swift + let directory = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("UITests") + + return try FileManager.default + .contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .filter { $0.pathExtension == "swift" } + } +} diff --git a/apps/macos/Tools/jpdrive/Package.swift b/apps/macos/Tools/jpdrive/Package.swift new file mode 100644 index 000000000..5e3cee840 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Package.swift @@ -0,0 +1,34 @@ +// swift-tools-version: 6.0 + +import PackageDescription + +// Mirrors the app's `project.yml`: an existential is spelled `any P`, and a +// warning that never fails a build is a warning nobody fixes. +// +// SwiftPM 6.0 has no first-class setting for warnings-as-errors, and +// `unsafeFlags` is rejected only for a package consumed as a dependency, which +// this one never is. +let strict: [SwiftSetting] = [ + .swiftLanguageMode(.v6), + .enableUpcomingFeature("ExistentialAny"), + .unsafeFlags(["-warnings-as-errors"]), +] + +let package = Package( + name: "jpdrive", + platforms: [.macOS(.v15)], + products: [ + .executable(name: "jpdrive", targets: ["jpdrive"]) + ], + targets: [ + // The driver's logic, in a library so it can be tested. SwiftPM cannot + // cleanly test an executable target, and the traversal is where the bugs + // are. + .target(name: "DriveKit", swiftSettings: strict), + + // One line, calling into the library. + .executableTarget(name: "jpdrive", dependencies: ["DriveKit"], swiftSettings: strict), + + .testTarget(name: "DriveKitTests", dependencies: ["DriveKit"], swiftSettings: strict), + ] +) diff --git a/apps/macos/Tools/jpdrive/README.md b/apps/macos/Tools/jpdrive/README.md new file mode 100644 index 000000000..98124d4f7 --- /dev/null +++ b/apps/macos/Tools/jpdrive/README.md @@ -0,0 +1,330 @@ +# jpdrive + +Reads and acts on a running macOS app's accessibility tree, speaking JSON. +The `debug_app_*` tools shell out to it; the Rust side stays the presenter, +parsing the JSON and rendering markdown. + +Swift rather than Rust because `AXUIElement` is CoreFoundation-shaped: ordinary +code here, unsafe bindings or a 784-download crate there. + +External rather than an in-app automation socket, deliberately. +Driving through `AXUIElement` means a broken accessibility tree breaks the +tooling, which is the pressure that keeps the app's accessibility honest. + +## Build + +```sh +just build-drive +``` + +The binary lands at `.build/release/jpdrive` under this directory. + +## The TCC question + +Everything downstream depends on one unknown: does a binary launched as a child +of `just serve-tools` inherit the Accessibility grant given to the terminal? + +macOS attributes TCC to the *responsible process*, which for a command-line tool +is normally the terminal rather than the tool. +That is the same mechanism behind the `sample(1)` note in +`.config/jp/tools/src/debug_jp/profile_sampling.rs` about granting Terminal +*Developer Tools*. +Apple documents neither the algorithm nor its stability, so the answer has to be +measured. + +`jpdrive doctor` measures it. +Run it three ways, with Accessibility granted to the terminal application and +the app running: + +Check the target first. +An empty `pgrep` means the app is not running, and a run without a target +reports the trust flag alone, which is the half of the answer that can be wrong: + +```sh +pgrep -f JP.app # must print exactly one pid +``` + +```sh +# 1. Directly from the terminal. +.build/release/jpdrive doctor --pid $(pgrep -f JP.app) + +# 2. Through just, which adds the process layer the tools will run under. +just drive-doctor $(pgrep -f JP.app) +``` + +The third case, a child of `jp-tools` under `just serve-tools`, needs a tool +that shells out to the driver. +Reaching it means writing the first `debug_app_*` tool, which is why cases 1 and +2 come first: if the grant already fails at case 2, nothing is learned by going +further. + +Compare `trusted` and `probe.axError` across the runs. +`trusted: true` with a window count means the grant inherits. +`trusted: false`, or `api_disabled` / `cannot_complete` from the probe, means it +does not, and the driver needs its own signed bundle or its own grant. + +The report lists the ancestor chain, so a `false` says which processes were +candidates for holding the grant. + +The check never prompts. +`AXIsProcessTrustedWithOptions` with `kAXTrustedCheckOptionPrompt` would raise +the system dialog and change the state being measured. + +### Result + +**The grant inherits.** With Accessibility granted to Ghostty, case 2 reports +`trusted: true` and a window count from a chain of six: + +``` +ghostty → login → fish → just → sh → jpdrive +``` + +So `tree`, `windows`, `menu`, and `act` need no signed bundle and no grant of +their own. +They can assume the terminal's. +Case 3, a child of `jp-tools` under `just serve-tools`, adds one more process of +the same kind and is still unmeasured. + +Observations across the runs, on macOS with Ghostty as the terminal: + +- Process depth is not the variable. + Run directly from the shell (chain of four, up to `ghostty`) and through + `just` (chain of six, adding `sh` and `just`), the report is identical. + Whatever governs the grant, it is not the number of processes between the + terminal and the driver. +- The trust flag and the probe agree. + `trusted: false` came with `ax_error: api_disabled` from a real read against a + running app, which is what the accessibility API returns to an untrusted + caller; `trusted: true` came with a window count. + No case has been seen where the two disagree. +- Untested: a terminal instance started *before* the grant. + The `false` runs and the `true` run may differ by the grant alone, by a + relaunch, or by both, so "the grant is not visible to this terminal instance" + is not yet ruled out as a separate failure mode. + +## Screen Recording is a second grant + +`windowid` answers the window server rather than the accessibility API, and the +two are governed by different TCC grants. +Enumerating windows needs neither, so the command works with nothing granted at +all; reading a window's *title*, and capturing its content with `screencapture +-l`, need Screen Recording. + +That is why the report pairs the list with a `screen_recording` flag rather than +refusing outright. +Missing the grant, a capture succeeds and returns the desktop where the window +should be, so the caller has to know before it writes a file. +An untitled window in the list is the same fact seen from the other side. + +The pane is System Settings ▸ Privacy & Security ▸ Screen & System Audio +Recording, and as with Accessibility it is the terminal application that needs +it, not the driver. + +### Result + +**The grant inherits.** Measured with Ghostty as the terminal, the driver run as +a child of `jp-tools` under `just serve-tools`: before the grant, +`screen_recording` came back `false` and `debug_app_screenshot` refused; after +granting Screen Recording to Ghostty and restarting it, the same call captured +the window. + +So the flag is worth trusting, and this grant reaches a driver six processes +deep from the terminal, same as Accessibility does. + +Untested: whether the restart was necessary. +The grant and the restart happened together, so nothing here separates them. + +## What the sidebar looks like through accessibility + +SwiftUI's `.accessibilityIdentifier` does not land on the element that owns +behaviour. +For a `List` row it lands two levels below it: + +``` +AXOutline AXIdentifier: sidebar.list AXRows: 1065, AXVisibleRows: 9 + AXRow AXSelected settable: true + AXCell AXSelected settable: false, AXScrollToVisible settable: true + AXUnknown AXIdentifier: sidebar.row. + AXAttributedDescription: ", <n> events" + no actions, no children +``` + +So addressing an element and acting on it are two different steps. +The identified element has no actions at all: no `AXPress`, nothing. +Selecting a row means walking up to the `AXRow` and writing `AXSelected`. + +That write is preferable to a synthesized click for a reason beyond determinism. +Every row exists as an accessibility element, but only nine are on screen: the +outline's frame is 41658pt tall against a 398pt viewport. +A click at `AXActivationPoint` would miss an off-screen row, or land on +whichever row occupies those coordinates instead. +An `AXSelected` write is independent of scroll position. + +`AXScrollToVisible` appears as a settable *attribute* on a sidebar cell and as +an *action* on a transcript event, so scrolling has to try both forms. + +The sidebar materialises every row; the transcript does not. +Only one `transcript.event.*` element exists at a time, so an identifier that +names an unrendered event cannot be waited for, only scrolled to. + +Writing `AXSelected` on a row 690 places down a thousand-row list selects it and +brings it into view, so selecting a row needs no scrolling step of its own. +The transcript still does. + +### The identified element cannot be walked upwards + +The `AXUnknown` carrying the identifier reports no `AXParent`, and no +`AXTopLevelUIElement` either, unlike the cell and row above it. +Climbing from it arrives nowhere. + +So resolving an identifier means keeping the chain the search descended through, +not finding the element and navigating from it afterwards. +Anything that acts on an ancestor of an identified element depends on this. + +### Cost + +An accessibility round-trip to this app costs roughly 3ms, and that number sets +every other budget: + +- Reading the first few rows under `sidebar.` takes 250ms. +- Finding one row 690 places down takes 5.8s, because the search reads about two + thousand elements to get there and cannot prune on the way: every identifier + in the sidebar sits on a leaf. + +Hence the batched reads and the match budget. +Anything that polls should resolve an element once and re-read that reference, +rather than searching each time. + +## Acting on an element + +Each step names exactly one mechanism, because the mechanism depends on what the +element is and guessing hides regressions: + +| step | addressed by | mechanism | +| --------- | ------------ | ------------------------------------------------------- | +| `select` | identifier | write `AXSelected` on the nearest ancestor accepting it | +| `press` | identifier | `AXPress` on the element itself | +| `type` | identifier | write `AXValue`, then `AXConfirm` | +| `perform` | identifier | a named action, for the verbs with no step of their own | +| `menu` | titled path | `AXPress` on the item the path resolves to | +| `click` | identifier | synthesized mouse event at `AXActivationPoint` | + +`press` and `menu` end in the same call and are not redundant: they differ in +what they address by, and that is what a test pins. +`closeAll:` is an `AppKit` selector name that survives the item moving to +another menu, so a script keyed on it cannot notice the menu bar being +rearranged. +`["File", "Close All"]` names the structure the user sees, and a path that stops +resolving reports how far it got and what that level holds instead — which is +the assertion failure a layout test wants to read. + +A step that names the wrong mechanism fails and says which actions the element +does accept. +There is no fallback chain: if a sidebar row stopped accepting `AXSelected`, a +driver that quietly fell back to a synthesized click would keep every script +green while the app's accessibility rotted, which is the failure this tool +exists to prevent. + +`select` and `type` read the attribute back afterwards, because a write can be +accepted and discarded. +`press` cannot: nothing observable says a button did anything, so its result +reports no confirmation rather than claiming one. + +### Some steps have to bring the app forward + +`menu`, `click` and `drag` write `AXFrontmost` on the application and wait for +it to take, which makes them the steps that take focus from whatever had it. +A step that cannot bring the application forward fails rather than acting +anyway. + +Without it almost nothing in the menu bar can be pressed. +AppKit disables every item that acts on the front window or the responder chain +while the application is in the background, and against a driven instance that +is most of the bar: `Close`, `Copy`, `Select All`, `Show Sidebar`, and every +`SwiftUI` command reading a `@FocusedValue` all report `AXEnabled: 0`. +`New Window` and `Close All` do not, which is what makes the difference easy to +miss — a first menu step against an app-level item works, and the next one +silently does nothing. + +So the item's enabled state is checked before it is pressed, rather than +trusting `AXPress` to report a refusal. +A disabled item accepts the press and answers success. + +An element that reports no `AXEnabled` at all is not disabled. +Plenty carry no such attribute, and reading its absence as a refusal would +reject them all. + +### Typing writes the value, and then has to commit it + +`type` writes `AXValue` and performs `AXConfirm`. +Both are needed, and the second one is the part that was not obvious. + +Writing `AXValue` on a `SwiftUI` `TextField` changes the text the field displays +and leaves the binding behind it untouched. +Measured against the conversation filter: after the write the field read back +`"accessibility"` and the list still showed all 1,066 rows. +Deleting one character by hand then filtered on `"accessibilit"` — the +keystroke made the binding resync from whatever the field held by then. +So a `type` that only wrote the value would report success while the application +carried on as though nothing had been typed. + +`AXConfirm` commits through the path the binding observes. +A field advertising no confirm action is not a failure — some publish every +change as it happens — so the result reports `committed` separately from +`confirmed`: the text being in the field and the application having seen it are +different facts. + +Synthesizing key events was rejected on three counts: the events go wherever +focus is, so a window activating mid-sequence types into it instead; posting +them fast enough to be useful means pauses between characters, which makes the +step flaky rather than deterministic; and event posting is global process state, +so it could not sit behind the element abstraction the rest of the driver is +tested through. + +The remaining cost is that per-character behaviour never runs. +A field that validates each keystroke, or completes as you type, sees one change +rather than a dozen. + +### Clicking is the last resort + +`click` brings the application forward, raises the element's window, and posts a +mouse event at its `AXActivationPoint`. +It is the only step whose effect is not addressed to an element: the event goes +to whatever occupies that screen coordinate. + +Both the activation and the raise are needed, and neither substitutes for the +other. +`AXRaise` orders a window *within* its own application; ordering between +applications follows activation. +The driver is normally started from a terminal, which leaves that terminal +frontmost, so a click that only raised would land in the terminal while +reporting the element it aimed at. +An occluding window from another application will still swallow it. + +An element reporting no activation point is refused rather than clicked at the +origin. +A sidebar row is exactly that case, and it wants `select`. + +Posting is behind an `EventPoster`, so where the driver aimed can be asserted in +a test even though where the event lands cannot. + +### Apple Events are a separate pathway, and that one does not inherit + +Reading the same tree through AppleScript fails from the terminal the driver +succeeds from: + +``` +System Events got an error: osascript is not allowed assistive access. (-1719) +``` + +Two different checks. +`AXIsProcessTrusted`, which the driver calls, resolves to the responsible +process and finds the terminal. +`System Events` requires the calling binary itself to be listed, and the calling +binary is `/usr/bin/osascript` — shared by everything on the machine, so +granting it grants far more than the driver needs. + +This is the second reason the driver is a binary of its own rather than a shell +script over `osascript`, alongside the one at the top of this file. +It also means AppleScript is not a fallback when the driver is missing a verb: +the verb has to be added here. diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift new file mode 100644 index 000000000..8d379c2d3 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXElement.swift @@ -0,0 +1,388 @@ +import ApplicationServices +import Foundation + +/// An element of a running application's accessibility tree. +/// +/// Every method here is one or more synchronous round-trips to the target's main +/// thread. That cost dominates everything the driver does, so callers batch reads +/// with ``values(_:)`` rather than reading attributes one at a time, and hold onto +/// an element they will read again instead of walking to it twice. +/// +/// A reference stays valid while the underlying element lives. Once it is gone, +/// reads answer `invalid_ui_element` rather than crashing. +struct AXElement { + let element: AXUIElement + + /// The root element of the application owning `pid`. + /// + /// Succeeds whether or not the process exists; the first read is what fails. + static func application(pid: pid_t) -> AXElement { + return AXElement(element: AXUIElementCreateApplication(pid)) + } + + /// `AXRole`, or `nil` when the element does not report one. + var role: String? { + return read(kAXRoleAttribute).flatMap { $0 as? String } + } + + /// `AXIdentifier`, or `nil` when the element carries none. + /// + /// SwiftUI's `.accessibilityIdentifier` surfaces here, but it also composites + /// with identifiers the framework generates itself, so a value like + /// `"workspace-AppWindow-1, SidebarNavigationSplitView"` is possible. + var identifier: String? { + return read(kAXIdentifierAttribute).flatMap { $0 as? String } + } + + /// The element's human-readable label. + /// + /// Tries `AXAttributedDescription`, then `AXDescription`, then `AXTitle`. + /// SwiftUI populates the first of those for list rows and leaves the others + /// empty, while AppKit controls tend to do the reverse. + var label: String? { + if let attributed = read(Self.attributedDescription) as? NSAttributedString { + return attributed.string + } + + for name in [kAXDescriptionAttribute, kAXTitleAttribute] { + guard let text = read(name).flatMap({ $0 as? String }), !text.isEmpty else { + continue + } + return text + } + + return nil + } + + /// `AXAttributedDescription`, which has no constant in the SDK headers. + static let attributedDescription = "AXAttributedDescription" + + /// `AXActivationPoint`, which has no constant in the SDK headers. + /// + /// Where the element says a click on it belongs, in screen coordinates. Not + /// always the middle of its frame. + static let activationPoint = "AXActivationPoint" + + /// The element's children, or an empty array when it has none. + /// + /// A round-trip of its own. A walk should take children from ``read(_:)``, + /// which fetches them alongside everything else it needs. + var children: [AXElement] { + guard let value = read(kAXChildrenAttribute), let raw = value as? [AXUIElement] else { + return [] + } + return raw.map { AXElement(element: $0) } + } + + /// Actions the element accepts, such as `AXPress`. + var actions: [String] { + var names: CFArray? + guard AXUIElementCopyActionNames(element, &names) == .success, + let names = names as? [String] + else { + return [] + } + return names + } + + /// Every attribute name the element advertises. + func names() -> [String] { + var names: CFArray? + guard AXUIElementCopyAttributeNames(element, &names) == .success, + let names = names as? [String] + else { + return [] + } + return names + } + + /// Read one attribute, or `nil` when the read fails or the value is absent. + /// + /// Use ``values(_:)`` when reading more than one: this costs a round-trip per + /// call, which is what makes a naive tree walk take seconds. + func read(_ name: String) -> CFTypeRef? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success + else { + return nil + } + guard let value, CFGetTypeID(value) != CFNullGetTypeID() else { return nil } + return value + } + + /// Read several attributes in one round-trip, or `nil` when the call fails. + /// + /// Results are positional and the same count as `names`. An attribute that + /// could not be read arrives as CoreFoundation's null or as an `AXValue` + /// boxing the error, both of which ``text(_:)`` reports rather than discards. + /// + /// A `nil` return is the whole call failing, which a target that is busy or + /// exiting answers with. Reporting that as a row of absent attributes would + /// make an element nothing could be read from look exactly like one that + /// reports nothing. + func values(_ names: [String]) -> [CFTypeRef?]? { + guard !names.isEmpty else { return [] } + + var raw: CFArray? + let status = AXUIElementCopyMultipleAttributeValues( + element, + names as CFArray, + AXCopyMultipleAttributeOptions(), + &raw + ) + + guard status == .success, + let values = raw as? [CFTypeRef], + values.count == names.count + else { + return nil + } + + return values + } + + /// Whether `name` can be written on this element. + /// + /// A failed query reports as not settable: the accessibility API answers this + /// for every attribute it advertises, so a failure means the element is gone + /// or the attribute is not really there. + func isSettable(_ name: String) -> Bool { + var settable = DarwinBoolean(false) + guard AXUIElementIsAttributeSettable(element, name as CFString, &settable) == .success + else { + return false + } + return settable.boolValue + } + + /// Perform `action`, returning the API's own status. + func perform(_ action: String) -> AXError { + return AXUIElementPerformAction(element, action as CFString) + } +} + +extension AXElement: Element { + /// Read the named attributes and the element's children in one round-trip. + /// + /// Children come back in the same batch as everything else: asking for them + /// separately would add a hop per element, and every walk asks for them. + func read(_ names: [String]) -> Reading<AXElement> { + guard let values = self.values(names + [kAXChildrenAttribute]) else { + return Reading( + text: Array(repeating: nil, count: names.count), + children: [], + failed: true + ) + } + + let children = (values.last.flatMap { $0 } as? [AXUIElement] ?? []) + .map { AXElement(element: $0) } + + return Reading( + text: values.dropLast().map(Self.optionalText), + children: children + ) + } + + /// Enumerate the element's attributes and read them all. + /// + /// Two round-trips: one to learn the names, one to read them. Only a dump + /// pays that, which is why it is not folded into ``read(_:)``. + func reportedAttributes(settable: Bool) -> [Attribute]? { + let names = self.names().sorted() + guard !names.isEmpty else { return [] } + guard let values = values(names) else { return nil } + + return zip(names, values).map { name, value in + Attribute( + name: name, + value: value.map(Self.text) ?? "<null>", + settable: settable ? isSettable(name) : nil + ) + } + } + + /// Read a boolean attribute. + /// + /// `CFBoolean` bridges to `NSNumber` rather than to `Bool`, so a direct cast + /// answers `nil` for a perfectly good `0` or `1`. + func flag(_ name: String) -> Bool? { + guard let value = read(name) as? NSNumber else { return nil } + return value.boolValue + } + + /// Write a boolean attribute, answering the API's own status. + func setFlag(_ name: String, _ value: Bool) -> AXError { + return AXUIElementSetAttributeValue( + element, + name as CFString, + value ? kCFBooleanTrue : kCFBooleanFalse + ) + } + + /// The point held in an attribute, in screen coordinates. + func point(_ name: String) -> CGPoint? { + guard let value = read(name), CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + + let boxed = unsafeDowncast(value, to: AXValue.self) + guard AXValueGetType(boxed) == .cgPoint else { return nil } + + var point = CGPoint.zero + guard AXValueGetValue(boxed, .cgPoint, &point) else { return nil } + + return point + } + + /// The size held in an attribute, in points. + func size(_ name: String) -> CGSize? { + guard let value = read(name), CFGetTypeID(value) == AXValueGetTypeID() else { + return nil + } + + let boxed = unsafeDowncast(value, to: AXValue.self) + guard AXValueGetType(boxed) == .cgSize else { return nil } + + var size = CGSize.zero + guard AXValueGetValue(boxed, .cgSize, &size) else { return nil } + + return size + } + + /// Write a size attribute, answering the API's own status. + /// + /// The value has to be boxed in an `AXValue`: the API takes `CFTypeRef` and a + /// bare `CGSize` is not one, so passing it any other way fails the write with + /// no indication of why. + func setSize(_ name: String, _ value: CGSize) -> AXError { + var size = value + guard let boxed = AXValueCreate(.cgSize, &size) else { + return .failure + } + + return AXUIElementSetAttributeValue(element, name as CFString, boxed) + } + + /// Write a string attribute, answering the API's own status. + func setText(_ name: String, _ value: String) -> AXError { + return AXUIElementSetAttributeValue(element, name as CFString, value as CFString) + } + + /// The elements held in an attribute. + func elements(_ name: String) -> [AXElement] { + guard let value = read(name) else { return [] } + + if let raw = value as? [AXUIElement] { + return raw.map { AXElement(element: $0) } + } + + guard CFGetTypeID(value) == AXUIElementGetTypeID() else { return [] } + return [AXElement(element: unsafeDowncast(value, to: AXUIElement.self))] + } +} + +extension AXElement { + /// Render an attribute value as text, or `nil` when there is no value. + /// + /// A batched read answers an absent attribute with CoreFoundation's null and an + /// unreadable one with a boxed error. Both are facts a dump wants to see and a + /// caller reading one attribute wants as nothing at all. + static func optionalText(_ value: CFTypeRef?) -> String? { + guard let value, CFGetTypeID(value) != CFNullGetTypeID() else { return nil } + + if CFGetTypeID(value) == AXValueGetTypeID(), + AXValueGetType(unsafeDowncast(value, to: AXValue.self)) == .axError + { + return nil + } + + return text(value) + } + + /// Render an attribute value as text. + /// + /// Values arrive as CoreFoundation types, including geometry boxed in + /// `AXValue` and references to other elements. Everything becomes a string so + /// that a reader can see which attributes exist and which carry identifiers + /// without this growing a case per boxed type. + static func text(_ value: CFTypeRef) -> String { + // An attribute the element advertises but cannot answer for, such as + // `AXSubrole` on an element that has none. + if CFGetTypeID(value) == CFNullGetTypeID() { + return "<null>" + } + if let text = value as? String { + return text + } + // Labels arrive as attributed strings more often than plain ones, and the + // attributes carry nothing the driver acts on. + if let attributed = value as? NSAttributedString { + return attributed.string + } + if let number = value as? NSNumber { + return number.stringValue + } + if let elements = value as? [AXUIElement] { + return "<\(elements.count) AXUIElement>" + } + if let array = value as? [Any] { + return "<array of \(array.count)>" + } + + let typeID = CFGetTypeID(value) + if typeID == AXUIElementGetTypeID() { + return "<AXUIElement>" + } + if typeID == AXValueGetTypeID() { + // The conditional form is rejected here: every CoreFoundation type is + // bridged as a class, so the compiler sees a cast that cannot fail. + // The type ID check above is the real test. + return text(unsafeDowncast(value, to: AXValue.self)) + } + return "<CFTypeID \(typeID)>" + } + + /// Render the geometry boxed in an `AXValue`. + /// + /// `AXActivationPoint` and `AXFrame` decide where a synthesized click lands, + /// so these arrive as numbers a reader can check against the screen rather + /// than as an opaque marker. + static func text(_ value: AXValue) -> String { + let type = AXValueGetType(value) + + switch type { + // A batched read reports a per-attribute failure by boxing the error + // rather than by failing the whole call. + case .axError: + var status = AXError.success + guard AXValueGetValue(value, .axError, &status) else { break } + return "<\(status.name)>" + + case .cgPoint: + var point = CGPoint.zero + guard AXValueGetValue(value, .cgPoint, &point) else { break } + return "\(point.x),\(point.y)" + + case .cgSize: + var size = CGSize.zero + guard AXValueGetValue(value, .cgSize, &size) else { break } + return "\(size.width)x\(size.height)" + + case .cgRect: + var rect = CGRect.zero + guard AXValueGetValue(value, .cgRect, &rect) else { break } + return "\(rect.origin.x),\(rect.origin.y) \(rect.size.width)x\(rect.size.height)" + + case .cfRange: + var range = CFRange() + guard AXValueGetValue(value, .cfRange, &range) else { break } + return "\(range.location)+\(range.length)" + + default: + break + } + + return "<AXValue \(type.rawValue)>" + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift new file mode 100644 index 000000000..68add5ffe --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/AXErrorName.swift @@ -0,0 +1,24 @@ +import ApplicationServices + +extension AXError { + /// A stable snake_case name for this error. + /// + /// Only the codes a read or an action can realistically produce are named. + /// Anything else keeps its numeric code rather than being flattened into + /// "unknown", so an unexpected failure stays traceable to a header. + var name: String { + switch self { + case .success: return "success" + case .apiDisabled: return "api_disabled" + case .cannotComplete: return "cannot_complete" + case .invalidUIElement: return "invalid_ui_element" + case .notImplemented: return "not_implemented" + case .attributeUnsupported: return "attribute_unsupported" + case .actionUnsupported: return "action_unsupported" + case .noValue: return "no_value" + case .illegalArgument: return "illegal_argument" + case .failure: return "failure" + default: return "ax_error_\(rawValue)" + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift new file mode 100644 index 000000000..bca2e1d60 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Act.swift @@ -0,0 +1,1266 @@ +import ApplicationServices +import Foundation + +/// One thing to do to one element. +/// +/// Each case names its own mechanism. There is no step that picks a mechanism +/// based on what the element supports: a script that says `select` against +/// something unselectable fails and says so, which is how a change in the app's +/// accessibility becomes visible instead of being absorbed by a fallback. +/// +/// Decoded from a single-key object, so a step reads as what it does: +/// +/// ```json +/// {"select": {"identifier": "sidebar.row.17855681129"}} +/// ``` +enum Step: Decodable { + /// Write `AXSelected` on the nearest ancestor that accepts it. + /// + /// The mechanism for list and outline rows, where the identified element is + /// below the one that owns selection. Independent of scroll position, so it + /// reaches a row that is not on screen. + case select(Target) + + /// Perform `AXPress` on the identified element itself. + /// + /// The mechanism for buttons and menu items, which advertise the action. + case press(Target) + + /// Synthesize a mouse click at the element's activation point. + /// + /// The last resort, and the only step that depends on the world outside the + /// accessibility tree: the window has to be frontmost and the element on + /// screen, or the click lands somewhere else entirely. + case click(Target) + + /// Perform a named accessibility action on the identified element. + /// + /// The long tail. `press` is this with `AXPress` and a better error message, + /// and is worth keeping because it is the overwhelmingly common case; anything + /// else an element advertises — `AXConfirm`, `AXShowMenu`, `AXScrollToVisible`, + /// `AXCancel` — is reached through here rather than by growing a step per verb. + case perform(ActionTarget) + + /// Put text into a text field. + case type(TypeTarget) + + /// Set an element's size, which for a window resizes it. + /// + /// The one step that changes the shape of what is on screen rather than what + /// is in it, and the only way to observe what a resize costs: a drag of a + /// window's edge cannot be synthesized against a background application, and + /// resizing is where a view that re-measures its contents shows up. + case resize(SizeTarget) + + /// Drag the pointer across an element, with the button held. + /// + /// The gesture no other step can stand in for. `resize` sets a window's size + /// in one write, which is not a drag: nothing enters live resize, and a view + /// that behaves differently *during* a gesture than after it looks correct to + /// every other step here. + /// + /// Not only for window edges. Any two points on any element — a split + /// divider's handle, a stretch of text to select, a row to drag out — is the + /// same gesture with different endpoints. + /// + /// Depends on the world outside the tree in the same way `click` does: the + /// events go to whatever occupies those coordinates, so the window is raised + /// first and has to be on screen. + case drag(DragTarget) + + /// What to drag across, and along what path. + struct DragTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// Names the coordinate space, not necessarily the thing that reacts. A + /// window's own frame is how its resize corner is addressed, and the + /// window is what reacts. + let identifier: String + + /// Where the button goes down, as a fraction of the element's frame. + let from: Offset + + /// Where it comes up. + let to: Offset + + /// How many moves to post between the two, not counting the press. + /// + /// Defaults to 24. The number is the point of the step: a drag posted as + /// one jump exercises a single frame, and the behaviour usually under + /// question is what happens across many. + let steps: Int? + + /// How long to pause between moves, in milliseconds. Defaults to 8. + let pauseMs: Int? + + private enum CodingKeys: String, CodingKey { + case identifier + case from + case to + case steps + case pauseMs = "pause_ms" + } + } + + /// A point on an element, as a fraction of its frame. + /// + /// Fractions rather than points, so a script says "the right edge, halfway + /// down" and keeps meaning it after the window is resized. + /// + /// `1.0` is the far edge exactly, and is what a window resize wants. The + /// region that resizes a window is a few points wide and straddles the frame + /// boundary, so aiming even five points inside lands in the content instead: + /// the gesture runs, the pointer moves, and whatever is under it gets dragged + /// rather than the window resized. Measured on a running window — `0.995` of + /// a 1070-point window grabs text, `1.0` grabs the edge. + struct Offset: Decodable { + let dx: Double + let dy: Double + } + + /// What to resize, and to what. + struct SizeTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// A window carries one, so it is addressed the same way as anything else + /// rather than through a step that means "the frontmost window". + let identifier: String + + /// The width to ask for, in points. + let width: Double + + /// The height to ask for, in points. + let height: Double + } + + /// What action to perform, and on what. + struct ActionTarget: Decodable { + /// The element's `AXIdentifier`, matched exactly. + let identifier: String + + /// The action's own name, spelled as the accessibility API spells it. + /// + /// Not translated from a friendlier vocabulary: a step that says + /// `AXConfirm` can be checked against what `jpdrive dump` reported for the + /// element, and a friendlier name could not. + let action: String + } + + /// Press a menu item, addressed by the titles leading to it. + case menu(MenuTarget) + + /// What to type, and where. + struct TypeTarget: Decodable { + /// The field's `AXIdentifier`, matched exactly. + let identifier: String + + /// The text to put in the field, replacing what is there. + /// + /// Written as a value and then confirmed, rather than typed a character at + /// a time. Two calls, neither of which can be derailed by focus moving to + /// another application halfway through, which a synthesized keystroke can. + /// + /// The confirm is not optional dressing. Writing `AXValue` on a `SwiftUI` + /// text field changes the text the field displays without the binding + /// behind it noticing, so the application carries on as though nothing was + /// typed. Confirming commits the edit through the path the binding does + /// observe. + /// + /// The cost is that per-character behaviour never runs. A field that + /// validates each keystroke, or completes as you type, sees one change + /// rather than a dozen. Assert the consequence — the list that narrowed, + /// the button that enabled — rather than assuming the field's own handlers + /// fired for every character. + let text: String + } + + /// Wait until an element with the given identifier exists. + case waitFor(WaitTarget) + + /// A path through a menu. + struct MenuTarget: Decodable { + /// Titles from the top of the menu downwards, such as `["File", "Close"]`. + /// + /// Titles rather than identifiers, because the structure is the thing worth + /// asserting. An identifier like `closeAll:` is an `AppKit` selector name: + /// it survives the item moving to a different menu, so a script keyed on it + /// cannot notice the menu bar being rearranged. A path cannot miss that. + /// + /// A path that does not resolve reports how far it got and what that level + /// holds, which is the assertion failure a layout test wants to read. + let path: [String] + + /// The element whose shown menu the path starts from. + /// + /// Absent, the path starts at the menu bar. Present, it starts at the menu + /// that element is currently displaying, which `AXShowMenu` puts up. + /// + /// A title is the only way to name a context menu item: `SwiftUI` does not + /// carry an accessibility identifier onto the `NSMenuItem` it bridges a + /// menu button to, so every item in one reports the same selector name. + let under: String? + + /// Spelled out so `under` can be left off, both here and on the wire. + init(path: [String], under: String? = nil) { + self.path = path + self.under = under + } + } + + /// What a wait addresses, and for how long. + struct WaitTarget: Decodable { + /// The `AXIdentifier` to wait for, matched exactly. + let identifier: String + + /// Identifier of a container to search inside, resolved once before + /// polling begins. + /// + /// Strongly worth setting. A search for something absent has no early exit + /// and reads every element in the application, which against a thousand-row + /// sidebar takes longer than a typical timeout allows for a single attempt. + /// Scoping to the container the element will appear in makes each poll + /// cheap. + /// + /// A container that does not exist fails immediately, rather than being + /// waited for. + let under: String? + + /// How long to keep trying. Defaults to 5000. + let timeoutMs: Int? + + /// How long to pause between attempts. Defaults to 100. + /// + /// Not the kind of sleep the driver avoids. Waiting a fixed duration and + /// assuming the work finished is a guess; pausing between two observations + /// of a condition is how polling stays off a busy loop that would flood the + /// target with accessibility traffic. + let intervalMs: Int? + + /// Spelled out, because the decoder converts no cases of its own: without + /// these two, a step naming `timeout_ms` decodes as though it had named + /// nothing and silently waits the default. + private enum CodingKeys: String, CodingKey { + case identifier + case under + case timeoutMs = "timeout_ms" + case intervalMs = "interval_ms" + } + } + + /// What a step addresses. + struct Target: Decodable { + /// The element's `AXIdentifier`, matched exactly. + /// + /// Exact rather than by prefix: `sidebar.row.1785` is a prefix of many + /// rows, and acting on whichever one happened to be found first is not a + /// thing a script can mean. + let identifier: String + } + + private enum CodingKeys: String, CodingKey { + case select + case press + case click + case perform + case type + case menu + case waitFor = "wait_for" + case resize + case drag + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + if let target = try container.decodeIfPresent(Target.self, forKey: .select) { + self = .select(target) + return + } + if let target = try container.decodeIfPresent(Target.self, forKey: .press) { + self = .press(target) + return + } + if let target = try container.decodeIfPresent(Target.self, forKey: .click) { + self = .click(target) + return + } + if let target = try container.decodeIfPresent(ActionTarget.self, forKey: .perform) { + self = .perform(target) + return + } + if let target = try container.decodeIfPresent(TypeTarget.self, forKey: .type) { + self = .type(target) + return + } + if let target = try container.decodeIfPresent(MenuTarget.self, forKey: .menu) { + self = .menu(target) + return + } + if let target = try container.decodeIfPresent(WaitTarget.self, forKey: .waitFor) { + self = .waitFor(target) + return + } + if let target = try container.decodeIfPresent(SizeTarget.self, forKey: .resize) { + self = .resize(target) + return + } + if let target = try container.decodeIfPresent(DragTarget.self, forKey: .drag) { + self = .drag(target) + return + } + + throw DecodingError.dataCorrupted( + .init( + codingPath: container.codingPath, + debugDescription: + "expected one of select, press, click, perform, type, menu, wait_for, " + + "resize, drag" + ) + ) + } +} + +/// What a step did. +struct StepResult: Encodable, Equatable { + /// The step that ran, named as it was written. + let step: String + + let identifier: String + + /// The role of the element the step acted on. + /// + /// Not always the identified element: `select` climbs to the ancestor that + /// owns selection, and reporting the role it reached is how a restructuring of + /// the view surfaces as a changed role rather than as a puzzling failure. + let role: String + + /// Whether the intended change was observed after the step ran. + /// + /// A write can succeed and change nothing, so this is read back from the + /// element rather than inferred from the API's status. + /// + /// Absent for a step with nothing to read back. Pressing a button runs + /// arbitrary code in the target and has no attribute that says it worked, so + /// reporting `true` there would be claiming more than was checked. + let confirmed: Bool? + + /// Where a click was aimed, in screen coordinates. + /// + /// Only `click` reports this. A click is the one step whose outcome depends on + /// a number the caller cannot otherwise see, and "it clicked the wrong thing" + /// is unanswerable without knowing where it clicked. + let point: String? + + /// Whether an edit was committed through the element's confirm action. + /// + /// Only `type` reports this. `false` means the field took the text but + /// advertises no `AXConfirm`, so whether the application noticed depends on it + /// watching the value directly — worth knowing, because the text being in the + /// field and the application having seen it are different facts. + let committed: Bool? + + /// The size the element ended at, as `WIDTHxHEIGHT` in points. + /// + /// Only `resize` reports this. A window clamps a size to its own limits, so + /// what was asked for and what happened are different facts and the second is + /// the one worth reading. + let size: String? + + /// How many moves a drag posted between pressing and releasing. + /// + /// Only `drag` reports this. It is what separates a gesture from a jump, and + /// a caller asking why a view did not react during one wants to know how many + /// chances it had. + let moves: Int? + + init( + step: String, + identifier: String, + role: String, + confirmed: Bool? = nil, + committed: Bool? = nil, + point: String? = nil, + size: String? = nil, + moves: Int? = nil + ) { + self.step = step + self.identifier = identifier + self.role = role + self.confirmed = confirmed + self.committed = committed + self.point = point + self.size = size + self.moves = moves + } +} + +/// Runs a single step against a running application. +enum Act { + /// How far `select` looks above the identified element for one that accepts + /// selection. + /// + /// The known chain is two levels, from the identified element through the cell + /// to the row. The cap is above that so an extra wrapper does not break the + /// step, and low enough that a miss fails rather than selecting the window. + private static let maxAncestors = 4 + + /// Resolve the step's target and act on it. + static func run(_ step: Step, pid: pid_t) throws(DriveError) -> StepResult { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return try run(step, in: AXElement.application(pid: pid), poster: SystemEventPoster()) + } + + /// Run a step against an already-resolved root. + /// + /// Split from ``run(_:pid:)`` so the part with the logic in it can be exercised + /// against a tree that is not a running application. `poster` is separate for + /// the same reason: a click is aimed using the tree but delivered outside it. + /// `activation` is how long a menu step waits for the application to come + /// forward and for the item to enable, and is a parameter so a test of either + /// wait does not have to sit through the real one. + static func run<E: Element>( + _ step: Step, + in root: E, + poster: any EventPoster = SystemEventPoster(), + activation: Duration = activationTimeout + ) throws(DriveError) -> StepResult { + switch step { + case .select(let target): + return try select(target, in: root) + + case .waitFor(let target): + return try waitFor(target, in: root) + + case .press(let target): + return try press(target, in: root) + + case .perform(let target): + return try perform(target.action, on: target.identifier, in: root, step: "perform") + + case .type(let target): + return try type(target, in: root) + + case .menu(let target): + return try menu(target, in: root, within: activation) + + case .click(let target): + return try click(target, in: root, poster: poster, activation: activation) + + case .resize(let target): + return try resize(target, in: root) + + case .drag(let target): + return try drag(target, in: root, poster: poster, activation: activation) + } + } + + /// How many moves a drag posts when it does not say. + private static let defaultDragSteps = 24 + + /// How long a drag pauses between moves when it does not say. + private static let defaultDragPause = Duration.milliseconds(8) + + /// The most moves a drag will post. + /// + /// Clamped rather than rejected, for the same reason the count is clamped + /// upwards from zero: a caller computing it from a distance should get a + /// gesture, not an error. The cap is what keeps an implausible count from + /// becoming a run that posts for minutes, or one whose route cannot be + /// allocated at all — either of which ends the process without the JSON + /// document every call promises. + private static let maxDragSteps = 1000 + + /// Drag the pointer from one point on an element to another. + private static func drag<E: Element>( + _ target: Step.DragTarget, + in root: E, + poster: any EventPoster, + activation: Duration + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard + let origin = element.point(kAXPositionAttribute), + let size = element.size(kAXSizeAttribute) + else { + throw DriveError( + kind: .notClickable, + message: "\(target.identifier) reports no frame to drag across", + hint: "an element with no position or size cannot be aimed at" + ) + } + + try check(target.from, named: "from") + try check(target.to, named: "to") + + let steps = min(max(target.steps ?? defaultDragSteps, 1), maxDragSteps) + let pause = target.pauseMs.map { Duration.milliseconds($0) } ?? defaultDragPause + + let start = point(target.from, in: origin, size) + let end = point(target.to, in: origin, size) + let route = (0...steps).map { step in + let progress = Double(step) / Double(steps) + return CGPoint( + x: start.x + (end.x - start.x) * progress, + y: start.y + (end.y - start.y) * progress + ) + } + + // Activated, and then raised, and both are needed. + // + // `AXRaise` orders a window forward *within its own application*. Global + // ordering between applications follows activation, so raising a + // background app's window leaves it under the active app's windows: the + // gesture lands on whatever is on top at those coordinates, which is + // whatever the person at the keyboard is using. Measured, not assumed — a + // drag posted without this was received by the frontmost terminal. + // + // The cost is that a gesture takes focus. Nothing here can give it back: + // this process handles one step and exits, so the restore belongs to + // whatever drives the whole list. + try front(root, within: activation) + raiseWindow(in: path) + + guard poster.drag(through: route, pausing: pause) else { + throw DriveError( + kind: .actionFailed, + message: + "could not post a drag from \(start.x),\(start.y) to \(end.x),\(end.y)", + hint: nil + ) + } + + return StepResult( + step: "drag", + identifier: target.identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + point: "\(start.x),\(start.y) -> \(end.x),\(end.y)", + moves: route.count - 1 + ) + } + + /// Check that an offset names a point on the element. + /// + /// A value outside `0...1` resolves to a screen coordinate outside the + /// element's frame, and the gesture is posted into global screen space, so + /// the press or release lands on whatever occupies that point instead — + /// another application's window, or the desktop. Reading `dx` as a + /// percentage rather than a fraction is the mistake this catches, and it + /// resolves a long way outside: `100` on an 800pt window aims 80,000 points + /// to the right of it. + private static func check(_ offset: Step.Offset, named name: String) throws(DriveError) { + for (axis, value) in [("dx", offset.dx), ("dy", offset.dy)] + where !value.isFinite || value < 0 || value > 1 { + throw DriveError( + kind: .badUsage, + message: "\(name).\(axis) is \(value), which is not a fraction of the frame", + hint: "0 is the near edge of the element and 1 the far one, so 0.5 is halfway" + ) + } + } + + /// One fractional offset as a screen coordinate inside a frame. + private static func point( + _ offset: Step.Offset, in origin: CGPoint, _ size: CGSize + ) + -> CGPoint + { + CGPoint( + x: origin.x + size.width * offset.dx, + y: origin.y + size.height * offset.dy + ) + } + + /// Ask the identified element to take a new size. + /// + /// The size it ends at is read back and reported rather than assumed: a window + /// clamps to its own minimum and maximum, so asking for something outside those + /// succeeds and lands somewhere else. `confirmed` says whether it landed on + /// what was asked for. + private static func resize<E: Element>( + _ target: Step.SizeTarget, in root: E + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: "list what is addressable with: jpdrive tree --identifier <prefix>" + ) + } + + let role = element.read([kAXRoleAttribute]).text[0] ?? "" + + guard element.isSettable(kAXSizeAttribute) else { + throw DriveError( + kind: .notEditable, + message: "\(target.identifier) does not accept a write to AXSize", + hint: "a window does; most elements inside one do not" + ) + } + + let wanted = CGSize(width: target.width, height: target.height) + let status = element.setSize(kAXSizeAttribute, wanted) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXSize on \(target.identifier) failed: \(status.name)", + hint: nil + ) + } + + let reached = element.size(kAXSizeAttribute) + + return StepResult( + step: "resize", + identifier: target.identifier, + role: role, + confirmed: reached == wanted, + size: reached.map { "\(Int($0.width))x\(Int($0.height))" } + ) + } + + /// Click where the identified element says a click belongs. + /// + /// The last resort among the steps, and the only one whose effect is not + /// addressed to the element: the event goes to whatever occupies that screen + /// coordinate. Prefer `select` for rows and `press` for controls, both of which + /// reach their target regardless of what is on top of it or whether it is + /// scrolled into view. + private static func click<E: Element>( + _ target: Step.Target, + in root: E, + poster: any EventPoster, + activation: Duration + ) throws(DriveError) -> StepResult { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard let point = element.point(AXElement.activationPoint) else { + throw DriveError( + kind: .notClickable, + message: "\(target.identifier) reports no AXActivationPoint", + hint: + "an element with no place to be clicked is usually one that wants `select` " + + "or `press` instead" + ) + } + + // Brought forward, and then raised, and both are needed — the same pair + // `drag` performs, for the same reason. `AXRaise` orders a window within + // its own application; ordering *between* applications follows + // activation. A driver invoked from a terminal leaves that terminal + // frontmost, so raising alone posts the click into the terminal while + // reporting the element it aimed at. + try front(root, within: activation) + raiseWindow(in: path) + + guard poster.click(at: point) else { + throw DriveError( + kind: .actionFailed, + message: "could not post a click at \(point.x),\(point.y)", + hint: nil + ) + } + + return StepResult( + step: "click", + identifier: target.identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + point: "\(point.x),\(point.y)" + ) + } + + /// Bring the window holding the addressed element to the front, if it has one. + /// + /// Found along the path the search descended, for the same reason the selection + /// owner is: the identified element does not report a parent to climb from. + private static func raiseWindow<E: Element>(in path: [E]) { + for element in path where element.read([kAXRoleAttribute]).text[0] == kAXWindowRole { + _ = element.perform(kAXRaiseAction) + return + } + } + + /// Press the identified element. + private static func press<E: Element>( + _ target: Step.Target, in root: E + ) throws(DriveError) + -> StepResult + { + return try perform(kAXPressAction, on: target.identifier, in: root, step: "press") + } + + /// Perform `action` on the element with `identifier`. + /// + /// `step` names the result, so `press` reports itself rather than the general + /// mechanism it is a shorthand for. + private static func perform<E: Element>( + _ action: String, + on identifier: String, + in root: E, + step: String + ) throws(DriveError) -> StepResult { + let path = try find(identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: nil + ) + } + + // Checked before performing, so the error can name what the element does + // accept. Performing an unsupported action answers `action_unsupported` + // with nothing to act on. + let actions = element.actions + guard actions.contains(action) else { + throw DriveError( + kind: .actionUnsupported, + message: "\(identifier) does not accept \(action)", + hint: actions.isEmpty + ? "it advertises no actions at all; a list row is activated with `select`" + : "it accepts: \(actions.joined(separator: ", "))" + ) + } + + let status = element.perform(action) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "performing \(action) on \(identifier) answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: step, + identifier: identifier, + role: element.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: nil + ) + } + + /// Put text into the identified field. + private static func type<E: Element>( + _ target: Step.TypeTarget, in root: E + ) throws(DriveError) + -> StepResult + { + let path = try find(target.identifier, from: root) + guard let element = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(target.identifier)", + hint: nil + ) + } + + guard element.isSettable(kAXValueAttribute) else { + throw DriveError( + kind: .notEditable, + message: "\(target.identifier) does not accept a write to AXValue", + hint: "a static label and a disabled field both look like this; check with " + + "`jpdrive dump --settable`" + ) + } + + let status = element.setText(kAXValueAttribute, target.text) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXValue to \(target.identifier) answered \(status.name)", + hint: nil + ) + } + + let committed = try confirm(element, identifier: target.identifier) + + // `confirmed` says the field holds the text, and nothing more. Whether the + // application reacted is the caller's assertion to make, against whatever + // the typing was supposed to change. + let after = element.read([kAXValueAttribute, kAXRoleAttribute]) + + return StepResult( + step: "type", + identifier: target.identifier, + role: after.text[1] ?? "<none>", + confirmed: after.text[0] == target.text, + committed: committed + ) + } + + /// Commit an edit, if the element offers a way to. + /// + /// Answers whether it did. An element with no confirm action is not a failure: + /// some fields publish every change as it happens and need nothing further. + private static func confirm<E: Element>( + _ element: E, identifier: String + ) throws(DriveError) -> Bool { + guard element.actions.contains(kAXConfirmAction) else { return false } + + let status = element.perform(kAXConfirmAction) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "confirming \(identifier) answered \(status.name)", + hint: + "the text was written but not committed, so the application has not seen it" + ) + } + + return true + } + + /// Attributes read while walking a menu path. + private static let menuBatch = [ + kAXRoleAttribute, + AXElement.attributedDescription, + kAXDescriptionAttribute, + kAXTitleAttribute, + ] + + /// How long to wait for the application to come forward, and for the item + /// addressed through it to be enabled. + static let activationTimeout = Duration.milliseconds(2000) + + /// Press the menu item at the end of a titled path. + private static func menu<E: Element>( + _ target: Step.MenuTarget, in root: E, within timeout: Duration + ) throws(DriveError) + -> StepResult + { + guard !target.path.isEmpty else { + throw DriveError( + kind: .badUsage, + message: "a menu step needs a path, such as [\"File\", \"Close\"]", + hint: nil + ) + } + + let start: E + let origin: String + + if let owner = target.under { + // A menu already on screen. No activation: showing it required the + // application to be active, and asking again would be a no-op at best. + start = try shownMenu(of: owner, in: root) + origin = "'\(owner)' is showing a menu that" + } else { + // The one step that takes focus from whatever had it. AppKit disables + // every menu item that acts on the front window or on the responder + // chain while the application is in the background, which is most of + // the menu bar: without this, a path resolves to an item that cannot + // be pressed. + try front(root, within: timeout) + + guard let bar = root.elements(kAXMenuBarAttribute).first else { + throw DriveError( + kind: .notFound, + message: "the application reports no menu bar", + hint: "an agent or accessory application has none" + ) + } + start = bar + origin = "the menu bar" + } + + var current = start + var reached: [String] = [] + + for title in target.path { + guard let next = child(titled: title, of: current) else { + throw DriveError( + kind: .notFound, + message: reached.isEmpty + ? "\(origin) holds no item titled '\(title)'" + : "'\(reached.joined(separator: " > "))' holds no item titled '\(title)'", + hint: "it holds: \(titles(of: current).joined(separator: ", "))" + ) + } + current = next + reached.append(title) + } + + let path = target.path.joined(separator: " > ") + try waitUntilEnabled(current, named: path, within: timeout) + + let actions = current.actions + guard actions.contains(kAXPressAction) else { + throw DriveError( + kind: .actionUnsupported, + message: "'\(path)' does not accept AXPress", + hint: actions.isEmpty + ? "the path names a submenu rather than an item; name the item inside it" + : "it accepts: \(actions.joined(separator: ", "))" + ) + } + + let status = current.perform(kAXPressAction) + guard status == .success else { + throw DriveError( + kind: .actionFailed, + message: "pressing '\(path)' answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: "menu", + identifier: path, + role: current.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: nil + ) + } + + /// The menu an element is currently displaying. + /// + /// A shown menu hangs off the element that opened it, after that element's + /// own children, which is why a capped or filtered read passes straight over + /// it. + private static func shownMenu<E: Element>( + of identifier: String, in root: E + ) throws(DriveError) -> E { + let path = try find(identifier, from: root) + guard let owner = path.last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: nil + ) + } + + let children = owner.read([kAXRoleAttribute]).children + for child in children where child.read([kAXRoleAttribute]).text[0] == kAXMenuRole { + return child + } + + throw DriveError( + kind: .notFound, + message: "\(identifier) is not showing a menu", + hint: """ + open one first, in an earlier step: \ + {"perform": {"identifier": "\(identifier)", "action": "AXShowMenu"}} + """ + ) + } + + /// Bring the application forward, and wait until it reports that it is. + /// + /// Writing `AXFrontmost` is a request. The window server grants it a moment + /// later, and whatever depends on it later still. + /// + /// Failing rather than carrying on is the point. Every caller does something + /// that only means what it says once the application is in front: a menu item + /// acting on the front window is disabled until then, and a synthesized + /// pointer event goes to whoever owns the screen coordinate, which is the + /// application the person at the keyboard is using. + private static func front<E: Element>( + _ root: E, within timeout: Duration + ) throws(DriveError) { + guard root.flag(kAXFrontmostAttribute) != true else { return } + + let status = root.setFlag(kAXFrontmostAttribute, true) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "bringing the application forward answered \(status.name)", + hint: "the step cannot be trusted to reach the app while it is behind another" + ) + } + + guard poll(untilTrue: { root.flag(kAXFrontmostAttribute) == true }, within: timeout) + else { + throw DriveError( + kind: .timeout, + message: + "the application did not come forward within \(timeout.milliseconds)ms", + hint: "another application may be holding focus with a modal panel" + ) + } + } + + /// Wait for an item to stop reporting itself disabled. + /// + /// An element that reports no `AXEnabled` at all is not disabled: plenty + /// carry no such attribute, and treating its absence as a refusal would + /// reject every one of them. + private static func waitUntilEnabled<E: Element>( + _ item: E, named path: String, within timeout: Duration + ) throws(DriveError) { + if poll(untilTrue: { item.flag(kAXEnabledAttribute) != false }, within: timeout) { + return + } + + throw DriveError( + kind: .disabled, + message: "'\(path)' is disabled", + hint: + "an item acting on a selection is disabled while nothing is selected, and one " + + "acting on the front window while no window has focus" + ) + } + + /// Poll `condition` until it holds, or `timeout` elapses. + private static func poll(untilTrue condition: () -> Bool, within timeout: Duration) -> Bool + { + let clock = ContinuousClock() + let started = clock.now + + while true { + if condition() { return true } + guard clock.now - started < timeout else { return false } + Thread.sleep(forTimeInterval: defaultInterval.seconds) + } + } + + /// The child of `parent` whose title is `title`. + /// + /// Descends through `AXMenu`, which carries no title of its own: a bar item's + /// items live inside one, so a path names `["File", "Close"]` rather than + /// spelling out the container between them. + private static func child<E: Element>(titled title: String, of parent: E) -> E? { + for child in parent.read([]).children { + let text = child.read(menuBatch).text + + if text[1] ?? text[2] ?? text[3] == title { + return child + } + + guard text[0] == "AXMenu", let found = self.child(titled: title, of: child) else { + continue + } + return found + } + + return nil + } + + /// The titles a level offers, for saying what a path could have named instead. + private static func titles<E: Element>(of parent: E) -> [String] { + var found: [String] = [] + + for child in parent.read([]).children { + let text = child.read(menuBatch).text + + if let title = text[1] ?? text[2] ?? text[3], !title.isEmpty { + found.append(title) + continue + } + + // An untitled `AXMenu` is the container a path skips, so what it holds + // is what this level effectively offers. + guard text[0] == "AXMenu" else { continue } + found.append(contentsOf: titles(of: child)) + } + + return found + } + + /// Select the row that owns the identified element. + private static func select<E: Element>( + _ target: Step.Target, in root: E + ) throws(DriveError) + -> StepResult + { + let path = try find(target.identifier, from: root) + + guard let owner = selectionOwner(in: path) else { + throw DriveError( + kind: .notSelectable, + message: + "neither \(target.identifier) nor its \(maxAncestors) nearest ancestors accept " + + "a write to AXSelected", + hint: "check what the element reports with: jpdrive dump --settable" + ) + } + + let status = owner.setFlag(kAXSelectedAttribute, true) + guard status == .success else { + throw DriveError( + kind: .writeFailed, + message: "writing AXSelected to \(target.identifier) answered \(status.name)", + hint: nil + ) + } + + return StepResult( + step: "select", + identifier: target.identifier, + role: owner.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: owner.flag(kAXSelectedAttribute) ?? false + ) + } + + /// Default time to keep polling for an element to appear. + private static let defaultTimeout = Duration.milliseconds(5000) + + /// Default pause between polling attempts. + private static let defaultInterval = Duration.milliseconds(100) + + /// Wait until an element with the target's identifier exists. + /// + /// Returns as soon as it is found, including on the first attempt when it was + /// already there. + private static func waitFor<E: Element>( + _ target: Step.WaitTarget, in root: E + ) throws(DriveError) + -> StepResult + { + // Resolved once, before the loop. This is the expensive search, and paying + // it on every attempt is what makes an unscoped wait useless. + let scope: E + if let under = target.under { + guard let container = try find(under, from: root).last else { + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(under) to wait inside", + hint: "`under` names a container that must already exist" + ) + } + scope = container + } else { + scope = root + } + + let timeout = target.timeoutMs.map { Duration.milliseconds($0) } ?? defaultTimeout + let interval = target.intervalMs.map { Duration.milliseconds($0) } ?? defaultInterval + + let clock = ContinuousClock() + let started = clock.now + var attempts = 0 + + while true { + attempts += 1 + + if let path = try? find(target.identifier, from: scope), let found = path.last { + return StepResult( + step: "wait_for", + identifier: target.identifier, + role: found.read([kAXRoleAttribute]).text[0] ?? "<none>", + confirmed: true + ) + } + + guard clock.now - started < timeout else { break } + Thread.sleep(forTimeInterval: interval.seconds) + } + + let elapsed = clock.now - started + throw DriveError( + kind: .timeout, + message: + "\(target.identifier) did not appear within \(timeout.milliseconds)ms " + + "(\(attempts) attempts over \(elapsed.milliseconds)ms)", + hint: attempts == 1 + ? "one attempt exhausted the timeout; scope the search with `under`" + : nil + ) + } + + /// The nearest element at or above the end of `path` that accepts a write to + /// `AXSelected`. + /// + /// Walks the chain the search descended rather than reading `AXParent`. The + /// identified element is a SwiftUI leaf that does not report a parent, so + /// climbing from it arrives nowhere, while the chain that reached it is known + /// for free and is not subject to that. + private static func selectionOwner<E: Element>(in path: [E]) -> E? { + for element in path.suffix(maxAncestors + 1).reversed() + where element.isSettable(kAXSelectedAttribute) { + return element + } + return nil + } + + /// What a search reads at each element. + /// + /// Children arrive alongside, so the identifier is all the search asks for. + private static let searchBatch = [kAXIdentifierAttribute] + + /// Find the element whose identifier is exactly `identifier`, and the chain of + /// elements that reached it. + /// + /// The path comes back rather than the element alone because acting on an + /// element often means acting on one of its ancestors, and this tree cannot + /// reliably be walked upwards. + /// + /// Depth-first with an early exit, reading only what the search needs. Reading + /// every attribute of each element on the way past would make a step against + /// this app's few thousand elements cost seconds. + private static func find<E: Element>( + _ identifier: String, from root: E + ) throws(DriveError) + -> [E] + { + var stack = [[root]] + var unreadable = 0 + + while let path = stack.popLast() { + guard let element = path.last else { continue } + let reading = element.read(searchBatch) + + if reading.failed { + unreadable += 1 + } + + if reading.text[0] == identifier { + return path + } + + // Reversed, so a depth-first walk visits siblings in the order the + // application reports them. + for child in reading.children.reversed() { + stack.append(path + [child]) + } + } + + // A miss with a gap in it is not a miss. The element may sit under one of + // the branches that could not be read, and reporting a clean + // `identifier_not_found` invites the caller to change an identifier that + // was right all along. + guard unreadable == 0 else { + throw DriveError( + kind: .readFailed, + message: + "\(identifier) was not found, and \(unreadable) element(s) could not be " + + "read", + hint: "the application may be busy or exiting; the identifier may exist" + ) + } + + throw DriveError( + kind: .identifierNotFound, + message: "no element has the identifier \(identifier)", + hint: "list what is addressable with: jpdrive tree --identifier <prefix>" + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift new file mode 100644 index 000000000..0ad381f26 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Ambient.swift @@ -0,0 +1,80 @@ +import AppKit +import Foundation + +/// The state a driven run borrows from whoever is at the keyboard. +/// +/// Which application is in front, and where the pointer is. Neither belongs to +/// the app under test: a synthesized gesture has to take both — mouse events go +/// to whatever is on top at a coordinate, and the ordering between applications +/// follows activation — and a run that takes them owes them back. +/// +/// Deliberately not window geometry. A step that resizes a window did the thing +/// it was asked to do, and putting the window back would undo the effect under +/// test. What a run borrows is restored; what it was told to change is not. +/// +/// Read and written separately rather than as one capture-and-restore pair, so a +/// caller composes what it needs and decides for itself when a restore is owed. +enum Ambient { + /// The bundle identifier of the frontmost application. + /// + /// `nil` when there is none, or when it has no identifier — a process + /// launched without a bundle has neither. + static func frontmost() -> FrontmostReport { + FrontmostReport(bundleID: NSWorkspace.shared.frontmostApplication?.bundleIdentifier) + } + + /// Bring the application with `bundleID` back to the front. + /// + /// Through `NSWorkspace`, which asks the application to activate itself, so + /// this needs no permission beyond launching one. Answers whether an + /// application with that identifier was found to ask. + static func activate(bundleID: String) -> FrontmostReport { + guard + let app = NSRunningApplication.runningApplications(withBundleIdentifier: bundleID) + .first + else { + return FrontmostReport(bundleID: nil) + } + + app.activate() + return FrontmostReport(bundleID: bundleID) + } + + /// Where the pointer is, in the coordinates a synthesized event uses. + /// + /// `NSEvent.mouseLocation` is bottom-left origin and screen coordinates are + /// top-left, so the y is flipped here rather than at each call site. The + /// height flipped against is the *main* screen's, which is what the window + /// server measures global coordinates from. + static func pointer() -> PointerReport { + let location = NSEvent.mouseLocation + let height = NSScreen.screens.first?.frame.height ?? 0 + + return PointerReport(x: location.x, y: height - location.y) + } + + /// Put the pointer back at `point`. + /// + /// Warped rather than moved: `CGWarpMouseCursorPosition` relocates the cursor + /// without synthesizing motion, so nothing under it takes a hover, and no + /// application sees a gesture it has to interpret. + static func movePointer(to point: CGPoint) -> PointerReport { + CGWarpMouseCursorPosition(point) + return PointerReport(x: point.x, y: point.y) + } +} + +/// Which application is in front. +struct FrontmostReport: Encodable, Equatable { + let bundleID: String? + + private enum CodingKeys: String, CodingKey { + case bundleID = "bundle_id" + } +} + +/// Where the pointer is, in top-left-origin screen coordinates. +struct PointerReport: Encodable, Equatable { + let x: CGFloat + let y: CGFloat +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift new file mode 100644 index 000000000..64fdbcfb0 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Arguments.swift @@ -0,0 +1,425 @@ +import Foundation + +/// A parsed command line. +enum Command { + /// Report whether this process may read another app's accessibility tree. + case doctor(pid: pid_t?) + + /// Print the elements and attributes under an application. + case dump(DumpOptions) + + /// Report the elements under an application, identified and described. + case tree(TreeOptions) + + /// List the application's windows. + case windows(pid: pid_t) + + /// Report the window-server identifiers of the application's windows. + case windowid(pid: pid_t) + + /// Report the application's menu bar. + case menu(pid: pid_t, options: TreeOptions) + + /// Do one thing to one element. + case act(step: Step, pid: pid_t) + + /// Report the colours along one row or column of a screenshot. + case pixels(PixelOptions) + + /// Report which application is in front, or put one there. + case frontmost(set: String?) + + /// Report where the pointer is, or put it somewhere. + case pointer(set: CGPoint?) +} + +/// What to walk, and how much of it. +struct DumpOptions { + let pid: pid_t + + /// How deep to recurse before reporting a node's children as elided. + let maxDepth: Int + + /// How many children to walk at each level, or `0` for all of them. + let maxSiblings: Int + + /// Whether to ask, per attribute, if it can be written. + let settable: Bool +} + +/// The command line the driver accepts. +/// +/// Hand-rolled rather than pulled from `swift-argument-parser`: the package has +/// no other dependency, and keeping it that way means the build needs no network +/// and no resolved manifest. +enum Arguments { + /// Usage text, embedded in every bad-usage error. + static let usage = """ + usage: jpdrive doctor [--pid <pid>] + jpdrive tree --pid <pid> [--identifier <prefix>] [--max-matches <n>] + [--frames] [--depth <n>] [--max-siblings <n>] + jpdrive windows --pid <pid> + jpdrive windowid --pid <pid> + jpdrive menu --pid <pid> [--depth <n>] [--max-siblings <n>] + jpdrive dump --pid <pid> [--depth <n>] [--max-siblings <n>] [--settable] + jpdrive act --pid <pid> --json '<step>' + a step is a single-key object, e.g. + {"resize":{"identifier":"w","width":1400,"height":900}} + jpdrive frontmost [--set <bundle-id>] + jpdrive pointer [--set <x>,<y>] + jpdrive pixels --image <path> --scan row|column --at <n> + [--from <n>] [--to <n>] + """ + + /// Depth cap for `dump` when `--depth` is not given. + /// + /// A SwiftUI window nests deeply: the wrapper groups between a `List` and its + /// rows are several levels on their own, so a cap low enough to be tidy hides + /// the elements worth seeing. + static let defaultDepth = 20 + + /// Sibling cap for `dump` when `--max-siblings` is not given. + /// + /// A thousand sidebar rows are a thousand copies of one shape, and walking + /// them all costs a round-trip per attribute per element. Five is enough to + /// see the shape and to tell a homogeneous list from a mixed one. + static let defaultSiblings = 5 + + /// Match budget for a filtered `tree` when `--max-matches` is not given. + /// + /// Identifiers sit on leaves, so a prefix search cannot prune on the way down + /// and an unbounded one reads every element in the application. Five answers + /// what a list looks like; looking up one known identifier wants `1`. + static let defaultMatches = 5 + + /// Parse `arguments`, which excludes the executable path. + static func parse(_ arguments: [String]) throws(DriveError) -> Command { + guard let subcommand = arguments.first else { + throw DriveError(kind: .badUsage, message: usage, hint: nil) + } + + let options = try options(arguments.dropFirst()) + + switch subcommand { + case "doctor": + return .doctor(pid: options.pid) + + case "dump": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, + message: "dump needs --pid <pid>", + hint: usage + ) + } + return .dump( + DumpOptions( + pid: pid, + maxDepth: options.depth ?? defaultDepth, + maxSiblings: options.siblings ?? defaultSiblings, + settable: options.settable + ) + ) + + case "tree": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "tree needs --pid <pid>", hint: usage) + } + return .tree( + TreeOptions( + pid: pid, + identifierPrefix: options.identifier, + maxMatches: options.matches ?? defaultMatches, + maxDepth: options.depth ?? defaultDepth, + maxSiblings: options.siblings ?? defaultSiblings, + frames: options.frames + ) + ) + + case "frontmost": + return .frontmost(set: options.set) + + case "pointer": + guard let raw = options.set else { + return .pointer(set: nil) + } + + let parts = raw.split(separator: ",") + guard + parts.count == 2, + let x = Double(parts[0].trimmingCharacters(in: .whitespaces)), + let y = Double(parts[1].trimmingCharacters(in: .whitespaces)) + else { + throw DriveError( + kind: .badUsage, + message: "pointer --set takes <x>,<y>", + hint: usage + ) + } + return .pointer(set: CGPoint(x: x, y: y)) + + case "windows": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "windows needs --pid <pid>", hint: usage) + } + return .windows(pid: pid) + + case "windowid": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "windowid needs --pid <pid>", hint: usage) + } + return .windowid(pid: pid) + + case "menu": + guard let pid = options.pid else { + throw DriveError( + kind: .badUsage, message: "menu needs --pid <pid>", hint: usage) + } + return .menu( + pid: pid, + options: TreeOptions( + pid: pid, + identifierPrefix: options.identifier, + maxMatches: options.matches ?? defaultMatches, + maxDepth: options.depth ?? defaultDepth, + // A menu bar is a couple of hundred elements and every one of + // them is a thing you might press, so the default that keeps a + // thousand-row list readable would only hide half the verbs. + maxSiblings: options.siblings ?? 0, + frames: options.frames + ) + ) + + case "act": + guard let pid = options.pid else { + throw DriveError(kind: .badUsage, message: "act needs --pid <pid>", hint: usage) + } + guard let json = options.json else { + throw DriveError( + kind: .badUsage, message: "act needs --json '<step>'", hint: usage) + } + + let step: Step + do { + step = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + } catch { + throw DriveError( + kind: .badUsage, + message: "could not read the step: \(error)", + hint: #"a step is a single-key object, e.g. {"select":{"identifier":"…"}}"# + ) + } + + return .act(step: step, pid: pid) + + case "pixels": + guard let image = options.image else { + throw DriveError( + kind: .badUsage, message: "pixels needs --image <path>", hint: usage) + } + guard let scan = options.scan else { + throw DriveError( + kind: .badUsage, + message: "pixels needs --scan row or --scan column", + hint: usage + ) + } + guard let at = options.at else { + throw DriveError( + kind: .badUsage, message: "pixels needs --at <n>", hint: usage) + } + + return .pixels( + PixelOptions( + image: image, + axis: scan, + at: at, + from: options.from, + to: options.to + ) + ) + + default: + throw DriveError( + kind: .badUsage, + message: "unknown subcommand '\(subcommand)'", + hint: usage + ) + } + } + + /// Flags accepted by any subcommand, whether or not that subcommand reads + /// them. Keeping one parser means `--pid` behaves identically everywhere. + private struct Options { + var pid: pid_t? + var depth: Int? + var siblings: Int? + var matches: Int? + var settable = false + var identifier: String? + var json: String? + var frames = false + var image: String? + var scan: PixelOptions.Axis? + var at: Int? + var from: Int? + var to: Int? + var set: String? + } + + private static func options(_ arguments: ArraySlice<String>) throws(DriveError) -> Options { + var options = Options() + var rest = arguments.makeIterator() + + while let argument = rest.next() { + switch argument { + case "--pid": + guard let value = rest.next(), + let raw = Int(value), + let pid = pid_t(exactly: raw) + else { + throw DriveError( + kind: .badUsage, + message: "--pid takes an integer process id", + hint: """ + \(usage). `--pid $(pgrep -f JP.app)` expands to nothing \ + when the app is not running, which lands here rather \ + than reporting app_not_running + """ + ) + } + options.pid = pid + + case "--depth": + guard let value = rest.next(), let depth = Int(value), depth > 0 else { + throw DriveError( + kind: .badUsage, + message: "--depth takes a positive integer", + hint: usage + ) + } + options.depth = depth + + case "--max-siblings": + guard let value = rest.next(), let siblings = Int(value), siblings >= 0 else { + throw DriveError( + kind: .badUsage, + message: + "--max-siblings takes a non-negative integer, where 0 means all", + hint: usage + ) + } + options.siblings = siblings + + case "--settable": + options.settable = true + + case "--max-matches": + guard let value = rest.next(), let matches = Int(value), matches > 0 else { + throw DriveError( + kind: .badUsage, + message: "--max-matches takes a positive integer", + hint: usage + ) + } + options.matches = matches + + case "--frames": + options.frames = true + + case "--identifier": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--identifier takes a value", + hint: usage + ) + } + options.identifier = value + + case "--set": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--set takes a value", + hint: usage + ) + } + options.set = value + + case "--json": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--json takes a value", + hint: usage + ) + } + options.json = value + + case "--image": + guard let value = rest.next() else { + throw DriveError( + kind: .badUsage, + message: "--image takes a path", + hint: usage + ) + } + options.image = value + + case "--scan": + guard let value = rest.next(), let axis = PixelOptions.Axis(rawValue: value) + else { + throw DriveError( + kind: .badUsage, + message: "--scan takes `row` or `column`", + hint: usage + ) + } + options.scan = axis + + case "--at": + guard let value = rest.next(), let at = Int(value), at >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--at takes a non-negative integer", + hint: usage + ) + } + options.at = at + + case "--from": + guard let value = rest.next(), let from = Int(value), from >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--from takes a non-negative integer", + hint: usage + ) + } + options.from = from + + case "--to": + guard let value = rest.next(), let to = Int(value), to >= 0 else { + throw DriveError( + kind: .badUsage, + message: "--to takes a non-negative integer", + hint: usage + ) + } + options.to = to + + default: + throw DriveError( + kind: .badUsage, + message: "unknown argument '\(argument)'", + hint: usage + ) + } + } + + return options + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift new file mode 100644 index 000000000..b6a474a2b --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Doctor.swift @@ -0,0 +1,95 @@ +import ApplicationServices +import Foundation + +/// What `jpdrive doctor` observed. +struct DoctorReport: Encodable { + /// `AXIsProcessTrusted()` for this process. + let trusted: Bool + + /// This process and its ancestors, nearest first. One of these holds the + /// Accessibility grant when `trusted` is true. + let processes: [ProcessLink] + + /// A real read against a target app, present when `--pid` was given. + let probe: WindowProbe? +} + +/// The outcome of reading a target application's window list. +struct WindowProbe: Encodable { + let pid: pid_t + + /// The target's short command name. + let command: String + + /// How many windows were read, when the read succeeded. + let windowCount: Int? + + /// The accessibility error, when it did not. + let axError: String? +} + +/// Answers whether this process may read another application's accessibility +/// tree, and records the evidence for why. +/// +/// `AXIsProcessTrusted()` alone is not enough: it reports what TCC believes +/// about the responsible process, which is not always the process making the +/// call. So the report pairs the flag with a real `AXUIElementCopyAttributeValue` +/// against a live app, and with the ancestor chain the grant might be attributed +/// to. Apple documents neither the attribution algorithm nor its stability, so +/// this records observations rather than asserting a rule. +enum Doctor { + /// Run every probe and collect the results. + /// + /// Throws only when `pid` names a process that is not running. A refused + /// accessibility read is an observation the report carries, not a failure of + /// the diagnostic. + static func run(targetPid pid: pid_t?) throws(DriveError) -> DoctorReport { + let probe: WindowProbe? + if let pid { + guard let record = ProcessTable.record(for: pid) else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + probe = windowProbe(pid: pid, command: ProcessTable.name(of: record)) + } else { + probe = nil + } + + return DoctorReport( + trusted: AXIsProcessTrusted(), + processes: ProcessTable.ancestry(from: getpid()), + probe: probe + ) + } + + /// Read the target's window list, reporting the accessibility error instead + /// of the count when the read is refused. + /// + /// Uses the non-prompting trust path throughout: a spike that raises the + /// system's "grant access" dialog changes the state it is measuring. + private static func windowProbe(pid: pid_t, command: String) -> WindowProbe { + let app = AXUIElementCreateApplication(pid) + var value: CFTypeRef? + let status = AXUIElementCopyAttributeValue(app, kAXWindowsAttribute as CFString, &value) + + guard status == .success else { + return WindowProbe( + pid: pid, + command: command, + windowCount: nil, + axError: status.name + ) + } + + let windows = value as? [AXUIElement] + return WindowProbe( + pid: pid, + command: command, + windowCount: windows?.count ?? 0, + axError: nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift new file mode 100644 index 000000000..2f95cd25a --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/DriveError.swift @@ -0,0 +1,85 @@ +import Foundation + +/// A failure reported as JSON on stdout, alongside a non-zero exit status. +/// +/// Every exit path produces either a result document or one of these, so a +/// caller never has to scrape prose off stderr to find out what happened. +struct DriveError: Error, Encodable { + /// Machine-readable discriminator. Callers switch on this; the message is + /// for humans and may be reworded freely. + enum Kind: String, Encodable { + /// The command line named an unknown subcommand or was missing a value. + case badUsage = "bad_usage" + + /// No process is running under the given pid. + case appNotRunning = "app_not_running" + + /// The accessibility API refused the request for want of a TCC grant. + case notPermitted = "not_permitted" + + /// No element carries the identifier the step addressed. + case identifierNotFound = "identifier_not_found" + + /// The addressed element and its nearest ancestors do not accept a write + /// to `AXSelected`. + case notSelectable = "not_selectable" + + /// An attribute write was refused by the accessibility API. + case writeFailed = "write_failed" + + /// The addressed element does not accept a write to its value. + case notEditable = "not_editable" + + /// The addressed element reports nowhere on screen to click. + case notClickable = "not_clickable" + + /// The addressed element does not accept the action the step performs. + case actionUnsupported = "action_unsupported" + + /// The addressed element is present but refuses to act while disabled. + case disabled = "disabled" + + /// An action was refused by the accessibility API. + case actionFailed = "action_failed" + + /// An element waited for did not appear in time. + case timeout = "timeout" + + /// The application reports no element of the requested kind. + case notFound = "not_found" + + /// The accessibility API refused to read part of the tree, so what it + /// holds is unknown rather than known to be absent. + case readFailed = "read_failed" + + /// The result could not be encoded as JSON. + case encodingFailed = "encoding_failed" + } + + let kind: Kind + + /// One sentence saying what went wrong. + let message: String + + /// What the operator can do about it, when there is something to do. + var hint: String? +} + +extension DriveError { + /// Names the System Settings pane that grants Accessibility. + /// + /// macOS attributes the grant to the responsible process, which for a + /// command-line tool is normally the terminal rather than the tool, so this + /// points at the terminal and not at `jpdrive`. + static let accessibilityHint = """ + grant Accessibility to the terminal application running this command, \ + under System Settings > Privacy & Security > Accessibility, then start \ + a new terminal session + """ +} + +/// Envelope that makes an error document distinguishable from a result document +/// by its top-level key alone. +struct ErrorDocument: Encodable { + let error: DriveError +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift new file mode 100644 index 000000000..d3e222961 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Driver.swift @@ -0,0 +1,68 @@ +import Foundation + +/// The driver's entry point. +/// +/// Everything below this is internal to the library, so the tests reach it with +/// `@testable import` and the executable target stays a single line. +public enum Driver { + /// Parse the process arguments, run the command, and exit. + /// + /// Writes one JSON document to stdout either way: a result, or an error with a + /// non-zero exit status. + public static func run() -> Never { + do throws(DriveError) { + try dispatch(Array(CommandLine.arguments.dropFirst())) + } catch { + Output.writeError(error) + exit(1) + } + + exit(0) + } + + /// Run one command and write its result. + static func dispatch(_ arguments: [String]) throws(DriveError) { + switch try Arguments.parse(arguments) { + case .doctor(let pid): + try Output.write(try Doctor.run(targetPid: pid)) + + case .dump(let options): + try Output.write(try Dump.walk(options)) + + case .tree(let options): + guard let tree = try Tree.read(options) else { + throw DriveError( + kind: .identifierNotFound, + message: + "no element's identifier begins with \(options.identifierPrefix ?? "")", + hint: "drop --identifier to see what the application reports" + ) + } + try Output.write(tree) + + case .windows(let pid): + try Output.write(try Windows.read(pid: pid)) + + case .windowid(let pid): + try Output.write(try WindowIDs.read(pid: pid)) + + case .menu(let pid, let options): + try Output.write(try Menu.read(pid: pid, options: options)) + + case .act(let step, let pid): + try Output.write(try Act.run(step, pid: pid)) + + case .pixels(let options): + try Output.write(try Pixels.read(options)) + + case .frontmost(let set): + let report = + if let set { Ambient.activate(bundleID: set) } else { Ambient.frontmost() } + try Output.write(report) + + case .pointer(let set): + let report = if let set { Ambient.movePointer(to: set) } else { Ambient.pointer() } + try Output.write(report) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift new file mode 100644 index 000000000..48ab5ec25 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Dump.swift @@ -0,0 +1,106 @@ +import ApplicationServices +import Foundation + +/// One element of an application's accessibility tree, with everything it reports. +struct DumpNode: Encodable, Equatable { + /// `AXRole`, lifted out of the attributes because it is what a reader scans + /// for. + let role: String + + /// Every attribute the element reports, minus the two that only lead back into + /// the tree, sorted by name. + let attributes: [Attribute] + + /// Actions the element accepts, such as `AXPress`. + let actions: [String] + + let children: [DumpNode] + + /// How many children were dropped to keep the walk bounded. + /// + /// Absent when every child was walked. A sidebar of a thousand conversations + /// repeats one row shape a thousand times, so the count is the useful part and + /// the repetition is not. + let elidedChildren: Int? + + /// Set when the accessibility API refused to read this element. + /// + /// Absent otherwise. Without it an element that could not be read reports no + /// attributes and no children, which is exactly how an element that has + /// neither reports — and a reader has no way to tell a gap in the walk from + /// a leaf. + let unreadable: Bool? +} + +/// Walks an application's accessibility tree and reports everything it finds. +/// +/// This is a design instrument. SwiftUI's mapping onto accessibility elements is +/// undocumented and not one-to-one, so decisions about how to address and act on +/// an element are made by reading a real dump rather than by predicting where a +/// `.accessibilityIdentifier` lands. +/// +/// Unfiltered by intent: every attribute of every element it visits, so nothing +/// that turns out to matter has been quietly dropped. [`Tree`](Tree) is the +/// filtered counterpart for everyday use. +enum Dump { + /// Walk the tree rooted at the application owning `options.pid`. + static func walk(_ options: DumpOptions) throws(DriveError) -> DumpNode { + guard ProcessTable.record(for: options.pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(options.pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + // Checked up front rather than reported per element: without the grant + // every read fails, and a tree of identical refusals says less than one + // error naming the pane that fixes it. + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return node(AXElement.application(pid: options.pid), depth: 0, options: options) + } + + /// Attributes that only lead back into the tree, and so are not recorded. + /// + /// `AXChildren` is what the walk recurses into, and `AXParent` points at the + /// element that just reported this one. + private static let structuralAttributes: Set<String> = [ + kAXChildrenAttribute, + kAXParentAttribute, + ] + + static func node<E: Element>(_ element: E, depth: Int, options: DumpOptions) -> DumpNode { + let reported = element.reportedAttributes(settable: options.settable) + let attributes = (reported ?? []) + .filter { !structuralAttributes.contains($0.name) } + + // Every child the element has, whether or not this walk descends into it. + // Counted before the depth cap is applied, because a node stopped at the + // cap is otherwise indistinguishable from a leaf. + let reading = element.read([]) + let available = reading.children + let descend = depth < options.maxDepth ? available : [] + let walked = + options.maxSiblings > 0 + ? Array(descend.prefix(options.maxSiblings)) + : descend + let elided = available.count - walked.count + let unreadable = reported == nil || reading.failed + + return DumpNode( + role: attributes.first { $0.name == kAXRoleAttribute }?.value ?? "<none>", + attributes: attributes, + actions: element.actions, + children: walked.map { node($0, depth: depth + 1, options: options) }, + elidedChildren: elided > 0 ? elided : nil, + unreadable: unreadable ? true : nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift new file mode 100644 index 000000000..09ef2dbce --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Duration.swift @@ -0,0 +1,15 @@ +import Foundation + +extension Duration { + /// The duration in whole milliseconds, for reporting. + var milliseconds: Int { + let (seconds, attoseconds) = components + return Int(seconds) * 1000 + Int(attoseconds / 1_000_000_000_000_000) + } + + /// The duration in seconds, for the APIs that take a `TimeInterval`. + var seconds: TimeInterval { + let (seconds, attoseconds) = components + return TimeInterval(seconds) + TimeInterval(attoseconds) / 1e18 + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift new file mode 100644 index 000000000..ae0bf24eb --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Element.swift @@ -0,0 +1,233 @@ +import ApplicationServices + +extension Optional where Wrapped == String { + /// The value read as a boolean. + /// + /// The accessibility API renders `AXEnabled`, `AXMain` and their like as `"0"` + /// or `"1"`. Anything else, including an absent attribute, is neither true nor + /// false. + var axFlag: Bool? { + switch self { + case "0": return false + case "1": return true + default: return nil + } + } +} + +/// Posts synthesized input to the window server. +/// +/// Behind a protocol because posting is the one thing the driver does that is not +/// addressed to an element. A click goes to whatever occupies a screen +/// coordinate, which is global state and cannot be exercised against a fake tree +/// the way every other step can. +protocol EventPoster { + /// Click once at `point`, in screen coordinates. + /// + /// Answers whether the events could be built and posted, which is not whether + /// anything received them. + func click(at point: CGPoint) -> Bool + + /// Press at the first point of `path`, move through the rest, release at the + /// last. + /// + /// `pause` separates one move from the next. Without it the moves are posted + /// faster than the target can consume them and the window server delivers a + /// coalesced few, which is the opposite of what a drag is usually being + /// synthesized to exercise: what a view does *during* the gesture, frame by + /// frame. + /// + /// Answers whether every event could be built and posted. + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool +} + +/// Posts through `CoreGraphics`. +struct SystemEventPoster: EventPoster { + func click(at point: CGPoint) -> Bool { + guard + let down = event(.leftMouseDown, at: point), + let up = event(.leftMouseUp, at: point) + else { + return false + } + + down.post(tap: .cghidEventTap) + up.post(tap: .cghidEventTap) + return true + } + + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool { + guard let first = path.first, let last = path.last else { return false } + guard let down = event(.leftMouseDown, at: first) else { return false } + + down.post(tap: .cghidEventTap) + + for point in path.dropFirst() { + guard let moved = event(.leftMouseDragged, at: point) else { + // Released wherever it got to rather than returned from. A drag + // abandoned with the button still down leaves the whole machine + // holding a mouse button nobody is pressing, which outlives this + // process and is not something a failed test should do to the + // person running it. + release(at: point) + return false + } + + moved.post(tap: .cghidEventTap) + Thread.sleep(forTimeInterval: pause.seconds) + } + + guard let up = event(.leftMouseUp, at: last) else { + release(at: last) + return false + } + + up.post(tap: .cghidEventTap) + return true + } + + /// Let the button go, on a path that could not be finished. + private func release(at point: CGPoint) { + event(.leftMouseUp, at: point)?.post(tap: .cghidEventTap) + } + + private func event(_ type: CGEventType, at point: CGPoint) -> CGEvent? { + CGEvent( + mouseEventSource: nil, + mouseType: type, + mouseCursorPosition: point, + mouseButton: .left + ) + } +} + +/// One attribute, as reported name and rendered value. +/// +/// A list of these rather than a dictionary, so attribute names reach the JSON +/// exactly as the accessibility API spells them. `JSONEncoder`'s snake-case key +/// strategy rewrites dictionary keys, which would turn `AXIdentifier` into +/// `ax_identifier` and make a dump a poor record of what the app reports. +struct Attribute: Encodable, Equatable { + let name: String + let value: String + + /// Whether the accessibility API reports this attribute as writable, when + /// settability was asked for. + /// + /// This decides how the driver changes state. Writing `AXSelected` on a row is + /// deterministic; synthesizing a click at a screen coordinate depends on the + /// window being frontmost and unobscured. + /// + /// Absent unless requested: answering it costs one round-trip per attribute, + /// which doubles the cost of a walk. + let settable: Bool? +} + +/// Attribute text and children, read together. +/// +/// The pair exists because reading them separately costs an extra round-trip per +/// element, and walking to a child is the most common read the driver makes. +struct Reading<E> { + /// One entry per requested name, positionally, `nil` where the element has no + /// value for that attribute. + let text: [String?] + + let children: [E] + + /// Whether the call that produced this failed as a whole. + /// + /// Distinct from a `nil` in ``text``, which says the element reports no value + /// for that one attribute. This says the accessibility API refused the read, + /// so every entry is `nil` and ``children`` is empty because nothing could be + /// asked — not because the element has none. A target that is busy or exiting + /// answers that way, and the two are indistinguishable without this. + let failed: Bool + + init(text: [String?], children: [E], failed: Bool = false) { + self.text = text + self.children = children + self.failed = failed + } +} + +/// One element of an accessibility tree, as the driver's traversal needs it. +/// +/// The traversal is where the driver's logic lives: pruning a filtered walk, +/// spending a match budget, finding which ancestor of an identified element owns +/// selection. None of that is about the accessibility API, and all of it has been +/// wrong at least once. Behind this protocol it can be tested against a fake tree +/// instead of against a running application. +/// +/// Deliberately narrow. Everything here is something a walk actually does, so a +/// fake stays small enough to read at a glance and cannot drift far from the real +/// implementation. +protocol Element { + /// Read the named attributes and the element's children. + /// + /// Implementations batch: this is one round-trip in the real one. + func read(_ names: [String]) -> Reading<Self> + + /// Every attribute the element reports, rendered and sorted by name. + /// + /// Separate from ``read(_:)``, which asks for a list of names known in + /// advance. A dump asks for whatever the element happens to have, which costs + /// an extra round-trip to enumerate and is why only a dump does it. + /// + /// `nil` when the read failed as a whole, which is not the same as an element + /// that reports no attributes. + func reportedAttributes(settable: Bool) -> [Attribute]? + + /// Actions the element accepts, such as `AXPress`. + /// + /// Separate from ``read(_:)`` because it costs its own round-trip and most + /// elements a filtered walk passes through are discarded unread. + var actions: [String] { get } + + /// Whether `name` can be written on this element. + func isSettable(_ name: String) -> Bool + + /// Read a boolean attribute, `nil` when it is absent or not a boolean. + func flag(_ name: String) -> Bool? + + /// Write a boolean attribute, answering the accessibility API's own status. + /// + /// A successful write is not a successful change: the target can accept the + /// value and do nothing with it. Read it back to find out. + func setFlag(_ name: String, _ value: Bool) -> AXError + + /// Write a string attribute, answering the accessibility API's own status. + func setText(_ name: String, _ value: String) -> AXError + + /// Perform an action, answering the accessibility API's own status. + /// + /// What the action did is not observable from here. Pressing a button runs + /// arbitrary code in the target, and success means the press was delivered, + /// not that anything came of it. + func perform(_ action: String) -> AXError + + /// The point held in an attribute, in screen coordinates. + /// + /// `nil` when the attribute is absent or holds something else. Separate from + /// ``read(_:)`` because a caller aiming a click needs the numbers, not the + /// text they render as. + func point(_ name: String) -> CGPoint? + + /// The size held in an attribute, in points. + /// + /// `nil` when the attribute is absent or holds something else. + func size(_ name: String) -> CGSize? + + /// Write a size attribute, answering the accessibility API's own status. + /// + /// As with every other write here, success is not change: a window clamps a + /// size to its own minimum and maximum, so what it ends up at has to be read + /// back. + func setSize(_ name: String, _ value: CGSize) -> AXError + + /// The elements held in an attribute, such as `AXWindows` or `AXMenuBar`. + /// + /// Answers a single element as a one-element array, since the accessibility + /// API spells "the menu bar" and "the windows" the same way apart from the + /// plural. + func elements(_ name: String) -> [Self] +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift new file mode 100644 index 000000000..a1c595b9d --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Menu.swift @@ -0,0 +1,52 @@ +import ApplicationServices +import Foundation + +/// Reads an application's menu bar. +/// +/// Reported as a tree, because a menu is one: bar, then bar items, then menus, +/// then items. What makes it worth its own subcommand is the root — reaching the +/// menu bar from the application element takes an attribute that holds it, not a +/// walk through the window hierarchy. +/// +/// Menu items are pressed with `act press`; they advertise `AXPress` where a list +/// row does not. +enum Menu { + /// Read the menu bar of the application owning `pid`. + static func read(pid: pid_t, options: TreeOptions) throws(DriveError) -> TreeNode { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + let app = AXElement.application(pid: pid) + + guard let bar = app.elements(kAXMenuBarAttribute).first else { + throw DriveError( + kind: .notFound, + message: "the application reports no menu bar", + hint: "an agent or accessory application has none" + ) + } + + guard let tree = Tree.walk(from: bar, options: options) else { + throw DriveError( + kind: .notFound, + message: "the menu bar held nothing matching", + hint: nil + ) + } + + return tree + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift new file mode 100644 index 000000000..47ca031d5 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Output.swift @@ -0,0 +1,44 @@ +import Foundation + +/// Writes the driver's JSON documents. +/// +/// Both results and errors go to stdout, so a caller reads one stream and +/// distinguishes the two by the top-level `error` key or by the exit status. +enum Output { + /// Encode `value` as pretty JSON on stdout, with a trailing newline. + static func write(_ value: some Encodable) throws(DriveError) { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + + // Snake case on the wire, matching every other JSON payload JP produces. + encoder.keyEncodingStrategy = .convertToSnakeCase + + let data: Data + do { + data = try encoder.encode(value) + } catch { + throw DriveError( + kind: .encodingFailed, + message: "could not encode the result as JSON: \(error)", + hint: nil + ) + } + + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data("\n".utf8)) + } + + /// Write an error document. + /// + /// Falls back to hand-built JSON, so a caller still gets something parseable + /// in the case where even the error will not encode. + static func writeError(_ error: DriveError) { + do throws(DriveError) { + try write(ErrorDocument(error: error)) + } catch { + let message = error.message.replacingOccurrences(of: "\"", with: "'") + let json = #"{"error":{"kind":"encoding_failed","message":"\#(message)"}}"# + "\n" + FileHandle.standardOutput.write(Data(json.utf8)) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift new file mode 100644 index 000000000..071752fd7 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Pixels.swift @@ -0,0 +1,286 @@ +import CoreGraphics +import Foundation +import ImageIO + +/// One stretch of identical pixels along a scanline. +struct PixelRun: Encodable, Equatable { + /// Where the run begins, in pixels along the scan. + let start: Int + + /// How many pixels it covers. + let count: Int + + /// The colour, `#RRGGBB` when opaque and `#RRGGBBAA` when it is not. + let color: String +} + +/// What one scan across an image found. +struct PixelReport: Encodable, Equatable { + /// The image's width in pixels, which on a retina display is twice its width + /// in points. + let width: Int + + /// The image's height in pixels. + let height: Int + + /// The colour space the values are reported in. + /// + /// Always sRGB. Stated anyway, because the numbers mean nothing without it: + /// the same screenshot read in the display's own profile and in sRGB gives two + /// different sets of values for the same pixels, and a light grey moves by + /// several steps between them. + /// + /// sRGB because that is the space colours are *written* in — a palette + /// constant, a value from a colour picker, a hex in a design note — so a + /// reading can be compared against the thing it was supposed to be. + let colorSpace: String + + /// Which way the scan ran: `row` or `column`. + let scan: String + + /// The row or column that was read, in pixels. + let at: Int + + /// The runs along it, in order, covering the scanned range without gaps. + let runs: [PixelRun] +} + +/// What to scan, and where. +struct PixelOptions { + /// Which way a scan runs. + enum Axis: String { + /// Left to right, across one row. + case row + + /// Top to bottom, down one column. + case column + } + + /// The PNG to read. + let image: String + + let axis: Axis + + /// The row or column to read, in pixels. + let at: Int + + /// Where along the scan to start, in pixels. The near edge when absent. + let from: Int? + + /// Where along the scan to stop, inclusive, in pixels. The far edge when + /// absent. + let to: Int? +} + +/// Reads the pixels of a screenshot. +/// +/// Answers the questions the accessibility tree cannot: what colour something is, +/// and how wide a drawn thing is. A hairline, a selection fill, a divider and a +/// row separator are all invisible to the tree, and all obvious in a scanline. +/// +/// Reads a file rather than capturing one. Capture already has a home +/// (`screencapture`, driven by `debug_app_screenshot`), and the only ways to +/// capture from inside this process are deprecated. It also makes this testable +/// against an image built by hand, with no window server and no grants. +enum Pixels { + /// Scan `options.image` and report the runs along the requested line. + static func read(_ options: PixelOptions) throws(DriveError) -> PixelReport { + let bitmap = try Bitmap(path: options.image) + let extent = options.axis == .row ? bitmap.width : bitmap.height + let across = options.axis == .row ? bitmap.height : bitmap.width + + guard options.at >= 0, options.at < across else { + throw DriveError( + kind: .notFound, + message: + "\(options.axis.rawValue) \(options.at) is outside the image, which is " + + "\(bitmap.width)x\(bitmap.height) pixels", + hint: "a row is indexed down from the top and a column across from the left" + ) + } + + // Checked against the image rather than against `to`, which is clamped + // below: a `--from` past the edge would otherwise be reported as being + // past a `--to` the caller never wrote. + let from = max(options.from ?? 0, 0) + guard from < extent else { + throw DriveError( + kind: .badUsage, + message: + "--from \(from) is outside the image, which is " + + "\(bitmap.width)x\(bitmap.height) pixels", + hint: "a scan along a \(options.axis.rawValue) runs from 0 to \(extent - 1)" + ) + } + + let to = min(options.to ?? extent - 1, extent - 1) + + guard from <= to else { + throw DriveError( + kind: .badUsage, + message: "--from \(from) is past --to \(to)", + hint: "both are pixel offsets along the scan, and --to is inclusive" + ) + } + + let line = (from...to).map { along in + options.axis == .row + ? bitmap.pixel(x: along, y: options.at) + : bitmap.pixel(x: options.at, y: along) + } + + return PixelReport( + width: bitmap.width, + height: bitmap.height, + colorSpace: bitmap.colorSpace, + scan: options.axis.rawValue, + at: options.at, + runs: runs(of: line, startingAt: from) + ) + } + + /// Collapse `line` into runs of one colour, the first starting at `start`. + /// + /// The whole point of the output shape: a scan across a window is thousands of + /// pixels and a handful of colours, and the edges between them are the + /// measurements a reader is after. + static func runs(of line: [Pixel], startingAt start: Int) -> [PixelRun] { + var runs: [PixelRun] = [] + + for (offset, pixel) in line.enumerated() { + if let last = runs.last, last.color == pixel.hex { + runs[runs.count - 1] = PixelRun( + start: last.start, count: last.count + 1, color: last.color) + continue + } + + runs.append(PixelRun(start: start + offset, count: 1, color: pixel.hex)) + } + + return runs + } +} + +/// One pixel, as read out of an image. +struct Pixel: Equatable { + let red: UInt8 + let green: UInt8 + let blue: UInt8 + let alpha: UInt8 + + /// `#RRGGBB` when opaque, `#RRGGBBAA` when not. + /// + /// Alpha is left off the common case so the values read the way a colour + /// picker reports them, and included when it is not 255 because a translucent + /// pixel that printed as opaque would be a lie about what is on screen. + var hex: String { + let rgb = String(format: "#%02X%02X%02X", red, green, blue) + return alpha == 255 ? rgb : rgb + String(format: "%02X", alpha) + } +} + +/// An image's pixels, in the image's own colour space. +private struct Bitmap { + let width: Int + let height: Int + let colorSpace: String + + /// RGBA, row-major, four bytes per pixel and no row padding. + private let bytes: [UInt8] + + /// Decode the PNG at `path`. + init(path: String) throws(DriveError) { + guard + let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw DriveError( + kind: .notFound, + message: "could not read an image at \(path)", + hint: "debug_app_screenshot writes one, and reports where it put it" + ) + } + + width = image.width + height = image.height + + // Converted rather than read raw. `screencapture` writes in the display's + // profile, which is often unnamed and never the space a palette was + // written in: a `#DBDBDB` divider comes back as `#D6D6D6` read that way, + // which looks like a bug in the app rather than a difference of space. + let target = CGColorSpace(name: CGColorSpace.sRGB) + + guard + let target, + let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: width * 4, + space: target, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + else { + throw DriveError( + kind: .notFound, + message: "could not open \(path) as an 8-bit RGBA image", + hint: nil + ) + } + + colorSpace = "sRGB" + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + + guard let data = context.data else { + throw DriveError( + kind: .notFound, + message: "the drawing context for \(path) reported no pixels", + hint: nil + ) + } + + bytes = [UInt8]( + UnsafeBufferPointer( + start: data.assumingMemoryBound(to: UInt8.self), + count: width * height * 4 + )) + } + + /// The pixel at `x`, `y`, counted from the top-left corner. + /// + /// The buffer runs in the same direction: a bitmap context's first row is the + /// top of what was drawn into it, so a screenshot's rows and this buffer's rows + /// are the same rows in the same order. + /// + /// Colour components are divided back out by alpha. The buffer holds them + /// premultiplied, which is the only 8-bit RGBA layout a bitmap context + /// accepts, and a window capture carries real partial alpha at its rounded + /// corners and antialiased edges. Read straight out, a half-transparent red + /// reports as a dark red — a colour that was never on screen, in the same + /// spelling as one that was. + func pixel(x: Int, y: Int) -> Pixel { + let offset = (y * width + x) * 4 + let alpha = bytes[offset + 3] + + return Pixel( + red: straight(bytes[offset], alpha), + green: straight(bytes[offset + 1], alpha), + blue: straight(bytes[offset + 2], alpha), + alpha: alpha + ) + } + + /// One premultiplied component, divided back out by `alpha`. + /// + /// A fully transparent pixel carries no colour to recover — every component + /// is zero whatever it was before — so it reports as zero rather than as a + /// division nobody can perform. + private func straight(_ component: UInt8, _ alpha: UInt8) -> UInt8 { + guard alpha != 0, alpha != 255 else { return component } + + let recovered = (Double(component) / Double(alpha) * 255).rounded() + + return UInt8(min(recovered, 255)) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift new file mode 100644 index 000000000..9aed316e4 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/ProcessTable.swift @@ -0,0 +1,58 @@ +import Darwin +import Foundation + +/// One process in the chain from the driver up towards `launchd`. +struct ProcessLink: Encodable { + let pid: pid_t + + /// Short command name from the kernel process table. The kernel truncates it + /// to 16 bytes, so `Terminal` and `iTerm2` arrive whole but a long binary + /// name does not. + let command: String +} + +/// Process identity read from the kernel through `sysctl(KERN_PROC_PID)`. +/// +/// The spike needs the ancestor chain because TCC attributes a grant to the +/// responsible process, and the chain is the list of candidates for that role. +enum ProcessTable { + /// Depth limit for the ancestor walk. A shell-to-`launchd` chain is a + /// handful of processes; the limit only guards against a process table that + /// changes underneath the walk. + private static let maxDepth = 32 + + /// `pid` and its ancestors, nearest first, stopping below `launchd`. + static func ancestry(from pid: pid_t) -> [ProcessLink] { + var links: [ProcessLink] = [] + var current = pid + + while current > 1, links.count < maxDepth { + guard let record = record(for: current) else { break } + links.append(ProcessLink(pid: current, command: name(of: record))) + current = record.kp_eproc.e_ppid + } + + return links + } + + /// The kernel's record for `pid`, or `nil` when no such process is running. + static func record(for pid: pid_t) -> kinfo_proc? { + var selector: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, pid] + var record = kinfo_proc() + var size = MemoryLayout<kinfo_proc>.stride + + let result = sysctl(&selector, u_int(selector.count), &record, &size, nil, 0) + + // Querying a pid that no longer exists succeeds and writes nothing, so + // the written size is what separates a dead pid from a live one. + guard result == 0, size > 0 else { return nil } + return record + } + + /// The short command name held in a kernel record. + static func name(of record: kinfo_proc) -> String { + return withUnsafeBytes(of: record.kp_proc.p_comm) { bytes in + return String(decoding: bytes.prefix { $0 != 0 }, as: UTF8.self) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift new file mode 100644 index 000000000..265149437 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Tree.swift @@ -0,0 +1,192 @@ +import ApplicationServices +import Foundation + +/// One element, as the driver reports it. +/// +/// An absent field means the element does not report that attribute. An +/// `AXUnknown` row carries a label and no value; a text field carries both. +struct TreeNode: Encodable, Equatable { + let role: String + let identifier: String? + let label: String? + let value: String? + let enabled: Bool? + let focused: Bool? + + /// The element's frame in screen coordinates, only when frames were asked for. + /// + /// Left out by default: coordinates change whenever a window moves or a list + /// scrolls, so including them turns every diff between two snapshots into + /// noise. + let frame: String? + + let actions: [String] + let children: [TreeNode] + + /// How many of this element's children are missing from ``children``. + /// + /// Every reason a child goes missing is counted the same, because they answer + /// one question: is there more here than I am looking at? The depth limit, the + /// per-level sibling cap, the match budget running out, and a filter discarding + /// a branch that held no match all leave the reader in the same position, and + /// the last two are the easiest to mistake for an element having no children at + /// all. + let elidedChildren: Int? + + /// Set when the accessibility API refused to read this element. + /// + /// Absent otherwise. An element that could not be read reports no identifier, + /// no label and no children, which is how a bare container reports too — so + /// without this a gap in the walk reads as a plain element. + let unreadable: Bool? +} + +/// What to walk, and what to keep. +struct TreeOptions { + let pid: pid_t + + /// Keep only elements whose identifier begins with this, along with the + /// ancestors that lead to them. `nil` keeps everything. + /// + /// A prefix rather than an exact match, because the useful question to ask of a + /// tree is "what is under `sidebar.`". Acting on an element is the opposite + /// case and matches exactly. + let identifierPrefix: String? + + /// How many matches to find before stopping. + /// + /// This is the bound that matters. Every identifier in this app's sidebar sits + /// on a leaf, so a prefix search cannot prune on the way down and an unbounded + /// one visits every element in the application. Stopping at a handful of + /// matches answers "what does the sidebar look like" for the cost of the first + /// handful rather than of all thousand. + /// + /// Set this to `1` when looking up one identifier already known, or the walk + /// continues past it looking for a second. + let maxMatches: Int + + let maxDepth: Int + let maxSiblings: Int + let frames: Bool +} + +/// Reads an application's accessibility tree into something a person can scan. +/// +/// Where [`Dump`](Dump) reports every attribute of every element for design work, +/// this reports the handful that identify and describe an element, and prunes +/// branches holding nothing that matched. +enum Tree { + /// Attributes read for every node, in one batch. Order matters, since values + /// come back positionally. + static let batch = [ + kAXRoleAttribute, + kAXIdentifierAttribute, + AXElement.attributedDescription, + kAXDescriptionAttribute, + kAXTitleAttribute, + kAXValueAttribute, + kAXEnabledAttribute, + kAXFocusedAttribute, + "AXFrame", + ] + + /// Walk the tree of the application owning `options.pid`. + /// + /// Returns `nil` when a prefix was given and nothing matched it. + static func read(_ options: TreeOptions) throws(DriveError) -> TreeNode? { + guard ProcessTable.record(for: options.pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(options.pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return walk(from: AXElement.application(pid: options.pid), options: options) + } + + /// Walk from `root`, spending a fresh match budget. + static func walk<E: Element>(from root: E, options: TreeOptions) -> TreeNode? { + // An unfiltered walk has no matches to count, so the budget only bounds a + // filtered one. + var remaining = options.identifierPrefix == nil ? Int.max : options.maxMatches + return node(root, depth: 0, options: options, remaining: &remaining) + } + + private static func node<E: Element>( + _ element: E, + depth: Int, + options: TreeOptions, + remaining: inout Int + ) -> TreeNode? { + let reading = element.read(batch) + let text = reading.text + + let identifier = text[1] + + let matches = + options.identifierPrefix.map { identifier?.hasPrefix($0) ?? false } ?? false + if matches { + remaining -= 1 + } + + // Every child the element has, whether or not this walk descends into it. + // The count is what tells a reader there is more here; without it a node + // stopped at the depth limit is indistinguishable from a leaf. + let available = reading.children + let all = depth < options.maxDepth ? available : [] + + // The sibling cap is for reading an unfiltered tree, where every level is + // worth seeing but a thousand copies of one row are not. Under a filter the + // match budget does the bounding instead: a cap here would hide the eight + // hundredth row from a search that named it. + let capped = options.identifierPrefix == nil && options.maxSiblings > 0 + + var children: [TreeNode] = [] + var visited = 0 + + for child in all { + guard remaining > 0 else { break } + guard !capped || visited < options.maxSiblings else { break } + visited += 1 + + guard + let node = node( + child, depth: depth + 1, options: options, remaining: &remaining) + else { continue } + children.append(node) + } + + // A branch is kept when it matches, or when something under it does. The + // ancestors are what make a match locatable rather than a bare hit. + guard matches || !children.isEmpty || options.identifierPrefix == nil else { + return nil + } + + return TreeNode( + role: text[0] ?? "<none>", + identifier: identifier, + label: text[2] ?? text[3] ?? text[4], + value: text[5], + enabled: text[6].axFlag, + focused: text[7].axFlag, + frame: options.frames ? text[8] : nil, + // Read only for a node being kept. Actions cost their own round-trip and + // most elements a filtered walk passes through are discarded. + actions: element.actions, + children: children, + elidedChildren: available.count > children.count + ? available.count - children.count + : nil, + unreadable: reading.failed ? true : nil + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift new file mode 100644 index 000000000..78952628e --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/WindowIDs.swift @@ -0,0 +1,138 @@ +import CoreGraphics +import Foundation + +/// A window the window server can be told to capture. +struct CaptureWindow: Encodable, Equatable { + /// The window server's identifier, in the form `screencapture -l` takes. + let id: CGWindowID + + /// The window's title, or `nil` when this process holds no Screen Recording + /// grant: the window server withholds other applications' titles until it + /// does. + let title: String? + + let width: Int + let height: Int +} + +/// What `jpdrive windowid` observed. +struct WindowIDReport: Encodable, Equatable { + /// Whether this process may read other applications' screen content. + /// + /// Enumerating windows needs no grant, so a report can list windows that + /// cannot be captured. A caller that acts on the list without reading this + /// gets a picture of the desktop where it expected a window. + let screenRecording: Bool + + /// The application's capturable windows, front to back. + let windows: [CaptureWindow] + + /// Windows the application has that the screen is not currently showing. + /// + /// Reported separately because such a window is absent from every on-screen + /// enumeration and from the accessibility tree, so an app that has one and + /// nothing else is indistinguishable from an app with no window at all — + /// except by asking for every window regardless, which is this list. + /// + /// Says nothing about *why* it is not showing. Minimized, hidden, and on + /// another Space all land here, and the window server's list carries no flag + /// separating them. `jpdrive windows` reads `AXMinimized` from the + /// accessibility tree, which distinguishes the first of the three. + let offScreen: [CaptureWindow] +} + +/// Resolves an application's window-server identifiers. +/// +/// Separate from `Windows`, which reads the accessibility tree: the two answer +/// different questions and neither identifier converts into the other. An +/// accessibility window has a title and a frame but no number the capture tools +/// accept, and a window-server window has that number but nothing structural. +enum WindowIDs { + /// The layer ordinary application windows sit on. + /// + /// Everything else the window server reports for an application is chrome — + /// tooltips, drag images, the shadow behind a menu — and capturing one of + /// those instead of the window is a silent wrong answer rather than a + /// failure. + static let normalLayer = 0 + + /// The capturable windows of the application owning `pid`. + static func read(pid: pid_t) throws(DriveError) -> WindowIDReport { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + let onScreen = + CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] ?? [] + + // Every window, not just the ones on screen. The difference between the + // two lists is what says a window exists somewhere the screen is not + // showing it. + let everywhere = + CGWindowListCopyWindowInfo( + [.excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] ?? [] + + let here = capturable(from: onScreen, pid: pid) + let all = capturable(from: everywhere, pid: pid) + let shown = Set(here.map(\.id)) + + // The preflight variant, never the requesting one: raising the system's + // permission dialog from a background tool leaves a prompt nobody is + // watching, in front of the app being measured. + return WindowIDReport( + screenRecording: CGPreflightScreenCaptureAccess(), + windows: here, + offScreen: all.filter { !shown.contains($0.id) } + ) + } + + /// The windows in `listed` that belong to `pid` and can be captured, + /// in the order the window server reported them, which is front to back. + /// + /// A window with no area is dropped: `AppKit` keeps zero-sized windows + /// around for panels that have never been shown, and capturing one produces + /// an empty file. + static func capturable(from listed: [[String: Any]], pid: pid_t) -> [CaptureWindow] { + return listed.compactMap { window -> CaptureWindow? in + guard integer(window[kCGWindowOwnerPID as String]) == Int(pid), + integer(window[kCGWindowLayer as String]) == normalLayer, + let number = integer(window[kCGWindowNumber as String]), + let id = CGWindowID(exactly: number), + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let width = integer(bounds["Width"]), + let height = integer(bounds["Height"]), + width > 0, height > 0 + else { + return nil + } + + return CaptureWindow( + id: id, + title: window[kCGWindowName as String] as? String, + width: width, + height: height + ) + } + } + + /// One of the window server's numbers, whichever numeric type it arrives as. + /// + /// The list holds `CFNumber`s in untyped dictionaries. Bridged, those cast to + /// `Int` while the value is whole and only to `Double` otherwise, which is a + /// distinction window bounds can cross: a window on a scaled display sits at + /// fractional points. + private static func integer(_ value: Any?) -> Int? { + if let int = value as? Int { return int } + if let double = value as? Double { return Int(double) } + return nil + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift b/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift new file mode 100644 index 000000000..a2a5ab144 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/DriveKit/Windows.swift @@ -0,0 +1,74 @@ +import ApplicationServices +import Foundation + +/// One of an application's windows. +struct WindowSummary: Encodable, Equatable { + let identifier: String? + let title: String? + + /// Whether this is the application's main window. + let main: Bool? + + let minimized: Bool? + + /// Position and size in screen coordinates. + /// + /// Included here, unlike in a tree, because a window's frame is what the + /// listing is for: which window is where, and how big. + let frame: String? +} + +/// Lists an application's windows. +/// +/// Separate from a tree walk because the useful facts about a window are its own — +/// which one is main, which is minimized, where it sits — rather than what it +/// contains. +enum Windows { + /// Attributes read for every window, in one batch. + static let batch = [ + kAXIdentifierAttribute, + kAXTitleAttribute, + kAXMainAttribute, + kAXMinimizedAttribute, + "AXFrame", + ] + + /// List the windows of the application owning `pid`. + static func read(pid: pid_t) throws(DriveError) -> [WindowSummary] { + guard ProcessTable.record(for: pid) != nil else { + throw DriveError( + kind: .appNotRunning, + message: "no process is running under pid \(pid)", + hint: "start the app, then pass its pid: --pid $(pgrep -f JP.app)" + ) + } + + guard AXIsProcessTrusted() else { + throw DriveError( + kind: .notPermitted, + message: "not trusted to read another application's accessibility tree", + hint: DriveError.accessibilityHint + ) + } + + return list(of: AXElement.application(pid: pid)) + } + + /// The windows an application element reports. + /// + /// An application with no windows answers an empty list, which is a state a + /// running app can legitimately be in. + static func list<E: Element>(of app: E) -> [WindowSummary] { + return app.elements(kAXWindowsAttribute).map { window in + let text = window.read(batch).text + + return WindowSummary( + identifier: text[0], + title: text[1], + main: text[2].axFlag, + minimized: text[3].axFlag, + frame: text[4] + ) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift b/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift new file mode 100644 index 000000000..7e1373e35 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Sources/jpdrive/main.swift @@ -0,0 +1,3 @@ +import DriveKit + +Driver.run() diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift new file mode 100644 index 000000000..4ed7ffd6c --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ActTests.swift @@ -0,0 +1,283 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act") +struct ActTests { + /// The case the driver exists for, and the one that was broken: the identifier + /// is on a leaf two levels below the element that owns selection. + @Test("select writes AXSelected on the row, not on the identified element") + func selectsTheOwningRow() throws { + let root = FakeElement.sidebar(rowCount: 3) + let step = Step.select(.init(identifier: "sidebar.row.1")) + + let result = try Act.run(step, in: root) + + #expect( + result + == StepResult( + step: "select", + identifier: "sidebar.row.1", + role: "AXRow", + confirmed: true + ) + ) + #expect(result.confirmed == true) + + let rows = try #require(root.children.first?.children) + #expect(rows[1].attributes[kAXSelectedAttribute] == "1") + + // The leaf that carried the identifier must not have been written to. It + // reports no AXSelected at all, and a driver that wrote there would report + // success while selecting nothing. + let leaf = try #require(rows[1].children.first?.children.first) + #expect(leaf.attributes[kAXSelectedAttribute] == nil) + } + + /// Selection reaches a row regardless of where it sits, which is what makes the + /// attribute write preferable to a synthesized click. + @Test("select reaches a row far down a long list") + func selectsADeepRow() throws { + let root = FakeElement.sidebar(rowCount: 1000) + + let result = try Act.run(.select(.init(identifier: "sidebar.row.987")), in: root) + + #expect(result.confirmed == true) + let rows = try #require(root.children.first?.children) + #expect(rows[987].attributes[kAXSelectedAttribute] == "1") + } + + @Test("select reports the identifier it could not find") + func reportsAMissingIdentifier() throws { + let root = FakeElement.sidebar(rowCount: 3) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.nope")), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// A search that could not read part of the tree has not established that the + /// element is absent — it may sit under the branch that failed. Reporting a + /// clean miss invites the caller to change an identifier that was right, so + /// the two answers have to be different. + @Test("a miss with an unreadable branch is not reported as a clean miss") + func distinguishesAGapFromAMiss() throws { + let broken = FakeElement(role: "AXGroup") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .readFailed) + #expect(error.message.contains("could not be read")) + } + + /// The other half of the pair: a tree that read cleanly throughout still + /// reports a plain miss, so the flag above cannot be firing on everything. + @Test("a miss in a fully readable tree stays a clean miss") + func aReadableTreeReportsACleanMiss() throws { + let readable = FakeElement(role: "AXGroup") + let root = FakeElement(role: "AXApplication", children: [readable]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// An element nothing in its chain can select is a failure, not a fallback onto + /// some other mechanism. + @Test("select fails when no ancestor accepts the write") + func failsWhenNothingIsSelectable() throws { + let leaf = FakeElement(role: "AXUnknown", identifier: "lonely") + let root = FakeElement(role: "AXApplication", children: [leaf]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "lonely")), in: root) + } + + #expect(error.kind == .notSelectable) + } + + /// A write the accessibility API refuses is reported, not silently treated as + /// an unconfirmed success. + @Test("select reports a refused write") + func reportsARefusedWrite() throws { + let root = FakeElement.sidebar(rowCount: 2) + let rows = try #require(root.children.first?.children) + rows[0].writeStatus = .cannotComplete + + let error = try #require(throws: DriveError.self) { + try Act.run(.select(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .writeFailed) + #expect(error.message.contains("cannot_complete")) + } + + /// A write can be accepted and do nothing. The step reports that as an + /// unconfirmed success rather than as a failure, because the distinction is + /// what tells a caller the mechanism stopped working. + @Test("select reports an accepted write that changed nothing") + func reportsAnIneffectiveWrite() throws { + let leaf = FakeElement(role: "AXUnknown", identifier: "row") + let row = FakeElement(role: "AXRow", settable: [kAXSelectedAttribute], children: [leaf]) + let root = FakeElement(role: "AXApplication", children: [row]) + row.ignoresWrites = true + + let result = try Act.run(.select(.init(identifier: "row")), in: root) + + #expect(result.role == "AXRow") + #expect( + result.confirmed == false, + "a write the target discarded must not report as confirmed" + ) + } + + /// A sidebar row is the case that makes `click` the wrong tool: the identified + /// element has no activation point, and `select` reaches it whether or not it + /// is on screen. + @Test("click fails on a row, which wants select instead") + func clickFailsOnARow() throws { + let root = FakeElement.sidebar(rowCount: 1) + + let error = try #require(throws: DriveError.self) { + try Act.run( + .click(.init(identifier: "sidebar.row.0")), in: root, poster: FakePoster()) + } + + #expect(error.kind == .notClickable) + } + + @Test("press performs AXPress on the identified element") + func pressesTheElement() throws { + let item = FakeElement( + role: "AXMenuItem", + identifier: "terminate:", + actions: ["AXCancel", "AXPress", "AXPick"] + ) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "terminate:")), in: root) + + #expect(item.performed == ["AXPress"]) + #expect(result.step == "press") + #expect(result.role == "AXMenuItem") + } + + /// Nothing readable says a press worked, so the step must not claim it did. + /// Reporting `true` here would be the one dishonest field in the output. + @Test("press reports no confirmation") + func pressDoesNotClaimConfirmation() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "go")), in: root) + + #expect(result.confirmed == nil) + } + + /// A sidebar row is the case this catches: it advertises no actions at all, so + /// the error points at the step that does work on it. + @Test("press fails on an element that does not accept it") + func pressFailsWithoutTheAction() throws { + let root = FakeElement.sidebar(rowCount: 1) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "sidebar.row.0")), in: root) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint?.contains("select") == true) + // The press must not have been attempted anyway. + let leaf = root.children.first?.children.first?.children.first?.children.first + #expect(leaf?.performed.isEmpty == true) + } + + /// An element with other actions gets told what it does accept, which is how a + /// script author finds the right verb without dumping the tree. + @Test("press names the actions an element does accept") + func pressNamesAvailableActions() throws { + let item = FakeElement(role: "AXRow", identifier: "row", actions: ["AXShowMenu"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "row")), in: root) + } + + #expect(error.hint?.contains("AXShowMenu") == true) + } + + /// `press` is a shorthand for `perform` with `AXPress`, and must keep saying so + /// in its result rather than reporting the mechanism underneath. + @Test("press names itself, not the general mechanism") + func pressNamesItself() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + let root = FakeElement(role: "AXApplication", children: [item]) + + let result = try Act.run(.press(.init(identifier: "go")), in: root) + + #expect(result.step == "press") + } + + /// The escape hatch for the actions with no step of their own. A text field + /// offers `AXConfirm` and no `AXPress`, so this is the only way to reach it. + @Test("perform runs any action the element advertises") + func performsANamedAction() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "sidebar.filter", + actions: ["AXShowMenu", "AXConfirm"] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .perform(.init(identifier: "sidebar.filter", action: "AXConfirm")), + in: root + ) + + #expect(field.performed == ["AXConfirm"]) + #expect(result.step == "perform") + } + + @Test("perform fails on an action the element does not advertise") + func performRejectsAnUnknownAction() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "sidebar.filter", + actions: ["AXConfirm"] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run( + .perform(.init(identifier: "sidebar.filter", action: "AXPress")), + in: root + ) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint == "it accepts: AXConfirm") + #expect(field.performed.isEmpty) + } + + @Test("press reports a refused action") + func pressReportsARefusal() throws { + let item = FakeElement(role: "AXButton", identifier: "go", actions: ["AXPress"]) + item.performStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [item]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.press(.init(identifier: "go")), in: root) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("cannot_complete")) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift new file mode 100644 index 000000000..41534c061 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ArgumentsTests.swift @@ -0,0 +1,201 @@ +import Testing + +@testable import DriveKit + +@Suite("Arguments") +struct ArgumentsTests { + @Test("doctor takes an optional pid") + func doctorPid() throws { + guard case .doctor(let pid) = try Arguments.parse(["doctor", "--pid", "42"]) else { + Issue.record("expected a doctor command") + return + } + #expect(pid == 42) + + guard case .doctor(let none) = try Arguments.parse(["doctor"]) else { + Issue.record("expected a doctor command") + return + } + #expect(none == nil) + } + + @Test("tree defaults its bounds") + func treeDefaults() throws { + guard case .tree(let options) = try Arguments.parse(["tree", "--pid", "42"]) else { + Issue.record("expected a tree command") + return + } + + #expect(options.pid == 42) + #expect(options.identifierPrefix == nil) + #expect(options.maxMatches == Arguments.defaultMatches) + #expect(options.maxDepth == Arguments.defaultDepth) + #expect(options.maxSiblings == Arguments.defaultSiblings) + #expect(!options.frames) + } + + @Test("tree takes every bound") + func treeFlags() throws { + let parsed = try Arguments.parse([ + "tree", "--pid", "42", "--identifier", "sidebar.", "--max-matches", "1", + "--depth", "3", "--max-siblings", "0", "--frames", + ]) + + guard case .tree(let options) = parsed else { + Issue.record("expected a tree command") + return + } + + #expect(options.identifierPrefix == "sidebar.") + #expect(options.maxMatches == 1) + #expect(options.maxDepth == 3) + #expect(options.maxSiblings == 0) + #expect(options.frames) + } + + /// Zero means "every sibling", which is a different thing from the cap being + /// unset, so it has to survive parsing rather than be rejected as non-positive. + @Test("max-siblings accepts zero for no cap") + func zeroSiblingsIsAllowed() throws { + guard + case .dump(let options) = try Arguments.parse([ + "dump", "--pid", "1", "--max-siblings", "0", + ]) + else { + Issue.record("expected a dump command") + return + } + + #expect(options.maxSiblings == 0) + } + + @Test("windows takes only a pid") + func windowsPid() throws { + guard case .windows(let pid) = try Arguments.parse(["windows", "--pid", "42"]) else { + Issue.record("expected a windows command") + return + } + #expect(pid == 42) + } + + @Test("windowid takes only a pid") + func windowidPid() throws { + guard case .windowid(let pid) = try Arguments.parse(["windowid", "--pid", "42"]) else { + Issue.record("expected a windowid command") + return + } + #expect(pid == 42) + } + + /// A menu bar is small and every item in it is a thing to press, so the sibling + /// cap that keeps a thousand-row list readable would only hide verbs here. + @Test("menu walks every sibling by default") + func menuHasNoSiblingCap() throws { + guard case .menu(let pid, let options) = try Arguments.parse(["menu", "--pid", "42"]) + else { + Issue.record("expected a menu command") + return + } + + #expect(pid == 42) + #expect(options.maxSiblings == 0) + } + + @Test("act decodes a step") + func actStep() throws { + let parsed = try Arguments.parse([ + "act", "--pid", "42", "--json", #"{"select":{"identifier":"sidebar.row.7"}}"#, + ]) + + guard case .act(let step, let pid) = parsed else { + Issue.record("expected an act command") + return + } + + #expect(pid == 42) + guard case .select(let target) = step else { + Issue.record("expected a select step") + return + } + #expect(target.identifier == "sidebar.row.7") + } + + /// Every field of a step is spelled the way the tool definition documents it, + /// and a mismatch is silent: an unrecognised key decodes as absent, so a wait + /// given a short timeout would wait the default instead and the run would look + /// merely slow. + @Test("act decodes every field of a wait") + func actWaitFields() throws { + let parsed = try Arguments.parse([ + "act", "--pid", "42", "--json", + #"{"wait_for":{"identifier":"transcript.scroll","under":"sidebar.list","timeout_ms":1500,"interval_ms":25}}"#, + ]) + + guard case .act(let step, _) = parsed, case .waitFor(let target) = step else { + Issue.record("expected a wait_for step") + return + } + + #expect(target.identifier == "transcript.scroll") + #expect(target.under == "sidebar.list") + #expect(target.timeoutMs == 1500) + #expect(target.intervalMs == 25) + } + + @Test( + "a step naming no known verb is rejected", + arguments: [ + #"{"nope":{"identifier":"x"}}"#, + #"{"wait":{"identifier":"x"}}"#, + #"{}"#, + #"not json"#, + #"{"select":{}}"#, + ] + ) + func rejectsAMalformedStep(json: String) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(["act", "--pid", "1", "--json", json]) + } + + #expect(error.kind == .badUsage) + } + + @Test( + "a command missing its pid is rejected", + arguments: [ + ["tree"], ["dump"], ["windows"], ["windowid"], ["menu"], ["act", "--json", "{}"], + ] + ) + func requiresAPid(arguments: [String]) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(arguments) + } + + #expect(error.kind == .badUsage) + } + + /// An empty command substitution is the shape this most often takes: + /// `--pid $(pgrep -f JP.app)` expands to nothing when the app is not running, + /// leaving the flag with no value. + @Test("a pid flag with no value is rejected with a usable hint") + func rejectsAMissingPidValue() throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(["doctor", "--pid"]) + } + + #expect(error.kind == .badUsage) + #expect(error.hint?.contains("pgrep") == true) + } + + @Test( + "unknown input is rejected", + arguments: [["fly", "--pid", "1"], ["tree", "--pid", "1", "--nope"], []] + ) + func rejectsUnknownInput(arguments: [String]) throws { + let error = try #require(throws: DriveError.self) { + try Arguments.parse(arguments) + } + + #expect(error.kind == .badUsage) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift new file mode 100644 index 000000000..ff8647b45 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ClickTests.swift @@ -0,0 +1,145 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.click") +struct ClickTests { + /// A button inside a window, which is the shape a click needs: something with a + /// point, under something that can be raised. + private func app(activationPoint: CGPoint? = CGPoint(x: 120, y: 340)) -> FakeElement { + let button = FakeElement( + role: "AXButton", identifier: "toolbar.open", actions: ["AXPress"]) + if let activationPoint { + button.points[AXElement.activationPoint] = activationPoint + } + + let window = FakeElement( + role: kAXWindowRole, + identifier: "workspace-AppWindow-1", + actions: [kAXRaiseAction], + children: [button] + ) + + return FakeElement(role: "AXApplication", children: [window]) + } + + @Test("clicks where the element says a click belongs") + func clicksTheActivationPoint() throws { + let root = app() + let poster = FakePoster() + + let result = try Act.run( + .click(.init(identifier: "toolbar.open")), + in: root, + poster: poster + ) + + #expect(poster.clicks == [CGPoint(x: 120, y: 340)]) + #expect(result.step == "click") + #expect(result.role == "AXButton") + #expect(result.point == "120.0,340.0") + } + + /// `AXRaise` orders a window within its own application; ordering *between* + /// applications follows activation. The driver is invoked from a terminal, so + /// without this the terminal stays frontmost and the click lands there while + /// the result names the element it aimed at. + @Test("brings the application forward before clicking") + func activatesTheApplication() throws { + let root = app() + + _ = try Act.run( + .click(.init(identifier: "toolbar.open")), in: root, poster: FakePoster()) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// Posting into another application's window is worse than not clicking, so a + /// refused activation stops the step rather than aiming anyway. + @Test("refuses to click when the application cannot be brought forward") + func refusesToClickWithoutActivating() throws { + let root = app() + root.writeStatus = .cannotComplete + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "toolbar.open")), in: root, poster: poster) + } + + #expect(error.kind == .writeFailed) + #expect(poster.clicks.isEmpty, "nothing may be posted at a background application") + } + + /// The event goes to whatever occupies the coordinate, so a window behind + /// another one would have its click swallowed. Raising is what makes the + /// coordinate mean the element that named it. + @Test("raises the window before clicking") + func raisesTheWindowFirst() throws { + let root = app() + let window = try #require(root.children.first) + + _ = try Act.run( + .click(.init(identifier: "toolbar.open")), in: root, poster: FakePoster()) + + #expect(window.performed == [kAXRaiseAction]) + } + + /// A sidebar row has no activation point of its own, and pointing at `select` + /// is more use than clicking at the origin would be. + @Test("fails on an element with nowhere to click") + func failsWithoutAnActivationPoint() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run( + .click(.init(identifier: "toolbar.open")), + in: app(activationPoint: nil), + poster: poster + ) + } + + #expect(error.kind == .notClickable) + #expect(error.hint?.contains("select") == true) + #expect(poster.clicks.isEmpty, "nothing may be clicked when there is no point to click") + } + + @Test("reports a click that could not be posted") + func reportsAFailedPost() throws { + let poster = FakePoster() + poster.succeeds = false + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "toolbar.open")), in: app(), poster: poster) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("120.0,340.0")) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(.click(.init(identifier: "nope")), in: app(), poster: poster) + } + + #expect(error.kind == .identifierNotFound) + #expect(poster.clicks.isEmpty) + } + + /// An element outside any window still has a point, and clicking it is better + /// than refusing because there was nothing to raise. + @Test("clicks without a window to raise") + func clicksWithoutAWindow() throws { + let element = FakeElement(role: "AXButton", identifier: "loose") + element.points[AXElement.activationPoint] = CGPoint(x: 1, y: 2) + let root = FakeElement(role: "AXApplication", children: [element]) + let poster = FakePoster() + + _ = try Act.run(.click(.init(identifier: "loose")), in: root, poster: poster) + + #expect(poster.clicks == [CGPoint(x: 1, y: 2)]) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift new file mode 100644 index 000000000..064bfc71c --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DragTests.swift @@ -0,0 +1,285 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.drag") +struct DragTests { + /// A window with a known frame, which is all a drag needs: somewhere to + /// measure fractions against, and something to raise. + private func app( + origin: CGPoint? = CGPoint(x: 100, y: 200), + size: CGSize? = CGSize(width: 800, height: 600) + ) -> FakeElement { + let window = FakeElement( + role: kAXWindowRole, + identifier: "workspace-AppWindow-1", + actions: [kAXRaiseAction] + ) + if let origin { + window.points[kAXPositionAttribute] = origin + } + if let size { + window.sizes[kAXSizeAttribute] = size + } + + return FakeElement(role: "AXApplication", children: [window]) + } + + private func step( + from: (Double, Double), + to: (Double, Double), + steps: Int? = nil, + pauseMs: Int? = nil + ) -> Step { + .drag( + .init( + identifier: "workspace-AppWindow-1", + from: .init(dx: from.0, dy: from.1), + to: .init(dx: to.0, dy: to.1), + steps: steps, + pauseMs: pauseMs + ) + ) + } + + /// Fractions are resolved against the element's own frame, so a script says + /// "the right edge, halfway down" rather than a screen coordinate that stops + /// being right the moment the window moves. + @Test("resolves fractional offsets against the element's frame") + func resolvesOffsets() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (1.0, 0.5), to: (0.5, 0.5), steps: 2), in: app(), poster: poster) + + // Right edge, halfway down: 100 + 800, 200 + 300. Halfway across: 100 + 400. + #expect( + poster.drags == [ + [ + CGPoint(x: 900, y: 500), + CGPoint(x: 700, y: 500), + CGPoint(x: 500, y: 500), + ] + ] + ) + } + + /// The step exists to produce many frames rather than one jump, so the count + /// is asserted rather than assumed: a drag delivered as a single move cannot + /// show what a view does *during* a gesture, which is the whole reason for it. + @Test("posts one move per step, plus the press") + func postsOneMovePerStep() throws { + let poster = FakePoster() + + let result = try Act.run( + step(from: (0, 0), to: (1, 0), steps: 12), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 13) + #expect(result.moves == 12) + #expect(result.step == "drag") + #expect(result.role == kAXWindowRole) + } + + @Test("defaults to enough moves to be a gesture") + func defaultsToAGesture() throws { + let poster = FakePoster() + + _ = try Act.run(step(from: (0, 0), to: (1, 1)), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 25) + #expect(poster.pauses == [.milliseconds(8)]) + } + + @Test("honours a stated pause between moves") + func honoursThePause() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 3, pauseMs: 40), in: app(), poster: poster) + + #expect(poster.pauses == [.milliseconds(40)]) + } + + /// A drag of zero steps is a click with extra words. Clamped rather than + /// rejected, so a caller computing the count from a distance cannot produce a + /// path with nothing in it. + @Test("clamps a step count below one") + func clampsZeroSteps() throws { + let poster = FakePoster() + + _ = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 0), in: app(), poster: poster) + + #expect(poster.drags.first?.count == 2) + } + + /// The events land on whatever occupies the coordinates, so a window behind + /// another would have the gesture swallowed. + @Test("raises the window before dragging") + func raisesTheWindowFirst() throws { + let root = app() + let window = try #require(root.children.first) + + _ = try Act.run(step(from: (1, 0.5), to: (0.5, 0.5)), in: root, poster: FakePoster()) + + #expect(window.performed == [kAXRaiseAction]) + } + + /// Raising alone is not enough, and this is the assertion that says so. + /// + /// `AXRaise` orders a window forward within its own application; the ordering + /// *between* applications follows activation. A drag posted at a background + /// window's coordinates without this was received by the frontmost terminal + /// instead — measured, and the reason the step takes focus. + @Test("brings the application forward before dragging") + func activatesTheApplication() throws { + let root = app() + + _ = try Act.run(step(from: (1, 0.5), to: (0.5, 0.5)), in: root, poster: FakePoster()) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// An application that will not come forward cannot be dragged in: the events + /// go to whatever owns those screen coordinates, which is the application the + /// person at the keyboard is using. Refusing is the only honest outcome, and + /// posting anyway is what this pins shut. + @Test("refuses to drag when the application cannot be brought forward") + func refusesToDragWithoutActivating() throws { + let root = app() + root.writeStatus = .cannotComplete + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1), steps: 2), in: root, poster: poster) + } + + #expect(error.kind == .writeFailed) + #expect(poster.drags.isEmpty, "nothing may be posted at a background application") + } + + /// An offset is a fraction of the element's frame. Read as a percentage — a + /// plausible misreading — `100` aims 80,000 points past an 800pt window, and + /// the gesture is posted into global screen space, so it lands on whatever is + /// there instead. + @Test( + "refuses an offset that is not a fraction of the frame", + arguments: [100.0, -0.5, 1.5, Double.nan, Double.infinity] + ) + func refusesAnOffsetOutsideTheFrame(dx: Double) throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (dx, 0.5), to: (0.5, 0.5)), in: app(), poster: poster) + } + + #expect(error.kind == .badUsage) + #expect(poster.drags.isEmpty, "nothing may be posted at a coordinate off the element") + } + + /// The far endpoint is checked too, and before anything is posted: a drag that + /// pressed on the element and released over another application would be worse + /// than one that never started. + @Test("refuses an out-of-frame destination before posting") + func refusesAnOutOfFrameDestination() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0.5, 0.5), to: (42, 0.5)), in: app(), poster: poster) + } + + #expect(error.kind == .badUsage) + #expect(poster.drags.isEmpty) + } + + /// A count nobody meant should still produce a gesture rather than an error, + /// but not one that posts for minutes or cannot allocate its route at all. + /// + /// Any value above the cap proves it, so this one is small enough that the + /// test survives its own failure: with the clamp reverted, a count in the + /// millions allocates its whole route before the assertion runs and takes the + /// runner down instead of going red. + @Test("clamps a step count above the cap") + func clampsAnEnormousStepCount() throws { + let poster = FakePoster() + + let result = try Act.run( + step(from: (0, 0), to: (1, 1), steps: 5000), in: app(), poster: poster) + + #expect(result.moves == 1000) + #expect(poster.drags.first?.count == 1001) + } + + @Test("fails on an element with no frame to measure") + func failsWithoutAFrame() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1)), in: app(size: nil), poster: poster) + } + + #expect(error.kind == .notClickable) + #expect(poster.drags.isEmpty, "nothing may be dragged across a frame that is not known") + } + + @Test("reports a drag that could not be posted") + func reportsAFailedPost() throws { + let poster = FakePoster() + poster.succeeds = false + + let error = try #require(throws: DriveError.self) { + try Act.run(step(from: (0, 0), to: (1, 1)), in: app(), poster: poster) + } + + #expect(error.kind == .actionFailed) + #expect(error.message.contains("100.0,200.0")) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let poster = FakePoster() + + let error = try #require(throws: DriveError.self) { + try Act.run( + .drag( + .init( + identifier: "nope", + from: .init(dx: 0, dy: 0), + to: .init(dx: 1, dy: 1), + steps: nil, + pauseMs: nil + ) + ), + in: app(), + poster: poster + ) + } + + #expect(error.kind == .identifierNotFound) + #expect(poster.drags.isEmpty) + } + + /// Decoded from the wire, because the snake-cased key is spelled by hand and a + /// mismatch there reads as the default silently applying. + @Test("decodes a step from its written form") + func decodesFromJSON() throws { + let json = """ + {"drag": {"identifier": "transcript.text", "from": {"dx": 0.1, "dy": 0.2}, + "to": {"dx": 0.8, "dy": 0.6}, "steps": 6, "pause_ms": 15}} + """ + + let decoded = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + + guard case .drag(let target) = decoded else { + Issue.record("expected a drag step, got \(decoded)") + return + } + + #expect(target.identifier == "transcript.text") + #expect(target.from.dx == 0.1) + #expect(target.to.dy == 0.6) + #expect(target.steps == 6) + #expect(target.pauseMs == 15) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift new file mode 100644 index 000000000..767d86ddf --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/DumpTests.swift @@ -0,0 +1,121 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Dump") +struct DumpTests { + /// Options with the bounds wide open, so a test names only what it is about. + private func options( + maxDepth: Int = 20, + maxSiblings: Int = 0, + settable: Bool = false + ) -> DumpOptions { + return DumpOptions( + pid: 0, + maxDepth: maxDepth, + maxSiblings: maxSiblings, + settable: settable + ) + } + + @Test("an unbounded walk reports every element and elides nothing") + func walksEverything() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let node = Dump.node(root, depth: 0, options: options()) + + #expect(node.role == "AXApplication") + #expect(node.elidedChildren == nil) + + let outline = try #require(node.children.first) + #expect(outline.role == "AXOutline") + #expect(outline.children.count == 2) + #expect(outline.elidedChildren == nil) + } + + /// The bug this file was written for: the depth cap emptied the child list + /// before the count was taken, so a node stopped at the cap reported no + /// children and no elision — identical to a leaf, which is the reading + /// `elidedChildren` exists to prevent. + @Test("a node stopped at the depth limit reports what it did not walk") + func depthLimitReportsElision() throws { + let root = FakeElement.sidebar(rowCount: 3) + + let node = Dump.node(root, depth: 0, options: options(maxDepth: 1)) + + let outline = try #require(node.children.first) + #expect(outline.children.isEmpty) + #expect(outline.elidedChildren == 3) + } + + @Test("a node under the sibling cap reports what it skipped") + func siblingCapReportsElision() throws { + let root = FakeElement.sidebar(rowCount: 10) + + let node = Dump.node(root, depth: 0, options: options(maxSiblings: 4)) + + let outline = try #require(node.children.first) + #expect(outline.children.count == 4) + #expect(outline.elidedChildren == 6) + } + + /// A genuine leaf and a truncated node must not render the same way, which is + /// the whole point of the field. + @Test("a leaf reports no elision") + func leafReportsNothing() { + let node = Dump.node(FakeElement(role: "AXButton"), depth: 0, options: options()) + + #expect(node.children.isEmpty) + #expect(node.elidedChildren == nil) + } + + /// `AXChildren` is what the walk recurses into and `AXParent` points back at + /// the element that reported this one, so neither is worth recording. + @Test("the attributes that only lead back into the tree are dropped") + func dropsStructuralAttributes() { + let element = FakeElement(role: "AXRow", identifier: "row") + element.attributes[kAXChildrenAttribute] = "<2 AXUIElement>" + element.attributes[kAXParentAttribute] = "<AXUIElement>" + + let node = Dump.node(element, depth: 0, options: options()) + let names = node.attributes.map(\.name) + + #expect(!names.contains(kAXChildrenAttribute)) + #expect(!names.contains(kAXParentAttribute)) + #expect(names.contains(kAXIdentifierAttribute)) + } + + @Test("settability is answered only when it was asked for") + func settabilityIsOptIn() throws { + let element = FakeElement( + role: "AXRow", settable: [kAXSelectedAttribute]) + element.attributes[kAXSelectedAttribute] = "0" + + let without = Dump.node(element, depth: 0, options: options()) + #expect(without.attributes.allSatisfy { $0.settable == nil }) + + let with = Dump.node(element, depth: 0, options: options(settable: true)) + let selected = try #require(with.attributes.first { $0.name == kAXSelectedAttribute }) + #expect(selected.settable == true) + } + + /// An element the accessibility API refused reports no attributes and no + /// children, which is exactly how an empty element reports. Without the flag + /// a gap in the walk is indistinguishable from a leaf that is really there. + @Test("an element that could not be read is marked rather than shown empty") + func marksAnUnreadableElement() throws { + let broken = FakeElement(role: "AXGroup", identifier: "gone") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let node = Dump.node(root, depth: 0, options: options()) + + #expect(node.unreadable == nil) + + let child = try #require(node.children.first) + #expect(child.unreadable == true) + #expect(child.attributes.isEmpty) + #expect(child.role == "<none>") + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift new file mode 100644 index 000000000..5afda8aec --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakeElement.swift @@ -0,0 +1,202 @@ +import ApplicationServices + +@testable import DriveKit + +/// An accessibility element that is not one. +/// +/// Reference semantics on purpose: a step writes to an element and then reads it +/// back, and a test asserts on what was written. With a value type the write would +/// land on a copy and every such assertion would pass vacuously. +final class FakeElement: Element { + /// Attribute values by name. A name that is absent here reads as `nil`, which + /// is what the real implementation answers for an attribute the element does + /// not report. + var attributes: [String: String] + + /// Attribute names this element accepts writes for. + let settable: Set<String> + + var actions: [String] + var children: [FakeElement] + + /// Counts every call to ``read(_:)``, so a test can pin how much of a tree a + /// walk touched rather than only what it returned. + private(set) var reads = 0 + + /// What `setFlag` should answer, for exercising a refused write. + var writeStatus: AXError = .success + + /// Make every read of this element fail as a whole. + /// + /// What the accessibility API answers for an element whose application is + /// busy or exiting. Distinct from an element that simply reports nothing: + /// that is the confusion the `failed` flag exists to prevent, so a fake has + /// to be able to produce both. + var readFails = false + + /// Accept writes and discard them. + /// + /// The accessibility API lets a target answer `success` and then do nothing, + /// which is why a step reads back rather than trusting the status. Without this + /// there is no way to tell a driver that reads back from one that pretends to. + var ignoresWrites = false + + /// Called during every ``read(_:)``, after ``reads`` is incremented and before + /// children are handed back. + /// + /// This is how a test makes an element appear partway through a wait. A fixture + /// that has the element from the start cannot tell polling from a single lucky + /// look. + var onRead: ((FakeElement) -> Void)? + + /// Element-valued attributes, such as `AXWindows` and `AXMenuBar`. + var related: [String: [FakeElement]] = [:] + + /// Point-valued attributes, such as `AXActivationPoint`. + var points: [String: CGPoint] = [:] + + /// Size-valued attributes, such as `AXSize`. + var sizes: [String: CGSize] = [:] + + /// Actions performed on this element, in order. + private(set) var performed: [String] = [] + + /// What `perform` should answer, for exercising a refused action. + var performStatus: AXError = .success + + init( + role: String, + identifier: String? = nil, + label: String? = nil, + settable: Set<String> = [], + actions: [String] = [], + children: [FakeElement] = [] + ) { + self.attributes = [kAXRoleAttribute: role] + self.attributes[kAXIdentifierAttribute] = identifier + self.attributes[AXElement.attributedDescription] = label + self.settable = settable + self.actions = actions + self.children = children + } + + func read(_ names: [String]) -> Reading<FakeElement> { + reads += 1 + onRead?(self) + + guard !readFails else { + return Reading( + text: Array(repeating: nil, count: names.count), + children: [], + failed: true + ) + } + + return Reading(text: names.map { attributes[$0] }, children: children) + } + + func reportedAttributes(settable: Bool) -> [Attribute]? { + guard !readFails else { return nil } + + return attributes.keys.sorted().map { name in + Attribute( + name: name, + value: attributes[name] ?? "<null>", + settable: settable ? isSettable(name) : nil + ) + } + } + + func isSettable(_ name: String) -> Bool { + return settable.contains(name) + } + + func flag(_ name: String) -> Bool? { + switch attributes[name] { + case "1": return true + case "0": return false + default: return nil + } + } + + func setFlag(_ name: String, _ value: Bool) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + attributes[name] = value ? "1" : "0" + return .success + } + + func setText(_ name: String, _ value: String) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + attributes[name] = value + return .success + } + + func perform(_ action: String) -> AXError { + guard performStatus == .success else { return performStatus } + performed.append(action) + return .success + } + + func point(_ name: String) -> CGPoint? { + return points[name] + } + + func size(_ name: String) -> CGSize? { + return sizes[name] + } + + func setSize(_ name: String, _ value: CGSize) -> AXError { + guard writeStatus == .success else { return writeStatus } + guard !ignoresWrites else { return .success } + sizes[name] = value + return .success + } + + func elements(_ name: String) -> [FakeElement] { + return related[name] ?? [] + } +} + +extension FakeElement { + /// The sidebar shape this app actually produces, at whatever size a test needs. + /// + /// Three elements per conversation, with the identifier on the leaf and + /// `AXSelected` writable only on the row. Reproducing that here is the point: + /// the driver has to address one element and act on another. + static func sidebar(rowCount: Int) -> FakeElement { + let rows = (0..<rowCount).map { index in + FakeElement( + role: "AXRow", + settable: [kAXSelectedAttribute], + actions: ["AXShowDefaultUI"], + children: [ + FakeElement( + role: "AXCell", + children: [ + FakeElement( + role: "AXUnknown", + identifier: "sidebar.row.\(index)", + label: "Conversation \(index), 4 events" + ) + ] + ) + ] + ) + } + + return FakeElement( + role: "AXApplication", + children: [ + FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + label: "Conversations", + actions: ["AXShowMenu"], + children: rows + ) + ] + ) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift new file mode 100644 index 000000000..808d79006 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/FakePoster.swift @@ -0,0 +1,35 @@ +import ApplicationServices + +@testable import DriveKit + +/// Records clicks and drags instead of posting them. +/// +/// A real click goes to the window server and lands on whatever occupies the +/// coordinate, so the only part a test can hold still is where the driver aimed. +final class FakePoster: EventPoster { + /// Every point clicked, in order. + private(set) var clicks: [CGPoint] = [] + + /// Every drag's path, in order. + private(set) var drags: [[CGPoint]] = [] + + /// The pause each drag was asked to wait between moves. + private(set) var pauses: [Duration] = [] + + /// What `click` and `drag` should answer, for exercising a post that could + /// not be built. + var succeeds = true + + func click(at point: CGPoint) -> Bool { + guard succeeds else { return false } + clicks.append(point) + return true + } + + func drag(through path: [CGPoint], pausing pause: Duration) -> Bool { + guard succeeds else { return false } + drags.append(path) + pauses.append(pause) + return true + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift new file mode 100644 index 000000000..002559049 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/MenuTests.swift @@ -0,0 +1,272 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +/// The menu bar's own walk is [`Tree`](Tree)'s, already covered by `TreeTests`. +/// What is specific here is the root: the menu bar hangs off an attribute of the +/// application rather than sitting in its children, and every level of it should be +/// reported rather than capped. +@Suite("Menu") +struct MenuTests { + /// The whole menu bar, with no cap, because every item in it is something a + /// script might press. + private func options() -> TreeOptions { + return TreeOptions( + pid: 0, + identifierPrefix: nil, + maxMatches: 100, + maxDepth: 20, + maxSiblings: 0, + frames: false + ) + } + + /// An application whose menu bar hangs off the attribute the real one uses. + private func app() -> FakeElement { + let app = FakeElement(role: "AXApplication") + app.related[kAXMenuBarAttribute] = [menuBar()] + return app + } + + private func menuBar() -> FakeElement { + return FakeElement( + role: "AXMenuBar", + children: [ + FakeElement( + role: "AXMenuBarItem", + label: "File", + children: [ + FakeElement( + role: "AXMenu", + children: [ + FakeElement( + role: "AXMenuItem", + identifier: "performClose:", + label: "Close", + actions: ["AXCancel", "AXPress", "AXPick"] + ), + FakeElement( + role: "AXMenuItem", + identifier: "closeAll:", + label: "Close All", + actions: ["AXCancel", "AXPress", "AXPick"] + ), + ] + ) + ] + ) + ] + ) + } + + @Test("every menu item is reported, with the action that activates it") + func reportsEveryItem() throws { + let tree = try #require(Tree.walk(from: menuBar(), options: options())) + + #expect(tree.role == "AXMenuBar") + let items = try #require(tree.children.first?.children.first?.children) + #expect(items.count == 2) + #expect(items.map(\.identifier) == ["performClose:", "closeAll:"]) + #expect(items[0].actions.contains("AXPress")) + #expect(tree.children.first?.children.first?.elidedChildren == nil) + } + + /// Menu items advertise `AXPress` where a list row advertises nothing, so they + /// are the case `press` was built for. + @Test("a menu item can be pressed by identifier") + func pressesAMenuItem() throws { + let bar = menuBar() + + let result = try Act.run(.press(.init(identifier: "closeAll:")), in: bar) + + #expect(result.role == "AXMenuItem") + let items = try #require(bar.children.first?.children.first?.children) + #expect(items[1].performed == ["AXPress"]) + #expect(items[0].performed.isEmpty, "only the addressed item may be pressed") + } + + /// The path names the two titled levels a user sees and skips the `AXMenu` + /// between them, because that container has no title to name. + @Test("a titled path resolves through the intervening menu") + func resolvesATitledPath() throws { + let root = app() + + let result = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(result.step == "menu") + #expect(result.identifier == "File > Close All") + #expect(result.role == "AXMenuItem") + + let bar = try #require(root.elements(kAXMenuBarAttribute).first) + let items = try #require(bar.children.first?.children.first?.children) + #expect(items[1].performed == ["AXPress"]) + #expect(items[0].performed.isEmpty) + } + + /// The point of addressing by title: an item that moved to another menu keeps + /// its identifier, so only a path notices. The failure has to say what the + /// level does hold, or the test that catches the move cannot say what changed. + @Test("a path that does not resolve names what the level holds") + func reportsWhatTheLevelHolds() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File", "Quit"])), in: app()) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("'File' holds no item titled 'Quit'")) + #expect(error.hint == "it holds: Close, Close All") + } + + /// A context menu's items cannot be addressed any other way: `SwiftUI` gives + /// every one of them the same selector name, so a title is all there is. + @Test("a path can start at the menu an element is showing") + func resolvesUnderAShownMenu() throws { + let item = FakeElement(role: "AXMenuItem", label: "Copy Link", actions: ["AXPress"]) + let owner = FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [ + FakeElement(role: "AXRow", identifier: "sidebar.row.0"), + FakeElement(role: "AXMenu", children: [item]), + ] + ) + let root = FakeElement(role: "AXApplication", children: [owner]) + + let result = try Act.run( + .menu(.init(path: ["Copy Link"], under: "sidebar.list")), in: root) + + #expect(result.step == "menu") + #expect(result.identifier == "Copy Link") + #expect(item.performed == ["AXPress"]) + } + + /// The menu closes as soon as the application deactivates, so "press an item + /// in it" fails far more often than "open it" does. The error has to name the + /// step that was missed. + @Test("a path under an element that shows no menu says how to open one") + func reportsAnUnshownMenu() throws { + let owner = FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [FakeElement(role: "AXRow", identifier: "sidebar.row.0")] + ) + let root = FakeElement(role: "AXApplication", children: [owner]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["Copy Link"], under: "sidebar.list")), in: root) + } + + #expect(error.kind == .notFound) + #expect(error.message == "sidebar.list is not showing a menu") + #expect(error.hint?.contains("AXShowMenu") == true) + } + + @Test("a missing top-level menu is reported against the bar") + func reportsAMissingTopLevelMenu() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["Edit", "Copy"])), in: app()) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("the menu bar holds no item titled 'Edit'")) + #expect(error.hint == "it holds: File") + } + + /// Stopping at a bar item addresses the menu, not an item in it, and pressing a + /// menu is not what the script meant. + @Test("a path stopping at a submenu is rejected") + func rejectsAPathToASubmenu() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File"])), in: app()) + } + + #expect(error.kind == .actionUnsupported) + #expect(error.hint?.contains("name the item inside it") == true) + } + + @Test("an empty path is rejected") + func rejectsAnEmptyPath() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: [])), in: app()) + } + + #expect(error.kind == .badUsage) + } + + /// The reason every menu test would otherwise pass while the real thing did + /// nothing: AppKit disables every item acting on the front window or the + /// responder chain while the application is in the background, which a driven + /// app always is. + @Test("a menu step brings the application forward first") + func bringsTheApplicationForward() throws { + let root = app() + + _ = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(root.flag(kAXFrontmostAttribute) == true) + } + + /// An application already in front must not be written to: the write is what + /// steals focus, and a run of several menu steps would take it repeatedly. + @Test("an application already in front is left alone") + func leavesAFrontApplicationAlone() throws { + let root = app() + root.attributes[kAXFrontmostAttribute] = "1" + // Any write from here on fails, so a needless one fails the step. + root.writeStatus = .failure + + let result = try Act.run(.menu(.init(path: ["File", "Close All"])), in: root) + + #expect(result.identifier == "File > Close All") + } + + /// A disabled item swallows `AXPress` and answers success, so a step that + /// pressed it anyway would report having done something it did not do. + @Test("a disabled item is refused rather than pressed") + func refusesADisabledItem() throws { + let root = app() + let bar = try #require(root.elements(kAXMenuBarAttribute).first) + let items = try #require(bar.children.first?.children.first?.children) + items[1].attributes[kAXEnabledAttribute] = "0" + + let error = try #require(throws: DriveError.self) { + try Act.run( + .menu(.init(path: ["File", "Close All"])), + in: root, + activation: .milliseconds(1) + ) + } + + #expect(error.kind == .disabled) + #expect(error.message == "'File > Close All' is disabled") + #expect(items[1].performed.isEmpty, "a disabled item must not be pressed") + } + + /// Most elements report no `AXEnabled` at all, and reading its absence as a + /// refusal would reject every one of them. + @Test("an item reporting no enabled state is pressed") + func pressesAnItemWithNoEnabledState() throws { + let root = app() + + let result = try Act.run( + .menu(.init(path: ["File", "Close All"])), + in: root, + activation: .milliseconds(1) + ) + + #expect(result.identifier == "File > Close All") + } + + /// The menu bar comes from the application's attribute. An app without one is a + /// real case, and it must not be reported as a missing menu item. + @Test("an application with no menu bar says so") + func reportsNoMenuBar() throws { + let error = try #require(throws: DriveError.self) { + try Act.run(.menu(.init(path: ["File"])), in: FakeElement(role: "AXApplication")) + } + + #expect(error.kind == .notFound) + #expect(error.message.contains("no menu bar")) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift new file mode 100644 index 000000000..f5a4b011c --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/PixelsTests.swift @@ -0,0 +1,299 @@ +import CoreGraphics +import Foundation +import ImageIO +import Testing +import UniformTypeIdentifiers + +@testable import DriveKit + +/// Tests for reading a screenshot's pixels. +/// +/// Everything here works against a PNG written by the test, so none of it needs a +/// window server, a running app or a Screen Recording grant. +@Suite("Pixels") +struct PixelsTests { + /// A four-by-two image, written to a temporary file and removed afterwards. + /// + /// Rows top to bottom, each row left to right, as `#RRGGBB` strings. Written + /// through `CGImageDestination` so the file is a real PNG decoded by the same + /// path a screenshot takes. + private func withImage( + rows: [[String]], + _ body: (String) throws -> Void + ) throws { + let height = rows.count + let width = try #require(rows.first?.count) + + var bytes: [UInt8] = [] + for row in rows { + for hex in row { + let value = try #require(UInt32(hex.dropFirst(), radix: 16)) + bytes.append(UInt8((value >> 16) & 0xFF)) + bytes.append(UInt8((value >> 8) & 0xFF)) + bytes.append(UInt8(value & 0xFF)) + bytes.append(255) + } + } + + let space = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: width, + height: height, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: width * 4, + space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let path = NSTemporaryDirectory() + "/jpdrive-pixels-\(UUID().uuidString).png" + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + defer { try? FileManager.default.removeItem(atPath: path) } + try body(path) + } + + /// Two colours across a row, which is the shape every real question takes: a + /// wide background, a narrow line, and the offset where one becomes the other. + @Test("collapses a row into runs of one colour") + func scansARow() throws { + try withImage(rows: [ + ["#FFFFFF", "#FFFFFF", "#DBDBDB", "#FFFFFF"], + ["#000000", "#000000", "#000000", "#000000"], + ]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect(report.width == 4) + #expect(report.height == 2) + #expect(report.colorSpace == "sRGB") + #expect( + report.runs == [ + PixelRun(start: 0, count: 2, color: "#FFFFFF"), + PixelRun(start: 2, count: 1, color: "#DBDBDB"), + PixelRun(start: 3, count: 1, color: "#FFFFFF"), + ] + ) + } + } + + /// Rows are indexed down from the top, the way a screenshot is read, not up + /// from the bottom the way CoreGraphics draws. + @Test("counts rows down from the top") + func rowsCountFromTheTop() throws { + try withImage(rows: [ + ["#FF0000", "#FF0000"], + ["#00FF00", "#00FF00"], + ["#0000FF", "#0000FF"], + ]) { path in + let top = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + let bottom = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 2, from: nil, to: nil)) + + #expect(top.runs == [PixelRun(start: 0, count: 2, color: "#FF0000")]) + #expect(bottom.runs == [PixelRun(start: 0, count: 2, color: "#0000FF")]) + } + } + + @Test("collapses a column into runs of one colour") + func scansAColumn() throws { + try withImage(rows: [ + ["#FFFFFF", "#111111"], + ["#FFFFFF", "#111111"], + ["#222222", "#111111"], + ]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .column, at: 0, from: nil, to: nil)) + + #expect(report.scan == "column") + #expect( + report.runs == [ + PixelRun(start: 0, count: 2, color: "#FFFFFF"), + PixelRun(start: 2, count: 1, color: "#222222"), + ] + ) + } + } + + /// A window is nine hundred points wide and the interesting part is a few of + /// them, so a scan can be bounded. The offsets stay absolute, because they are + /// what gets compared against a frame from the accessibility tree. + @Test("bounds a scan and keeps the offsets absolute") + func boundsAScan() throws { + try withImage(rows: [["#FFFFFF", "#AAAAAA", "#BBBBBB", "#FFFFFF"]]) { path in + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: 1, to: 2)) + + #expect( + report.runs == [ + PixelRun(start: 1, count: 1, color: "#AAAAAA"), + PixelRun(start: 2, count: 1, color: "#BBBBBB"), + ] + ) + } + } + + /// Reading past the edge is a mistake worth reporting rather than clamping: a + /// silently moved scan answers a question nobody asked. + @Test("refuses a line outside the image") + func refusesALineOutside() throws { + try withImage(rows: [["#FFFFFF"]]) { path in + #expect(throws: DriveError.self) { + try Pixels.read( + PixelOptions(image: path, axis: .row, at: 7, from: nil, to: nil)) + } + } + } + + @Test("reports an image it cannot read") + func reportsAMissingImage() { + #expect(throws: DriveError.self) { + try Pixels.read( + PixelOptions( + image: "/no/such/screenshot.png", axis: .row, at: 0, from: nil, to: nil)) + } + } + + /// A translucent pixel carries its alpha, so it cannot be mistaken for an + /// opaque one of the same colour. + @Test("spells an opaque colour without alpha and a translucent one with it") + func spellsAlphaOnlyWhenItMatters() { + #expect(Pixel(red: 0xDB, green: 0xDB, blue: 0xDB, alpha: 255).hex == "#DBDBDB") + #expect(Pixel(red: 0xDB, green: 0xDB, blue: 0xDB, alpha: 128).hex == "#DBDBDB80") + } + + @Test("collapses an empty line into no runs") + func emptyLine() { + #expect(Pixels.runs(of: [], startingAt: 0).isEmpty) + } + + /// A screenshot is written in the display's profile, which is not the space a + /// palette constant was written in. Read raw, a `#DBDBDB` divider comes back + /// as something several steps off and looks like a bug in the app. + /// + /// The image here is tagged Display P3 and holds the P3 encoding of sRGB + /// `#DBDBDB`, so a reader that converts reports the value the palette names + /// and a reader that does not reports `#DBDBDB` itself — which is the wrong + /// answer, arrived at by leaving the numbers alone. + @Test("reports colours in sRGB whatever space the image is tagged with") + func convertsToSRGB() throws { + let p3 = try #require(CGColorSpace(name: CGColorSpace.displayP3)) + let sRGB = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let level = CGFloat(0xDB) / 255 + let grey = try #require( + CGColor(colorSpace: sRGB, components: [level, level, level, 1])) + let converted = try #require( + grey.converted(to: p3, intent: CGColorRenderingIntent.defaultIntent, options: nil)) + let parts = try #require(converted.components) + + let path = NSTemporaryDirectory() + "/jpdrive-p3-\(UUID().uuidString).png" + defer { try? FileManager.default.removeItem(atPath: path) } + + let bytes = parts.prefix(3).map { UInt8(($0 * 255).rounded()) } + [255] + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: 1, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 4, + space: p3, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect(report.runs.first?.color == "#DBDBDB") + } + + /// A window capture carries real partial alpha at its rounded corners and + /// antialiased edges, and the bitmap behind it is premultiplied — the only + /// 8-bit RGBA layout a context accepts. Read straight out, a half-transparent + /// red reports as a dark red: a colour that was never on screen, spelled the + /// same way as one that was. + /// + /// Round-tripped through a PNG rather than asserted on a hand-built `Pixel`, + /// because the premultiplication happens in the decode path and a test that + /// constructs the pixel itself never reaches it. + @Test("recovers the colour behind a partially transparent pixel") + func unpremultipliesPartialAlpha() throws { + // Left: `#FF0000` at half alpha, stored premultiplied as 128,0,0,128. + // Right: fully transparent, which carries no colour to recover. + let bytes: [UInt8] = [128, 0, 0, 128, 0, 0, 0, 0] + + let space = try #require(CGColorSpace(name: CGColorSpace.sRGB)) + let provider = try #require(CGDataProvider(data: Data(bytes) as CFData)) + let image = try #require( + CGImage( + width: 2, + height: 1, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: 8, + space: space, + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + )) + + let path = NSTemporaryDirectory() + "/jpdrive-alpha-\(UUID().uuidString).png" + defer { try? FileManager.default.removeItem(atPath: path) } + + let url = URL(fileURLWithPath: path) as CFURL + let destination = try #require( + CGImageDestinationCreateWithURL(url, UTType.png.identifier as CFString, 1, nil)) + CGImageDestinationAddImage(destination, image, nil) + #expect(CGImageDestinationFinalize(destination)) + + let report = try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: nil, to: nil)) + + #expect( + report.runs == [ + PixelRun(start: 0, count: 1, color: "#FF000080"), + PixelRun(start: 1, count: 1, color: "#00000000"), + ] + ) + } + + /// A `--from` past the edge used to be reported as being past a `--to` the + /// caller never wrote, because `to` is clamped to the image before the two + /// are compared. The message has to name the bound that was actually wrong. + @Test("refuses a start offset outside the image") + func refusesAStartOutsideTheImage() throws { + try withImage(rows: [["#FFFFFF", "#FFFFFF"]]) { path in + let error = try #require(throws: DriveError.self) { + try Pixels.read( + PixelOptions(image: path, axis: .row, at: 0, from: 150, to: nil)) + } + + #expect(error.kind == .badUsage) + #expect(error.message.contains("--from 150 is outside the image")) + } + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift new file mode 100644 index 000000000..6240dcf08 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/ResizeTests.swift @@ -0,0 +1,103 @@ +import ApplicationServices +import Foundation +import Testing + +@testable import DriveKit + +/// Tests for the `resize` step. +/// +/// A window is the only thing that accepts a write to `AXSize`, and resizing is +/// the one interaction a driver cannot reach any other way: a drag of a window's +/// edge has to be synthesized, and a synthesized drag needs the window frontmost. +@Suite("Resize") +struct ResizeTests { + /// A window that accepts a size, starting at `size`. + private func window(_ size: CGSize, settable: Bool = true) -> FakeElement { + let window = FakeElement( + role: "AXWindow", + identifier: "the-window", + settable: settable ? [kAXSizeAttribute] : [] + ) + window.sizes[kAXSizeAttribute] = size + return window + } + + private func step(width: Double, height: Double) -> Step { + .resize(Step.SizeTarget(identifier: "the-window", width: width, height: height)) + } + + @Test("writes the size it was asked for") + func writesTheSize() throws { + let window = self.window(CGSize(width: 900, height: 450)) + + let result = try Act.run(step(width: 1400, height: 900), in: window) + + #expect(window.sizes[kAXSizeAttribute] == CGSize(width: 1400, height: 900)) + #expect(result.step == "resize") + #expect(result.role == "AXWindow") + #expect(result.confirmed == true) + #expect(result.size == "1400x900") + } + + /// A window clamps to its own minimum and maximum, so the write succeeds and + /// the window lands somewhere else. Reporting what it reached is the whole + /// reason the step reads the size back instead of echoing the request. + @Test("reports the size it reached when the window clamps the request") + func reportsAClampedSize() throws { + let window = self.window(CGSize(width: 900, height: 450)) + window.ignoresWrites = true + + let result = try Act.run(step(width: 200, height: 100), in: window) + + #expect(result.confirmed == false) + #expect(result.size == "900x450") + } + + /// Most elements inside a window do not accept a size, and a step that asked + /// anyway would report success having changed nothing. + @Test("refuses an element that does not accept a size") + func refusesAnUnsizableElement() { + let element = window(CGSize(width: 900, height: 450), settable: false) + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: element) + } + } + + @Test("reports an identifier that is not in the tree") + func reportsAMissingIdentifier() { + let other = FakeElement(role: "AXWindow", identifier: "something-else") + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: other) + } + } + + @Test("surfaces a write the accessibility API refused") + func surfacesARefusedWrite() { + let window = self.window(CGSize(width: 900, height: 450)) + window.writeStatus = .cannotComplete + + #expect(throws: DriveError.self) { + try Act.run(step(width: 1400, height: 900), in: window) + } + } + + /// The step arrives as JSON from the driver's caller, so the spelling of its + /// keys is part of the contract. + @Test("decodes the step a caller writes") + func decodesTheStep() throws { + let json = #"{"resize":{"identifier":"w","width":1400,"height":900}}"# + + let decoded = try JSONDecoder().decode(Step.self, from: Data(json.utf8)) + + guard case .resize(let target) = decoded else { + Issue.record("expected a resize step, got \(decoded)") + return + } + + #expect(target.identifier == "w") + #expect(target.width == 1400) + #expect(target.height == 900) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift new file mode 100644 index 000000000..350b55bff --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TreeTests.swift @@ -0,0 +1,183 @@ +import Testing + +@testable import DriveKit + +@Suite("Tree") +struct TreeTests { + /// Options with the bounds wide open, so a test names only what it is about. + private func options( + prefix: String? = nil, + maxMatches: Int = 100, + maxDepth: Int = 20, + maxSiblings: Int = 0, + frames: Bool = false + ) -> TreeOptions { + return TreeOptions( + pid: 0, + identifierPrefix: prefix, + maxMatches: maxMatches, + maxDepth: maxDepth, + maxSiblings: maxSiblings, + frames: frames + ) + } + + @Test("an unfiltered walk keeps every element") + func keepsEverythingUnfiltered() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.role == "AXApplication") + let outline = try #require(tree.children.first) + #expect(outline.identifier == "sidebar.list") + #expect(outline.children.count == 2) + } + + /// The bug this replaced: with a sibling cap in force, a search for a row past + /// the cap found nothing, because the cap dropped it before the filter saw it. + @Test("a filtered walk finds a match past the sibling cap") + func filterOutrunsTheSiblingCap() throws { + let root = FakeElement.sidebar(rowCount: 50) + + let tree = try #require( + Tree.walk(from: root, options: options(prefix: "sidebar.row.42", maxSiblings: 5)) + ) + + let leaf = tree.children.first?.children.first?.children.first?.children.first + #expect(leaf?.identifier == "sidebar.row.42") + } + + /// Ancestors are kept so a match can be located, but only the ones leading to it. + @Test("a filtered walk drops branches holding no match") + func prunesUnmatchedBranches() throws { + let root = FakeElement( + role: "AXApplication", + children: [ + FakeElement(role: "AXWindow", identifier: "other.window"), + FakeElement( + role: "AXOutline", + identifier: "sidebar.list", + children: [FakeElement(role: "AXRow", identifier: "sidebar.row.0")] + ), + ] + ) + + let tree = try #require(Tree.walk(from: root, options: options(prefix: "sidebar."))) + + #expect(tree.children.count == 1) + #expect(tree.children.first?.identifier == "sidebar.list") + + // The dropped window is still counted. A filtered read that silently + // showed one child of two would have the reader believe the application + // has one. + #expect(tree.elidedChildren == 1) + } + + @Test("a filtered walk with no match returns nothing") + func returnsNothingWhenNothingMatches() { + let root = FakeElement.sidebar(rowCount: 3) + + #expect(Tree.walk(from: root, options: options(prefix: "nope.")) == nil) + } + + /// The budget is the bound that keeps a prefix search off the whole tree, so it + /// has to actually stop the walk rather than only trim the output. + @Test("the match budget stops the walk") + func budgetStopsTheWalk() throws { + let root = FakeElement.sidebar(rowCount: 500) + + let tree = try #require( + Tree.walk(from: root, options: options(prefix: "sidebar.", maxMatches: 3)) + ) + + // One match is the outline itself, leaving two rows. + let outline = try #require(tree.children.first) + #expect(outline.children.count == 2) + + // Reads, not results: a budget that trimmed the output while still visiting + // every element would pass an assertion on the tree alone. + let rows = try #require(root.children.first?.children) + #expect(rows.dropFirst(3).allSatisfy { $0.reads == 0 }) + } + + @Test("the sibling cap reports what it skipped") + func capReportsElidedChildren() throws { + let root = FakeElement.sidebar(rowCount: 10) + + let tree = try #require(Tree.walk(from: root, options: options(maxSiblings: 4))) + + let outline = try #require(tree.children.first) + #expect(outline.children.count == 4) + #expect(outline.elidedChildren == 6) + } + + @Test("a complete level reports no elision") + func noElisionWhenComplete() throws { + let root = FakeElement.sidebar(rowCount: 3) + + let tree = try #require(Tree.walk(from: root, options: options(maxSiblings: 10))) + + #expect(tree.children.first?.elidedChildren == nil) + } + + /// The count is what separates a node stopped at the depth limit from a leaf. + /// Without it the two render identically and a reader concludes the element + /// has no children. + @Test("the depth limit stops the descent and reports what it did not reach") + func depthLimitStopsDescent() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let tree = try #require(Tree.walk(from: root, options: options(maxDepth: 1))) + + #expect(tree.children.first?.children.isEmpty == true) + #expect(tree.children.first?.elidedChildren == 2) + } + + /// Frames move whenever a window moves or a list scrolls, so they stay out + /// unless asked for. + @Test("frames are omitted by default") + func framesAreOptIn() throws { + let root = FakeElement(role: "AXWindow") + root.attributes["AXFrame"] = "0.0,0.0 100.0x100.0" + + let without = try #require(Tree.walk(from: root, options: options())) + #expect(without.frame == nil) + + let with = try #require(Tree.walk(from: root, options: options(frames: true))) + #expect(with.frame == "0.0,0.0 100.0x100.0") + } + + /// An element the accessibility API refused reports no identifier, no label + /// and no children — which is how a bare container reports too. Without the + /// marker a gap in the walk reads as a plain element that is really there. + @Test("an element that could not be read is marked") + func marksAnUnreadableElement() throws { + let broken = FakeElement(role: "AXGroup", identifier: "gone") + broken.readFails = true + let root = FakeElement(role: "AXApplication", children: [broken]) + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.unreadable == nil) + + let child = try #require(tree.children.first) + #expect(child.unreadable == true) + #expect(child.identifier == nil) + #expect(child.role == "<none>") + } + + /// An attribute the element does not report must arrive as absent, not as the + /// text of whatever error the accessibility API answered with. + @Test("absent attributes are absent, not error text") + func absentAttributesAreNull() throws { + let root = FakeElement(role: "AXCell") + + let tree = try #require(Tree.walk(from: root, options: options())) + + #expect(tree.identifier == nil) + #expect(tree.label == nil) + #expect(tree.value == nil) + #expect(tree.enabled == nil) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift new file mode 100644 index 000000000..436ce5ab2 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/TypeTests.swift @@ -0,0 +1,186 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Act.type") +struct TypeTests { + /// A text field shaped like the one SwiftUI produces: `AXValue` writable, + /// `AXPress` absent. + private func field(identifier: String = "sidebar.filter") -> FakeElement { + return FakeElement( + role: "AXTextField", + identifier: identifier, + settable: [kAXValueAttribute, kAXFocusedAttribute], + actions: ["AXShowMenu", "AXConfirm"] + ) + } + + @Test("writes the text into the field") + func writesTheText() throws { + let field = field() + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect(field.attributes[kAXValueAttribute] == "driving") + #expect(result.step == "type") + #expect(result.role == "AXTextField") + #expect(result.confirmed == true) + } + + /// Writing the value alone changes the text a `SwiftUI` field shows without the + /// binding behind it noticing, so the application carries on as though nothing + /// was typed. The confirm is what the application actually observes, and a + /// `type` that skipped it would report success having done nothing. + @Test("commits the edit through the field's confirm action") + func commitsTheEdit() throws { + let field = field() + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect(field.performed == ["AXConfirm"]) + #expect(result.committed == true) + } + + /// A field that publishes every change as it happens needs nothing committing, + /// so this is reported rather than treated as a failure. + @Test("reports a field with no confirm action as uncommitted") + func reportsAnUncommittedWrite() throws { + let field = FakeElement( + role: "AXTextField", + identifier: "live", + settable: [kAXValueAttribute], + actions: [] + ) + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run(.type(.init(identifier: "live", text: "x")), in: root) + + #expect(result.confirmed == true) + #expect(result.committed == false) + #expect(field.performed.isEmpty) + } + + /// Text in the field that the application never saw is the worst outcome to + /// report as success, so a refused confirm fails the step. + @Test("fails when the edit cannot be committed") + func failsOnARefusedConfirm() throws { + let field = field() + field.performStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "sidebar.filter", text: "x")), in: root) + } + + #expect(error.kind == .actionFailed) + #expect(error.hint?.contains("not committed") == true) + } + + /// Typing replaces rather than appends, so a script does not have to clear the + /// field first and a second step cannot silently concatenate. + @Test("replaces what the field already held") + func replacesExistingText() throws { + let field = field() + field.attributes[kAXValueAttribute] = "old" + let root = FakeElement(role: "AXApplication", children: [field]) + + _ = try Act.run(.type(.init(identifier: "sidebar.filter", text: "new")), in: root) + + #expect(field.attributes[kAXValueAttribute] == "new") + } + + /// Clearing is typing nothing, not a step of its own. + @Test("an empty string clears the field") + func clearsTheField() throws { + let field = field() + field.attributes[kAXValueAttribute] = "something" + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run(.type(.init(identifier: "sidebar.filter", text: "")), in: root) + + #expect(field.attributes[kAXValueAttribute] == "") + #expect(result.confirmed == true) + } + + /// A static label and a disabled field both resolve by identifier and both + /// refuse the write. Failing here beats reporting a write that went nowhere. + @Test("fails on an element whose value is not writable") + func failsOnAReadOnlyElement() throws { + let label = FakeElement(role: "AXStaticText", identifier: "subtitle") + let root = FakeElement(role: "AXApplication", children: [label]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "subtitle", text: "x")), in: root) + } + + #expect(error.kind == .notEditable) + #expect(label.attributes[kAXValueAttribute] == nil) + } + + @Test("reports a refused write") + func reportsARefusedWrite() throws { + let field = field() + field.writeStatus = .cannotComplete + let root = FakeElement(role: "AXApplication", children: [field]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "sidebar.filter", text: "x")), in: root) + } + + #expect(error.kind == .writeFailed) + #expect(error.message.contains("cannot_complete")) + } + + /// The accessibility API lets a target accept a write and discard it, which is + /// the whole reason the step reads back instead of trusting the status. + @Test("reports an accepted write that did not take") + func reportsAnIneffectiveWrite() throws { + let field = field() + field.ignoresWrites = true + let root = FakeElement(role: "AXApplication", children: [field]) + + let result = try Act.run( + .type(.init(identifier: "sidebar.filter", text: "driving")), + in: root + ) + + #expect( + result.confirmed == false, + "a field that discarded the text must not report as confirmed" + ) + } + + @Test("reports an identifier it could not find") + func reportsAMissingIdentifier() throws { + let root = FakeElement(role: "AXApplication", children: [field()]) + + let error = try #require(throws: DriveError.self) { + try Act.run(.type(.init(identifier: "nope", text: "x")), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// Only the addressed field is written to, so a step cannot quietly clobber a + /// second field that happens to sit nearby. + @Test("leaves other fields alone") + func leavesOtherFieldsAlone() throws { + let first = field(identifier: "one") + let second = field(identifier: "two") + let root = FakeElement(role: "AXApplication", children: [first, second]) + + _ = try Act.run(.type(.init(identifier: "two", text: "x")), in: root) + + #expect(first.attributes[kAXValueAttribute] == nil) + #expect(second.attributes[kAXValueAttribute] == "x") + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift new file mode 100644 index 000000000..7510632c2 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WaitForTests.swift @@ -0,0 +1,137 @@ +import Testing + +@testable import DriveKit + +@Suite("Act.waitFor") +struct WaitForTests { + /// A container that produces the awaited element on its third read, and not + /// before. + /// + /// The delay is what makes a wait test mean anything: against a tree that + /// already holds the element, polling and not polling look identical. + private func appearsOnThirdRead() -> FakeElement { + let container = FakeElement(role: "AXScrollArea", identifier: "transcript.scroll") + container.onRead = { element in + guard element.reads == 3 else { return } + element.children = [ + FakeElement(role: "AXGroup", identifier: "transcript.event.1") + ] + } + return container + } + + private func step( + _ identifier: String, + under: String? = nil, + timeoutMs: Int? = nil, + intervalMs: Int? = 1 + ) -> Step { + return .waitFor( + .init( + identifier: identifier, under: under, timeoutMs: timeoutMs, + intervalMs: intervalMs) + ) + } + + @Test("an element already present is returned on the first attempt") + func returnsImmediately() throws { + let root = FakeElement.sidebar(rowCount: 2) + + // A zero timeout permits exactly one attempt, so a pass here cannot have + // come from a retry. + let result = try Act.run(step("sidebar.row.1", timeoutMs: 0), in: root) + + #expect(result.step == "wait_for") + #expect(result.role == "AXUnknown") + #expect(result.confirmed == true) + } + + /// Half of a pair. This one proves the fixture genuinely withholds the element, + /// so that the passing case below is evidence of retrying rather than of the + /// element having been there all along. + @Test("one attempt is not enough for an element that appears later") + func oneAttemptIsNotEnough() throws { + let container = appearsOnThirdRead() + let root = FakeElement(role: "AXApplication", children: [container]) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("transcript.event.1", timeoutMs: 0), in: root) + } + + #expect(error.kind == .timeout) + } + + @Test("polling finds an element that appears later") + func findsAnElementThatAppearsLater() throws { + let container = appearsOnThirdRead() + let root = FakeElement(role: "AXApplication", children: [container]) + + let result = try Act.run(step("transcript.event.1", timeoutMs: 2000), in: root) + + #expect(result.confirmed == true) + #expect(result.role == "AXGroup") + #expect( + container.reads >= 3, "the element cannot have been found before its third read") + } + + @Test("an element that never appears times out") + func timesOut() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("never.appears", timeoutMs: 20), in: root) + } + + #expect(error.kind == .timeout) + #expect(error.message.contains("never.appears")) + } + + /// A single attempt eating the whole timeout is the failure mode that makes an + /// unscoped wait useless, so the error says what to do about it. + @Test("a timeout after one attempt suggests scoping") + func suggestsScopingAfterOneAttempt() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("never.appears", timeoutMs: 0), in: root) + } + + #expect(error.hint?.contains("under") == true) + } + + /// Waiting inside something that does not exist is a mistake in the script, not + /// a condition that might come true. + @Test("a missing container fails at once rather than being waited for") + func missingContainerFailsFast() throws { + let root = FakeElement.sidebar(rowCount: 2) + + let error = try #require(throws: DriveError.self) { + try Act.run(step("anything", under: "no.such.container", timeoutMs: 5000), in: root) + } + + #expect(error.kind == .identifierNotFound) + } + + /// The reason `under` exists. Without it every attempt re-reads the whole + /// application, and against this app's sidebar one attempt outlasts a typical + /// timeout. + @Test("scoping keeps polling off the rest of the tree") + func scopingBoundsThePolling() throws { + let sidebar = FakeElement.sidebar(rowCount: 20) + let outline = try #require(sidebar.children.first) + let container = FakeElement(role: "AXScrollArea", identifier: "transcript.scroll") + let root = FakeElement(role: "AXApplication", children: [outline, container]) + + let error = try #require(throws: DriveError.self) { + try Act.run( + step("transcript.event.1", under: "transcript.scroll", timeoutMs: 30), + in: root + ) + } + #expect(error.kind == .timeout) + + // Read once while resolving the container, and never again. Repeated reads + // here would mean each poll was walking the sidebar. + #expect(outline.children.allSatisfy { $0.reads == 1 }) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift new file mode 100644 index 000000000..701110dc4 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowIDsTests.swift @@ -0,0 +1,129 @@ +import CoreGraphics +import Foundation +import Testing + +@testable import DriveKit + +@Suite("WindowIDs") +struct WindowIDsTests { + /// One entry shaped the way the window server reports it: every number a + /// `CFNumber`, with the bounds arriving as doubles. + private func entry( + id: Int, + pid: Int, + layer: Int = 0, + title: String? = "JP", + width: Double = 1200, + height: Double = 800 + ) -> [String: Any] { + var window: [String: Any] = [ + kCGWindowNumber as String: id, + kCGWindowOwnerPID as String: pid, + kCGWindowLayer as String: layer, + kCGWindowBounds as String: ["X": 0.0, "Y": 0.0, "Width": width, "Height": height], + ] + window[kCGWindowName as String] = title + return window + } + + @Test("reports the window server's number and the window's size") + func reportsIdentifiers() { + let listed = [entry(id: 7412, pid: 4321)] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: "JP", width: 1200, height: 800) + ] + ) + } + + /// Every application on the desktop is in the list, so a capture that took the + /// first entry would photograph whatever happened to be frontmost. + @Test("keeps only the windows the pid owns") + func filtersByOwner() { + let listed = [ + entry(id: 1, pid: 999, title: "Terminal"), + entry(id: 2, pid: 4321), + entry(id: 3, pid: 111, title: "Finder"), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// Tooltips, drag images and menu shadows are the app's too, and capturing one + /// in place of the window is a wrong answer rather than a failure. + @Test("keeps only windows on the normal layer") + func dropsChrome() { + let listed = [ + entry(id: 1, pid: 4321, layer: 25, title: "tooltip"), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// A panel that has never been shown sits in the list at zero size, and + /// capturing it produces an empty file. + @Test("drops windows with no area") + func dropsEmptyWindows() { + let listed = [ + entry(id: 1, pid: 4321, width: 0, height: 0), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [2]) + } + + /// The window server withholds other applications' titles until the Screen + /// Recording grant is given, which is the state a first run is in. + @Test("a window with no readable title is still reported") + func toleratesAMissingTitle() { + let listed = [entry(id: 7412, pid: 4321, title: nil)] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: nil, width: 1200, height: 800) + ] + ) + } + + /// What actually arrives is a `CFArray` of `CFDictionary`, so every number in it + /// is an `NSNumber` once bridged, and a reader that only understood Swift's own + /// numeric types would report an application with no windows at all. + @Test("reads the numbers as the bridged types the window server hands over") + func readsBridgedNumbers() { + let listed: [[String: Any]] = [ + [ + kCGWindowNumber as String: NSNumber(value: 7412), + kCGWindowOwnerPID as String: NSNumber(value: 4321), + kCGWindowLayer as String: NSNumber(value: 0), + kCGWindowName as String: "JP", + kCGWindowBounds as String: [ + "X": NSNumber(value: 0.0), + "Y": NSNumber(value: 0.0), + "Width": NSNumber(value: 1200.0), + "Height": NSNumber(value: 800.0), + ], + ] + ] + + #expect( + WindowIDs.capturable(from: listed, pid: 4321) == [ + CaptureWindow(id: 7412, title: "JP", width: 1200, height: 800) + ] + ) + } + + /// Front-to-back is the window server's own order, and it is the only thing + /// telling a caller which of two windows to capture. + @Test("preserves the order the window server reported") + func preservesOrder() { + let listed = [ + entry(id: 3, pid: 4321), + entry(id: 1, pid: 4321), + entry(id: 2, pid: 4321), + ] + + #expect(WindowIDs.capturable(from: listed, pid: 4321).map(\.id) == [3, 1, 2]) + } +} diff --git a/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift new file mode 100644 index 000000000..437fa98e0 --- /dev/null +++ b/apps/macos/Tools/jpdrive/Tests/DriveKitTests/WindowsTests.swift @@ -0,0 +1,61 @@ +import ApplicationServices +import Testing + +@testable import DriveKit + +@Suite("Windows") +struct WindowsTests { + @Test("reports each window's own facts") + func reportsWindowFacts() { + let main = FakeElement(role: "AXWindow", identifier: "workspace-AppWindow-1") + main.attributes[kAXTitleAttribute] = "JP" + main.attributes[kAXMainAttribute] = "1" + main.attributes[kAXMinimizedAttribute] = "0" + main.attributes["AXFrame"] = "0.0,0.0 1200.0x800.0" + + let other = FakeElement(role: "AXWindow", identifier: "workspace-AppWindow-2") + other.attributes[kAXMainAttribute] = "0" + other.attributes[kAXMinimizedAttribute] = "1" + + let app = FakeElement(role: "AXApplication") + app.related[kAXWindowsAttribute] = [main, other] + + #expect( + Windows.list(of: app) == [ + WindowSummary( + identifier: "workspace-AppWindow-1", + title: "JP", + main: true, + minimized: false, + frame: "0.0,0.0 1200.0x800.0" + ), + WindowSummary( + identifier: "workspace-AppWindow-2", + title: nil, + main: false, + minimized: true, + frame: nil + ), + ] + ) + } + + /// A running application with every window closed is a normal state, not an + /// error. + @Test("an application with no windows reports an empty list") + func noWindows() { + #expect(Windows.list(of: FakeElement(role: "AXApplication")).isEmpty) + } + + /// Windows are read from the application's own attribute, not found by walking + /// into the hierarchy. A listing that descended would pick up sheets and popups + /// as if they were windows. + @Test("windows are read from the attribute, not from the children") + func doesNotWalkChildren() { + let child = FakeElement(role: "AXWindow", identifier: "not.a.window") + let app = FakeElement(role: "AXApplication", children: [child]) + + #expect(Windows.list(of: app).isEmpty) + #expect(child.reads == 0) + } +} diff --git a/justfile b/justfile index 01ec2a8d4..9981f9d3f 100644 --- a/justfile +++ b/justfile @@ -194,6 +194,52 @@ build-ffi PROFILE="debug": (_install "cbindgen@" + cbindgen_version) echo "library: $out/libjp_ffi.a" >&2 echo "header: $out/include/jp_ffi.h" >&2 +# Build the `jpdrive` accessibility driver that the `debug_app_*` tools shell out +# to. +# +# A standalone SwiftPM package rather than a target in the app's Xcode project, +# so the binary lands at a predictable path with no derived-data lookup. +[group('build')] +[macos] +build-drive CONFIG="release": + #!/usr/bin/env sh + set -eu + + swift build --package-path apps/macos/Tools/jpdrive -c {{CONFIG}} + + bin=$(swift build --package-path apps/macos/Tools/jpdrive -c {{CONFIG}} --show-bin-path) + echo "binary: $bin/jpdrive" >&2 + +# Run the `jpdrive` test suite. +# +# Covers the driver's traversal against a fake accessibility tree, so it needs no +# running app and no accessibility grant. +[group('test')] +[macos] +test-drive *ARGS: + swift test --package-path apps/macos/Tools/jpdrive {{ARGS}} + +# Report whether this process may read another app's accessibility tree. +# +# Run under the terminal, under `just`, and under `serve-tools` to find out +# whether a TCC grant given to the terminal reaches a tool it started. See +# `apps/macos/Tools/jpdrive/README.md`. +# +# PID is the target application's process id, e.g. `$(pgrep -f JP.app)`. +[group('debug')] +[macos] +drive-doctor PID="": build-drive + #!/usr/bin/env sh + set -eu + + bin=$(swift build --package-path apps/macos/Tools/jpdrive -c release --show-bin-path) + + if [ -n "{{PID}}" ]; then + "$bin/jpdrive" doctor --pid "{{PID}}" + else + "$bin/jpdrive" doctor + fi + # Generate the macOS app's Xcode project from `apps/macos/project.yml`. # # The project file is generated rather than committed, so `project.yml` stays the