From 5c750f2565ec48833ae0dd01b932256e0c214f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 7 Jul 2026 17:12:47 +0200 Subject: [PATCH 1/6] feat: add TV remote command --- scripts/integration-progress-model.ts | 2 +- src/__tests__/cli-grammar.test.ts | 2 +- src/__tests__/contracts-schema-public.test.ts | 9 +++ src/__tests__/runtime-public.test.ts | 1 + src/backend.ts | 10 +++ .../__tests__/args-parse-interaction.test.ts | 13 ++++ src/cli/parser/args.ts | 2 + src/cli/parser/cli-help.ts | 4 +- src/client/client-types.ts | 10 ++- src/client/client.ts | 2 + src/cloud-webdriver/capabilities.ts | 2 + src/cloud-webdriver/webdriver-interactor.ts | 5 ++ src/command-catalog.ts | 1 + src/commands/interaction/metadata.ts | 2 +- .../interaction/runtime/selector-read.test.ts | 67 +++++++++++++++++ .../interaction/runtime/selector-read.ts | 19 ++++- src/commands/interaction/selectors.ts | 3 +- src/commands/system/index.test.ts | 27 +++++++ src/commands/system/index.ts | 74 +++++++++++++++++- src/commands/system/output.ts | 1 + src/commands/system/runtime/index.ts | 7 ++ src/commands/system/runtime/system.test.ts | 12 +++ src/commands/system/runtime/system.ts | 36 +++++++++ src/contracts/navigation.ts | 9 +++ src/core/__tests__/capabilities.test.ts | 19 +++++ .../capability-plugin-routing-parity.test.ts | 24 +++++- .../__tests__/dispatch-interactions.test.ts | 1 + src/core/__tests__/dispatch-tv-remote.test.ts | 75 +++++++++++++++++++ .../__tests__/command-result.test.ts | 6 +- src/core/command-descriptor/command-result.ts | 2 + src/core/command-descriptor/registry.ts | 11 +++ src/core/dispatch.ts | 33 ++++++++ src/core/interactor-types.ts | 2 + src/core/interactors/android.ts | 2 + src/core/interactors/linux.ts | 3 + src/core/interactors/register-builtins.ts | 9 ++- src/core/interactors/web.ts | 1 + .../apple-os-capability-table-parity.test.ts | 15 ++++ src/core/tv-remote.ts | 33 ++++++++ .../__tests__/request-handler-catalog.test.ts | 1 + src/daemon/__tests__/selectors.test.ts | 27 +++++++ src/daemon/selectors-match.ts | 2 + src/mcp/__tests__/command-tools.test.ts | 8 ++ src/mcp/command-output-schemas.ts | 10 +++ src/platforms/android/input-actions.ts | 22 ++++++ src/platforms/apple/interactor.ts | 12 +++ src/platforms/apple/plugin.ts | 22 +++++- .../__tests__/selector-is-predicates.test.ts | 21 ++++++ src/utils/selector-is-predicates.ts | 22 ++++-- src/utils/selectors-parse.ts | 2 + .../apple-platform-output-guard.test.ts | 1 + .../suites/agent-device-smoke-suite.ts | 19 +++++ 52 files changed, 704 insertions(+), 21 deletions(-) create mode 100644 src/core/__tests__/dispatch-tv-remote.test.ts create mode 100644 src/core/tv-remote.ts diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 299d78a7d..1b4c4511f 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -152,7 +152,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['hideTouches', 'recording without touch overlays'], ['intervalMs', 'repeated press interval'], ['delayMs', 'typing/fill delay'], - ['durationMs', 'scroll and gesture duration'], + ['durationMs', 'scroll, gesture, and TV remote duration'], ['holdMs', 'press hold duration'], ['jitterPx', 'press jitter'], ['pixels', 'scroll distance'], diff --git a/src/__tests__/cli-grammar.test.ts b/src/__tests__/cli-grammar.test.ts index 5b62e5868..a2b9e42e9 100644 --- a/src/__tests__/cli-grammar.test.ts +++ b/src/__tests__/cli-grammar.test.ts @@ -164,7 +164,7 @@ test('is grammar explains the predicate/selector-key collision on invalid predic assert.equal(err.code, 'INVALID_ARGS'); assert.match( err.message, - /is requires predicate: visible\|hidden\|exists\|editable\|selected\|text/, + /is requires predicate: visible\|hidden\|exists\|editable\|selected\|focused\|text/, ); assert.match(err.details?.hint ?? '', /is /); assert.match(err.details?.hint ?? '', /visible=true/); diff --git a/src/__tests__/contracts-schema-public.test.ts b/src/__tests__/contracts-schema-public.test.ts index b22506e68..be88f7bcd 100644 --- a/src/__tests__/contracts-schema-public.test.ts +++ b/src/__tests__/contracts-schema-public.test.ts @@ -12,6 +12,7 @@ import type { BackCommandResult, HomeCommandResult, RotateCommandResult, + TvRemoteCommandResult, } from '../contracts/navigation.ts'; import type { ViewportCommandResult } from '../contracts/viewport.ts'; import { centerOfRect, defaultHintForCode, normalizeError } from '../sdk/contracts.ts'; @@ -166,6 +167,13 @@ test('command result contracts are assignable to command result map', () => { } satisfies RotateCommandResult; const rotateFromMap: CommandResult<'rotate'> = rotate; + const tvRemote = { + action: 'tv-remote', + button: 'select', + message: 'Pressed TV remote select', + } satisfies TvRemoteCommandResult; + const tvRemoteFromMap: CommandResult<'tv-remote'> = tvRemote; + const clipboard = { action: 'write', textLength: 11, @@ -187,6 +195,7 @@ test('command result contracts are assignable to command result map', () => { assert.equal(homeFromMap.action, 'home'); assert.equal(appSwitcherFromMap.action, 'app-switcher'); assert.equal(rotateFromMap.orientation, 'portrait'); + assert.equal(tvRemoteFromMap.button, 'select'); assert.equal(clipboardFromMap.action === 'write' ? clipboardFromMap.textLength : -1, 11); assert.equal( appstateFromMap.platform === 'android' ? appstateFromMap.package : '', diff --git a/src/__tests__/runtime-public.test.ts b/src/__tests__/runtime-public.test.ts index 7c8559438..6aace159d 100644 --- a/src/__tests__/runtime-public.test.ts +++ b/src/__tests__/runtime-public.test.ts @@ -265,6 +265,7 @@ test('internal backend, commands, and io modules are usable', () => { assert.equal(typeof commands.system.settings, 'function'); assert.equal(typeof commands.system.alert, 'function'); assert.equal(typeof commands.system.appSwitcher, 'function'); + assert.equal(typeof commands.system.tvRemote, 'function'); assert.equal(typeof commands.admin.devices, 'function'); assert.equal(typeof commands.admin.install, 'function'); assert.equal(typeof commands.recording.record, 'function'); diff --git a/src/backend.ts b/src/backend.ts index e58e6db11..b619a2ce2 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -9,6 +9,7 @@ import type { ClickButton } from './core/click-button.ts'; import type { DeviceRotation } from './core/device-rotation.ts'; import type { ScrollDirection } from './core/scroll-gesture.ts'; import type { SessionSurface } from './core/session-surface.ts'; +import type { TvRemoteButton } from './core/tv-remote.ts'; import type { RecordingExportQuality } from './core/recording-export-quality.ts'; import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts'; import type { @@ -87,6 +88,11 @@ export type BackendBackOptions = { mode?: BackMode; }; +export type BackendTvRemoteOptions = { + button: TvRemoteButton; + durationMs?: number; +}; + export type BackendKeyboardOptions = { action: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; @@ -491,6 +497,10 @@ export type AgentDeviceBackend = { options?: BackendBackOptions, ): Promise; pressHome?(context: BackendCommandContext): Promise; + pressTvRemote?( + context: BackendCommandContext, + options: BackendTvRemoteOptions, + ): Promise; rotate?( context: BackendCommandContext, orientation: BackendDeviceOrientation, diff --git a/src/cli/parser/__tests__/args-parse-interaction.test.ts b/src/cli/parser/__tests__/args-parse-interaction.test.ts index 1942f09b5..56b8d68ab 100644 --- a/src/cli/parser/__tests__/args-parse-interaction.test.ts +++ b/src/cli/parser/__tests__/args-parse-interaction.test.ts @@ -26,6 +26,19 @@ test('parseArgs accepts keyboard subcommands', () => { assert.deepEqual(enter.positionals, ['enter']); }); +test('parseArgs accepts tv-remote and d-pad alias', () => { + const remote = parseArgs(['tv-remote', 'press', 'select', '--duration-ms', '250'], { + strictFlags: true, + }); + assert.equal(remote.command, 'tv-remote'); + assert.deepEqual(remote.positionals, ['press', 'select']); + assert.equal(remote.flags.durationMs, 250); + + const dpad = parseArgs(['d-pad', 'down'], { strictFlags: true }); + assert.equal(dpad.command, 'tv-remote'); + assert.deepEqual(dpad.positionals, ['down']); +}); + test('parseArgs accepts scroll pixel distance and duration flags', () => { const parsed = parseArgs(['scroll', 'down', '--pixels', '240', '--duration-ms', '50'], { strictFlags: true, diff --git a/src/cli/parser/args.ts b/src/cli/parser/args.ts index cf37275ec..b7689fdf9 100644 --- a/src/cli/parser/args.ts +++ b/src/cli/parser/args.ts @@ -346,6 +346,7 @@ function normalizeParsedCommandAliases(parsed: ParsedArgs): ParsedArgs { } const COMMAND_ALIAS_SUGGESTIONS: Record = { + 'd-pad': 'tv-remote', tap: 'press or click', }; @@ -377,6 +378,7 @@ export async function usageForCommand(command: string): Promise { } function normalizeCommandAlias(command: string): string { + if (command === 'd-pad') return 'tv-remote'; if (command === 'long-press') return 'longpress'; if (command === 'metrics') return 'perf'; if (command === 'tap') return 'press'; diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index c148300cd..a8cd0d724 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -94,6 +94,7 @@ const AGENT_QUICKSTART_LINES = [ 'Direct proxy: run agent-device connect proxy --daemon-base-url before using a shared Mac proxy. Device leases are automatic on open and expire after five minutes of inactivity.', 'Batch JSON steps use "command" and structured "input"; legacy "positionals"/"flags" steps still run in CLI but are deprecated until the next major version.', 'Navigation: app-owned back uses back; system back uses back --system.', + 'TV targets are focus-first: use tv-remote press up|down|left|right|select (d-pad is a CLI alias) to move focus and activate focused controls. Do not assume press/click @ref works on Android TV or tvOS until the desired element is focused.', 'Web browser sessions: read help web; first slice is web setup if needed -> web doctor -> open --platform web -> snapshot -i -> click/fill/get/is/find/wait/screenshot -> close.', 'Verification commands must name the expected text/selector; bare screenshots/snapshots are not enough.', 'Debug evidence: Session state contains request diagnostics and runner.log; use logs clear --restart/mark/path, trace, and network dump --include headers for app evidence.', @@ -156,7 +157,7 @@ Command shape: Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output. If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @, @ref, @Label_Name, or @eN placeholders. Close means agent-device close. App-owned back means back; system back means back --system. - Taps are press or click; tap is an alias for press. Gestures use swipe, longpress, or gesture . Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. Android swipe, pinch, rotate, and transform use provider-native touch injection when available, then the bundled touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION. + Taps are press or click; tap is an alias for press. On Android TV and tvOS, use tv-remote press up|down|left|right|select (or the d-pad alias) to move focus before activating controls. Gestures use swipe, longpress, or gesture . Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. Android swipe, pinch, rotate, and transform use provider-native touch injection when available, then the bundled touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION. Bootstrap: agent-device devices --platform ios @@ -194,6 +195,7 @@ Snapshots and refs: Missing target in a long list: use a short manual scroll + snapshot loop with a max attempt count. If a named target is summarized as off-screen below/above, use scroll down/up, then snapshot -i; do not use scroll bottom/top because the target may appear before the absolute list edge. Use scroll bottom/top only when the task explicitly asks for the list edge. Edge scrolls verify hidden content with snapshots and stop when no matching hidden content remains. Truncated text/input previews: do not use get text first; expand with snapshot -s @ref (for example snapshot -s @e7), then read the scoped output. Rare iOS accessibility gaps: if a row ref is shown disabled/hittable:false and press @ref reports success but no UI change, or a horizontal tab/filter bar is collapsed into one composite/seekbar with no child refs, run agent-device snapshot -i --json to read rects, compute the target center, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them. + TV focus gaps: if a fresh snapshot exposes focused nodes, verify with is focused ; use wait focused=true only on apps where repeated snapshots preserve focus metadata. If the app exposes only a surface view or focus metadata is transient, use screenshot/snapshot diff as visual truth and tv-remote press directions/select; do not switch to raw adb keyevent in command plans. Selectors: Use selectors as positional targets: id="field-email" or label="Allow". diff --git a/src/client/client-types.ts b/src/client/client-types.ts index b154c2585..51955d2aa 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -23,6 +23,7 @@ import type { BackMode } from '../core/back-mode.ts'; import type { ClickButton } from '../core/click-button.ts'; import type { RecordingExportQuality } from '../core/recording-export-quality.ts'; import type { DeviceRotation } from '../core/device-rotation.ts'; +import type { TvRemoteButton } from '../core/tv-remote.ts'; import type { ScrollDirection, SwipePattern, @@ -70,6 +71,7 @@ export type { BackCommandResult, HomeCommandResult, RotateCommandResult, + TvRemoteCommandResult, } from '../contracts/navigation.ts'; export type { ClipboardCommandResult } from '../contracts/clipboard.ts'; export type { AppStateCommandResult } from '../contracts/app-state.ts'; @@ -534,6 +536,11 @@ export type ClipboardCommandOptions = text: string; }); +export type TvRemoteCommandOptions = DeviceCommandBaseOptions & { + button: TvRemoteButton; + durationMs?: number; +}; + export type ReactNativeCommandOptions = DeviceCommandBaseOptions & { action: 'dismiss-overlay'; }; @@ -563,6 +570,7 @@ export type AgentDeviceCommandClient = { appSwitcher: (options?: AppSwitcherCommandOptions) => Promise>; keyboard: (options?: KeyboardCommandOptions) => Promise>; clipboard: (options: ClipboardCommandOptions) => Promise>; + tvRemote: (options: TvRemoteCommandOptions) => Promise>; reactNative: (options: ReactNativeCommandOptions) => Promise; doctor: (options?: DoctorCommandOptions) => Promise; /** @@ -763,7 +771,7 @@ type IsTextPredicateOptions = DeviceCommandBaseOptions & type IsStatePredicateOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & { - predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected'; + predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused'; selector: string; value?: never; }; diff --git a/src/client/client.ts b/src/client/client.ts index 10f3181aa..a69bfea48 100644 --- a/src/client/client.ts +++ b/src/client/client.ts @@ -122,6 +122,8 @@ export function createAgentDeviceClient( await executeCommand>('keyboard', options), clipboard: async (options) => await executeCommand>('clipboard', options), + tvRemote: async (options) => + await executeCommand>('tv-remote', options), reactNative: async (options) => await executeCommand('react-native', options), doctor: async (options = {}) => await executeCommand('doctor', options), prepare: async (options) => await executeCommand('prepare', options), diff --git a/src/cloud-webdriver/capabilities.ts b/src/cloud-webdriver/capabilities.ts index 9fa8b9930..356790876 100644 --- a/src/cloud-webdriver/capabilities.ts +++ b/src/cloud-webdriver/capabilities.ts @@ -20,6 +20,7 @@ export type CloudWebDriverOperation = | 'home' | 'rotate' | 'appSwitcher' + | 'tvRemote' | 'clipboard.read' | 'clipboard.write' | 'settings' @@ -99,6 +100,7 @@ const BASE_WEBDRIVER_CAPABILITIES: CloudWebDriverCapabilityMap = { support: 'partial', note: 'Uses provider/Appium mobile pressButton support where available.', }, + tvRemote: unsupported, 'clipboard.read': { support: 'partial', note: 'Uses provider/Appium clipboard extension support where available.', diff --git a/src/cloud-webdriver/webdriver-interactor.ts b/src/cloud-webdriver/webdriver-interactor.ts index 5e9d87cd7..428fcd8d5 100644 --- a/src/cloud-webdriver/webdriver-interactor.ts +++ b/src/cloud-webdriver/webdriver-interactor.ts @@ -7,6 +7,7 @@ import type { import type { BackMode } from '../core/back-mode.ts'; import type { DeviceRotation } from '../core/device-rotation.ts'; import type { ScrollDirection, TransformGestureParams } from '../core/scroll-gesture.ts'; +import type { TvRemoteButton } from '../core/tv-remote.ts'; import type { SettingOptions } from '../platforms/permission-utils.ts'; import { AppError } from '../kernel/errors.ts'; import { buildScrollGesturePlan } from '../core/scroll-gesture.ts'; @@ -253,6 +254,10 @@ class WebDriverInteractor implements Interactor { await this.client.executeScript('mobile: pressButton', [{ name: 'appSwitch' }]); } + async tvRemote(_button: TvRemoteButton, _durationMs?: number): Promise { + this.unsupported('tvRemote'); + } + async readClipboard(): Promise { this.requireSupport('clipboard.read'); const value = await this.client.executeScript('mobile: getClipboard', [{}]); diff --git a/src/command-catalog.ts b/src/command-catalog.ts index 5d051c575..3baa637a1 100644 --- a/src/command-catalog.ts +++ b/src/command-catalog.ts @@ -48,6 +48,7 @@ export const PUBLIC_COMMANDS = { trace: 'trace', triggerAppEvent: 'trigger-app-event', type: 'type', + tvRemote: 'tv-remote', viewport: 'viewport', wait: 'wait', } as const; diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 757184d4c..d62c78b44 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -169,7 +169,7 @@ const getFields = { const isFields = { predicate: requiredField( - enumField(['visible', 'hidden', 'exists', 'editable', 'selected', 'text'] as const), + enumField(['visible', 'hidden', 'exists', 'editable', 'selected', 'focused', 'text'] as const), ), selector: requiredField(stringField()), value: stringField(), diff --git a/src/commands/interaction/runtime/selector-read.test.ts b/src/commands/interaction/runtime/selector-read.test.ts index 6142ffd4d..9fb9d7553 100644 --- a/src/commands/interaction/runtime/selector-read.test.ts +++ b/src/commands/interaction/runtime/selector-read.test.ts @@ -168,6 +168,73 @@ test('runtime visibility predicates request snapshot rects', async () => { assert.equal(captureOptions?.includeRects, true); }); +test('runtime focused predicate requests an interactive snapshot', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Cell', + label: 'Profiles and Accounts', + focused: true, + }, + ]); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.is({ + session: 'default', + predicate: 'focused', + selector: 'role=cell label="Profiles and Accounts"', + }); + + assert.equal(result.pass, true); + assert.equal(captureOptions?.interactiveOnly, true); +}); + +test('runtime focused selector waits against an interactive snapshot', async () => { + const snapshot = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Cell', + label: 'Profiles and Accounts', + focused: true, + }, + ]); + let captureOptions: BackendSnapshotOptions | undefined; + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async (_context, options) => { + captureOptions = options; + return { snapshot }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default', snapshot }]), + policy: localCommandPolicy(), + }); + + const result = await device.selectors.wait({ + session: 'default', + target: { kind: 'selector', selector: 'focused=true', timeoutMs: 1000 }, + }); + + assert.deepEqual(result, { kind: 'selector', selector: 'focused=true', waitedMs: 0 }); + assert.equal(captureOptions?.interactiveOnly, true); +}); + test('runtime is validates selector predicates', async () => { const device = createSelectorDevice(selectorReadSnapshot()); diff --git a/src/commands/interaction/runtime/selector-read.ts b/src/commands/interaction/runtime/selector-read.ts index 88dc204ac..5c56dd2df 100644 --- a/src/commands/interaction/runtime/selector-read.ts +++ b/src/commands/interaction/runtime/selector-read.ts @@ -98,7 +98,7 @@ export type GetAttrsCommandOptions = CommandContext & export type IsCommandOptions = CommandContext & SelectorSnapshotOptions & { - predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'text'; + predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused' | 'text'; selector: string; expectedText?: string; }; @@ -274,12 +274,13 @@ export const isCommand: RuntimeCommand = asyn if (options.predicate === 'text' && !options.expectedText) { throw new AppError('INVALID_ARGS', 'is text requires expected text value'); } + const chain = parseSelectorChain(options.selector); const includeRects = predicateNeedsRects(options.predicate); const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true, includeRects, + interactiveOnly: options.predicate === 'focused' || selectorChainReadsFocus(chain), }); - const chain = parseSelectorChain(options.selector); if (options.predicate === 'exists') { const matched = findSelectorChainMatch(capture.snapshot.nodes, chain, { @@ -435,6 +436,7 @@ async function findFirstLocatorMatch( const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true, scope: findSnapshotScope(runtime, locator, options.query, selectorChain), + interactiveOnly: selectorChain ? selectorChainReadsFocus(selectorChain) : false, }); if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) { throw sparseSelectorSnapshotError(capture.snapshot.snapshotQuality); @@ -475,6 +477,10 @@ function predicateNeedsRects(predicate: IsPredicate): boolean { return predicate === 'visible' || predicate === 'hidden'; } +function selectorChainReadsFocus(chain: SelectorChain): boolean { + return chain.selectors.some((selector) => selector.terms.some((term) => term.key === 'focused')); +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, @@ -484,8 +490,12 @@ async function waitForSelector( const timeout = timeoutMs ?? DEFAULT_TIMEOUT_MS; const start = now(runtime); const chain = parseSelectorChain(selectorExpression); + const interactiveOnly = selectorChainReadsFocus(chain); while (now(runtime) - start < timeout) { - const capture = await captureSelectorSnapshot(runtime, options, { updateSession: true }); + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + interactiveOnly, + }); const match = findSelectorChainMatch(capture.snapshot.nodes, chain, { platform: runtime.backend.platform, }); @@ -568,14 +578,15 @@ async function resolveSelectorNode( sessionName: string, params: { selector: string; disambiguateAmbiguous: boolean }, ): Promise<{ capture: CapturedSnapshot; node: SnapshotNode; selector: string; ref: string }> { + const chain = parseSelectorChain(params.selector); const capture = await captureSelectorSnapshot( runtime, { ...options, session: sessionName }, { updateSession: true, + interactiveOnly: selectorChainReadsFocus(chain), }, ); - const chain = parseSelectorChain(params.selector); const resolved = resolveSelectorChain(capture.snapshot.nodes, chain, { platform: runtime.backend.platform, requireRect: false, diff --git a/src/commands/interaction/selectors.ts b/src/commands/interaction/selectors.ts index 0b8555cd6..761190079 100644 --- a/src/commands/interaction/selectors.ts +++ b/src/commands/interaction/selectors.ts @@ -120,7 +120,8 @@ function readIsOptionsFromPositionals(positionals: string[], flags: CliFlags): I predicate !== 'hidden' && predicate !== 'exists' && predicate !== 'editable' && - predicate !== 'selected' + predicate !== 'selected' && + predicate !== 'focused' ) { throw new AppError('INVALID_ARGS', IS_PREDICATE_REQUIRED_MESSAGE, { hint: IS_PREDICATE_USAGE_HINT, diff --git a/src/commands/system/index.test.ts b/src/commands/system/index.test.ts index 4164363b9..dccb090ea 100644 --- a/src/commands/system/index.test.ts +++ b/src/commands/system/index.test.ts @@ -15,6 +15,8 @@ import { keyboardDaemonWriter, rotateCliReader, rotateDaemonWriter, + tvRemoteCliReader, + tvRemoteDaemonWriter, } from './index.ts'; function flags(overrides: Partial = {}): CliFlags { @@ -117,4 +119,29 @@ describe('system command interface', () => { 'copied', ]); }); + + test('tv-remote reader parses button and optional press subcommand', () => { + expect(tvRemoteCliReader(['down'], flags({ durationMs: 250 }))).toMatchObject({ + button: 'down', + durationMs: 250, + }); + expect(tvRemoteCliReader(['press', 'select'], flags())).toMatchObject({ + button: 'select', + }); + }); + + test('tv-remote reader and writer validate button arguments', () => { + expect( + tvRemoteDaemonWriter({ button: 'right' } as Record).positionals, + ).toEqual(['right']); + expectInvalidArgs( + () => tvRemoteCliReader([], flags()), + 'tv-remote requires exactly one button', + ); + expectInvalidArgs( + () => tvRemoteCliReader(['press', 'left', 'extra'], flags()), + 'tv-remote requires exactly one button', + ); + expectInvalidArgs(() => tvRemoteCliReader(['blue'], flags()), 'button must be one of'); + }); }); diff --git a/src/commands/system/index.ts b/src/commands/system/index.ts index c921b9607..1f8c83631 100644 --- a/src/commands/system/index.ts +++ b/src/commands/system/index.ts @@ -2,11 +2,18 @@ import type { ClipboardCommandOptions } from '../../client/client-types.ts'; import type { BackMode } from '../../core/back-mode.ts'; import { BACK_MODES } from '../../core/back-mode.ts'; import { parseDeviceRotation, DEVICE_ROTATIONS } from '../../core/device-rotation.ts'; +import { parseTvRemoteButton, TV_REMOTE_BUTTONS } from '../../core/tv-remote.ts'; import { AppError } from '../../kernel/errors.ts'; import type { CommandSchemaOverride } from '../../utils/cli-command-schema-types.ts'; import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; import { defineExecutableCommand } from '../command-contract.ts'; -import { compactRecord, enumField, requiredField, stringField } from '../command-input.ts'; +import { + compactRecord, + enumField, + integerField, + requiredField, + stringField, +} from '../command-input.ts'; import { defineFieldCommandMetadata } from '../field-command-contract.ts'; import { commonInputFromFlags, @@ -25,6 +32,7 @@ const ROTATE_COMMAND_NAME = 'rotate'; const APP_SWITCHER_COMMAND_NAME = 'app-switcher'; const KEYBOARD_COMMAND_NAME = 'keyboard'; const CLIPBOARD_COMMAND_NAME = 'clipboard'; +const TV_REMOTE_COMMAND_NAME = 'tv-remote'; const CLIPBOARD_ACTION_VALUES = ['read', 'write'] as const; const KEYBOARD_METADATA_ACTION_VALUES = ['status', 'dismiss'] as const; @@ -36,6 +44,7 @@ const rotateCommandDescription = 'Rotate device orientation.'; const appSwitcherCommandDescription = 'Open the app switcher.'; const keyboardCommandDescription = 'Inspect or dismiss the keyboard.'; const clipboardCommandDescription = 'Read or write clipboard text.'; +const tvRemoteCommandDescription = 'Press a TV remote/D-pad button.'; const appStateCommandMetadata = defineFieldCommandMetadata( APPSTATE_COMMAND_NAME, @@ -84,6 +93,17 @@ const clipboardCommandMetadata = defineFieldCommandMetadata( }, ); +const tvRemoteCommandMetadata = defineFieldCommandMetadata( + TV_REMOTE_COMMAND_NAME, + tvRemoteCommandDescription, + { + button: requiredField(enumField(TV_REMOTE_BUTTONS)), + durationMs: integerField('Press duration in milliseconds when the backend supports it.', { + min: 0, + }), + }, +); + const appStateCommandDefinition = defineExecutableCommand( appStateCommandMetadata, (client, input) => client.command.appState(input), @@ -116,6 +136,11 @@ const clipboardCommandDefinition = defineExecutableCommand( (client, input) => client.command.clipboard(input as ClipboardCommandOptions), ); +const tvRemoteCommandDefinition = defineExecutableCommand( + tvRemoteCommandMetadata, + (client, input) => client.command.tvRemote(input), +); + const appStateCliSchema = { helpDescription: 'Show foreground app/activity', } as const satisfies CommandSchemaOverride; @@ -147,6 +172,16 @@ const clipboardCliSchema = { allowsExtraPositionals: true, } as const satisfies CommandSchemaOverride; +const tvRemoteCliSchema = { + usageOverride: 'tv-remote [press] ', + listUsageOverride: 'tv-remote press