Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions CONTEXT.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,9 @@ The perfect-shape refactor is complete and merged. Its end-state:
(`src/core/command-descriptor/registry.ts`) is the single declaration site from which the
capability matrix, daemon command registry, batch allowlist, timeout policy, MCP exposure list, and
capability-checked CLI command list are *derived* by parity-tested projection. The command catalog
still owns identity, command families still own surface metadata/CLI schema, and the Node client
surface remains a deferred derivation target. One
still owns identity, command families still own surface metadata/CLI schema, and system command
facets now project their simple Node client command methods; the rest of the Node client surface
remains a deferred derivation target. One
`PlatformPlugin` per platform family (`src/core/platform-plugin/`) stops core/daemon from branching
on platform, with the Apple plugin the first instance. See
[ADR 0008](docs/adr/0008-command-descriptor-registry.md).
Expand Down
2 changes: 1 addition & 1 deletion scripts/integration-progress-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ function summarizeProviderScenarioFlagCoverage(files) {
['hideTouches', 'recording without touch overlays'],
['intervalMs', 'repeated press interval'],
['delayMs', 'typing/fill delay'],
['durationMs', 'scroll and gesture duration'],
['durationMs', 'scroll, gesture, and TV remote duration'],
['holdMs', 'press hold duration'],
['jitterPx', 'press jitter'],
['pixels', 'scroll distance'],
Expand Down
2 changes: 1 addition & 1 deletion src/__tests__/cli-grammar.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ test('is grammar explains the predicate/selector-key collision on invalid predic
assert.equal(err.code, 'INVALID_ARGS');
assert.match(
err.message,
/is requires predicate: visible\|hidden\|exists\|editable\|selected\|text/,
/is requires predicate: visible\|hidden\|exists\|editable\|selected\|focused\|text/,
);
assert.match(err.details?.hint ?? '', /is <selector> <predicate>/);
assert.match(err.details?.hint ?? '', /visible=true/);
Expand Down
9 changes: 9 additions & 0 deletions src/__tests__/contracts-schema-public.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type {
BackCommandResult,
HomeCommandResult,
RotateCommandResult,
TvRemoteCommandResult,
} from '../contracts/navigation.ts';
import type { ViewportCommandResult } from '../contracts/viewport.ts';
import { centerOfRect, defaultHintForCode, normalizeError } from '../sdk/contracts.ts';
Expand Down Expand Up @@ -166,6 +167,13 @@ test('command result contracts are assignable to command result map', () => {
} satisfies RotateCommandResult;
const rotateFromMap: CommandResult<'rotate'> = rotate;

const tvRemote = {
action: 'tv-remote',
button: 'select',
message: 'Pressed TV remote select',
} satisfies TvRemoteCommandResult;
const tvRemoteFromMap: CommandResult<'tv-remote'> = tvRemote;

const clipboard = {
action: 'write',
textLength: 11,
Expand All @@ -187,6 +195,7 @@ test('command result contracts are assignable to command result map', () => {
assert.equal(homeFromMap.action, 'home');
assert.equal(appSwitcherFromMap.action, 'app-switcher');
assert.equal(rotateFromMap.orientation, 'portrait');
assert.equal(tvRemoteFromMap.button, 'select');
assert.equal(clipboardFromMap.action === 'write' ? clipboardFromMap.textLength : -1, 11);
assert.equal(
appstateFromMap.platform === 'android' ? appstateFromMap.package : '',
Expand Down
1 change: 1 addition & 0 deletions src/__tests__/runtime-public.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ test('internal backend, commands, and io modules are usable', () => {
assert.equal(typeof commands.system.settings, 'function');
assert.equal(typeof commands.system.alert, 'function');
assert.equal(typeof commands.system.appSwitcher, 'function');
assert.equal(typeof commands.system.tvRemote, 'function');
assert.equal(typeof commands.admin.devices, 'function');
assert.equal(typeof commands.admin.install, 'function');
assert.equal(typeof commands.recording.record, 'function');
Expand Down
10 changes: 10 additions & 0 deletions src/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { ClickButton } from './core/click-button.ts';
import type { DeviceRotation } from './core/device-rotation.ts';
import type { ScrollDirection } from './core/scroll-gesture.ts';
import type { SessionSurface } from './core/session-surface.ts';
import type { TvRemoteButton } from './core/tv-remote.ts';
import type { RecordingExportQuality } from './core/recording-export-quality.ts';
import type { SnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts';
import type {
Expand Down Expand Up @@ -87,6 +88,11 @@ export type BackendBackOptions = {
mode?: BackMode;
};

export type BackendTvRemoteOptions = {
button: TvRemoteButton;
durationMs?: number;
};

export type BackendKeyboardOptions = {
action: 'status' | 'get' | 'dismiss' | 'enter' | 'return';
};
Expand Down Expand Up @@ -491,6 +497,10 @@ export type AgentDeviceBackend = {
options?: BackendBackOptions,
): Promise<BackendActionResult>;
pressHome?(context: BackendCommandContext): Promise<BackendActionResult>;
pressTvRemote?(
context: BackendCommandContext,
options: BackendTvRemoteOptions,
): Promise<BackendActionResult>;
rotate?(
context: BackendCommandContext,
orientation: BackendDeviceOrientation,
Expand Down
13 changes: 13 additions & 0 deletions src/cli/parser/__tests__/args-parse-interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,19 @@ test('parseArgs accepts keyboard subcommands', () => {
assert.deepEqual(enter.positionals, ['enter']);
});

test('parseArgs accepts tv-remote and d-pad alias', () => {
const remote = parseArgs(['tv-remote', 'press', 'select', '--duration-ms', '250'], {
strictFlags: true,
});
assert.equal(remote.command, 'tv-remote');
assert.deepEqual(remote.positionals, ['press', 'select']);
assert.equal(remote.flags.durationMs, 250);

const dpad = parseArgs(['d-pad', 'down'], { strictFlags: true });
assert.equal(dpad.command, 'tv-remote');
assert.deepEqual(dpad.positionals, ['down']);
});

test('parseArgs accepts scroll pixel distance and duration flags', () => {
const parsed = parseArgs(['scroll', 'down', '--pixels', '240', '--duration-ms', '50'], {
strictFlags: true,
Expand Down
16 changes: 16 additions & 0 deletions src/cli/parser/__tests__/cli-help-topics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ test('usage includes agent workflows, config, environment, and examples footers'
usageText,
/agent-device help debugging\s+Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter/,
);
assert.match(
usageText,
/agent-device help tv\s+Use when navigating Android TV or tvOS focus-first surfaces/,
);
assert.match(
usageText,
/agent-device help react-devtools\s+Use when inspecting components, props\/state\/hooks, renders, or profiles/,
Expand Down Expand Up @@ -296,6 +300,18 @@ test('usageForCommand resolves workflow help topic', async () => {
assert.doesNotMatch(help, /agent-device react-devtools profile/);
});

test('usageForCommand resolves tv help topic', async () => {
const help = await usageForCommand('tv');
if (help === null) throw new Error('Expected tv help text');
assert.match(help, /agent-device help tv/);
assert.match(help, /agent-device tv-remote press down/);
assert.match(help, /agent-device screenshot \.\/tv-focus\.png --overlay-refs/);
assert.match(help, /d-pad is a CLI alias for tv-remote/);
assert.match(help, /ok, center, and enter are input aliases for select/);
assert.match(help, /do not switch to raw adb keyevent/);
assert.match(help, /Use --platform ios --target tv/);
});

test('usageForCommand resolves web help topic', async () => {
const help = await usageForCommand('web');
if (help === null) throw new Error('Expected web help text');
Expand Down
2 changes: 2 additions & 0 deletions src/cli/parser/args.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ function normalizeParsedCommandAliases(parsed: ParsedArgs): ParsedArgs {
}

const COMMAND_ALIAS_SUGGESTIONS: Record<string, string> = {
'd-pad': 'tv-remote',
tap: 'press or click',
};

Expand Down Expand Up @@ -377,6 +378,7 @@ export async function usageForCommand(command: string): Promise<string | null> {
}

function normalizeCommandAlias(command: string): string {
if (command === 'd-pad') return 'tv-remote';
if (command === 'long-press') return 'longpress';
if (command === 'metrics') return 'perf';
if (command === 'tap') return 'press';
Expand Down
44 changes: 43 additions & 1 deletion src/cli/parser/cli-help.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ const AGENT_WORKFLOWS = [
description:
'Use when logs, network, audio, perf memory, traces, alerts, or diagnostics matter',
},
{
label: 'agent-device help tv',
description: 'Use when navigating Android TV or tvOS focus-first surfaces',
},
{
label: 'agent-device help react-native',
description: 'Use when the target app is React Native, Expo, or a dev client',
Expand Down Expand Up @@ -94,6 +98,7 @@ const AGENT_QUICKSTART_LINES = [
'Direct proxy: run agent-device connect proxy --daemon-base-url <proxy-agent-device-url> before using a shared Mac proxy. Device leases are automatic on open and expire after five minutes of inactivity.',
'Batch JSON steps use "command" and structured "input"; legacy "positionals"/"flags" steps still run in CLI but are deprecated until the next major version.',
'Navigation: app-owned back uses back; system back uses back --system.',
'TV targets are focus-first: read help tv; use tv-remote press up|down|left|right|select (d-pad is a CLI alias) to move focus and activate focused controls before assuming press/click @ref works.',
'Web browser sessions: read help web; first slice is web setup if needed -> web doctor -> open <url> --platform web -> snapshot -i -> click/fill/get/is/find/wait/screenshot -> close.',
'Verification commands must name the expected text/selector; bare screenshots/snapshots are not enough.',
'Debug evidence: Session state contains request diagnostics and runner.log; use logs clear --restart/mark/path, trace, and network dump --include headers for app evidence.',
Expand Down Expand Up @@ -156,7 +161,7 @@ Command shape:
Snapshot refs look like @e12. After snapshot -i, use the exact @eN ref from that output.
If the exact ref is not known yet, first output snapshot -i, then use a concrete example shape like press @e12 in the next command; do not write @<ref>, @ref, @Label_Name, or @eN placeholders.
Close means agent-device close. App-owned back means back; system back means back --system.
Taps are press or click; tap is an alias for press. Gestures use swipe, longpress, or gesture <pan|fling|swipe|pinch|rotate|transform>. Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. Android swipe, pinch, rotate, and transform use provider-native touch injection when available, then the bundled touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION.
Taps are press or click; tap is an alias for press. On Android TV and tvOS, read help tv and use tv-remote press up|down|left|right|select (or the d-pad alias) to move focus before activating controls. Gestures use swipe, longpress, or gesture <pan|fling|swipe|pinch|rotate|transform>. Use gesture swipe left|right for reliable in-page horizontal swipes, and gesture swipe right-edge for left-edge navigation/back gestures. Android swipe, pinch, rotate, and transform use provider-native touch injection when available, then the bundled touch helper. iOS simulator transform uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path; otherwise it reports UNSUPPORTED_OPERATION.

Bootstrap:
agent-device devices --platform ios
Expand Down Expand Up @@ -194,6 +199,7 @@ Snapshots and refs:
Missing target in a long list: use a short manual scroll + snapshot loop with a max attempt count. If a named target is summarized as off-screen below/above, use scroll down/up, then snapshot -i; do not use scroll bottom/top because the target may appear before the absolute list edge. Use scroll bottom/top only when the task explicitly asks for the list edge. Edge scrolls verify hidden content with snapshots and stop when no matching hidden content remains.
Truncated text/input previews: do not use get text first; expand with snapshot -s @ref (for example snapshot -s @e7), then read the scoped output.
Rare iOS accessibility gaps: if a row ref is shown disabled/hittable:false and press @ref reports success but no UI change, or a horizontal tab/filter bar is collapsed into one composite/seekbar with no child refs, run agent-device snapshot -i --json to read rects, compute the target center, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them.
TV focus gaps: read help tv. If a fresh snapshot exposes focused nodes, verify with is focused <selector>; use wait focused=true only on apps where repeated snapshots preserve focus metadata. If the app exposes only a surface view or focus metadata is transient, use screenshot/snapshot diff as visual truth and tv-remote press directions/select; do not switch to raw adb keyevent in command plans.

Selectors:
Use selectors as positional targets: id="field-email" or label="Allow".
Expand Down Expand Up @@ -351,12 +357,48 @@ Guarantees:

Escalate:
help debugging logs, network, alerts, traces, flaky runtime failures
help tv Android TV and tvOS focus-first remote navigation
help react-devtools React Native performance, profiling, props/state/hooks, slow renders, rerenders
help react-native React Native app automation hazards, overlays, Metro, and routing
help remote remote/cloud config, tenant, lease, local service tunnels
help macos desktop, frontmost-app, menu bar surfaces
help dogfood exploratory QA report workflow`,
},
tv: {
summary: 'Android TV and tvOS focus-first remote navigation',
body: `agent-device help tv

Use this when the target is Android TV, Apple TV, or tvOS. TV surfaces are focus-first: move focus with remote/D-pad buttons, then activate the focused control.

Core loop:
agent-device open Settings --platform android --target tv --session tv
agent-device snapshot -i --platform android --target tv --session tv
agent-device tv-remote press down --platform android --target tv --session tv
agent-device is focused 'label="Profiles"' --platform android --target tv --session tv
agent-device tv-remote press select --platform android --target tv --session tv
agent-device screenshot ./tv-focus.png --overlay-refs --platform android --target tv --session tv

Buttons:
tv-remote press up|down|left|right|select|menu|home|back
d-pad is a CLI alias for tv-remote, not a separate Node/MCP command.
ok, center, and enter are input aliases for select; command output still reports button: "select".
--duration-ms holds a tvOS remote button for that duration. On Android TV, any positive duration maps to the ADB longpress form because Android input keyevent has no exact hold duration.

Android TV:
Android TV uses ADB keyevents behind agent-device tv-remote. Keep command plans on agent-device; do not switch to raw adb keyevent.
Use --target tv when a host has both phone/tablet and TV emulators/devices.

tvOS:
tvOS is driven by the Siri Remote focus engine, not coordinate taps.
back maps to the Menu remote button; home maps to the Home remote button.
Use --platform ios --target tv for Apple TV simulators and devices.

Focus and visual truth:
If snapshot -i exposes a focused node, verify it with is focused <selector>.
Use wait focused=true only when repeated snapshots preserve focus metadata for the app.
If the app exposes only a surface view, or focus metadata is transient, use screenshot --overlay-refs, screenshot, or diff snapshot as visual truth and keep moving focus with tv-remote.
Do not assume press/click @ref works on Android TV or tvOS until the desired element is focused.`,
},
debugging: {
summary: 'Targeted failure evidence without dumping stale context',
body: `agent-device help debugging
Expand Down
10 changes: 9 additions & 1 deletion src/client/client-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import type { BackMode } from '../core/back-mode.ts';
import type { ClickButton } from '../core/click-button.ts';
import type { RecordingExportQuality } from '../core/recording-export-quality.ts';
import type { DeviceRotation } from '../core/device-rotation.ts';
import type { TvRemoteButton } from '../core/tv-remote.ts';
import type {
ScrollDirection,
SwipePattern,
Expand Down Expand Up @@ -70,6 +71,7 @@ export type {
BackCommandResult,
HomeCommandResult,
RotateCommandResult,
TvRemoteCommandResult,
} from '../contracts/navigation.ts';
export type { ClipboardCommandResult } from '../contracts/clipboard.ts';
export type { AppStateCommandResult } from '../contracts/app-state.ts';
Expand Down Expand Up @@ -534,6 +536,11 @@ export type ClipboardCommandOptions =
text: string;
});

export type TvRemoteCommandOptions = DeviceCommandBaseOptions & {
button: TvRemoteButton;
durationMs?: number;
};

export type ReactNativeCommandOptions = DeviceCommandBaseOptions & {
action: 'dismiss-overlay';
};
Expand Down Expand Up @@ -563,6 +570,7 @@ export type AgentDeviceCommandClient = {
appSwitcher: (options?: AppSwitcherCommandOptions) => Promise<CommandResult<'app-switcher'>>;
keyboard: (options?: KeyboardCommandOptions) => Promise<CommandResult<'keyboard'>>;
clipboard: (options: ClipboardCommandOptions) => Promise<CommandResult<'clipboard'>>;
tvRemote: (options: TvRemoteCommandOptions) => Promise<CommandResult<'tv-remote'>>;
reactNative: (options: ReactNativeCommandOptions) => Promise<CommandRequestResult>;
doctor: (options?: DoctorCommandOptions) => Promise<CommandRequestResult>;
/**
Expand Down Expand Up @@ -763,7 +771,7 @@ type IsTextPredicateOptions = DeviceCommandBaseOptions &

type IsStatePredicateOptions = DeviceCommandBaseOptions &
SelectorSnapshotCommandOptions & {
predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected';
predicate: 'visible' | 'hidden' | 'exists' | 'editable' | 'selected' | 'focused';
selector: string;
value?: never;
};
Expand Down
Loading
Loading