From a6ed330a3f9fbb3c3c82bfcb5137cefccd51e943 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 16 Jul 2026 09:08:40 +0300 Subject: [PATCH] feat: route press_key chords through the portal keyboard session On Wayland, press_key previously went straight to ydotool even when an XDG RemoteDesktop keyboard session was already active (the path type_text already uses). Plasma users therefore needed ydotoold and /dev/uinput solely for key combos. press_key now parses the chord into evdev codes (new key_chord helper, shared with the ydotool key_sequence encoding) and sends it through the portal via the existing press_keycode_chord when the chord policy prefers the portal: any Wayland session where a keyboard session is already cached (reusing granted consent even if ydotool is present), where ydotool is absent, or where the portal is forced via COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD. ydotool remains the fallback and COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD still pins it. Portal send failures clear the cached session and report the error explicitly. Closes #50. --- src/server.rs | 88 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/src/server.rs b/src/server.rs index fd197c3..8bf7078 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1155,7 +1155,7 @@ impl ComputerUseLinux { #[tool( name = "press_key", - description = "Press a key or key-combination on the keyboard, optionally after focusing a target window or terminal selector. Key grammar (case-insensitive; hyphens/spaces ignored): combos join with '+', e.g. Ctrl+L or Ctrl+Shift+T. Modifiers: ctrl/control, alt/option, shift, meta/super/cmd/command. Named keys: enter/return, escape/esc, tab, backspace, delete/del, space, home, end, pageup, pagedown, arrowleft/left, arrowright/right, arrowup/up, arrowdown/down, f1-f12. Plus single US letters a-z and digits 0-9. Anything else returns an error (never silently dropped). Note: compositor-level shortcuts (e.g. Super+Up) may be consumed by GNOME before reaching the app.", + description = "Press a key or key-combination on the keyboard, optionally after focusing a target window or terminal selector. Key grammar (case-insensitive; hyphens/spaces ignored): combos join with '+', e.g. Ctrl+L or Ctrl+Shift+T. Modifiers: ctrl/control, alt/option, shift, meta/super/cmd/command. Named keys: enter/return, escape/esc, tab, backspace, delete/del, space, home, end, pageup, pagedown, arrowleft/left, arrowright/right, arrowup/up, arrowdown/down, f1-f12. Plus single US letters a-z and digits 0-9. Anything else returns an error (never silently dropped). On Wayland, chords are sent through an active remote desktop portal keyboard session when one is available (or when ydotool is absent), falling back to ydotool otherwise. Note: compositor-level shortcuts (e.g. Super+Up) may be consumed by GNOME before reaching the app.", annotations( read_only_hint = false, destructive_hint = true, @@ -1180,6 +1180,48 @@ impl ComputerUseLinux { }); } }; + let Some((chord_modifiers, chord_key)) = key_chord(¶ms.key) else { + return Json(ActionOutput { + ok: false, + implemented: true, + action: "press_key".to_string(), + message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(), + received, + }); + }; + if self.should_prefer_portal_keyboard_for_chords() { + match self.ensure_portal_keyboard_session().await { + Ok(Some(session)) => { + let modifiers: Vec = + chord_modifiers.iter().map(|m| i32::from(*m)).collect(); + match press_keycode_chord(&session, &modifiers, i32::from(chord_key)).await { + Ok(()) => { + let notes = self.input_landing_notes(focus.as_ref(), false).await; + return Json(with_notes( + successful_action_with_focus( + "press_key", + "Action sent through the remote desktop portal.", + received, + focus, + ), + notes, + )); + } + Err(error) => { + self.clear_portal_keyboard_session(); + return Json(action_result_with_focus( + "press_key", + Err(format!("{error:#}")), + received, + focus, + )); + } + } + } + Ok(None) => {} + Err(_) => {} + } + } let Some(key_events) = key_sequence(¶ms.key) else { return Json(ActionOutput { ok: false, @@ -1986,6 +2028,29 @@ impl ComputerUseLinux { self.is_wayland_session() && !self.is_kde_wayland_session() && ydotool_socket().is_none() } + /// Portal keyboard policy for `press_key` chords. Unlike literal text + /// (where KDE prefers the clipboard paste backend), key chords have no + /// clipboard route, so the portal keyboard session is preferred on ANY + /// Wayland session — including Plasma. An already-active keyboard + /// session (e.g. established by a KDE clipboard paste) is reused even + /// when ydotool is available, so the consent the user already granted + /// keeps covering key chords; otherwise the portal is preferred only + /// when ydotool is absent or the portal is forced. + fn should_prefer_portal_keyboard_for_chords(&self) -> bool { + if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD") { + return false; + } + if !self.is_wayland_session() { + return false; + } + if self.cached_portal_keyboard_session().is_some() + || env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD") + { + return true; + } + ydotool_socket().is_none() + } + fn should_prefer_kde_clipboard_text_backend(&self) -> bool { !env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD") && self.is_kde_wayland_session() @@ -3673,7 +3738,10 @@ fn mouse_button_code(button: Option<&str>) -> String { .to_string() } -fn key_sequence(key: &str) -> Option> { +/// Parse a chord like `Ctrl+Shift+P` into raw evdev codes: the held +/// modifiers plus the final key. A bare modifier (`Super`) parses as a +/// chord with no held modifiers whose key is the modifier itself. +fn key_chord(key: &str) -> Option<(Vec, u16)> { let parts = key .split('+') .map(str::trim) @@ -3682,7 +3750,7 @@ fn key_sequence(key: &str) -> Option> { let (key_part, modifier_parts) = parts.split_last()?; if modifier_parts.is_empty() { if let Some(modifier) = modifier_keycode(key_part) { - return Some(vec![format!("{modifier}:1"), format!("{modifier}:0")]); + return Some((Vec::new(), modifier)); } } let mut modifiers = Vec::new(); @@ -3690,7 +3758,11 @@ fn key_sequence(key: &str) -> Option> { modifiers.push(modifier_keycode(part)?); } let keycode = keycode(key_part)?; + Some((modifiers, keycode)) +} +fn key_sequence(key: &str) -> Option> { + let (modifiers, keycode) = key_chord(key)?; let mut events = Vec::new(); for modifier in &modifiers { events.push(format!("{modifier}:1")); @@ -4712,6 +4784,16 @@ mod tests { ); } + #[test] + fn key_chord_splits_modifiers_and_key() { + assert_eq!(key_chord("Ctrl+Shift+P"), Some((vec![29, 42], 25))); + assert_eq!(key_chord("Ctrl+S"), Some((vec![29], 31))); + assert_eq!(key_chord("Enter"), Some((vec![], 28))); + // A bare modifier is a chord with no held modifiers. + assert_eq!(key_chord("Super"), Some((vec![], 125))); + assert_eq!(key_chord("NotAKey"), None); + } + #[test] fn key_sequence_presses_modifiers_around_key() { assert_eq!(