From 57b95d3167b815b5f0139d4dbd8a9ebaf9d36846 Mon Sep 17 00:00:00 2001 From: abonneth Date: Mon, 13 Jul 2026 14:07:00 +0200 Subject: [PATCH 1/4] feat(local): scale and encode desktop screenshots at the source Co-authored-by: Cursor --- pyproject.toml | 2 +- src/hai_agents_cli/app.py | 23 +++++++++++++- src/hai_agents_local/desktop.py | 53 ++++++++++++++++++++++++++++----- tests/test_local.py | 24 ++++++++++++++- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b72d04..828a93b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ cli = [ "typer>=0.12,<1", ] desktop = [ - "hai-drivers[desktop]>=0.1.0", + "hai-drivers[desktop]>=0.1.2", ] browser = [ "hai-drivers[web]>=0.1.0", diff --git a/src/hai_agents_cli/app.py b/src/hai_agents_cli/app.py index e8c26c6..da1d9d6 100644 --- a/src/hai_agents_cli/app.py +++ b/src/hai_agents_cli/app.py @@ -21,6 +21,7 @@ from hai_agents_common import credentials from hai_agents_common.credentials import absolute_share_url, make_client from hai_agents_common.jsonable import to_jsonable +from hai_agents_local import desktop as desktop_defaults from . import auth, mcp_hosts @@ -586,11 +587,31 @@ def local_browser( def local_desktop( ctx: typer.Context, session_id: str | None = typer.Option(None, "--session-id", help="Session id to serve. Generated when omitted."), + max_width: int = typer.Option( + desktop_defaults.DEFAULT_MAX_WIDTH, "--max-width", help="Cap screenshot width in pixels. 0 disables." + ), + max_height: int = typer.Option(0, "--max-height", help="Cap screenshot height in pixels. 0 disables."), + image_format: str = typer.Option( + desktop_defaults.DEFAULT_IMAGE_FORMAT, "--image-format", help="Screenshot encoding: png, jpeg, or webp." + ), + quality: int = typer.Option( + desktop_defaults.DEFAULT_QUALITY, "--quality", help="Encoding quality (1-100) for jpeg/webp." + ), ) -> None: """Serve desktop commands on this machine's mouse, keyboard, and screen.""" from hai_agents_local import PyautoguiDesktopBridge - _run_bridge(_state(ctx), PyautoguiDesktopBridge, session_id) + if image_format not in ("png", "jpeg", "webp"): + _raise_cli_error(ValueError(f"--image-format must be png, jpeg, or webp; got {image_format!r}")) + _run_bridge( + _state(ctx), + PyautoguiDesktopBridge, + session_id, + max_width=max_width or None, + max_height=max_height or None, + image_format=image_format, + quality=quality, + ) def _run_bridge(state: AppState, bridge_type: type[LocalBridge], session_id: str | None, **options: Any) -> None: diff --git a/src/hai_agents_local/desktop.py b/src/hai_agents_local/desktop.py index 85363cb..2fcce8b 100644 --- a/src/hai_agents_local/desktop.py +++ b/src/hai_agents_local/desktop.py @@ -1,19 +1,47 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal -from .bridge import LocalBridge +from .bridge import LocalBridge, TokenSource if TYPE_CHECKING: - from hai_drivers.desktop.local import LocalDesktopDriver + from hai_drivers.desktop.interface import DesktopDriverInterface +ImageFormat = Literal["png", "jpeg", "webp"] -class PyautoguiDesktopBridge(LocalBridge["LocalDesktopDriver"]): - """Serves desktop environments (mouse, keyboard, screen, files, shell) on this machine via pyautogui.""" +DEFAULT_MAX_WIDTH = 1920 +DEFAULT_IMAGE_FORMAT: ImageFormat = "jpeg" +DEFAULT_QUALITY = 85 + + +class PyautoguiDesktopBridge(LocalBridge["DesktopDriverInterface"]): + """Serves desktop environments (mouse, keyboard, screen, files, shell) on this machine via pyautogui. + + Screenshots are downscaled and encoded here, before they cross the network; set every + knob to None to serve raw native-resolution captures. + """ environment_kind = "desktop" - def create_driver(self) -> LocalDesktopDriver: + def __init__( + self, + environment_id: str | None = None, + *, + api_key: TokenSource, + base_url: str | None = None, + session_id: str | None = None, + max_width: int | None = DEFAULT_MAX_WIDTH, + max_height: int | None = None, + image_format: ImageFormat | None = DEFAULT_IMAGE_FORMAT, + quality: int = DEFAULT_QUALITY, + ) -> None: + super().__init__(environment_id, api_key=api_key, base_url=base_url, session_id=session_id) + self.max_width = max_width + self.max_height = max_height + self.image_format = image_format + self.quality = quality + + def create_driver(self) -> DesktopDriverInterface: # Runtime import: hai-drivers is absent unless installed with hai-agents[desktop]. try: from hai_drivers.desktop.local import LocalDesktopDriver @@ -21,7 +49,18 @@ def create_driver(self) -> LocalDesktopDriver: raise ImportError( "Local desktop control requires extra deps. Install with: pip install 'hai-agents[desktop]'" ) from exc - return LocalDesktopDriver() + driver = LocalDesktopDriver() + if self.max_width is None and self.max_height is None and self.image_format is None: + return driver + from hai_drivers.desktop.scaled import ScaledDesktopDriver + + return ScaledDesktopDriver( + driver, + max_width=self.max_width, + max_height=self.max_height, + image_format=self.image_format, + quality=self.quality, + ) def driver_interface(self) -> type: # Runtime import: hai-drivers is absent unless installed with hai-agents[desktop]. diff --git a/tests/test_local.py b/tests/test_local.py index 1a02ccc..1266d47 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -119,6 +119,28 @@ def test_bridge_rejects_non_uuid_session_id(self): with pytest.raises(ValueError, match="must be a UUID"): PyautoguiDesktopBridge(api_key=API_KEY, session_id="my-laptop-1") + def test_desktop_screenshots_are_scaled_at_the_source(self, monkeypatch): + from hai_drivers.desktop import local as local_module + from hai_drivers.desktop.scaled import ScaledDesktopDriver + + monkeypatch.setattr(local_module, "LocalDesktopDriver", _FakeLocalDriver) + bridge = PyautoguiDesktopBridge(api_key=API_KEY, max_width=1280, image_format="webp", quality=70) + driver = bridge.create_driver() + assert isinstance(driver, ScaledDesktopDriver) + assert (driver._max_width, driver._image_format, driver._quality) == (1280, "webp", 70) + assert isinstance(driver._driver, _FakeLocalDriver) + + def test_desktop_bridge_with_all_knobs_off_serves_the_raw_driver(self, monkeypatch): + from hai_drivers.desktop import local as local_module + + monkeypatch.setattr(local_module, "LocalDesktopDriver", _FakeLocalDriver) + bridge = PyautoguiDesktopBridge(api_key=API_KEY, max_width=None, image_format=None) + assert isinstance(bridge.create_driver(), _FakeLocalDriver) + + +class _FakeLocalDriver: + """Stands in for LocalDesktopDriver, whose constructor needs a display.""" + class TestAutoStart: def test_create_session_starts_bridges_and_stamps_matching_session_ids(self, monkeypatch): @@ -396,7 +418,7 @@ def test_browser_commands_match_hai_drivers_interface(self): def test_desktop_commands_match_hai_drivers_interface(self): pytest.importorskip("hai_drivers.desktop.interface") commands = PyautoguiDesktopBridge(api_key="k").commands - assert {"click", "write", "run_command", "read_file", "screenshot_b64"} <= commands + assert {"click", "write", "run_command", "read_file", "screenshot"} <= commands assert not any(name.startswith("_") for name in commands) From 1b78c5acdb365062db9d680f67767708f7f345b6 Mon Sep 17 00:00:00 2001 From: abonneth Date: Mon, 13 Jul 2026 18:46:02 +0200 Subject: [PATCH 2/4] Match the screenshot_b64 driver command surface Co-authored-by: Cursor --- tests/test_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_local.py b/tests/test_local.py index 1266d47..4ed3084 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -418,7 +418,7 @@ def test_browser_commands_match_hai_drivers_interface(self): def test_desktop_commands_match_hai_drivers_interface(self): pytest.importorskip("hai_drivers.desktop.interface") commands = PyautoguiDesktopBridge(api_key="k").commands - assert {"click", "write", "run_command", "read_file", "screenshot"} <= commands + assert {"click", "write", "run_command", "read_file", "screenshot_b64"} <= commands assert not any(name.startswith("_") for name in commands) From fb13e7ece088e7d5bf1ef033320ec5f35d3062f9 Mon Sep 17 00:00:00 2001 From: abonneth Date: Tue, 14 Jul 2026 01:09:28 +0200 Subject: [PATCH 3/4] Guard the scaled driver import with the install hint Co-authored-by: Cursor --- src/hai_agents_local/desktop.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hai_agents_local/desktop.py b/src/hai_agents_local/desktop.py index 2fcce8b..1f96592 100644 --- a/src/hai_agents_local/desktop.py +++ b/src/hai_agents_local/desktop.py @@ -45,15 +45,14 @@ def create_driver(self) -> DesktopDriverInterface: # Runtime import: hai-drivers is absent unless installed with hai-agents[desktop]. try: from hai_drivers.desktop.local import LocalDesktopDriver + from hai_drivers.desktop.scaled import ScaledDesktopDriver except ImportError as exc: raise ImportError( - "Local desktop control requires extra deps. Install with: pip install 'hai-agents[desktop]'" + "Local desktop control requires hai-drivers>=0.1.2. Install with: pip install 'hai-agents[desktop]'" ) from exc driver = LocalDesktopDriver() if self.max_width is None and self.max_height is None and self.image_format is None: return driver - from hai_drivers.desktop.scaled import ScaledDesktopDriver - return ScaledDesktopDriver( driver, max_width=self.max_width, From 8d82af5744748335002684d472b35c60c7aa46a9 Mon Sep 17 00:00:00 2001 From: abonneth Date: Wed, 15 Jul 2026 18:14:57 +0200 Subject: [PATCH 4/4] Pin hai-drivers 0.1.2 from PyPI Co-authored-by: Cursor --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 88c9b7d..bb3127b 100644 --- a/uv.lock +++ b/uv.lock @@ -184,7 +184,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "hai-agents", extras = ["browser", "cli", "desktop"], marker = "extra == 'all'" }, - { name = "hai-drivers", extras = ["desktop"], marker = "extra == 'desktop'", specifier = ">=0.1.0" }, + { name = "hai-drivers", extras = ["desktop"], marker = "extra == 'desktop'", specifier = ">=0.1.2" }, { name = "hai-drivers", extras = ["web"], marker = "extra == 'browser'", specifier = ">=0.1.0" }, { name = "httpx", specifier = ">=0.23.0,<0.29.0" }, { name = "pydantic", specifier = ">=2.0,<3" }, @@ -207,19 +207,19 @@ dev = [ [[package]] name = "hai-drivers" -version = "0.1.1" +version = "0.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdownify" }, + { name = "pillow" }, { name = "pydantic" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/37/b9/0deb075f1d76f0b4c470ad0be7bb1476a224c375b6d98da95978e46025a8/hai_drivers-0.1.1-py3-none-any.whl", hash = "sha256:bdf6207e2226c488a432cf064acd49943c71e9ac4d90a2529e2e5f2f3cda59fb", size = 31073, upload-time = "2026-07-07T13:27:09Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/f4bf3bbd6340f1a24f5b760028d5370bdf2180d03fcc3bc274279750285e/hai_drivers-0.1.2-py3-none-any.whl", hash = "sha256:586d93678ad225d6face74c0bcab40b91deb8a5ca63258cbee0010b215c0568d", size = 34347, upload-time = "2026-07-15T16:11:31.487Z" }, ] [package.optional-dependencies] desktop = [ - { name = "pillow" }, { name = "pyautogui" }, { name = "pynput" }, ]