From e92a4846d7fca68f89fadcc4bbe1511806ffad2d Mon Sep 17 00:00:00 2001 From: harshitboots Date: Mon, 31 Aug 2026 22:57:44 +0530 Subject: [PATCH] feat: add audio capture support (#4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends ScreenSight with loopback (system-audio) capture — lets agents hear what the user is playing, gated by the master switch and a separate opt-in env var (SCREENSIGHT_ENABLE_AUDIO=1) for privacy. - New capture/audio.py: record_system_audio() via soundcard WASAPI/Pulse loopback → 16-bit PCM WAV with stdlib wave; deps lazily imported so the screenshot path is unaffected when [audio] extras aren't installed. - New MCP tool screen_capture_audio(duration, question) — 9th tool, returns a FastMCP Audio content block + text context. - New CLI command: screensight capture-audio [--duration N]. - config.py: AUDIO_PATH, duration/sample-rate constants, audio_enabled(). - core.py: capture_audio() + CaptureAudioOutcome, double-gated (master switch + SCREENSIGHT_ENABLE_AUDIO), duration clamped to 1–30s. - state.py: turn_off() deletes audio.wav alongside frame.jpg. - pyproject.toml: optional [audio] extra (soundcard>=0.4.3, numpy>=1.24). - Tests: 6 new tests in test_audio.py; test_mcp_server updated 8→9 tools with schema and content-block tests for screen_capture_audio. - Docs: cli.md, mcp-tools.md, configuration.md updated with per-OS loopback notes (Windows WASAPI ✅, Linux PulseAudio ✅, macOS/WSL ⚠️). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HGJmVMdVAbyTnUV2KYWfF7 --- CHANGELOG.md | 14 ++++ docs/reference/cli.md | 31 +++++++ docs/reference/configuration.md | 36 +++++++- docs/reference/mcp-tools.md | 33 +++++++- pyproject.toml | 4 + src/screensight/__main__.py | 27 ++++++ src/screensight/capture/audio.py | 85 +++++++++++++++++++ src/screensight/config.py | 14 ++++ src/screensight/core.py | 51 +++++++++++- src/screensight/mcp_server.py | 81 ++++++++++++++++-- src/screensight/state.py | 5 +- tests/test_audio.py | 138 +++++++++++++++++++++++++++++++ tests/test_mcp_server.py | 54 +++++++++++- 13 files changed, 559 insertions(+), 14 deletions(-) create mode 100644 src/screensight/capture/audio.py create mode 100644 tests/test_audio.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 55300d1..dbaa56f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **System-audio capture** ([#4](https://github.com/himanshu231204/ScreenSight/issues/4)) — + record the system's audio output (loopback) as a WAV: + - New `screen_capture_audio` MCP tool (9 tools total) returning the recording as an + audio content block. + - New `screensight capture-audio [--duration N]` CLI command. + - New optional dependency group `screensight[audio]` (`soundcard`, `numpy`). + - Opt-in and off by default: requires both the master switch on **and** + `SCREENSIGHT_ENABLE_AUDIO=1`. Duration is capped at 30s. + - `audio.wav` is cleaned up on `screensight off`, alongside `frame.jpg`. + - Loopback via `soundcard`: Windows (WASAPI) and Linux (PulseAudio monitor) work out of + the box; macOS/WSL need a virtual output device (BlackHole/SoundFlower). + ## [0.1.0] - 2026-08-11 Initial release. ScreenSight generalizes agent screen-awareness beyond a single diff --git a/docs/reference/cli.md b/docs/reference/cli.md index ad08bbf..9612232 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -12,6 +12,8 @@ screensight off # Disable capture (deletes frame) screensight status # Check switch state screensight capture # Capture primary display screensight capture --display 1 # Capture specific display +screensight capture-audio # Record 5s of system audio (opt-in) +screensight capture-audio --duration 10 # Record a specific duration screensight watch --interval 5 # Watch every 5s (default) screensight watch --interval 3 --max-frames 20 screensight watch-stop # Stop watch daemon @@ -54,6 +56,35 @@ screensight capture --display 1 # a specific display index When `active_window_title` is `null`, the backend could not determine the foreground window. Callers must treat that as *not verified safe*, never as *safe*. +## `screensight capture-audio` + +Records the system's audio output (loopback — what's playing through the speakers) and +prints a JSON object. **Opt-in:** the master switch must be on **and** the +`SCREENSIGHT_ENABLE_AUDIO=1` environment variable must be set, otherwise the command exits +with code `3`. Requires the audio extras: `pip install 'screensight[audio]'`. + +```bash +export SCREENSIGHT_ENABLE_AUDIO=1 +screensight capture-audio # 5s (default) +screensight capture-audio --duration 10 # 1–30s +``` + +```json +{ + "path": "C:\\Users\\you\\.screensight\\audio.wav", + "duration": 5.0, + "sample_rate": 44100 +} +``` + +| Flag | Default | Meaning | +|---|---|---| +| `--duration` | `5` | Seconds of audio to record (clamped to 1–30) | + +Loopback support: Windows (WASAPI) and Linux (PulseAudio/PipeWire monitor) work out of the +box. macOS and WSL need a virtual output device (BlackHole or SoundFlower) installed and set +as the default output. The `audio.wav` file is deleted on `screensight off`. + ## `screensight displays` ```bash diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index abbd630..c1d6668 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,12 +1,14 @@ # Configuration Everything ScreenSight stores lives under `~/.screensight/`. There is no global config -file elsewhere, no environment-variable layer, and no remote state. +file elsewhere and no remote state. The only environment variable is the opt-in audio gate +described in [Audio capture](#audio-capture) below. ```text ~/.screensight/ state.json # Master on/off switch frame.jpg # Latest captured frame + audio.wav # Latest captured system audio (if audio is used) daemon.json # Watch daemon status daemon.pid # Watch daemon process ID redact_zones.json # Blocklist + redaction zones @@ -17,6 +19,7 @@ file elsewhere, no environment-variable layer, and no remote state. |---|---|---|---| | `state.json` | `state.py` | `core.py` | Master on/off switch | | `frame.jpg` | `core.py` via `privacy.process_frame` | CLI, MCP tools | The one current screenshot, overwritten each capture | +| `audio.wav` | `core.py` via `capture.audio` | CLI, MCP tools | The one current audio recording, overwritten each capture | | `redact_zones.json` | You (or your agent) | `privacy.py` | Blocklist terms + redaction rectangles | | `daemon.json` | `watch.py` | CLI `watch-status`, MCP `screen_watch_latest` | Running state, frame count, last change | | `daemon.pid` | `watch.py` on start | `watch.py` on stop | Lets `watch-stop` find the process | @@ -93,6 +96,37 @@ screensight on # enable screensight off # disable + delete frame.jpg ``` +## Audio capture + +System-audio capture is **opt-in and off by default**. Two independent gates must both be +satisfied before any audio is recorded: + +1. The master switch is on (`screensight on`). +2. The `SCREENSIGHT_ENABLE_AUDIO` environment variable is set to `1` (or `true`/`yes`). + +```bash +pip install 'screensight[audio]' # soundcard + numpy +export SCREENSIGHT_ENABLE_AUDIO=1 +screensight capture-audio --duration 5 +``` + +| Setting | Default | Meaning | +|---|---|---| +| `SCREENSIGHT_ENABLE_AUDIO` | unset (off) | Must be `1`/`true`/`yes` to allow audio capture | +| duration | `5` | Seconds per capture, clamped to `1`–`30` | +| sample rate | `44100` Hz | Fixed | + +The recording is written to `~/.screensight/audio.wav` (a single reused file, like +`frame.jpg`) and deleted on `screensight off`. + +**Loopback device support** — capture records what's playing through the default speaker: + +| Platform | Status | +|---|---| +| Windows | ✅ WASAPI loopback, works out of the box | +| Linux | ✅ PulseAudio/PipeWire monitor source, works out of the box | +| macOS / WSL | ⚠️ requires a virtual output device (BlackHole or SoundFlower) set as the default output | + ## Resetting ```bash diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 9465d2e..9b52e08 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -1,8 +1,9 @@ # MCP tools reference -The MCP server (`screensight-mcp`) exposes **eight tools** over stdio. Unlike the CLI, the -capture tool returns an MCP `Image` content block rather than a file path, so the calling -agent looks at the frame in its own context instead of reading a file off disk. +The MCP server (`screensight-mcp`) exposes **nine tools** over stdio. Unlike the CLI, the +capture tools return MCP `Image`/`Audio` content blocks rather than file paths, so the +calling agent looks at the frame (or listens to the audio) in its own context instead of +reading a file off disk. | Tool | Description | |------|-------------| @@ -10,6 +11,7 @@ agent looks at the frame in its own context instead of reading a file off disk. | [`screen_disable`](#screen_disable) | Turn OFF the master switch | | [`screen_status`](#screen_status) | Check whether capture is enabled | | [`screen_capture`](#screen_capture) | Capture screen, returns image + window title | +| [`screen_capture_audio`](#screen_capture_audio) | Record system audio, returns audio block | | [`screen_watch_start`](#screen_watch_start) | Start the bounded watch daemon | | [`screen_watch_stop`](#screen_watch_stop) | Stop the watch daemon | | [`screen_watch_latest`](#screen_watch_latest) | Daemon status and frame count | @@ -63,6 +65,31 @@ describe what it sees. blocklist itself. No tool description, prompt or injected instruction can talk the server into capturing while the switch is off. +## screen_capture_audio + +Record the system's audio output (loopback) and return it to the agent. + +```text +Input: + - duration (default 5): seconds to record (clamped to 1–30) + - question (optional): text echoed back for context +Output: Audio content block (WAV) + text with duration, sample rate and path +``` + +**Opt-in and off by default.** Beyond the master switch, audio requires the +`SCREENSIGHT_ENABLE_AUDIO=1` environment variable, and the server must be installed with the +audio extras (`pip install 'screensight[audio]'`). If either is missing, the tool returns a +clear `Audio capture failed: ...` message rather than recording. + +Loopback works out of the box on Windows (WASAPI) and Linux (PulseAudio/PipeWire monitor); +macOS and WSL need a virtual output device (BlackHole/SoundFlower). The `audio.wav` file is +deleted on `screen_disable`. + +!!! info "The switch is enforced below this tool" + Like `screen_capture`, this tool calls into `core.capture_audio()`, which re-checks the + master switch and the `SCREENSIGHT_ENABLE_AUDIO` gate itself — no prompt can talk the + server into recording while either is off. + ## screen_watch_start Start a detached background daemon that captures on an interval. diff --git a/pyproject.toml b/pyproject.toml index fa13aaf..f5320ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,10 @@ dependencies = [ ] [project.optional-dependencies] +audio = [ + "soundcard>=0.4.3", + "numpy>=1.24", +] dev = [ "pytest>=8.0", "ruff>=0.16,<0.17", diff --git a/src/screensight/__main__.py b/src/screensight/__main__.py index 480ef80..4ec0143 100644 --- a/src/screensight/__main__.py +++ b/src/screensight/__main__.py @@ -6,6 +6,7 @@ screensight off screensight status screensight capture [--display N] + screensight capture-audio [--duration N] screensight watch [--interval N] [--max-frames N] screensight watch-stop screensight watch-status @@ -19,6 +20,7 @@ import sys from . import core, state, watch +from .config import DEFAULT_AUDIO_DURATION def cmd_on(args: argparse.Namespace) -> None: @@ -51,6 +53,22 @@ def cmd_capture(args: argparse.Namespace) -> None: ) +def cmd_capture_audio(args: argparse.Namespace) -> None: + outcome = core.capture_audio(duration=args.duration) + if not outcome.ok: + print(json.dumps({"error": outcome.error}), file=sys.stderr) + sys.exit(3) + print( + json.dumps( + { + "path": outcome.path, + "duration": outcome.duration, + "sample_rate": outcome.sample_rate, + } + ) + ) + + def cmd_watch(args: argparse.Namespace) -> None: result = watch.start_daemon( interval=args.interval, @@ -88,6 +106,14 @@ def main() -> None: cap = sub.add_parser("capture", help="Capture the current screen") cap.add_argument("--display", type=int, default=None, help="Display index (omit for primary)") + cap_audio = sub.add_parser("capture-audio", help="Record system audio output (loopback)") + cap_audio.add_argument( + "--duration", + type=int, + default=DEFAULT_AUDIO_DURATION, + help="Seconds of audio to record (default: %(default)s, max 30)", + ) + w = sub.add_parser("watch", help="Start a bounded watch session") w.add_argument( "--interval", @@ -113,6 +139,7 @@ def main() -> None: "off": cmd_off, "status": cmd_status, "capture": cmd_capture, + "capture-audio": cmd_capture_audio, "watch": cmd_watch, "watch-stop": cmd_watch_stop, "watch-status": cmd_watch_status, diff --git a/src/screensight/capture/audio.py b/src/screensight/capture/audio.py new file mode 100644 index 0000000..f049dc0 --- /dev/null +++ b/src/screensight/capture/audio.py @@ -0,0 +1,85 @@ +"""System-audio (loopback) capture — the audio counterpart to the screenshot +backends. Kept in its own module because audio is time-based (a duration), +not instantaneous like a screenshot, and because its dependencies +(``soundcard``, ``numpy``) are optional extras that the core screenshot path +must never import. + +Platform support via ``soundcard``: +- Windows: WASAPI loopback works out of the box (records the default speaker). +- Linux: PulseAudio/PipeWire monitor source works out of the box. +- macOS / WSL: there is no OS loopback device; a virtual one + (BlackHole, SoundFlower) must be installed and selected as the default + output, otherwise no loopback microphone exists and capture fails with an + actionable error. + +Every failure path returns ``AudioResult(ok=False, error=...)`` — capture must +degrade gracefully rather than raise, matching the rest of the package. +""" + +from __future__ import annotations + +import wave +from dataclasses import dataclass + +_DEPS_HINT = "audio deps not installed — run: pip install 'screensight[audio]'" +_NO_LOOPBACK_HINT = ( + "no loopback audio device found — on macOS/WSL install a virtual output " + "device (BlackHole or SoundFlower) and set it as the default output" +) + + +@dataclass +class AudioResult: + ok: bool + path: str | None = None + error: str | None = None + duration: float | None = None + sample_rate: int | None = None + channels: int | None = None + + +def record_system_audio(out_path: str, duration: float, sample_rate: int) -> AudioResult: + """Record ``duration`` seconds of system audio output to ``out_path`` as a + 16-bit PCM WAV. Returns an :class:`AudioResult`; never raises.""" + try: + import numpy as np + import soundcard as sc # type: ignore[import-untyped] + except ImportError: + return AudioResult(ok=False, error=_DEPS_HINT) + + try: + speaker = sc.default_speaker() + if speaker is None: + return AudioResult(ok=False, error=_NO_LOOPBACK_HINT) + + # include_loopback=True returns a microphone that records the speaker's + # output rather than a physical input. + loopback = sc.get_microphone(speaker.name, include_loopback=True) + if loopback is None: + return AudioResult(ok=False, error=_NO_LOOPBACK_HINT) + + frames = int(duration * sample_rate) + with loopback.recorder(samplerate=sample_rate) as rec: + data = rec.record(numframes=frames) # float32 array, shape (frames, channels) + + channels = data.shape[1] if data.ndim > 1 else 1 + + # Convert float32 [-1.0, 1.0] to int16 PCM, clipping out-of-range values. + clipped = np.clip(data, -1.0, 1.0) + pcm16 = (clipped * 32767).astype(" None: BASE_DIR.mkdir(parents=True, exist_ok=True) +def audio_enabled() -> bool: + """Whether system-audio capture is allowed. Off unless the user opts in via + the SCREENSIGHT_ENABLE_AUDIO env var — a separate gate on top of the master + switch, so audio is never recorded by default.""" + return os.environ.get("SCREENSIGHT_ENABLE_AUDIO", "").strip().lower() in ("1", "true", "yes") + + def get_os() -> str: system = platform.system().lower() if system == "linux" and "microsoft" in platform.uname().release.lower(): diff --git a/src/screensight/core.py b/src/screensight/core.py index 3f3f2dd..281a802 100644 --- a/src/screensight/core.py +++ b/src/screensight/core.py @@ -7,8 +7,17 @@ from dataclasses import dataclass +from .capture.audio import record_system_audio from .capture.base import get_backend -from .config import FRAME_PATH, ensure_base_dir +from .config import ( + AUDIO_PATH, + AUDIO_SAMPLE_RATE, + DEFAULT_AUDIO_DURATION, + FRAME_PATH, + MAX_AUDIO_DURATION, + audio_enabled, + ensure_base_dir, +) from .diff import sha256_of_file from .privacy import process_frame, title_is_blocked from .state import is_on @@ -23,6 +32,15 @@ class CaptureOutcome: active_window_title: str | None = None +@dataclass +class CaptureAudioOutcome: + ok: bool + path: str | None = None + error: str | None = None + duration: float | None = None + sample_rate: int | None = None + + def capture_once(display: int | None = None) -> CaptureOutcome: """Returns exit-code-3 semantics as ok=False, error='off' when the master switch is off — checked here, not just in a prompt.""" @@ -53,5 +71,36 @@ def capture_once(display: int | None = None) -> CaptureOutcome: ) +def capture_audio(duration: int | None = None) -> CaptureAudioOutcome: + """Record system audio output (loopback) to AUDIO_PATH as a WAV. + + Gated twice: the master switch must be on (same as capture_once) AND audio + must be explicitly enabled via SCREENSIGHT_ENABLE_AUDIO — audio is never + recorded by default. Duration is clamped to [1, MAX_AUDIO_DURATION].""" + if not is_on(): + return CaptureAudioOutcome(ok=False, error="off") + if not audio_enabled(): + return CaptureAudioOutcome( + ok=False, + error="audio capture disabled (set SCREENSIGHT_ENABLE_AUDIO=1 to enable)", + ) + + if duration is None: + duration = DEFAULT_AUDIO_DURATION + duration = max(1, min(int(duration), MAX_AUDIO_DURATION)) + + ensure_base_dir() + result = record_system_audio(str(AUDIO_PATH), duration, AUDIO_SAMPLE_RATE) + if not result.ok: + return CaptureAudioOutcome(ok=False, error=result.error) + + return CaptureAudioOutcome( + ok=True, + path=result.path, + duration=result.duration, + sample_rate=result.sample_rate, + ) + + def list_displays() -> list[dict]: return get_backend().list_displays() diff --git a/src/screensight/mcp_server.py b/src/screensight/mcp_server.py index db3a87f..731442f 100644 --- a/src/screensight/mcp_server.py +++ b/src/screensight/mcp_server.py @@ -1,4 +1,4 @@ -"""FastMCP server exposing ScreenSight as 8 tools for any MCP-capable agent. +"""FastMCP server exposing ScreenSight as 9 tools for any MCP-capable agent. Every tool docstring is written for the *calling agent* (AGENTS.md rule 8) — treat them as man-page entries, not code comments. @@ -6,11 +6,17 @@ from __future__ import annotations +import base64 from pathlib import Path from fastmcp import FastMCP from fastmcp.utilities.types import Image +try: # Audio helper added in newer FastMCP; fall back to a raw resource block if absent. + from fastmcp.utilities.types import Audio as _Audio +except ImportError: # pragma: no cover - depends on installed FastMCP version + _Audio = None # type: ignore[assignment,misc] + from . import core, state, watch mcp = FastMCP( @@ -127,7 +133,72 @@ def screen_capture( return result_parts -# ── 5. screen_watch_start ───────────────────────────────────────────── +# ── 5. screen_capture_audio ─────────────────────────────────────────── + + +@mcp.tool(output_schema=None) +def screen_capture_audio( + duration: int = 5, + question: str = "", +) -> list: + """Record the system's audio output (what's playing through the speakers). + + Captures loopback audio — the sound the user is hearing (music, a video, + a call) — for `duration` seconds and returns it as an audio content block + you can listen to, plus text context. + + This is separate from screen_capture: audio is time-based, so this call + blocks for roughly `duration` seconds while recording. + + Two gates must be satisfied: + - The master switch must be ON (screen_enable). + - Audio must be enabled by the user via the SCREENSIGHT_ENABLE_AUDIO=1 + environment variable — it is OFF by default for privacy. + + Platform notes: Windows (WASAPI) and Linux (PulseAudio monitor) work out of + the box; macOS and WSL require a virtual loopback device (BlackHole / + SoundFlower) set as the default output. + + Args: + duration: Seconds of audio to record (1–30, default 5). + question: Optional question to echo back for your context. + + Returns: + Audio content block + text context (+ echoed question if given). + """ + outcome = core.capture_audio(duration=duration) + if not outcome.ok: + return [f"Audio capture failed: {outcome.error}"] + + result_parts: list = [] + + audio_path = Path(outcome.path) + audio_bytes = audio_path.read_bytes() + if _Audio is not None: + result_parts.append(_Audio(data=audio_bytes, format="wav")) + else: + # Fallback for FastMCP builds without the Audio helper: embed as a + # base64 audio resource block. + result_parts.append( + { + "type": "audio", + "data": base64.b64encode(audio_bytes).decode("ascii"), + "mimeType": "audio/wav", + } + ) + + text_parts = [ + f"Recorded {outcome.duration}s of system audio at {outcome.sample_rate} Hz", + f"Audio saved to: {outcome.path}", + ] + if question: + text_parts.append(f"Your question: {question}") + result_parts.append("\n".join(text_parts)) + + return result_parts + + +# ── 6. screen_watch_start ───────────────────────────────────────────── @mcp.tool() @@ -152,7 +223,7 @@ def screen_watch_start( return f"Watch daemon started: {result}" -# ── 6. screen_watch_stop ────────────────────────────────────────────── +# ── 7. screen_watch_stop ────────────────────────────────────────────── @mcp.tool() @@ -166,7 +237,7 @@ def screen_watch_stop() -> str: return f"Watch daemon stopped: {result}" -# ── 7. screen_watch_latest ──────────────────────────────────────────── +# ── 8. screen_watch_latest ──────────────────────────────────────────── @mcp.tool() @@ -200,7 +271,7 @@ def screen_watch_latest() -> str: return "\n".join(lines) -# ── 8. screen_list_displays ─────────────────────────────────────────── +# ── 9. screen_list_displays ─────────────────────────────────────────── @mcp.tool() diff --git a/src/screensight/state.py b/src/screensight/state.py index 02defa8..ea0c346 100644 --- a/src/screensight/state.py +++ b/src/screensight/state.py @@ -8,7 +8,7 @@ import json import time -from .config import FRAME_PATH, STATE_FILE, ensure_base_dir +from .config import AUDIO_PATH, FRAME_PATH, STATE_FILE, ensure_base_dir def is_on() -> bool: @@ -30,8 +30,9 @@ def turn_on() -> None: def turn_off() -> None: ensure_base_dir() STATE_FILE.write_text(json.dumps({"enabled": False, "since": time.time()})) - # Frame hygiene: delete the frame file on off. + # Frame hygiene: delete the frame and audio files on off. FRAME_PATH.unlink(missing_ok=True) + AUDIO_PATH.unlink(missing_ok=True) def status() -> dict: diff --git a/tests/test_audio.py b/tests/test_audio.py new file mode 100644 index 0000000..37214aa --- /dev/null +++ b/tests/test_audio.py @@ -0,0 +1,138 @@ +"""Tests for the capture_audio() pipeline and audio file cleanup, with the +soundcard/numpy hardware path mocked out.""" + +from __future__ import annotations + +import wave +from pathlib import Path + +from screensight import core, state +from screensight.capture.audio import AudioResult + + +def _write_silent_wav(path: Path, duration: float = 1.0, sample_rate: int = 44100) -> None: + """Write a valid, silent mono 16-bit WAV so no hardware is touched.""" + frames = int(duration * sample_rate) + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(b"\x00\x00" * frames) + + +def _mock_record(**overrides): + """Return a fake record_system_audio that writes a silent WAV and records + the args it was called with in `calls`.""" + calls: list = [] + + def _record(out_path, duration, sample_rate): + calls.append({"out_path": out_path, "duration": duration, "sample_rate": sample_rate}) + _write_silent_wav(Path(out_path), duration=1.0, sample_rate=sample_rate) + return AudioResult( + ok=True, + path=out_path, + duration=float(duration), + sample_rate=sample_rate, + channels=1, + **overrides, + ) + + return _record, calls + + +def test_switch_off_short_circuits(tmp_path, monkeypatch): + """capture_audio() returns error='off' when the master switch is off, + without ever recording.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.setenv("SCREENSIGHT_ENABLE_AUDIO", "1") + + record, calls = _mock_record() + monkeypatch.setattr(core, "record_system_audio", record) + + result = core.capture_audio() + assert result.ok is False + assert result.error == "off" + assert calls == [] + + +def test_disabled_without_env(tmp_path, monkeypatch): + """With the master switch on but SCREENSIGHT_ENABLE_AUDIO unset, capture is + refused and nothing is recorded.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.delenv("SCREENSIGHT_ENABLE_AUDIO", raising=False) + state.turn_on() + + record, calls = _mock_record() + monkeypatch.setattr(core, "record_system_audio", record) + + result = core.capture_audio() + assert result.ok is False + assert "disabled" in result.error + assert calls == [] + + +def test_successful_capture(tmp_path, monkeypatch): + """A normal capture returns ok=True with path, duration, and sample rate.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.setenv("SCREENSIGHT_ENABLE_AUDIO", "1") + state.turn_on() + + record, calls = _mock_record() + monkeypatch.setattr(core, "record_system_audio", record) + + result = core.capture_audio(duration=3) + assert result.ok is True + assert result.path == str(tmp_path / "audio.wav") + assert result.duration == 3.0 + assert result.sample_rate == core.AUDIO_SAMPLE_RATE + assert (tmp_path / "audio.wav").exists() + assert calls[0]["duration"] == 3 + + +def test_duration_is_clamped(tmp_path, monkeypatch): + """Durations above MAX_AUDIO_DURATION are clamped before recording.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.setenv("SCREENSIGHT_ENABLE_AUDIO", "1") + state.turn_on() + + record, calls = _mock_record() + monkeypatch.setattr(core, "record_system_audio", record) + + core.capture_audio(duration=999) + assert calls[0]["duration"] == core.MAX_AUDIO_DURATION + + +def test_record_failure_propagates(tmp_path, monkeypatch): + """A failing recorder surfaces ok=False with the error message.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.setenv("SCREENSIGHT_ENABLE_AUDIO", "1") + state.turn_on() + + monkeypatch.setattr( + core, + "record_system_audio", + lambda *a, **k: AudioResult(ok=False, error="no loopback device"), + ) + + result = core.capture_audio() + assert result.ok is False + assert result.error == "no loopback device" + + +def test_turn_off_deletes_audio(tmp_path, monkeypatch): + """state.turn_off() removes the audio file, satisfying the cleanup + acceptance criterion.""" + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(state, "FRAME_PATH", tmp_path / "frame.jpg") + monkeypatch.setattr(state, "AUDIO_PATH", tmp_path / "audio.wav") + + _write_silent_wav(tmp_path / "audio.wav") + assert (tmp_path / "audio.wav").exists() + + state.turn_off() + assert not (tmp_path / "audio.wav").exists() diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 058fa7b..2e17d6c 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -64,8 +64,8 @@ def test_other_tools_have_no_output_schema_conflict(): # ── tool count ──────────────────────────────────────────────────────── -def test_all_eight_tools_registered(): - """ScreenSight must expose exactly 8 tools.""" +def test_all_nine_tools_registered(): + """ScreenSight must expose exactly 9 tools.""" tools = _get_tools() names = sorted(t.name for t in tools) expected = sorted( @@ -74,6 +74,7 @@ def test_all_eight_tools_registered(): "screen_disable", "screen_status", "screen_capture", + "screen_capture_audio", "screen_watch_start", "screen_watch_stop", "screen_watch_latest", @@ -83,6 +84,15 @@ def test_all_eight_tools_registered(): assert names == expected, f"Expected tools {expected}, got {names}" +def test_screen_capture_audio_output_schema_is_none(): + """screen_capture_audio returns audio + text content blocks, so like + screen_capture it must have output_schema=None.""" + tool = _get_tool("screen_capture_audio") + assert tool.output_schema is None, ( + f"screen_capture_audio.output_schema should be None, got {tool.output_schema!r}." + ) + + # ── screen_capture content blocks (requires mocking) ────────────────── @@ -129,3 +139,43 @@ def list_displays(self): assert result.structured_content is None, ( f"structured_content should be None, got {result.structured_content!r}" ) + + +def test_screen_capture_audio_returns_content_blocks(tmp_path, monkeypatch): + """screen_capture_audio should return content blocks (audio + text) with no + structured_content. Mock the audio pipeline so no hardware is touched.""" + import wave + from pathlib import Path + + from screensight import core, state + from screensight.capture.audio import AudioResult + from screensight.mcp_server import mcp as mcp_server + + def _write_wav(path: Path) -> None: + with wave.open(str(path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(44100) + wav.writeframes(b"\x00\x00" * 44100) + + def _fake_record(out_path, duration, sample_rate): + _write_wav(Path(out_path)) + return AudioResult( + ok=True, path=out_path, duration=float(duration), sample_rate=sample_rate, channels=1 + ) + + monkeypatch.setattr(state, "STATE_FILE", tmp_path / "state.json") + monkeypatch.setattr(core, "AUDIO_PATH", tmp_path / "audio.wav") + monkeypatch.setattr(core, "record_system_audio", _fake_record) + monkeypatch.setenv("SCREENSIGHT_ENABLE_AUDIO", "1") + state.turn_on() + + tools = asyncio.run(mcp_server.list_tools()) + audio_tool = next(t for t in tools if t.name == "screen_capture_audio") + result = asyncio.run(audio_tool.run({"duration": 2, "question": "what is playing?"})) + + assert result.content is not None, "content should not be None" + assert len(result.content) >= 2, f"Expected >=2 content blocks, got {len(result.content)}" + assert result.structured_content is None, ( + f"structured_content should be None, got {result.structured_content!r}" + )