From c7ef63ba98ba8ca2feef21e235f32cd5f147fc29 Mon Sep 17 00:00:00 2001 From: Daliys Date: Tue, 1 Sep 2026 23:43:40 +0200 Subject: [PATCH] fix: capture valid Unity editor screenshots --- API_REFERENCE.MD | 4 + CHANGELOG.md | 4 + DOCUMENTATION.MD | 4 +- .../MCPServerMethods.HighValue.Screenshots.cs | 181 ++++++++++-------- Editor/MCPServerMethods.Tools.cs | 4 +- Editor/UIVerification.cs | 23 +++ README.md | 1 + Tests~/Editor/AgentToolingTests.cs | 65 +++++++ scripts/agent-tooling-smoke.py | 46 ++++- 9 files changed, 252 insertions(+), 80 deletions(-) diff --git a/API_REFERENCE.MD b/API_REFERENCE.MD index dfabc36..32bb6b3 100644 --- a/API_REFERENCE.MD +++ b/API_REFERENCE.MD @@ -159,6 +159,10 @@ Schema compatibility notes: `batch_execute` accepts at most 50 requests and rejects nested `batch_execute` calls. +### Screenshot results + +`capture_game_view_screenshot` and `capture_inspector_screenshot` return `success`, `message`, and `duration_ms`, plus a `data` object containing `width`, `height`, `format: "png"`, and `image_base64`. The legacy top-level `status`, `format`, and successful `image_base64` fields remain available for existing raw clients; Inspector responses also retain `ui_layout`. + ## MCP Bridge Tools The MCP bridge is the recommended AI-client surface. It exposes 14 tools and two static read-only resources through `resources/list` and `resources/read`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 847121f..d1cbfc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable public changes to Nexus Unity are documented here. ## [Unreleased] +### Fixed +- Reworked Game View and Inspector screenshot capture to use Unity's native editor surface readback instead of macOS `screencapture`, with one repaint/frame retry and structured PNG metadata including dimensions and capture duration. Legacy successful response fields remain compatible. + +## [1.6.0] - 2026-08-23 ### Security - Sanitize and redact sensitive exception messages in tool usage stats (`get_tool_usage_stats` and `RecordToolUsage`), replacing absolute filesystem paths, user directories, and multiline frames with generic placeholders and safe truncated summaries, and exposing `last_error_type` (#142). - Require explicit confirmation (`confirm: true`) for `delete_asset`, enforce `AssetDatabase.MoveAssetToTrash` for OS trash recovery, and block creation, modification, moving, or deletion of `ProjectSettings/` and `Packages/` paths across all asset tools (#140). diff --git a/DOCUMENTATION.MD b/DOCUMENTATION.MD index 7cab974..ecff393 100644 --- a/DOCUMENTATION.MD +++ b/DOCUMENTATION.MD @@ -104,7 +104,7 @@ Raw HTTP JSON-RPC: - Intended for diagnostics, custom integrations, and direct automation. - The complete raw catalog is returned by `list_tools`. - Runtime schemas intentionally include compatibility fields where needed; for example `update_component` accepts both `properties` and the legacy `json_data` payload, `create_scene` accepts `path` / `open_if_exists`, `create_primitive` accepts transform/material fields, and `create_material` accepts an optional explicit asset `path` plus visible color fields. -- Agent diagnostics include `get_test_results` for Unity `TestResults*.xml` summaries with scoped non-passing test messages, `get_tool_usage_stats` / `reset_tool_usage_stats` for scoped in-memory call counters and timing, and `ui_get_window_rect` / `ui_set_window_rect` / `ui_capture_window_snapshot` for resize-based editor UI QA. +- Agent diagnostics include `get_test_results` for Unity `TestResults*.xml` summaries with scoped non-passing test messages, `get_tool_usage_stats` / `reset_tool_usage_stats` for scoped in-memory call counters and timing, and `ui_get_window_rect` / `ui_set_window_rect` / `ui_capture_window_snapshot` for resize-based editor UI QA. `capture_game_view_screenshot` and `capture_inspector_screenshot` return structured PNG data with dimensions, base64 image bytes, and capture duration while preserving legacy successful fields. - `batch_execute` runs requests serially, accepts at most 50 requests, and rejects nested `batch_execute` calls. MCP bridge: @@ -186,7 +186,7 @@ bash scripts/prepush-validate.sh --integration Integration validation requires the Unity project to be open and the Nexus Unity server to be running on `http://127.0.0.1:8081/`. -For a focused agent tooling check, run `python3 scripts/agent-tooling-smoke.py`. It exercises server status, raw catalog discovery, scoped usage stats, Nexus Unity window UI automation, snapshot capture, and bridge-side `run_tests_wait` without writing repo-tracked files. +For a focused agent tooling check, run `python3 scripts/agent-tooling-smoke.py`. It exercises server status, raw catalog discovery, scoped usage stats, Nexus Unity window UI automation, snapshot capture, 20 consecutive Game View and Inspector PNG captures, and bridge-side `run_tests_wait` without writing repo-tracked files. Before release, maintainers should run a public API stress audit that compares raw `list_tools` output with the MCP bridge catalog, exercises read-only and mutating tool groups in a disposable namespace, and verifies cleanup of generated assets and PlayerPrefs keys. diff --git a/Editor/MCPServerMethods.HighValue.Screenshots.cs b/Editor/MCPServerMethods.HighValue.Screenshots.cs index ceb7a52..af7d3f8 100644 --- a/Editor/MCPServerMethods.HighValue.Screenshots.cs +++ b/Editor/MCPServerMethods.HighValue.Screenshots.cs @@ -1,96 +1,141 @@ using System; -using System.IO; +using System.Diagnostics; using System.Linq; +using Newtonsoft.Json.Linq; using UnityEditor; +using UnityEditorInternal; using UnityEngine; -using Newtonsoft.Json.Linq; namespace UnityMCP.Editor { public static partial class MCPServerMethods { + private const int ScreenshotAttempts = 2; + private static JToken CaptureGameViewScreenshot(JToken p) { var gameView = Resources.FindObjectsOfTypeAll() .FirstOrDefault(window => window.GetType().Name == "GameView"); if (gameView == null) throw new Exception("Game View window not found or not open."); - gameView.Focus(); - gameView.Repaint(); - Rect position = gameView.position; - string tempPath = Path.Combine(Path.GetTempPath(), $"unity_gameview_{DateTime.Now.Ticks}.png"); - NexusEditorLog.Log(NexusLogCategory.UiAutomation, $"[MCP_SCREENSHOT] Capturing GameView at {position} to {tempPath}"); - CaptureGameViewImage(tempPath, position); - return ReadGameViewImage(tempPath); + return CaptureEditorWindowScreenshot(gameView, "Game View"); + } + + private static JToken CaptureInspectorScreenshot(JToken p) + { + SelectInspectorTarget(p); + var inspector = Resources.FindObjectsOfTypeAll() + .FirstOrDefault(window => window.titleContent.text == "Inspector"); + if (inspector == null) throw new Exception("Inspector window not found or not open."); + + return CaptureEditorWindowScreenshot(inspector, "Inspector", SerializeVisualElement(inspector.rootVisualElement, true)); } - private static void CaptureGameViewImage(string tempPath, Rect position) + private static JObject CaptureEditorWindowScreenshot(EditorWindow window, string windowName, JToken layout = null) { -#if UNITY_EDITOR_OSX - var startInfo = new System.Diagnostics.ProcessStartInfo + var stopwatch = Stopwatch.StartNew(); + window.Focus(); + window.Repaint(); + InternalEditorUtility.RepaintAllViews(); + + var size = new Vector2Int(Mathf.RoundToInt(window.position.width), Mathf.RoundToInt(window.position.height)); + if (size.x <= 0 || size.y <= 0) { - FileName = "screencapture", - Arguments = $"-x -R{(int)position.x},{(int)position.y},{(int)position.width},{(int)position.height} \"{tempPath}\"", - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardError = true, - RedirectStandardOutput = true - }; - using (var process = System.Diagnostics.Process.Start(startInfo)) + stopwatch.Stop(); + var fail = CreateScreenshotResult(false, windowName + " window has no capturable area.", null, + new Vector2Int(Mathf.Max(0, size.x), Mathf.Max(0, size.y)), stopwatch.Elapsed.TotalMilliseconds); + if (layout != null) fail["ui_layout"] = layout; + return fail; + } + + for (int attempt = 0; attempt < ScreenshotAttempts; attempt++) { - process.WaitForExit(); - string error = process.StandardError.ReadToEnd(); - if (process.ExitCode != 0) - NexusEditorLog.Error(NexusLogCategory.UiAutomation, $"[MCP_SCREENSHOT] screencapture failed with exit code {process.ExitCode}. Error: {error}"); + if (attempt > 0) WaitForCaptureFrame(); + + byte[] png = TryReadSurfacePixels(window.position.position, size, windowName, attempt); + if (png == null) continue; + + stopwatch.Stop(); + var success = CreateScreenshotResult(true, windowName + " screenshot captured.", png, size, + stopwatch.Elapsed.TotalMilliseconds); + if (layout != null) success["ui_layout"] = layout; + return success; } -#else - ScreenCapture.CaptureScreenshot(tempPath); - for (int retries = 0; !File.Exists(tempPath) && retries < 20; retries++) - System.Threading.Thread.Sleep(100); -#endif + + stopwatch.Stop(); + var result = CreateScreenshotResult(false, windowName + " screenshot could not be read from the editor surface.", + null, size, stopwatch.Elapsed.TotalMilliseconds); + if (layout != null) result["ui_layout"] = layout; + return result; } - private static JObject ReadGameViewImage(string tempPath) + private static byte[] TryReadSurfacePixels(Vector2 screenPosition, Vector2Int size, string windowName, int attempt) { - if (!File.Exists(tempPath)) throw new Exception("Failed to capture Game View screenshot."); - byte[] bytes = File.ReadAllBytes(tempPath); - File.Delete(tempPath); - return new JObject + try { - ["status"] = "Success", - ["image_base64"] = Convert.ToBase64String(bytes), - ["format"] = "png" - }; + Color[] pixels = InternalEditorUtility.ReadScreenPixel(screenPosition, size.x, size.y); + if (pixels == null || pixels.Length != size.x * size.y) return null; + + var texture = new Texture2D(size.x, size.y, TextureFormat.RGBA32, false); + try + { + texture.SetPixels(pixels); + byte[] png = texture.EncodeToPNG(); + return png != null && png.Length >= 8 && IsPng(png) ? png : null; + } + finally + { + UnityEngine.Object.DestroyImmediate(texture); + } + } + catch (Exception e) + { + NexusEditorLog.Warning(NexusLogCategory.UiAutomation, + $"[MCP_SCREENSHOT] {windowName} capture attempt {attempt + 1} failed: {e.Message}"); + return null; + } } - private static JToken CaptureInspectorScreenshot(JToken p) + private static void WaitForCaptureFrame() { -#if !UNITY_EDITOR_OSX - throw new Exception("Inspector screenshot is currently only supported on macOS."); -#else - return CaptureInspectorScreenshotOnMac(p); -#endif + EditorApplication.QueuePlayerLoopUpdate(); + InternalEditorUtility.RepaintAllViews(); + System.Threading.Thread.Sleep(16); } -#if UNITY_EDITOR_OSX - private static JObject CaptureInspectorScreenshotOnMac(JToken p) + private static JObject CreateScreenshotResult(bool success, string message, byte[] png, Vector2Int size, + double durationMs) { - SelectInspectorTarget(p); - var inspector = Resources.FindObjectsOfTypeAll() - .FirstOrDefault(window => window.titleContent.text == "Inspector"); - if (inspector == null) throw new Exception("Inspector window not found or not open."); + string imageBase64 = png == null ? string.Empty : Convert.ToBase64String(png); + var data = new JObject + { + ["width"] = size.x, + ["height"] = size.y, + ["format"] = "png", + ["image_base64"] = imageBase64 + }; + var result = new JObject + { + ["status"] = success ? "Success" : "PartialSuccess", + ["success"] = success, + ["message"] = message, + ["duration_ms"] = Math.Round(durationMs, 3), + ["data"] = data + }; - inspector.Focus(); - inspector.Repaint(); - var layout = SerializeVisualElement(inspector.rootVisualElement, true); - string tempPath = Path.Combine(Path.GetTempPath(), $"unity_inspector_{DateTime.Now.Ticks}.png"); - CaptureInspectorImage(tempPath, inspector.position); - if (!File.Exists(tempPath)) - return new JObject { ["status"] = "PartialSuccess", ["message"] = "Screenshot failed (permissions?), but UI layout was captured.", ["ui_layout"] = layout }; - - byte[] bytes = File.ReadAllBytes(tempPath); - File.Delete(tempPath); - return new JObject { ["status"] = "Success", ["image_base64"] = Convert.ToBase64String(bytes), ["format"] = "png", ["ui_layout"] = layout }; + // Keep the original top-level fields for existing raw JSON-RPC clients. + if (success) + { + result["image_base64"] = imageBase64; + result["format"] = "png"; + } + return result; + } + + private static bool IsPng(byte[] bytes) + { + return bytes.Length >= 8 && bytes[0] == 0x89 && bytes[1] == 0x50 && bytes[2] == 0x4e && + bytes[3] == 0x47 && bytes[4] == 0x0d && bytes[5] == 0x0a && bytes[6] == 0x1a && bytes[7] == 0x0a; } private static void SelectInspectorTarget(JToken p) @@ -99,19 +144,5 @@ private static void SelectInspectorTarget(JToken p) var target = MCPServerMethods.IdToObject(MCPServerMethods.ExtractId(p)); if (target != null) Selection.activeObject = target; } - - private static void CaptureInspectorImage(string tempPath, Rect position) - { - var startInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = "screencapture", - Arguments = $"-x -R{(int)position.x},{(int)position.y},{(int)position.width},{(int)position.height} \"{tempPath}\"", - UseShellExecute = false, - CreateNoWindow = true - }; - using (var process = System.Diagnostics.Process.Start(startInfo)) - process.WaitForExit(); - } -#endif } } diff --git a/Editor/MCPServerMethods.Tools.cs b/Editor/MCPServerMethods.Tools.cs index 964af76..fe789f3 100644 --- a/Editor/MCPServerMethods.Tools.cs +++ b/Editor/MCPServerMethods.Tools.cs @@ -172,8 +172,8 @@ private static void AddPlayerPrefsTools(JArray tools) private static void AddHighValueTools(JArray tools) { - tools.Add(CreateTool("capture_inspector_screenshot", "Capture PNG of Inspector (macOS only)", new JObject { ["instance_id"] = new JObject { ["type"] = "integer" } })); - tools.Add(CreateTool("capture_game_view_screenshot", "Capture PNG of Game View", new JObject { })); + tools.Add(CreateTool("capture_inspector_screenshot", "Capture Inspector as a structured PNG result", new JObject { ["instance_id"] = new JObject { ["type"] = "integer" } })); + tools.Add(CreateTool("capture_game_view_screenshot", "Capture Game View as a structured PNG result", new JObject { })); tools.Add(CreateTool("generate_mermaid_diagram", "Generate Mermaid diagram of scene", new JObject { })); tools.Add(CreateTool("semantic_find", "Find objects by semantic meaning", new JObject { ["query"] = new JObject { ["type"] = "string" } }, "query")); } diff --git a/Editor/UIVerification.cs b/Editor/UIVerification.cs index 848e882..3774892 100644 --- a/Editor/UIVerification.cs +++ b/Editor/UIVerification.cs @@ -27,6 +27,7 @@ public static void Verify() TestListAndHierarchy(); TestInputAndClick(wnd); + TestScreenshots(); NexusEditorLog.Log(NexusLogCategory.Diagnostics, "VERIFICATION SUCCESS", true); } @@ -47,6 +48,28 @@ private static void TestInputAndClick(MCPTestWindow wnd) if (!wnd.ButtonClicked) throw new System.Exception("Click failed"); } + private static void TestScreenshots() + { + Call("execute_menu_item", new JObject { ["item_path"] = "Window/General/Game" }); + AssertScreenshot(Call("capture_game_view_screenshot", null), "Game View"); + + Call("execute_menu_item", new JObject { ["item_path"] = "Window/General/Inspector" }); + AssertScreenshot(Call("capture_inspector_screenshot", null), "Inspector"); + } + + private static void AssertScreenshot(string response, string windowName) + { + JObject result = JObject.Parse(response); + if (result["success"]?.Value() != true) + throw new System.Exception($"{windowName} screenshot failed: {result.ToString(Newtonsoft.Json.Formatting.None)}"); + + JObject data = (JObject)result["data"]; + byte[] image = System.Convert.FromBase64String(data?["image_base64"]?.ToString() ?? string.Empty); + byte[] signature = { 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a }; + if (image.Length <= 5 * 1024 || image.Length < signature.Length || !image.Take(signature.Length).SequenceEqual(signature)) + throw new System.Exception($"{windowName} screenshot is not a non-trivial PNG."); + } + private static string Call(string m, JObject p) { string resp = MCPServerMethods.ProcessJsonRpc(new JObject { ["jsonrpc"] = "2.0", ["method"] = m, ["params"] = p, ["id"] = 1 }.ToString()); diff --git a/README.md b/README.md index 62a2bcc..be74b65 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ Root-deployed path: - `unity_asset_manager`: search, import, refresh, and manage prefab assets. - `unity_editor_controller`: play mode, menus, undo/redo, logs, editor state, asset refresh, and test-result polling. - `unity_ui_automation`: query and operate Unity Editor UI Toolkit windows, including window rects for resize QA. +- `capture_game_view_screenshot` and `capture_inspector_screenshot`: structured PNG data with dimensions, base64 image bytes, and capture duration. See `API_REFERENCE.MD` for the complete raw and MCP tool catalogs. diff --git a/Tests~/Editor/AgentToolingTests.cs b/Tests~/Editor/AgentToolingTests.cs index 5195ac6..1698212 100644 --- a/Tests~/Editor/AgentToolingTests.cs +++ b/Tests~/Editor/AgentToolingTests.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -236,6 +237,44 @@ public void UiWindowRectMethodsRoundTrip() Assert.GreaterOrEqual(getResult["rect"]?["height"]?.Value() ?? 0, window.minSize.y); } + [Test] + public void CaptureGameViewScreenshotReturnsStructuredPng() + { + Assume.That(Application.platform, Is.EqualTo(RuntimePlatform.OSXEditor), "Screenshot acceptance is validated on macOS."); + Assume.That(!Application.isBatchMode, "Screenshot capture requires a visible editor window."); + EditorWindow gameView = OpenEditorWindow("UnityEditor.GameView", "Game"); + + try + { + gameView.position = new Rect(80, 80, 640, 360); + JObject result = RpcResult("capture_game_view_screenshot"); + AssertStructuredPng(result); + } + finally + { + gameView.Close(); + } + } + + [Test] + public void CaptureInspectorScreenshotReturnsStructuredPng() + { + Assume.That(Application.platform, Is.EqualTo(RuntimePlatform.OSXEditor), "Screenshot acceptance is validated on macOS."); + Assume.That(!Application.isBatchMode, "Screenshot capture requires a visible editor window."); + EditorWindow inspector = OpenEditorWindow("UnityEditor.InspectorWindow", "Inspector"); + + try + { + inspector.position = new Rect(80, 80, 420, 620); + JObject result = RpcResult("capture_inspector_screenshot"); + AssertStructuredPng(result); + } + finally + { + inspector.Close(); + } + } + [Test] public void UiCaptureWindowSnapshotReturnsRectHierarchyAndBestEffortImage() { @@ -381,6 +420,32 @@ private static JObject RpcResult(string method, JObject parameters = null) return (JObject)response["result"]; } + private static EditorWindow OpenEditorWindow(string typeName, string title) + { + System.Type windowType = typeof(EditorWindow).Assembly.GetType(typeName); + Assert.IsNotNull(windowType, $"Unity editor window type not found: {typeName}"); + EditorWindow window = EditorWindow.GetWindow(windowType, false, title, true); + Assert.IsNotNull(window, $"Unable to open editor window: {typeName}"); + return window; + } + + private static void AssertStructuredPng(JObject result) + { + Assert.IsTrue(result["success"]?.Value() ?? false, result.ToString(Formatting.None)); + Assert.IsFalse(string.IsNullOrEmpty(result["message"]?.ToString())); + Assert.GreaterOrEqual(result["duration_ms"]?.Value() ?? -1, 0); + + JObject data = (JObject)result["data"]; + Assert.IsNotNull(data); + Assert.AreEqual("png", data["format"]?.ToString()); + Assert.Greater(data["width"]?.Value() ?? 0, 0); + Assert.Greater(data["height"]?.Value() ?? 0, 0); + + byte[] image = Convert.FromBase64String(data["image_base64"]?.ToString() ?? string.Empty); + CollectionAssert.AreEqual(new byte[] { 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a }, image.Take(8)); + Assert.Greater(image.Length, 5 * 1024); + } + private static JObject Rpc(string method, JObject parameters = null) { var request = new JObject diff --git a/scripts/agent-tooling-smoke.py b/scripts/agent-tooling-smoke.py index 4403479..0707050 100644 --- a/scripts/agent-tooling-smoke.py +++ b/scripts/agent-tooling-smoke.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import base64 import json import os import sys @@ -40,6 +41,39 @@ def summarize_snapshot(snapshot: dict) -> dict: } +def validate_png(result: dict) -> dict: + data = result.get("data", result) + try: + image = base64.b64decode(data.get("image_base64", ""), validate=True) + except (ValueError, TypeError): + image = b"" + return { + "success": result.get("success") is True, + "png": image.startswith(b"\x89PNG\r\n\x1a\n"), + "bytes": len(image), + "width": data.get("width"), + "height": data.get("height"), + "duration_ms": result.get("duration_ms"), + "message": result.get("message"), + } + + +def capture_screenshot_series(method: str, attempts: int = 20) -> dict: + captures = [] + for _ in range(attempts): + try: + captures.append(validate_png(rpc(method))) + except RuntimeError as error: + captures.append({"success": False, "png": False, "bytes": 0, "message": str(error)}) + valid = [capture for capture in captures if capture["success"] and capture["png"] and capture["bytes"] > 5 * 1024] + return { + "attempts": attempts, + "valid": len(valid), + "ok": len(valid) >= attempts - 1, + "captures": captures, + } + + def _run_ui_smoke() -> tuple[dict, dict, dict]: rpc("execute_menu_item", {"item_path": "Window/Nexus Unity"}) time.sleep(0.5) @@ -83,10 +117,18 @@ def main() -> int: rpc("reset_tool_usage_stats") query, rect, snapshot = _run_ui_smoke() + rpc("execute_menu_item", {"item_path": "Window/General/Game"}) + game_screenshots = capture_screenshot_series("capture_game_view_screenshot") + rpc("execute_menu_item", {"item_path": "Window/General/Inspector"}) + inspector_screenshots = capture_screenshot_series("capture_inspector_screenshot") editor_state = bridge_result("editor_controller", {"action": "get_state"}) stats = rpc("get_tool_usage_stats") success = ( - bool(tools) and server.get("state") and snapshot.get("status") in {None, "Success", "success", "PartialSuccess"} + bool(tools) + and server.get("state") + and snapshot.get("status") in {None, "Success", "success", "PartialSuccess"} + and game_screenshots["ok"] + and inspector_screenshots["ok"] ) summary = { @@ -99,6 +141,8 @@ def main() -> int: "window_query_count": len(query) if isinstance(query, list) else 0, "rect": rect.get("rect"), "snapshot": summarize_snapshot(snapshot), + "game_screenshots": game_screenshots, + "inspector_screenshots": inspector_screenshots, "usage_method_count": len(stats.get("tools", [])), }