diff --git a/docs/ui-automation.md b/docs/ui-automation.md index d666de1aa..a6746da24 100644 --- a/docs/ui-automation.md +++ b/docs/ui-automation.md @@ -910,7 +910,40 @@ if ($result.matchCount -ne 1) { throw "Expected 1 Submit button, found $($result $tree = winapp ui inspect "Counter Display" -a $pid --json | ConvertFrom-Json $counter = $tree.windows[0].elements[0] if ($counter.name -ne "Count: 3") { throw "Counter value wrong: $($counter.name)" } -``` + +# Read typed element state while preserving the legacy string property map +$property = winapp ui get-property "Counter Display" -a $pid --json | ConvertFrom-Json +if ($property.element.type -ne "Text") { throw "Unexpected type: $($property.element.type)" } +if ($property.element.isOffscreen) { throw "Counter is offscreen" } +``` + +The JSON envelopes are: + +- `inspect`: `{ "depth", "interactive", "hideDisabled", "hideOffscreen", "windows": [...] }` +- `search`: `{ "matchCount", "hasMore", "matches": [...] }` +- `wait-for`: `{ "found", "waitedMs", "element"?, "timedOut" }` +- `get-property`: `{ "elementId", "element", "properties": { ... } }` + +Typed elements use `type` and numeric `x`, `y`, `width`, and `height`. +Geometry is in physical screen pixels. `0,0,0,0` is UI Automation's +empty/no-displayed-UI rectangle in this projection; `isOffscreen` is separate, +so an offscreen element can still have nonzero bounds. + +Each `inspect --json` `windows[]` entry and the `status --json` result include +`windowDpi`, `scale` (`windowDpi / 96`), `dpiAwareness`, and +`coordinateSpace: "physical-screen-pixels"`. These describe the target +window's DPI context, not unconditional monitor DPI: Windows reports 96 for an +unaware window, system DPI for a system-aware window, and current monitor DPI +for a per-monitor-aware window. If the HWND or DPI context cannot be read, +the command fails rather than silently substituting 96. When `status` resolves +a process before it has a top-level window, `hwnd` is `0` and the DPI fields are +omitted until a window exists. For process-wide `inspect`, the selected target +window remains fail-fast; if a later popup disappears after its tree was read, +its `windows[]` entry carries `dpiError` and omits the DPI fields while the +remaining window trees are still returned. + +See the shipped `winapp-ui-automation` skill's +`references/ui-json-envelope.md` for complete examples of each envelope. ### Full smoke test example ```powershell diff --git a/plugins/winapp/skills/winapp-ui-automation/SKILL.md b/plugins/winapp/skills/winapp-ui-automation/SKILL.md index 6287b53c7..a2d00fc9f 100644 --- a/plugins/winapp/skills/winapp-ui-automation/SKILL.md +++ b/plugins/winapp/skills/winapp-ui-automation/SKILL.md @@ -360,13 +360,17 @@ winapp ui invoke btn-open-e6f7 -w ``` Note: The filename input in standard file dialogs typically has AutomationId `1148`. Use `inspect -w --interactive` to discover the actual slugs. -## JSON output envelopes (v0.3.1+) +## JSON output envelopes -The `--json` envelope for `ui inspect`, `ui get-focused`, `ui search`, and `ui wait-for` was reshaped in v0.3.1. Pre-0.3.1 parsers will silently break — most fields were renamed, removed, or moved into envelopes. Highlights: +The `--json` envelope for `ui inspect`, `ui get-focused`, `ui search`, and `ui wait-for` was reshaped in v0.3.1. The DPI context and typed `get-property` element are available in v0.6.3+. Highlights: - `ui inspect --json` now nests elements under `windows[].elements[]` (was a flat `elements[]`). +- Each inspected window and the `ui status --json` target reports `windowDpi`, `scale`, `dpiAwareness`, and `coordinateSpace: "physical-screen-pixels"`. This is the target window's DPI context. The selected target fails fast on an unreadable DPI instead of defaulting to 96; a secondary window that disappears mid-walk carries `dpiError` and omits the four context fields. - `ui get-focused --json` always emits an envelope — `{ "hasFocus": false }` or `{ "hasFocus": true, "element": {...} }` (was bare `null`). -- `ui search --json` / `ui wait-for --json` may include an `invokableAncestor` field (element-shaped) on each match. +- `ui search --json` returns `{ "matchCount", "hasMore", "matches" }`; `ui wait-for --json` returns `{ "found", "waitedMs", "element"?, "timedOut" }`. +- `ui get-property --json` preserves `elementId` and its string-valued `properties` map, and adds a typed, scrubbed `element`. +- Typed elements use `type` (not `controlType`) and numeric `x`, `y`, `width`, and `height` in physical screen pixels. `0,0,0,0` is UIA's empty/no-displayed-UI rectangle; `isOffscreen` remains independent. +- Search and wait-for elements may include an `invokableAncestor` field (element-shaped). - Per-element `id`, `parentSelector`, and `windowHandle` are **removed** — use `selector` as the public handle. Full schemas with examples: `references/ui-json-envelope.md`. diff --git a/plugins/winapp/skills/winapp-ui-automation/references/ui-json-envelope.md b/plugins/winapp/skills/winapp-ui-automation/references/ui-json-envelope.md index 6a0fa1988..0d91fdc07 100644 --- a/plugins/winapp/skills/winapp-ui-automation/references/ui-json-envelope.md +++ b/plugins/winapp/skills/winapp-ui-automation/references/ui-json-envelope.md @@ -1,8 +1,8 @@ -# `winapp ui --json` envelope (v0.3.1+) +# `winapp ui --json` envelopes -The `--json` output for the `winapp ui` command group was reshaped in v0.3.1. -Generate parsers against these shapes — pre-0.3.1 parsers will silently break -because most fields were renamed, removed, or moved into envelopes. +The `--json` output for the `winapp ui` command group uses the envelopes below. +The inspect, search, wait-for, and get-focused envelopes were reshaped in +v0.3.1; the DPI context and typed get-property element are available in v0.6.3+. ## `ui inspect --json` @@ -16,16 +16,26 @@ Top-level shape (elements are now nested under `windows[]`, not flat): "hideOffscreen": false, "windows": [ { - "hwnd": "0x...", + "hwnd": 123456, "title": "...", "className": "...", + "windowDpi": 144, + "scale": 1.5, + "dpiAwareness": "per-monitor-aware", + "coordinateSpace": "physical-screen-pixels", "elementCount": 0, "elements": [ { - "selector": "...", - "name": "...", - "controlType": "...", - "children": [ ... ] + "selector": "btn-save-c3d4", + "name": "Save", + "type": "Button", + "isEnabled": true, + "isOffscreen": false, + "x": 100, + "y": 200, + "width": 120, + "height": 32, + "isInvokable": true } ] } @@ -37,6 +47,24 @@ Pre-0.3.1 the shape was `{ "elements": [...] }`. Per-element `id`, `parentSelector`, and `windowHandle` fields have been **removed** — `selector` is the public handle. +`windowDpi` is the target window's effective DPI from `GetDpiForWindow(hwnd)`, +not unconditional monitor DPI. `scale` is `windowDpi / 96`. +`dpiAwareness` is `unaware`, `system-aware`, or `per-monitor-aware`: +`GetDpiForWindow` reports 96 for an unaware window, system DPI for a +system-aware window, and the current monitor DPI for a per-monitor-aware +window. If the HWND or DPI context cannot be read, the command fails with an +error instead of substituting 96. + +The selected target window remains fail-fast. If a later popup or secondary +window disappears after its UIA tree was collected, that window entry remains +in `windows[]` with a `dpiError` message and without the four DPI context fields; +the other window trees remain available. + +Element `x`, `y`, `width`, and `height` values are numbers in physical screen +pixels. `0,0,0,0` is UI Automation's empty/no-displayed-UI rectangle in this +projection. `isOffscreen` is independent: an offscreen element can still have +nonzero bounds. + ## `ui inspect --ancestors --json` Ancestors are now nested as a parent → child chain keyed by `Depth=i` @@ -57,34 +85,149 @@ Always emits an envelope (never a bare value): Pre-0.3.1 emitted bare `null` when nothing was focused. -## `ui search --json` / `ui wait-for --json` +## `ui search --json` + +Search returns an envelope, not a bare array: + +```json +{ + "matchCount": 1, + "hasMore": false, + "matches": [ + { + "selector": "txt-save-label-a1b2", + "name": "Save", + "type": "Text", + "isEnabled": true, + "isOffscreen": false, + "x": 100, + "y": 200, + "width": 80, + "height": 24, + "isInvokable": false, + "invokableAncestor": { + "selector": "btn-save-c3d4", + "name": "Save button", + "type": "Button", + "isEnabled": false, + "isOffscreen": false, + "x": 0, + "y": 0, + "width": 0, + "height": 0, + "isInvokable": true + } + } + ] +} +``` -Both commands return matching elements using the same element shape as -`ui inspect` (so `selector`, `name`, `controlType`, `children`, etc.). -Each match may also include an `invokableAncestor` field — itself an +Each match may include an `invokableAncestor` field — itself an element-shaped object — pointing to the nearest parent that supports `InvokePattern` (useful when a search hits a non-invokable element like a label inside a button). +## `ui wait-for --json` + +When the condition succeeds: + +```json +{ + "found": true, + "waitedMs": 125, + "element": { + "selector": "txt-status-a1b2", + "name": "Ready", + "type": "Text", + "isEnabled": true, + "isOffscreen": false, + "x": 100, + "y": 200, + "width": 80, + "height": 24, + "isInvokable": false + }, + "timedOut": false +} +``` + +On timeout, stdout still contains a parseable result and the process exits 1: + +```json +{ + "found": false, + "waitedMs": 5000, + "timedOut": true +} +``` + +With `--gone`, success after the element disappears is: + +```json +{ + "found": false, + "waitedMs": 125, + "timedOut": false +} +``` + +## `ui get-property --json` + +`elementId` and the string-valued `properties` map remain available. The +additive `element` field contains the same scrubbed typed element projection +used by search and wait-for: + ```json -[ - { - "selector": "txt-save-label-a1b2", +{ + "elementId": "btn-save-c3d4", + "element": { + "selector": "btn-save-c3d4", "name": "Save", - "controlType": "Text", - "children": [ ... ], - "invokableAncestor": { - "selector": "btn-save-c3d4", - "name": "Save button", - "controlType": "Button" - } + "type": "Button", + "isEnabled": true, + "isOffscreen": false, + "x": 100, + "y": 200, + "width": 80, + "height": 24, + "isInvokable": true + }, + "properties": { + "Name": "Save", + "IsEnabled": "True", + "BoundingRectangle": "100,200,80,24" } -] +} ``` +The typed `element` object is the canonical way to consume geometry and boolean +state. The existing `properties` values intentionally remain strings for +backward compatibility. + +## `ui status --json` + +The resolved target also reports its window DPI context: + +```json +{ + "processId": 1234, + "processName": "MyApp", + "windowTitle": "My App", + "hwnd": 123456, + "windowDpi": 144, + "scale": 1.5, + "dpiAwareness": "per-monitor-aware", + "coordinateSpace": "physical-screen-pixels" +} +``` + +If a process resolves before it has a top-level window, `hwnd` remains `0` and +the four DPI fields are omitted. A failed DPI read for a nonzero HWND is an +error rather than a silent 96-DPI fallback. + The internal `id`, `parentSelector`, and `windowHandle` fields are -**scrubbed** from results — both at the top level and inside any nested -`invokableAncestor`. Don't depend on them; use `selector` as the handle. +**scrubbed** from typed element results — both at the top level and inside any +nested `invokableAncestor`. Don't depend on them; use `selector` as the handle. ## Error envelope diff --git a/src/winapp-CLI/WinApp.Cli.Tests/FakeWindowDpiContextProvider.cs b/src/winapp-CLI/WinApp.Cli.Tests/FakeWindowDpiContextProvider.cs new file mode 100644 index 000000000..bdcf6e6aa --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli.Tests/FakeWindowDpiContextProvider.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using WinApp.Cli.Helpers; + +namespace WinApp.Cli.Tests; + +internal sealed class FakeWindowDpiContextProvider : IWindowDpiContextProvider +{ + public WindowDpiContext Result { get; set; } = + new(144, 1.5, "per-monitor-aware", WindowDpiContextProvider.PhysicalScreenPixels); + + public Dictionary ResultsByHwnd { get; } = []; + public Dictionary ThrowsByHwnd { get; } = []; + + public Exception? Throw { get; set; } + + public List RequestedHwnds { get; } = []; + + public WindowDpiContext GetForWindow(long hwnd) + { + RequestedHwnds.Add(hwnd); + if (ThrowsByHwnd.TryGetValue(hwnd, out var hwndException)) + { + throw hwndException; + } + if (Throw is not null) + { + throw Throw; + } + + return ResultsByHwnd.TryGetValue(hwnd, out var result) ? result : Result; + } +} diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Inspect.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Inspect.cs index 89ead85f0..3247266b7 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Inspect.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Inspect.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using WinApp.Cli.Commands; +using WinApp.Cli.Helpers; using WinApp.Cli.Models; namespace WinApp.Cli.Tests; @@ -157,4 +158,52 @@ public async Task Inspect_Generic_ReturnsError() var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp"]); Assert.AreEqual(1, exitCode); } + + [TestMethod] + public async Task Inspect_DpiReadFailure_ReturnsExplicitJsonError() + { + _fakeUia.InspectResult = + [ + new UiElement { Type = "Window", Depth = 0, WindowHandle = 321 }, + ]; + _fakeWindowDpiContextProvider.Throw = + new InvalidOperationException("GetDpiForWindow failed for HWND 321."); + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); + + Assert.AreEqual(1, exitCode); + StringAssert.Contains(ConsoleStdErr.ToString(), "GetDpiForWindow failed for HWND 321"); + } + + [TestMethod] + public async Task Inspect_SecondaryDpiReadFailure_PreservesTreeAndSurfacesWindowError() + { + _fakeWindowDpiContextProvider.ResultsByHwnd[100] = + new(192, 2, "per-monitor-aware", WindowDpiContextProvider.PhysicalScreenPixels); + _fakeWindowDpiContextProvider.ThrowsByHwnd[200] = + new InvalidOperationException("GetDpiForWindow failed for HWND 200."); + _fakeUia.InspectResult = + [ + new UiElement { Type = "---", Name = "HWND 100: \"Main\" (window, MainClass)", WindowHandle = 100 }, + new UiElement { Type = "Button", Depth = 0, Selector = "btn-main" }, + new UiElement { Type = "---", Name = "HWND 200: \"Closing\" (popup, PopupClass)", WindowHandle = 200 }, + new UiElement { Type = "Text", Depth = 0, Selector = "txt-closing" }, + ]; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); + + Assert.AreEqual(0, exitCode); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + var windows = document.RootElement.GetProperty("windows"); + Assert.AreEqual(2, windows.GetArrayLength()); + Assert.AreEqual((uint)192, windows[0].GetProperty("windowDpi").GetUInt32()); + Assert.AreEqual("txt-closing", windows[1].GetProperty("elements")[0].GetProperty("selector").GetString()); + Assert.AreEqual( + "GetDpiForWindow failed for HWND 200.", + windows[1].GetProperty("dpiError").GetString()); + Assert.IsFalse(windows[1].TryGetProperty("windowDpi", out _)); + Assert.IsFalse(windows[1].TryGetProperty("scale", out _)); + } } diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.SimpleVerbs.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.SimpleVerbs.cs index 13152decb..9a1f98470 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.SimpleVerbs.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.SimpleVerbs.cs @@ -83,6 +83,20 @@ public async Task Status_Generic_ReturnsError() Assert.AreEqual(1, exitCode); } + [TestMethod] + public async Task Status_DpiReadFailure_ReturnsExplicitJsonError() + { + _fakeTargetResolver.TargetResult.WindowHandle = 123; + _fakeWindowDpiContextProvider.Throw = + new InvalidOperationException("GetDpiForWindow failed for HWND 123."); + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); + + Assert.AreEqual(1, exitCode); + StringAssert.Contains(ConsoleStdErr.ToString(), "GetDpiForWindow failed for HWND 123"); + } + // ---------- focus ---------- [TestMethod] diff --git a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs index bce562652..2953b7420 100644 --- a/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs +++ b/src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.cs @@ -27,6 +27,7 @@ public partial class UiCommandTests : BaseCommandTests private FakeInteractiveDesktopLock _fakeDesktopLock = null!; private FakeDesktopForegroundService _fakeDesktopForeground = null!; private FakeWindowCapture _fakeWindowCapture = null!; + private FakeWindowDpiContextProvider _fakeWindowDpiContextProvider = null!; private void AssertJsonErrorCode(string expectedCode) => AssertJsonErrorCodeIn(ConsoleStdErr.ToString(), expectedCode); @@ -55,6 +56,7 @@ protected override IServiceCollection ConfigureServices(IServiceCollection servi _fakeDesktopLock = new FakeInteractiveDesktopLock(); _fakeDesktopForeground = new FakeDesktopForegroundService(); _fakeWindowCapture = new FakeWindowCapture(); + _fakeWindowDpiContextProvider = new FakeWindowDpiContextProvider(); return services .AddSingleton(_fakeUia) .AddSingleton(_fakeRecording) @@ -68,16 +70,26 @@ protected override IServiceCollection ConfigureServices(IServiceCollection servi .AddSingleton(_fakePollDelay) .AddSingleton(_fakeDesktopLock) .AddSingleton(_fakeDesktopForeground) - .AddSingleton(_fakeWindowCapture); + .AddSingleton(_fakeWindowCapture) + .AddSingleton(_fakeWindowDpiContextProvider); } [TestMethod] public async Task Status_WithApp_ReturnsSuccess() { + _fakeTargetResolver.TargetResult.WindowHandle = 123; var command = GetRequiredService(); var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); Assert.AreEqual(0, exitCode); - StringAssert.Contains(TestAnsiConsole.Output, "\"processId\": 1234"); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + var root = document.RootElement; + Assert.AreEqual(1234, root.GetProperty("processId").GetInt32()); + Assert.AreEqual(123, root.GetProperty("hwnd").GetInt64()); + Assert.AreEqual((uint)144, root.GetProperty("windowDpi").GetUInt32()); + Assert.AreEqual(1.5, root.GetProperty("scale").GetDouble()); + Assert.AreEqual("per-monitor-aware", root.GetProperty("dpiAwareness").GetString()); + Assert.AreEqual("physical-screen-pixels", root.GetProperty("coordinateSpace").GetString()); + CollectionAssert.AreEqual(new long[] { 123 }, _fakeWindowDpiContextProvider.RequestedHwnds); } [TestMethod] @@ -88,6 +100,25 @@ public async Task Status_WithoutApp_ReturnsError() Assert.AreEqual(1, exitCode); } + [TestMethod] + public async Task Status_Json_ProcessWithoutWindow_PreservesExistingSuccessShape() + { + _fakeTargetResolver.TargetResult.WindowHandle = 0; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); + + Assert.AreEqual(0, exitCode); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + var root = document.RootElement; + Assert.AreEqual(0, root.GetProperty("hwnd").GetInt64()); + Assert.IsFalse(root.TryGetProperty("windowDpi", out _)); + Assert.IsFalse(root.TryGetProperty("scale", out _)); + Assert.IsFalse(root.TryGetProperty("dpiAwareness", out _)); + Assert.IsFalse(root.TryGetProperty("coordinateSpace", out _)); + Assert.AreEqual(0, _fakeWindowDpiContextProvider.RequestedHwnds.Count); + } + [TestMethod] public async Task Inspect_ReturnsElements() { @@ -110,6 +141,41 @@ public async Task Inspect_ReturnsElements() StringAssert.Contains(TestAnsiConsole.Output, "\"type\": \"Button\""); } + [TestMethod] + public async Task Inspect_Json_AddsDpiContextToEveryWindow() + { + _fakeWindowDpiContextProvider.ResultsByHwnd[100] = + new(192, 2, "per-monitor-aware", WindowDpiContextProvider.PhysicalScreenPixels); + _fakeWindowDpiContextProvider.ResultsByHwnd[200] = + new(120, 1.25, "per-monitor-aware", WindowDpiContextProvider.PhysicalScreenPixels); + _fakeUia.InspectResult = + [ + new UiElement { Type = "---", Name = "HWND 100: \"Main\" (window, MainClass)", WindowHandle = 100 }, + new UiElement { Type = "Button", Depth = 0, Selector = "btn-main" }, + new UiElement { Type = "---", Name = "HWND 200: \"Popup\" (popup, PopupClass)", WindowHandle = 200 }, + new UiElement { Type = "MenuItem", Depth = 0, Selector = "menu-popup" }, + ]; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["-a", "TestApp", "--json"]); + + Assert.AreEqual(0, exitCode); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + var windows = document.RootElement.GetProperty("windows"); + Assert.AreEqual(2, windows.GetArrayLength()); + var main = windows[0]; + Assert.AreEqual((uint)192, main.GetProperty("windowDpi").GetUInt32()); + Assert.AreEqual(2, main.GetProperty("scale").GetDouble()); + Assert.AreEqual("per-monitor-aware", main.GetProperty("dpiAwareness").GetString()); + Assert.AreEqual("physical-screen-pixels", main.GetProperty("coordinateSpace").GetString()); + var popup = windows[1]; + Assert.AreEqual((uint)120, popup.GetProperty("windowDpi").GetUInt32()); + Assert.AreEqual(1.25, popup.GetProperty("scale").GetDouble()); + Assert.AreEqual("per-monitor-aware", popup.GetProperty("dpiAwareness").GetString()); + Assert.AreEqual("physical-screen-pixels", popup.GetProperty("coordinateSpace").GetString()); + CollectionAssert.AreEqual(new long[] { 100, 200 }, _fakeWindowDpiContextProvider.RequestedHwnds); + } + [TestMethod] public async Task Inspect_Json_OmitsRedundantFields() { @@ -248,13 +314,44 @@ public async Task Invoke_ByElementId_ReturnsSuccess() [TestMethod] public async Task GetProperty_ReturnsProperties() { - _fakeUia.FindSingleResult = new UiElement { Id = "e0", Type = "Button", Name = "OK", IsEnabled = true }; - _fakeUia.PropertiesResult = new Dictionary { ["IsEnabled"] = true, ["Name"] = "OK" }; + _fakeUia.FindSingleResult = new UiElement + { + Id = "e0", + Selector = "btn-ok-a1b2", + Type = "Button", + Name = "OK", + IsEnabled = true, + IsOffscreen = true, + X = 0, + Y = 0, + Width = 0, + Height = 0, + WindowHandle = 123, + }; + _fakeUia.PropertiesResult = new Dictionary + { + ["IsEnabled"] = true, + ["Name"] = "OK", + }; var command = GetRequiredService(); var exitCode = await ParseAndInvokeWithCaptureAsync(command, ["e0", "-a", "TestApp", "--json"]); Assert.AreEqual(0, exitCode); - StringAssert.Contains(TestAnsiConsole.Output, "\"elementId\": \"e0\""); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + var root = document.RootElement; + Assert.AreEqual("btn-ok-a1b2", root.GetProperty("elementId").GetString()); + Assert.AreEqual("True", root.GetProperty("properties").GetProperty("IsEnabled").GetString()); + + var element = root.GetProperty("element"); + Assert.AreEqual("Button", element.GetProperty("type").GetString()); + Assert.AreEqual("OK", element.GetProperty("name").GetString()); + Assert.AreEqual(0, element.GetProperty("x").GetDouble()); + Assert.AreEqual(0, element.GetProperty("y").GetDouble()); + Assert.AreEqual(0, element.GetProperty("width").GetDouble()); + Assert.AreEqual(0, element.GetProperty("height").GetDouble()); + Assert.IsTrue(element.GetProperty("isOffscreen").GetBoolean()); + Assert.IsFalse(element.TryGetProperty("id", out _)); + Assert.IsFalse(element.TryGetProperty("windowHandle", out _)); } [TestMethod] @@ -595,6 +692,27 @@ public async Task Inspect_Ancestors_Json_NestsChain() StringAssert.Contains(output, "\"elementCount\": 4"); } + [TestMethod] + public async Task Inspect_Ancestors_Json_UsesResolvedElementWindowForDpi() + { + _fakeTargetResolver.TargetResult.WindowHandle = 0; + _fakeUia.InspectResult = + [ + new UiElement { Type = "Window", WindowHandle = 321 }, + new UiElement { Type = "Button", WindowHandle = 321 }, + ]; + + var command = GetRequiredService(); + var exitCode = await ParseAndInvokeWithCaptureAsync( + command, + ["btn-target", "-a", "TestApp", "--ancestors", "--json"]); + + Assert.AreEqual(0, exitCode); + CollectionAssert.AreEqual(new long[] { 321 }, _fakeWindowDpiContextProvider.RequestedHwnds); + using var document = System.Text.Json.JsonDocument.Parse(TestAnsiConsole.Output); + Assert.AreEqual(321, document.RootElement.GetProperty("windows")[0].GetProperty("hwnd").GetInt64()); + } + // --------------------------------------------------------------------- // NativeAOT smoke tests (M10): exercise each result-type registration in // UiJsonContext at least once via --json so a missing/incorrect registration diff --git a/src/winapp-CLI/WinApp.Cli.Tests/WindowDpiContextProviderTests.cs b/src/winapp-CLI/WinApp.Cli.Tests/WindowDpiContextProviderTests.cs new file mode 100644 index 000000000..a5ae62333 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli.Tests/WindowDpiContextProviderTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using WinApp.Cli.Helpers; + +namespace WinApp.Cli.Tests; + +[TestClass] +public class WindowDpiContextProviderTests +{ + [TestMethod] + [DataRow(0, "unaware")] + [DataRow(1, "system-aware")] + [DataRow(2, "per-monitor-aware")] + public void GetForWindow_MapsAwarenessAndCalculatesScale(int awareness, string expected) + { + var provider = new WindowDpiContextProvider(_ => 144, _ => awareness); + + var result = provider.GetForWindow(123); + + Assert.AreEqual((uint)144, result.WindowDpi); + Assert.AreEqual(1.5, result.Scale); + Assert.AreEqual(expected, result.DpiAwareness); + Assert.AreEqual("physical-screen-pixels", result.CoordinateSpace); + } + [TestMethod] + public void GetForWindow_ZeroHwnd_ThrowsExplicitError() + { + var provider = new WindowDpiContextProvider(_ => 96, _ => 0); + + var exception = Assert.ThrowsExactly( + () => provider.GetForWindow(0)); + + StringAssert.Contains(exception.Message, "HWND is zero"); + } + + [TestMethod] + public void GetForWindow_ZeroDpi_ThrowsExplicitError() + { + var provider = new WindowDpiContextProvider(_ => 0, _ => 0); + + var exception = Assert.ThrowsExactly( + () => provider.GetForWindow(123)); + + StringAssert.Contains(exception.Message, "GetDpiForWindow failed"); + StringAssert.Contains(exception.Message, "123"); + } + + [TestMethod] + public void GetForWindow_InvalidAwareness_ThrowsExplicitError() + { + var provider = new WindowDpiContextProvider(_ => 96, _ => -1); + + var exception = Assert.ThrowsExactly( + () => provider.GetForWindow(123)); + + StringAssert.Contains(exception.Message, "invalid value (-1)"); + StringAssert.Contains(exception.Message, "123"); + } +} diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiGetPropertyCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiGetPropertyCommand.cs index aa16c3a30..58998e9b5 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/UiGetPropertyCommand.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiGetPropertyCommand.cs @@ -89,13 +89,19 @@ protected override async Task ExecuteAsync(ParseResult parseResult, IUiTurn if (json) { - // Convert to string values for JSON serialization (source-gen can't handle object?) var stringProps = new Dictionary(); foreach (var kvp in props) { stringProps[kvp.Key] = kvp.Value?.ToString(); } - var result = new UiPropertyResult { ElementId = (element.Selector ?? element.Id ?? ""), Properties = stringProps }; + var elementId = element.Selector ?? element.Id ?? ""; + UiElementScrubber.Scrub(element); + var result = new UiPropertyResult + { + ElementId = elementId, + Element = element, + Properties = stringProps, + }; ansiConsole.Profile.Out.Writer.WriteLine( JsonSerializer.Serialize(result, UiJsonContext.Default.UiPropertyResult)); } diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiInspectCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiInspectCommand.cs index 3b18221cc..4e492c382 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/UiInspectCommand.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiInspectCommand.cs @@ -46,6 +46,7 @@ public UiInspectCommand() public partial class Handler( IUiTargetResolver targetResolver, IUiAutomation uiAutomation, + IWindowDpiContextProvider windowDpiContextProvider, IAnsiConsole ansiConsole, IInteractiveDesktopLock desktopLock, ILogger logger) : UiCoordinatedAction(desktopLock, logger) @@ -143,6 +144,25 @@ protected override async Task ExecuteAsync(ParseResult parseResult, IUiTurn // and surface them as ancestorPath breadcrumbs on the surviving descendants. var jsonElements = interactive ? allElements : elements; var windows = BuildWindows(jsonElements, uiTarget, interactive); + for (var i = 0; i < windows.Length; i++) + { + var windowInfo = windows[i]; + try + { + var dpiContext = windowDpiContextProvider.GetForWindow(windowInfo.Hwnd); + windowInfo.WindowDpi = dpiContext.WindowDpi; + windowInfo.Scale = dpiContext.Scale; + windowInfo.DpiAwareness = dpiContext.DpiAwareness; + windowInfo.CoordinateSpace = dpiContext.CoordinateSpace; + } + catch (InvalidOperationException ex) when (i > 0) + { + // The first group is the selected target and remains fail-fast. A + // secondary transient window can disappear after its UIA walk; preserve + // the other groups and surface that window's missing context explicitly. + windowInfo.DpiError = ex.Message; + } + } var result = new UiInspectResult { diff --git a/src/winapp-CLI/WinApp.Cli/Commands/UiStatusCommand.cs b/src/winapp-CLI/WinApp.Cli/Commands/UiStatusCommand.cs index ad6a5c365..c43692634 100644 --- a/src/winapp-CLI/WinApp.Cli/Commands/UiStatusCommand.cs +++ b/src/winapp-CLI/WinApp.Cli/Commands/UiStatusCommand.cs @@ -29,6 +29,7 @@ public UiStatusCommand() public class Handler( IUiTargetResolver targetResolver, + IWindowDpiContextProvider windowDpiContextProvider, IAnsiConsole ansiConsole, IInteractiveDesktopLock desktopLock, ILogger logger) : UiCoordinatedAction(desktopLock, logger) @@ -65,12 +66,19 @@ protected override async Task ExecuteAsync(ParseResult parseResult, IUiTurn if (json) { + var dpiContext = uiTarget.WindowHandle != 0 + ? windowDpiContextProvider.GetForWindow(uiTarget.WindowHandle) + : null; var result = new UiStatusResult { ProcessId = uiTarget.ProcessId, ProcessName = uiTarget.ProcessName, WindowTitle = uiTarget.WindowTitle, Hwnd = uiTarget.WindowHandle, + WindowDpi = dpiContext?.WindowDpi, + Scale = dpiContext?.Scale, + DpiAwareness = dpiContext?.DpiAwareness, + CoordinateSpace = dpiContext?.CoordinateSpace, }; ansiConsole.Profile.Out.Writer.WriteLine( JsonSerializer.Serialize(result, UiJsonContext.Default.UiStatusResult)); diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs index 6cc541d3e..965829f2a 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/HostBuilderExtensions.cs @@ -86,6 +86,7 @@ public static IServiceCollection ConfigureServices(this IServiceCollection servi .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() // Execution targets (Windows Sandbox and any future target) .AddSingleton(_ => new TargetStateDirectoryProvider()) diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/IWindowDpiContextProvider.cs b/src/winapp-CLI/WinApp.Cli/Helpers/IWindowDpiContextProvider.cs new file mode 100644 index 000000000..fe5c277bb --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/IWindowDpiContextProvider.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +namespace WinApp.Cli.Helpers; + +internal interface IWindowDpiContextProvider +{ + WindowDpiContext GetForWindow(long hwnd); +} + +internal sealed record WindowDpiContext( + uint WindowDpi, + double Scale, + string DpiAwareness, + string CoordinateSpace); diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs index e74e441bf..629bd2537 100644 --- a/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs +++ b/src/winapp-CLI/WinApp.Cli/Helpers/UiJsonContext.cs @@ -77,6 +77,10 @@ internal sealed class UiStatusResult public string ProcessName { get; set; } = ""; public string? WindowTitle { get; set; } public long Hwnd { get; set; } + public uint? WindowDpi { get; set; } + public double? Scale { get; set; } + public string? DpiAwareness { get; set; } + public string? CoordinateSpace { get; set; } } internal sealed class UiInspectResult @@ -102,6 +106,11 @@ internal sealed class UiInspectWindowInfo public long Hwnd { get; set; } public string? Title { get; set; } public string? ClassName { get; set; } + public uint? WindowDpi { get; set; } + public double? Scale { get; set; } + public string? DpiAwareness { get; set; } + public string? CoordinateSpace { get; set; } + public string? DpiError { get; set; } /// Total real elements (counting nested children) belonging to this window. public int ElementCount { get; set; } /// Root elements for this window. Children are nested via UiElement.Children. @@ -118,6 +127,7 @@ internal sealed class UiSearchResult internal sealed class UiPropertyResult { public string ElementId { get; set; } = ""; + public UiElement Element { get; set; } = new(); public Dictionary Properties { get; set; } = []; } diff --git a/src/winapp-CLI/WinApp.Cli/Helpers/WindowDpiContextProvider.cs b/src/winapp-CLI/WinApp.Cli/Helpers/WindowDpiContextProvider.cs new file mode 100644 index 000000000..933b813e4 --- /dev/null +++ b/src/winapp-CLI/WinApp.Cli/Helpers/WindowDpiContextProvider.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation and Contributors. All rights reserved. +// Licensed under the MIT License. + +using Windows.Win32.Foundation; + +namespace WinApp.Cli.Helpers; + +internal sealed class WindowDpiContextProvider : IWindowDpiContextProvider +{ + internal const string PhysicalScreenPixels = "physical-screen-pixels"; + + private readonly Func _getDpiForWindow; + private readonly Func _getAwarenessForWindow; + + public WindowDpiContextProvider() + : this(GetDpiForWindow, GetAwarenessForWindow) + { + } + internal WindowDpiContextProvider( + Func getDpiForWindow, + Func getAwarenessForWindow) + { + _getDpiForWindow = getDpiForWindow; + _getAwarenessForWindow = getAwarenessForWindow; + } + + public WindowDpiContext GetForWindow(long hwnd) + { + if (hwnd == 0) + { + throw new InvalidOperationException("Cannot read the target window DPI context because its HWND is zero."); + } + + var windowDpi = _getDpiForWindow(hwnd); + if (windowDpi == 0) + { + throw new InvalidOperationException($"GetDpiForWindow failed for HWND {hwnd}; the handle may no longer be valid."); + } + + var dpiAwareness = _getAwarenessForWindow(hwnd) switch + { + 0 => "unaware", + 1 => "system-aware", + 2 => "per-monitor-aware", + var value => throw new InvalidOperationException( + $"GetAwarenessFromDpiAwarenessContext returned an invalid value ({value}) for HWND {hwnd}."), + }; + + return new WindowDpiContext( + windowDpi, + windowDpi / 96d, + dpiAwareness, + PhysicalScreenPixels); + } + + private static uint GetDpiForWindow(long hwnd) + => Windows.Win32.PInvoke.GetDpiForWindow(new HWND((nint)hwnd)); + + private static int GetAwarenessForWindow(long hwnd) + { + var context = Windows.Win32.PInvoke.GetWindowDpiAwarenessContext(new HWND((nint)hwnd)); + return (int)Windows.Win32.PInvoke.GetAwarenessFromDpiAwarenessContext(context); + } +} diff --git a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt index 3a369bee4..b3467b350 100644 --- a/src/winapp-CLI/WinApp.Cli/NativeMethods.txt +++ b/src/winapp-CLI/WinApp.Cli/NativeMethods.txt @@ -45,6 +45,9 @@ DBG_EXCEPTION_NOT_HANDLED GetShortPathName GetFullPathName GetForegroundWindow +GetDpiForWindow +GetWindowDpiAwarenessContext +GetAwarenessFromDpiAwarenessContext SetForegroundWindow GetCurrentThreadId SetWindowPos diff --git a/src/winapp-CLI/WinApp.UIAutomation.TestSupport/UiaTestFixture.cs b/src/winapp-CLI/WinApp.UIAutomation.TestSupport/UiaTestFixture.cs index f97853baf..97a3d497c 100644 --- a/src/winapp-CLI/WinApp.UIAutomation.TestSupport/UiaTestFixture.cs +++ b/src/winapp-CLI/WinApp.UIAutomation.TestSupport/UiaTestFixture.cs @@ -621,6 +621,14 @@ private void BuildExtraControls(Form form) Width = 160, Height = 30, }); + var ownedMenu = new MenuStrip { Name = "ownedMenu" }; + ownedMenu.Items.Add(new ToolStripMenuItem + { + Name = "mnuOwnedWindowless", + Text = "Windowless Owned Item", + AccessibleName = "Windowless Owned Item", + }); + _ownedWindow.Controls.Add(ownedMenu); _ownedWindow.Show(); } diff --git a/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.Coverage.cs b/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.Coverage.cs index 353950aba..8a144e5e0 100644 --- a/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.Coverage.cs +++ b/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.Coverage.cs @@ -256,19 +256,179 @@ public async Task ScrollErrors_NonScrollableLabelReportsNoSupportedPattern() } [TestMethod] - public async Task InspectAsync_NonExplicitSessionSkipsElementsAlreadyInMainTree() + public async Task InspectAsync_NonExplicitSessionSeparatesEnumeratedWindowAlreadyInMainTree() { using var fx = new UiaTestFixture(); - var logger = new CapturingLogger(); - var svc = new UiAutomationService(logger, new UiSelectorParser()); + var svc = NewService(); + var uiTarget = NonExplicitSession(fx); + var (ownedHwnd, ownedTitle) = fx.OpenOwnedWindow( + "SeparatedOwned_" + Guid.NewGuid().ToString("N")[..6], + ownedByMain: true); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (ownedHwnd, fx.ProcessId, ownedTitle)]; + + var elements = await svc.InspectAsync(uiTarget, null, 3, CancellationToken.None); + + Assert.AreEqual(1, elements.Count(e => e.AutomationId == "btnOwned"), + "the separately enumerated HWND must not remain duplicated in the main tree"); + Assert.AreEqual("btnOwned", elements.Single(e => e.AutomationId == "btnOwned").Selector, + "a separated window must preserve its stable AutomationId selector"); + Assert.IsTrue(elements.Any(e => e.Type == "---" && e.WindowHandle == ownedHwnd), + "the nested HWND must receive its own window group and context"); + } + + [TestMethod] + public async Task PromotedOwnedWindowSelectorResolvesWithOwnedWindowHandle() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var (ownedHwnd, ownedTitle) = fx.OpenOwnedWindow( + "SelectorRoundTrip_" + Guid.NewGuid().ToString("N")[..6], + ownedByMain: true); + var uiTarget = NonExplicitSession(fx); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (ownedHwnd, fx.ProcessId, ownedTitle)]; + + var elements = await svc.InspectAsync(uiTarget, null, 3, CancellationToken.None); + var inspected = elements.Single(element => element.AutomationId == "btnOwned"); + Assert.AreEqual("btnOwned", inspected.Selector); + + var resolved = await svc.FindSingleElementAsync( + uiTarget, + new UiSelector { Query = inspected.Selector }, + CancellationToken.None); + + Assert.IsNotNull(resolved); + Assert.AreEqual(ownedHwnd, resolved.WindowHandle, + "a selector emitted for an owned window must resolve back to that HWND"); + } + + [TestMethod] + public async Task InspectAsync_DuplicateAutomationIdsAcrossWindowsRemainSlugs() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + fx.OnUiThread(() => fx.InvokeButton.Name = "btnOwned"); + var (ownedHwnd, ownedTitle) = fx.OpenOwnedWindow( + "DuplicateAid_" + Guid.NewGuid().ToString("N")[..6], + ownedByMain: true); + var uiTarget = NonExplicitSession(fx); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (ownedHwnd, fx.ProcessId, ownedTitle)]; + + var elements = await svc.InspectAsync(uiTarget, null, 3, CancellationToken.None); + + var duplicates = elements.Where(element => element.AutomationId == "btnOwned").ToArray(); + Assert.AreEqual(2, duplicates.Length); + Assert.IsTrue(duplicates.All(element => element.Selector != "btnOwned"), + "an AutomationId shared by distinct windows must not become an unscoped selector"); + } + + [TestMethod] + public async Task InspectAsync_IndependentWindowAutomationIdRemainsSlug() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var (windowHwnd, windowTitle) = fx.OpenOwnedWindow( + "IndependentAid_" + Guid.NewGuid().ToString("N")[..6]); + var uiTarget = NonExplicitSession(fx); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (windowHwnd, fx.ProcessId, windowTitle)]; + + var elements = await svc.InspectAsync(uiTarget, null, 3, CancellationToken.None); + + var independentButton = elements.Single(element => element.AutomationId == "btnOwned"); + Assert.AreNotEqual("btnOwned", independentButton.Selector, + "an independent-window AutomationId must keep a slug for precise unscoped resolution"); + StringAssert.StartsWith(independentButton.Selector, "btn-btnowned-"); + } + + [TestMethod] + public async Task InspectAsync_PidOnlySessionDoesNotDuplicateRecoveredWindowInsideOwnerTree() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var (selectedHwnd, selectedTitle) = fx.OpenOwnedWindow( + "SelectedOwned_" + Guid.NewGuid().ToString("N")[..6], + ownedByMain: true); + var uiTarget = NonExplicitSession(fx); + uiTarget.WindowHandle = 0; + uiTarget.WindowTitle = selectedTitle; + UiAutomationService.s_getRootElement = (service, _) => + UiAutomationService.s_elementFromHandle(service, selectedHwnd); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(selectedHwnd, fx.ProcessId, selectedTitle), (fx.Hwnd, fx.ProcessId, fx.Title)]; + + var elements = await svc.InspectAsync(uiTarget, null, 3, CancellationToken.None); + + Assert.AreEqual(1, elements.Count(e => e.AutomationId == "btnOwned"), + "the selected HWND must not be repeated inside its separately emitted owner tree"); + Assert.AreEqual(2, elements.Count(e => e.Type == "---"), + "both selected and owner HWNDs must retain distinct window groups"); + } + + [TestMethod] + public async Task InspectAsync_UnresolvableSeparateWindowRemainsInMainTree() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var uiTarget = NonExplicitSession(fx); + var childHwnd = fx.OnUiThread(() => (nint)fx.InvokeButton.Handle); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (childHwnd, fx.ProcessId, "child")]; + UiAutomationService.s_getRootElementForHwnd = (_, _) => null; + + var elements = await svc.InspectAsync(uiTarget, null, 1, CancellationToken.None); + + Assert.AreEqual(1, elements.Count(e => e.AutomationId == "btnInvoke"), + "a window that cannot be re-rooted must remain available through the main UIA tree"); + Assert.IsFalse(elements.Any(e => e.Type == "---"), + "an unresolvable window must not create an empty separate group"); + } + + [TestMethod] + public async Task InspectAsync_LateRootRefreshFailureUsesInitiallyResolvedWindow() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); var uiTarget = NonExplicitSession(fx); var childHwnd = fx.OnUiThread(() => (nint)fx.InvokeButton.Handle); - UiAutomationService.s_getAllAppWindows = (_, _) => [(fx.Hwnd, fx.ProcessId, fx.Title), (childHwnd, fx.ProcessId, "child")]; + var calls = 0; + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (childHwnd, fx.ProcessId, "child")]; + UiAutomationService.s_getRootElementForHwnd = (service, hwnd) => + hwnd == childHwnd && ++calls == 1 + ? UiAutomationService.s_elementFromHandle(service, hwnd) + : null; var elements = await svc.InspectAsync(uiTarget, null, 1, CancellationToken.None); - Assert.IsTrue(elements.Any(e => e.AutomationId == "btnInvoke")); - Assert.IsTrue(logger.Has(Microsoft.Extensions.Logging.LogLevel.Debug, "already in main window tree")); + Assert.AreEqual(1, elements.Count(e => e.AutomationId == "btnInvoke"), + "a transient refresh failure must not lose the already-pruned window subtree"); + Assert.IsTrue(elements.Any(e => e.Type == "---" && e.WindowHandle == childHwnd)); + } + + [TestMethod] + public async Task InspectAsync_StaleInitialPopupRootDoesNotAbortMainTree() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var uiTarget = NonExplicitSession(fx); + var staleHwnd = fx.Hwnd + 1000; + var staleRoot = ComProxy((_, _) => + throw new COMException("window closed", unchecked((int)0x80040201))); + var calls = 0; + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title), (staleHwnd, fx.ProcessId, "closed")]; + UiAutomationService.s_getRootElementForHwnd = (_, hwnd) => + hwnd == staleHwnd && ++calls == 1 ? staleRoot : null; + + var elements = await svc.InspectAsync(uiTarget, null, 1, CancellationToken.None); + + Assert.IsTrue(elements.Any(e => e.AutomationId == "txtValue"), + "a stale secondary window must not discard the selected window tree"); + Assert.IsFalse(elements.Any(e => e.Type == "---" && e.WindowHandle == staleHwnd), + "a stale secondary window must not leave an empty group"); } [TestMethod] @@ -283,7 +443,30 @@ public async Task PidOnlySessionWithMultipleWindowsFallsBackToLargestBounds() var elements = await svc.InspectAsync(uiTarget, null, 0, CancellationToken.None); Assert.IsTrue(elements.Length > 0); - Assert.IsTrue(elements.Any(e => e.Name == fx.Title), "PID-only largest fallback should inspect the fixture window tree"); + Assert.AreEqual(1, elements.Count(e => e.Name == fx.Title), + "PID-only largest fallback should inspect the fixture window tree once"); + Assert.IsFalse(elements.Any(e => e.Type == "---" && e.WindowHandle == 0), + "the resolved root HWND must replace the PID-only target's initial zero handle"); + } + + [TestMethod] + public async Task InspectAsync_StaleStoredHwndUsesRecoveredRootHwnd() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var staleHwnd = fx.Hwnd + 1000; + var uiTarget = NonExplicitSession(fx); + uiTarget.WindowHandle = staleHwnd; + UiAutomationService.s_getRootElement = (service, _) => + UiAutomationService.s_elementFromHandle(service, fx.Hwnd); + UiAutomationService.s_getAllAppWindows = (_, _) => + [(fx.Hwnd, fx.ProcessId, fx.Title)]; + + var elements = await svc.InspectAsync(uiTarget, null, 1, CancellationToken.None); + + Assert.IsTrue(elements.Length > 0); + Assert.IsTrue(elements.All(element => element.WindowHandle == fx.Hwnd), + "a recovered live root must replace a stale nonzero session HWND"); } [TestMethod] diff --git a/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.cs b/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.cs index 47105fc0e..4cab63411 100644 --- a/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.cs +++ b/src/winapp-CLI/WinApp.UIAutomation.Tests/RealUiAutomationTests.cs @@ -191,6 +191,23 @@ public async Task InspectAsync_ScopedToElement_ReturnsSubtree() Assert.IsFalse(tree.Any(e => e.AutomationId == "btnInvoke"), "controls outside the panel must be excluded"); } + [TestMethod] + public async Task InspectAsync_ScopedOwnedElementUsesOwnedWindowHandle() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var (ownedHwnd, _) = fx.OpenOwnedWindow( + "ScopedOwned_" + Guid.NewGuid().ToString("N")[..6], + ownedByMain: true); + var uiTarget = NonExplicitSession(fx); + + var tree = await svc.InspectAsync(uiTarget, "Windowless Owned Item", 0, CancellationToken.None); + + var ownedItem = tree.Single(element => element.Name == "Windowless Owned Item"); + Assert.AreEqual(ownedHwnd, ownedItem.WindowHandle, + "a windowless scoped element must inherit its owned top-level HWND for DPI context"); + } + [TestMethod] public async Task InspectAncestorsAsync_ReturnsRootToTargetChain() { @@ -206,6 +223,21 @@ public async Task InspectAncestorsAsync_ReturnsRootToTargetChain() Assert.IsTrue(chain.Any(e => e.Type == "Window"), "the window ancestor should be present"); } + [TestMethod] + public async Task InspectAncestorsAsync_PidOnlyTargetSetsResolvedWindowHandle() + { + using var fx = new UiaTestFixture(); + var svc = NewService(); + var uiTarget = PidOnlySession(fx); + await ResolveAsync(svc, uiTarget, "btnInvoke"); + + var chain = await svc.InspectAncestorsAsync(uiTarget, "btnInvoke", CancellationToken.None); + + Assert.IsTrue(chain.Length >= 2); + Assert.IsTrue(chain.All(element => element.WindowHandle == fx.Hwnd), + "ancestor JSON must use the target element's real top-level HWND"); + } + [TestMethod] public async Task InspectAncestorsAsync_NotFound_Throws() { diff --git a/src/winapp-CLI/WinApp.UIAutomation/Models/UiElement.cs b/src/winapp-CLI/WinApp.UIAutomation/Models/UiElement.cs index 3b6687c8c..196805694 100644 --- a/src/winapp-CLI/WinApp.UIAutomation/Models/UiElement.cs +++ b/src/winapp-CLI/WinApp.UIAutomation/Models/UiElement.cs @@ -30,16 +30,16 @@ public sealed class UiElement /// when the element is scrolled out of view or otherwise not on screen. public bool IsOffscreen { get; set; } - /// Left edge of the element in screen coordinates. + /// Left edge of the element in physical screen pixels. public double X { get; set; } - /// Top edge of the element in screen coordinates. + /// Top edge of the element in physical screen pixels. public double Y { get; set; } - /// Width of the element in pixels. + /// Width of the element in physical screen pixels. public double Width { get; set; } - /// Height of the element in pixels. + /// Height of the element in physical screen pixels. A 0,0,0,0 rectangle is UIA's empty/no-displayed-UI rectangle. public double Height { get; set; } /// Child elements, when the result is a nested tree. on flat result lists. diff --git a/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.CaptureSupport.cs b/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.CaptureSupport.cs index cff6a18f5..d28df5ec3 100644 --- a/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.CaptureSupport.cs +++ b/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.CaptureSupport.cs @@ -51,23 +51,7 @@ public nint ResolveElementTopLevelWindow(UiTarget target, UiElement element) return 0; } - var walker = _automation.get_ControlViewWalker(); - var current = comElement; - var maxWalk = 40; - while (current is not null && maxWalk-- > 0) - { - var native = current.get_CurrentNativeWindowHandle(); - if (!native.IsNull) - { - var root = global::Windows.Win32.PInvoke.GetAncestor( - native, - global::Windows.Win32.UI.WindowsAndMessaging.GET_ANCESTOR_FLAGS.GA_ROOT); - return root.IsNull ? (nint)native : (nint)root; - } - current = walker.GetParentElement(current); - } - - return 0; + return ResolveTopLevelWindowHandle(comElement); } catch (System.Runtime.InteropServices.COMException ex) { diff --git a/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.cs b/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.cs index 695bc44cb..75e446d60 100644 --- a/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.cs +++ b/src/winapp-CLI/WinApp.UIAutomation/Services/UiAutomationService.cs @@ -174,6 +174,11 @@ public Task InspectAsync(UiTarget uiTarget, string? elementId, int return Task.FromResult([]); } + var resolvedRootHwnd = GetTopLevelWindowHandle(root); + var mainHwnd = resolvedRootHwnd != 0 + ? resolvedRootHwnd + : (nint)uiTarget.WindowHandle; + // If a selector is provided, scope the tree walk to that element IUIAutomationElement startElement = root; if (!string.IsNullOrEmpty(elementId)) @@ -205,69 +210,115 @@ public Task InspectAsync(UiTarget uiTarget, string? elementId, int if (target is not null) { startElement = target; + var scopedHwnd = ResolveTopLevelWindowHandle(startElement); + if (scopedHwnd != 0) + { + mainHwnd = scopedHwnd; + } } } - var elements = new List(); - WalkTree(startElement, depth, 0, "", elements, ref nextElementId); - - // Set WindowHandle on all elements from main window - foreach (var el in elements) - { - el.WindowHandle = uiTarget.WindowHandle; - } - - // Also walk popup/owned windows (when inspecting full tree, not scoped to element, - // and the user did not explicitly target a single HWND — see issue #472). + // A process-target inspect must keep each top-level HWND in its own output group. UIA can + // expose an owned window as a descendant of the selected root; prune those roots from the + // main walk and add them through the existing per-window path below. + var independentWindows = new List<(nint Hwnd, int Pid, string Title, IUIAutomationElement Root, bool IsInSelectedTree)>(); if (string.IsNullOrEmpty(elementId) && !uiTarget.IsExplicitWindow) { - var mainHwnd = (nint)uiTarget.WindowHandle; - var allWindows = GetAllAppWindows(uiTarget); - - // Filter out windows whose UIA root is already in the main tree (e.g., modal dialogs) - var independentWindows = new List<(nint Hwnd, int Pid, string Title)>(); - foreach (var (hwnd, pid, title) in allWindows) + foreach (var (hwnd, pid, title) in GetAllAppWindows(uiTarget)) { if (hwnd == mainHwnd) { continue; } - // Skip internal system windows (PseudoConsoleWindow, IME, etc.) var className = UiTargetResolver.GetWindowClassName(hwnd); if (IsInternalWindow(className)) { continue; } - try + var windowRoot = GetRootElementForHwnd(hwnd); + if (windowRoot is not null) { - var hwndCondition = _automation.CreatePropertyCondition( - UIA_PROPERTY_ID.UIA_NativeWindowHandlePropertyId, ComVariant.Create((int)hwnd)); - var alreadyInMain = root!.FindFirst(TreeScope.TreeScope_Descendants, hwndCondition); - if (alreadyInMain is not null) + var isInSelectedTree = false; + try + { + var hwndCondition = _automation.CreatePropertyCondition( + UIA_PROPERTY_ID.UIA_NativeWindowHandlePropertyId, + ComVariant.Create((int)hwnd)); + isInSelectedTree = + root.FindFirst(TreeScope.TreeScope_Descendants, hwndCondition) is not null; + } + catch (COMException) { - _logger.LogDebug("Skipping HWND {Hwnd} \"{Title}\" — already in main window tree", hwnd, title); - continue; + // If reachability cannot be proven, keep the precise slug selector. } + independentWindows.Add((hwnd, pid, title, windowRoot, isInSelectedTree)); } - catch { /* COM errors are non-fatal, include the window */ } - independentWindows.Add((hwnd, pid, title)); } + } + + var topLevelWindowHandles = independentWindows.Count > 0 + ? independentWindows.Select(window => window.Hwnd).ToHashSet() + : null; + topLevelWindowHandles?.Add(mainHwnd); + var promotableWindowHandles = new HashSet { mainHwnd }; + foreach (var window in independentWindows.Where(window => window.IsInSelectedTree)) + { + promotableWindowHandles.Add(window.Hwnd); + } + var elements = new List(); + WalkTree( + startElement, + depth, + 0, + "", + elements, + ref nextElementId, + topLevelWindowHandles: topLevelWindowHandles, + currentWindowHandle: mainHwnd); + + // Set WindowHandle on all elements from main window + foreach (var el in elements) + { + el.WindowHandle = mainHwnd; + } + // Also walk popup/owned windows (when inspecting full tree, not scoped to element, + // and the user did not explicitly target a single HWND — see issue #472). + if (independentWindows.Count > 0) + { // Add header for main window when there are other independent windows - if (independentWindows.Count > 0) - { - var mainInfo = UiTargetResolver.GetWindowInfo(mainHwnd); - var mainTitle = uiTarget.WindowTitle ?? ""; - elements.Insert(0, new UiElement - { - Id = $"--- HWND {mainHwnd}", - Type = "---", - Name = $"HWND {mainHwnd}: \"{mainTitle}\" ({mainInfo.Label}, {mainInfo.ClassName})", - Depth = 0, - WindowHandle = mainHwnd - }); - } + var mainInfo = UiTargetResolver.GetWindowInfo(mainHwnd); + var mainTitle = uiTarget.WindowTitle ?? ""; + elements.Insert(0, new UiElement + { + Id = $"--- HWND {mainHwnd}", + Type = "---", + Name = $"HWND {mainHwnd}: \"{mainTitle}\" ({mainInfo.Label}, {mainInfo.ClassName})", + Depth = 0, + WindowHandle = mainHwnd + }); - foreach (var (hwnd, pid, title) in independentWindows) + foreach (var (hwnd, pid, title, initialRoot, _) in independentWindows) { - var windowRoot = GetRootElementForHwnd(hwnd); - if (windowRoot is null) { continue; } + // Re-resolve immediately before walking so a transient window is not held through + // the potentially long selected-window walk. The initial root preserves the + // already-pruned subtree if the refresh transiently fails. + var windowRoot = GetRootElementForHwnd(hwnd) ?? initialRoot; + + var popupElements = new List(); + try + { + WalkTree( + windowRoot, + depth, + 0, + "", + popupElements, + ref nextElementId, + topLevelWindowHandles: topLevelWindowHandles, + currentWindowHandle: hwnd); + } + catch (COMException ex) + { + _logger.LogDebug(ex, "Skipping unavailable popup/owned window HWND {Hwnd}", hwnd); + continue; + } // Add a separator element to visually distinguish windows var info = UiTargetResolver.GetWindowInfo(hwnd); @@ -281,8 +332,6 @@ public Task InspectAsync(UiTarget uiTarget, string? elementId, int WindowHandle = hwnd }); - var popupElements = new List(); - WalkTree(windowRoot, depth, 0, "", popupElements, ref nextElementId); foreach (var el in popupElements) { el.WindowHandle = hwnd; @@ -292,7 +341,7 @@ public Task InspectAsync(UiTarget uiTarget, string? elementId, int } // Promote unique AutomationIds to selectors (more stable than slugs) - PromoteUniqueAutomationIds(root, elements, uiTarget.WindowHandle); + PromoteUniqueAutomationIds(root, elements, mainHwnd, promotableWindowHandles); var result = elements.ToArray(); return Task.FromResult(result); @@ -344,6 +393,7 @@ public Task InspectAncestorsAsync(UiTarget uiTarget, string element var current = target; // Add the target element itself first + var windowHandle = ResolveTopLevelWindowHandle(current); ancestors.Add(ToUiElement(current, "", ref nextElementId)); while (true) @@ -385,6 +435,12 @@ public Task InspectAncestorsAsync(UiTarget uiTarget, string element // Reverse so root is first, target is last ancestors.Reverse(); + windowHandle = windowHandle != 0 ? windowHandle : GetTopLevelWindowHandle(root); + windowHandle = windowHandle != 0 ? windowHandle : (nint)uiTarget.WindowHandle; + foreach (var ancestor in ancestors) + { + ancestor.WindowHandle = windowHandle; + } // Promote unique AutomationIds to selectors (more stable than slugs) PromoteUniqueAutomationIds(root, ancestors); @@ -569,10 +625,10 @@ public Task SearchAsync(UiTarget uiTarget, UiSelector selector, int // Slug resolution: walk tree, regenerate slugs, match and validate hash if (selector.IsSlug) { - var slugResult = FindElementBySlug(selector.Slug!, root); - if (slugResult is not null) + var (slugResult, slugElement) = FindElementBySlugWithCom(selector.Slug!, root); + if (slugResult is not null && slugElement is not null) { - slugResult.WindowHandle = uiTarget.WindowHandle; + SetResolvedWindowHandle(slugResult, slugElement, uiTarget.WindowHandle); return Task.FromResult(slugResult); } // Not found on main window — search other windows (unless --window scoped us to one) @@ -598,7 +654,7 @@ public Task SearchAsync(UiTarget uiTarget, UiSelector selector, int { var nextId = 0; var exactResult = ToUiElement(exactMatch, "", ref nextId); - exactResult.WindowHandle = uiTarget.WindowHandle; + SetResolvedWindowHandle(exactResult, exactMatch, uiTarget.WindowHandle); return Task.FromResult(exactResult); } } @@ -630,7 +686,7 @@ public Task SearchAsync(UiTarget uiTarget, UiSelector selector, int { var nextId = 0; var manualResult = ToUiElement(manualResults[0], "", ref nextId); - manualResult.WindowHandle = uiTarget.WindowHandle; + SetResolvedWindowHandle(manualResult, manualResults[0], uiTarget.WindowHandle); return Task.FromResult(manualResult); } } @@ -669,7 +725,7 @@ public Task SearchAsync(UiTarget uiTarget, UiSelector selector, int _logger.LogDebug("Disambiguated {Count} matches by picking the only invokable element", found.get_Length()); var nextId = 0; var invokableResult = ToUiElement(invokableMatch, "", ref nextId); - invokableResult.WindowHandle = uiTarget.WindowHandle; + SetResolvedWindowHandle(invokableResult, invokableMatch, uiTarget.WindowHandle); return Task.FromResult(invokableResult); } @@ -709,7 +765,7 @@ public Task SearchAsync(UiTarget uiTarget, UiSelector selector, int var element = found.GetElement(0); var nextElementId = 0; var result = ToUiElement(element, "", ref nextElementId); - result.WindowHandle = uiTarget.WindowHandle; + SetResolvedWindowHandle(result, element, uiTarget.WindowHandle); // Surface invokable ancestor for non-invokable elements if (!IsInvokable(element)) @@ -1901,8 +1957,26 @@ private static bool IsInvokable(IUIAutomationElement element) } private void WalkTree(IUIAutomationElement element, int maxDepth, int currentDepth, string path, List results, ref int nextElementId, - string? parentSelector = null, List? ancestorTypes = null) + string? parentSelector = null, List? ancestorTypes = null, + HashSet? topLevelWindowHandles = null, nint currentWindowHandle = 0) { + if (currentDepth > 0 && topLevelWindowHandles is not null) + { + try + { + var hwnd = (nint)element.get_CurrentNativeWindowHandle(); + if (hwnd != 0 && hwnd != currentWindowHandle && topLevelWindowHandles.Contains(hwnd)) + { + return; + } + } + catch (COMException) + { + // Keep walking when a provider cannot report the native handle. The independent + // HWND is still emitted below, matching the previous best-effort COM behavior. + } + } + var uiElement = ToUiElement(element, path, ref nextElementId); uiElement.Depth = currentDepth; uiElement.ParentSelector = parentSelector; @@ -1939,7 +2013,17 @@ private void WalkTree(IUIAutomationElement element, int maxDepth, int currentDep while (child is not null) { var childPath = string.IsNullOrEmpty(path) ? $"/{childIndex}" : $"{path}/{childIndex}"; - WalkTree(child, maxDepth, currentDepth + 1, childPath, results, ref nextElementId, childParentSelector, childAncestors); + WalkTree( + child, + maxDepth, + currentDepth + 1, + childPath, + results, + ref nextElementId, + childParentSelector, + childAncestors, + topLevelWindowHandles, + currentWindowHandle); IUIAutomationElement? next; try @@ -2066,6 +2150,56 @@ private static UiElement ToUiElement(IUIAutomationElement element, string path, }; } + private static nint GetTopLevelWindowHandle(IUIAutomationElement element) + { + try + { + var native = element.get_CurrentNativeWindowHandle(); + if (native.IsNull) { return 0; } + + var root = global::Windows.Win32.PInvoke.GetAncestor( + native, + global::Windows.Win32.UI.WindowsAndMessaging.GET_ANCESTOR_FLAGS.GA_ROOT); + return root.IsNull ? (nint)native : (nint)root; + } + catch (COMException) + { + return 0; + } + } + + private nint ResolveTopLevelWindowHandle(IUIAutomationElement element) + { + var walker = _automation.get_ControlViewWalker(); + IUIAutomationElement? current = element; + var remaining = 40; + while (current is not null && remaining-- > 0) + { + var hwnd = GetTopLevelWindowHandle(current); + if (hwnd != 0) { return hwnd; } + + try + { + current = walker.GetParentElement(current); + } + catch (COMException) + { + return 0; + } + } + + return 0; + } + + private void SetResolvedWindowHandle( + UiElement model, + IUIAutomationElement element, + long fallbackWindowHandle) + { + var hwnd = ResolveTopLevelWindowHandle(element); + model.WindowHandle = hwnd != 0 ? hwnd : fallbackWindowHandle; + } + private static bool HasPattern(IUIAutomationElement element, UIA_PATTERN_ID patternId) { try @@ -2083,7 +2217,11 @@ private static bool HasPattern(IUIAutomationElement element, UIA_PATTERN_ID patt /// across the full UIA tree, use it directly as the selector instead of a generated slug. /// AutomationIds are developer-set, stable across layout changes, and more readable. /// - private void PromoteUniqueAutomationIds(IUIAutomationElement root, IList elements, long mainWindowHandle = 0) + private void PromoteUniqueAutomationIds( + IUIAutomationElement root, + IList elements, + long mainWindowHandle = 0, + HashSet? promotableWindowHandles = null) { // Collect AutomationIds from the inspected elements that could be promoted var candidateAids = new HashSet(); @@ -2136,7 +2274,9 @@ private void PromoteUniqueAutomationIds(IUIAutomationElement root, IList getter) { try