diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 9a82ffd55..a574f957d 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -31,6 +31,7 @@ export function createCursorRecordingSession( getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, sampleIntervalMs: options.sampleIntervalMs, + sourceId: options.sourceId, startTimeMs: options.startTimeMs, }); } diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts new file mode 100644 index 000000000..598021f6c --- /dev/null +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { normalizeCursorToBounds } from "./macNativeCursorRecordingSession"; + +describe("normalizeCursorToBounds", () => { + it("maps a point to [0,1] within the given bounds", () => { + const bounds = { x: 0, y: 0, width: 1000, height: 500 }; + expect(normalizeCursorToBounds({ x: 0, y: 0 }, bounds)).toMatchObject({ + normalizedX: 0, + normalizedY: 0, + isOutsideBounds: false, + }); + expect(normalizeCursorToBounds({ x: 500, y: 250 }, bounds)).toMatchObject({ + normalizedX: 0.5, + normalizedY: 0.5, + isOutsideBounds: false, + }); + }); + + it("accounts for the bounds origin offset", () => { + const bounds = { x: 200, y: 100, width: 800, height: 600 }; + const { normalizedX, normalizedY, isOutsideBounds } = normalizeCursorToBounds( + { x: 600, y: 400 }, + bounds, + ); + expect(normalizedX).toBeCloseTo(0.5); + expect(normalizedY).toBeCloseTo(0.5); + expect(isOutsideBounds).toBe(false); + }); + + it("flags points outside the bounds", () => { + const bounds = { x: 200, y: 100, width: 800, height: 600 }; + expect(normalizeCursorToBounds({ x: 100, y: 50 }, bounds).isOutsideBounds).toBe(true); + expect(normalizeCursorToBounds({ x: 1200, y: 800 }, bounds).isOutsideBounds).toBe(true); + }); + + // The window-capture bug: the recorded video is just the window, but the cursor + // used to be normalized against the whole display, putting the overlay cursor far + // from where the click actually landed. Normalizing against the window frame fixes it. + it("normalizes against the window frame, not the display, for window capture", () => { + // A 1920x1080 display with a window offset to (600, 300) sized 800x600. + const display = { x: 0, y: 0, width: 1920, height: 1080 }; + const windowFrame = { x: 600, y: 300, width: 800, height: 600 }; + // Cursor sitting at the exact center of the window. + const cursor = { x: windowFrame.x + 400, y: windowFrame.y + 300 }; + + const againstWindow = normalizeCursorToBounds(cursor, windowFrame); + expect(againstWindow.normalizedX).toBeCloseTo(0.5); + expect(againstWindow.normalizedY).toBeCloseTo(0.5); + + const againstDisplay = normalizeCursorToBounds(cursor, display); + // Against the display the same physical point maps to a very different spot, + // which is the visible misalignment users reported. + expect(againstDisplay.normalizedX).toBeCloseTo(1000 / 1920); + expect(againstDisplay.normalizedY).toBeCloseTo(600 / 1080); + expect(Math.abs(againstWindow.normalizedX - againstDisplay.normalizedX)).toBeGreaterThan(0.02); + }); +}); diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index 27a8a870c..623cf587a 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -3,6 +3,7 @@ import { accessSync, constants as fsConstants } from "node:fs"; import path from "node:path"; import type { Readable } from "node:stream"; import { type Rectangle, screen, systemPreferences } from "electron"; +import { parseMacWindowIdFromSourceId } from "../../../../src/lib/nativeMacRecording"; import type { CursorRecordingData, CursorRecordingSample, @@ -11,6 +12,13 @@ import type { } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; +interface MacCursorBounds { + x: number; + y: number; + width: number; + height: number; +} + interface MacCursorAssetPayload { id: string; imageDataUrl: string; @@ -26,6 +34,8 @@ interface MacNativeCursorRecordingSessionOptions { maxSamples: number; sampleIntervalMs: number; startTimeMs?: number; + /** Source id of the recording target; used to detect window capture. */ + sourceId?: string | null; } type MacCursorEvent = @@ -41,6 +51,8 @@ type MacCursorEvent = cursorType?: NativeCursorType | null; assetId?: string | null; asset?: MacCursorAssetPayload | null; + /** On-screen frame of the captured window, when recording a single window. */ + bounds?: MacCursorBounds | null; leftButtonDown?: boolean; leftButtonPressed?: boolean; leftButtonReleased?: boolean; @@ -181,6 +193,24 @@ function normalizeCursorType(value: unknown): NativeCursorType | null { return value === "arrow" || value === "pointer" || value === "text" ? value : null; } +/** + * Normalize an absolute screen cursor point to [0,1] within a captured region's + * bounds. For window capture the region is the window's frame; for full-screen + * it's the display. Returned values may fall outside [0,1] when the cursor is + * outside the region, which the caller uses for visibility/clipping decisions. + */ +export function normalizeCursorToBounds( + cursor: { x: number; y: number }, + bounds: { x: number; y: number; width: number; height: number }, +): { normalizedX: number; normalizedY: number; isOutsideBounds: boolean } { + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + const normalizedX = (cursor.x - bounds.x) / width; + const normalizedY = (cursor.y - bounds.y) / height; + const isOutsideBounds = normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + return { normalizedX, normalizedY, isOutsideBounds }; +} + export class MacNativeCursorRecordingSession implements CursorRecordingSession { private samples: CursorRecordingSample[] = []; private assets = new Map(); @@ -220,11 +250,13 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { return; } + const windowId = parseMacWindowIdFromSourceId(this.options.sourceId); const child = spawn( helperPath, [ JSON.stringify({ sampleIntervalMs: this.options.sampleIntervalMs, + ...(windowId != null ? { windowId } : {}), }), ], { @@ -286,9 +318,9 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { } private startPositionOnlyFallback() { - this.captureSample(Date.now(), null, null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false, null); this.fallbackInterval = setInterval(() => { - this.captureSample(Date.now(), null, null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false, null); }, this.options.sampleIntervalMs); } @@ -350,6 +382,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { payload.leftButtonDown === true, payload.leftButtonPressed === true, payload.leftButtonReleased === true, + payload.bounds ?? null, ); } } @@ -361,15 +394,21 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { leftButtonDown: boolean, leftButtonPressed: boolean, leftButtonReleased: boolean, + windowBounds: MacCursorBounds | null, ) { const cursor = screen.getCursorScreenPoint(); - const bounds = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - const normalizedX = (cursor.x - bounds.x) / width; - const normalizedY = (cursor.y - bounds.y) / height; - const isOutsideDisplay = - normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; + // For window capture, the helper reports the window's on-screen frame so we + // normalize against the window (the recorded video is just that window). + // Otherwise fall back to the captured display's bounds. + const bounds = + windowBounds ?? + this.options.getDisplayBounds() ?? + screen.getDisplayNearestPoint(cursor).bounds; + const { + normalizedX, + normalizedY, + isOutsideBounds: isOutsideDisplay, + } = normalizeCursorToBounds(cursor, bounds); // Brief exits (under THRESHOLD samples) clip to the canvas edge via clip-path instead // of snapping invisible. Sustained exits (>=THRESHOLD, ~100ms) mark visible=false to // avoid ghost cursors and motion trails from multi-display movement. diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift index ace6827ca..b348a6436 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -5,6 +5,31 @@ import Foundation struct CursorHelperRequest: Decodable { let sampleIntervalMs: Int? + // When capturing a single window, the CGWindowID of that window. Cursor + // positions are then normalized against the window's on-screen frame instead + // of the whole display, so the editable cursor overlay lines up with the + // window-only recording. + let windowId: UInt32? +} + +/// Current on-screen frame of a window in global display points (top-left origin, +/// y increasing downward) — the same coordinate space as Electron's +/// `screen.getCursorScreenPoint()`. Re-read every sample so a window that is moved +/// mid-recording stays aligned. Returns nil if the window can't be found. +func windowBounds(for windowId: UInt32) -> CGRect? { + guard + let infoList = CGWindowListCopyWindowInfo([.optionIncludingWindow], windowId) + as? [[String: Any]], + let info = infoList.first, + let boundsDict = info[kCGWindowBounds as String] as? NSDictionary, + let rect = CGRect(dictionaryRepresentation: boundsDict as CFDictionary), + rect.width > 0, + rect.height > 0 + else { + return nil + } + + return rect } struct CapturedCursorAsset { @@ -300,7 +325,7 @@ if CommandLine.arguments.count >= 2, { request = decoded } else { - request = CursorHelperRequest(sampleIntervalMs: nil) + request = CursorHelperRequest(sampleIntervalMs: nil, windowId: nil) } let intervalMs = max(8, request.sampleIntervalMs ?? 33) @@ -337,12 +362,22 @@ while true { "scaleFactor": asset.scaleFactor, ] } + var boundsPayload: [String: Double]? + if let windowId = request.windowId, let frame = windowBounds(for: windowId) { + boundsPayload = [ + "x": Double(frame.origin.x), + "y": Double(frame.origin.y), + "width": Double(frame.size.width), + "height": Double(frame.size.height), + ] + } emit([ "type": "sample", "timestampMs": timestampMs(), "cursorType": currentCursorType(), "assetId": asset?.id, "asset": assetPayload, + "bounds": boundsPayload, "leftButtonDown": leftButtonDown(), "leftButtonPressed": mouseEvents.leftDownCount > 0, "leftButtonReleased": mouseEvents.leftUpCount > 0,