diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f85736c2..73029f6bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,14 @@ jobs: - name: Install dependencies run: npm ci + # Cache the downloaded caption model so we don't re-fetch from HuggingFace every run + # (and to avoid 429s when the platform matrix builds hit it at once). + - name: Cache caption assets + uses: actions/cache@v4 + with: + path: caption-assets + key: caption-assets-${{ hashFiles('scripts/fetch-caption-model.mjs') }} + - name: Build Windows app run: npm run build:win env: @@ -69,6 +77,23 @@ jobs: - name: Install dependencies run: npm ci + # ─── Ensure sharp prebuilt binary (+ bundled libvips) ───── + # `npm ci` can leave sharp without its prebuilt binary/vendored libvips, + # which makes electron-builder's native rebuild fail ("vips-cpp.42 not + # found"). Re-fetch the prebuilt (never compile from source). + - name: Ensure sharp prebuilt + run: npm rebuild sharp + env: + npm_config_build_from_source: "false" + + # ─── Cache caption assets ───────────────────────────────── + # Avoid re-fetching the Whisper model from HuggingFace every run (and 429s under the matrix). + - name: Cache caption assets + uses: actions/cache@v4 + with: + path: caption-assets + key: caption-assets-${{ hashFiles('scripts/fetch-caption-model.mjs') }} + # ─── Import Code Signing Certificate ────────────────────── # This is the KEY step that makes CI signing work. # We create a temporary keychain, import the .p12 cert into it, @@ -111,6 +136,17 @@ jobs: - name: Build Vite + Electron run: npx tsc && npx vite build + # ─── Build native macOS helpers ─────────────────────────── + # The package step below calls electron-builder directly (not `npm run build:mac`), + # so the Swift screencapturekit + cursor helpers must be compiled here first or the + # packaged app ships without them (no recording, "cursor helper couldn't be found"). + # Build the matrix arch into electron/native/bin/darwin- so each DMG gets its + # own native binary. + - name: Build native macOS helpers + run: npm run build:native:mac + env: + OPENSCREEN_MAC_HELPER_ARCHS: ${{ matrix.arch }} + # ─── Package with electron-builder ──────────────────────── # electron-builder handles deep codesigning the .app bundle # "notarize: false" in electron-builder.json5 prevents it from @@ -236,6 +272,14 @@ jobs: - name: Install pacman build dependencies run: sudo apt-get update && sudo apt-get install -y libarchive-tools + # Cache the downloaded caption model so we don't re-fetch from HuggingFace every run + # (and to avoid 429s when the platform matrix builds hit it at once). + - name: Cache caption assets + uses: actions/cache@v4 + with: + path: caption-assets + key: caption-assets-${{ hashFiles('scripts/fetch-caption-model.mjs') }} + - name: Build Linux app run: npm run build:linux env: diff --git a/.gitignore b/.gitignore index 82fc468b7..61dc9d2fc 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ result-* #others **/*.import + +# Auto-caption model + ORT wasm — regenerated at build by scripts/fetch-caption-model.mjs +/caption-assets/ diff --git a/README.md b/README.md index 7009a2209..867f21634 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,51 @@ > [!WARNING] -> This started as a side project that took off — it's not production grade and you'll hit bugs, but hopefully it covers what you need. +> This started as a side project that blew up; not production grade and you'll hit bugs, but hopefully it covers what you need. **This project will soon be archived.** +

OpenScreen Logo

- siddharthvaddem%2Fopenscreen | Trendshift -
-
- - Ask DeepWiki - -   - - Join Discord + siddharthvaddem%2Fopenscreen | Trendshift + +

#

OpenScreen

-

OpenScreen is your free, open-source alternative to Screen Studio (sort of).

+

OpenScreen is your free, open-source alternative to Screen Studio.

-If you don't want to pay $29/month for Screen Studio but want a much simpler version that does what most people seem to need - quick, polished product demos and walkthroughs you'd post on X, Reddit. OpenScreen does not offer all Screen Studio features, but covers the basics well! +If you don't want to pay $29/month for Screen Studio but want a version that does what most people seem to need - quick, polished product demos and walkthroughs you'd post on X, Reddit or Youtube. OpenScreen does not offer every Screen Studio feature, but covers a lot of the core functionality. -Screen Studio is an awesome product and this is definitely not a 1:1 clone. OpenScreen is a much simpler take, just the basics for folks who want control and don't want to pay. If you need all the fancy features, your best bet is to support Screen Studio (they really do a great job, haha). But if you just want something free (no gotchas) and open, this project does the job! +Screen Studio is an awesome product and this is definitely not a 1:1 clone. If you just want something fully free and open source, this project should cover most of your needs. -**100% free** for both **personal** and **commercial** use. Use it, modify it, distribute it — just be cool 😁 and shout out the project if you feel like it. +**100% free** for both **personal** and **commercial** use. Use it, modify it, distribute it. Please respect the License. + +> [!NOTE] +>Software should be accessible. OpenScreen has no paid tiers, premium features, upsells, or functionality locked behind a paywall.

- OpenScreen App Preview 3 - OpenScreen App Preview 4 + +

## Core Features -- Record a specific window, region, or your whole screen. +- Record a specific window, or your whole screen. - Record microphone and system audio. -- Webcam overlay with picture-in-picture, drag-to-position, and shape options. -- Auto or manual zooms with adjustable depth, duration, easing, and pixel-precise position. -- Wallpapers, solid colors, gradients, or a custom background. -- Motion blur for smoother pan and zoom transitions. +- Webcam overlay with picture-in-picture, drag-to-position, mirroring, and shape options. +- Auto or manual zooms with adjustable depth, duration, easing, and pixel-precise position; auto-zoom follows your cursor as you work. +- Custom cursor size, smoothing, and click effects, with cursor themes and post-recording path smoothing. +- Automatic captions for voiceovers, generated on-device with no upload (works offline). +- Wallpapers, solid colors, gradients, or your own background image. +- Motion blur. - Crop, trim, and per-segment speed control on the timeline. -- Blur effects to hide sensitive parts of the screen. -- Cursor and click highlighting. -- Text, arrow, and image annotations. -- Save and reopen projects without re-recording. +- Text, arrow, and image annotations, with text animation presets. +- Timeline snapping guides and an audio waveform to make trimming easier. +- Customizable keyboard shortcuts. - Export to MP4 or GIF in multiple aspect ratios and resolutions. -- Translated into Arabic, English, Spanish, French, Japanese, Korean, Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. +- Languages supported: Arabic, English, Spanish, French, Italian, Japanese, Korean, Portuguese (Brazil), Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. + ## Installation @@ -76,6 +76,9 @@ Note: Give your terminal Full Disk Access in **System Settings > Privacy & Secur After running this command, proceed to **System Preferences > Security & Privacy** to grant the necessary permissions for "screen recording" and "accessibility". Once permissions are granted, you can launch the app. +> [!NOTE] +> **Upgrading from an older version and hitting permission issues?** If you already had OpenScreen installed and the new version won't record (Screen Recording or Accessibility keep failing even after you grant them), uninstall the old version, remove OpenScreen's existing entries under **System Settings > Privacy & Security** (both Screen Recording and Accessibility), then do a fresh install and grant the permissions again when prompted. + ### Windows Install via [winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/): @@ -146,45 +149,20 @@ You may need to grant screen recording permissions depending on your desktop env ./Openscreen-Linux-*.AppImage --no-sandbox ``` -### Limitations - -System audio capture relies on Electron's [desktopCapturer](https://www.electronjs.org/docs/latest/api/desktop-capturer) and has some platform-specific quirks: +### Platform differences -- **macOS**: Requires macOS 13+. On macOS 14.2+ you'll be prompted to grant audio capture permission. macOS 12 and below does not support system audio (mic still works). -- **Windows**: Works out of the box. -- **Linux**: Needs PipeWire (default on Ubuntu 22.04+, Fedora 34+). Older PulseAudio-only setups may not support system audio (mic should still work). +Everything in the editor and export is the same on macOS, Windows, and Linux: zooms, backgrounds, motion blur, crop/trim/speed, blur regions, annotations, auto-captions, projects, export, and all languages. The differences are in **capture**, where macOS and Windows use a native pipeline that Linux doesn't have: -## Built with -- Electron -- React -- TypeScript -- Vite -- PixiJS -- dnd-timeline +- **Native recording**: macOS (ScreenCaptureKit) and Windows (Windows Graphics Capture) record through a native pipeline for higher quality and clean window-level capture. Linux records through the browser pipeline instead. +- **Custom cursors**: on macOS and Windows the real cursor is captured (shape, type, and clicks), which powers the cursor themes, click effects, and editable cursor overlay. On Linux only the cursor position is captured (used for auto-zoom), so those cursor options aren't available. +- **Webcam**: captured natively on macOS and Windows; on Linux it's recorded through the browser, but still works as a picture-in-picture overlay. +- **System audio** support varies by OS: + - **macOS**: requires macOS 13+. On macOS 14.2+ you'll be prompted to grant audio capture permission. macOS 12 and below can't capture system audio (mic still works). + - **Windows**: works out of the box. + - **Linux**: needs PipeWire (default on Ubuntu 22.04+, Fedora 34+). Older PulseAudio-only setups may not capture system audio (mic should still work). --- - -## Documentation - -See the documentation here: -[OpenScreen Docs](https://deepwiki.com/siddharthvaddem/openscreen) -Refresh if outdated. - -## Contributing - -Contributions are welcome - please **include screenshots or a short video** for any UI change or new user-facing feature. If it touches what users see or do, show it. Skip only when it genuinely doesn't apply. PRs that don't follow this will be closed. - -## Star History - - - - - - Star History Chart - - - ## License This project is licensed under the [MIT License](./LICENSE). By using this software, you agree that the authors are not liable for any issues, damages, or claims arising from its use. diff --git a/docs/testing/macos-native-cursor.md b/docs/testing/macos-native-cursor.md new file mode 100644 index 000000000..b79064f8c --- /dev/null +++ b/docs/testing/macos-native-cursor.md @@ -0,0 +1,183 @@ +# macOS native cursor test pipeline + +This document covers manual and diagnostic testing for macOS native cursor capture — the path that records real system cursor bitmaps via `NSCursor.currentSystem` and surfaces them through the OpenScreen editor and export pipeline. + +## How the macOS cursor helper works + +The helper binary (`openscreen-macos-cursor-helper`) runs as a child process of Electron during recording. It: + +- polls `NSCursor.currentSystem` at the configured sample interval +- converts each cursor image to PNG and computes a SHA-256 content hash as a stable asset id +- emits the full base64 bitmap payload **once** per unique cursor shape per session; subsequent samples carry only the `assetId` so stdout stays small +- tracks left-button down/up events via `CGEventTap` and tags each sample with `interactionType` +- uses the Accessibility API to detect `text` and `pointer` affordances (link/button/input roles) when Accessibility is granted; these shapes use the bundled high-quality SVG replacements instead of the raw bitmap + +Each sample line is newline-delimited JSON: + +```json +{ "type": "ready", "timestampMs": 1234567890, "accessibilityTrusted": true, "mouseTapReady": true } +{ "type": "sample", "timestampMs": 1234567891, "assetId": "a7472...", "asset": { "id": "a7472...", "imageDataUrl": "data:image/png;base64,...", "width": 64, "height": 64, "hotspotX": 16, "hotspotY": 16, "scaleFactor": 2.0 }, "cursorType": null, "leftButtonDown": false, "leftButtonPressed": false, "leftButtonReleased": false } +{ "type": "sample", "timestampMs": 1234567924, "assetId": "a7472...", "cursorType": null, "leftButtonDown": false, "leftButtonPressed": false, "leftButtonReleased": false } +``` + +`asset` is present only the first time a given `assetId` appears. The TypeScript session (`MacNativeCursorRecordingSession`) collects unique assets into a map and sets `provider: "native"` in the final `CursorRecordingData` when at least one bitmap was captured. + +## Build the helper + +```bash +npm run build:native:mac +``` + +This builds both Swift helpers (`openscreen-screencapturekit-helper` and `openscreen-macos-cursor-helper`) and copies them to: + +- `electron/native/screencapturekit/build/` — used by the local dev server +- `electron/native/bin/darwin-arm64/` or `darwin-x64/` — used by packaged builds + +Requires Xcode (not just Command Line Tools). If you see a build error about missing SDK metadata, run: + +```bash +sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer +sudo xcodebuild -license accept +``` + +## Smoke-test the helper directly + +You can run the cursor helper standalone to inspect its raw output before involving the full app: + +```bash +BIN=electron/native/screencapturekit/build/openscreen-macos-cursor-helper +("$BIN" '{"sampleIntervalMs":100}' & PID=$!; sleep 2; kill $PID) | head -20 +``` + +Expected first line: + +```json +{"type":"ready","mouseTapReady":true,"accessibilityTrusted":false,"timestampMs":...} +``` + +`accessibilityTrusted: false` is normal in dev/unsigned builds. It means text/pointer affordance detection is disabled; native bitmap capture still works. + +Expected sample lines: + +```json +{"type":"sample","assetId":"a7472...","asset":{"id":"a7472...","imageDataUrl":"data:image/png;base64,...","width":64,"height":64,"hotspotX":26,"hotspotY":16,"scaleFactor":2.0},...} +{"type":"sample","assetId":"a7472...",...} +``` + +Move the cursor over a text input while the helper is running and check that a new `assetId` appears with a different bitmap (if Accessibility is granted — see below). + +## Point the app at a custom helper binary + +```bash +export OPENSCREEN_MAC_CURSOR_HELPER_EXE=/path/to/openscreen-macos-cursor-helper +npm run dev +``` + +## macOS permissions + +Two separate permissions are needed: + +| Permission | What it enables | Where to grant | +|---|---|---| +| Screen Recording | ScreenCaptureKit video capture | System Settings → Privacy & Security → Screen & System Audio Recording → Electron ✅ | +| Accessibility | `text` / `pointer` cursor type detection (affordance hints) | System Settings → Privacy & Security → Accessibility → Electron ✅ | + +**Screen Recording** is required to record. Without it the recording never starts. + +**Accessibility** is optional. Without it, `cursorType` will always be `null` and all cursors render from their captured bitmaps (no SVG substitution). This is the expected fallback and does not degrade cursor quality for non-text/pointer shapes. + +After granting either permission in System Settings, **fully quit and relaunch** the dev server — `getMediaAccessStatus` caches the result per-process. + +## Manual test checklist + +### P0 — core bitmap capture + +- [ ] Record a short clip. Open the editor. Confirm the default arrow cursor is the real system arrow (not the bundled SVG approximation). +- [ ] Record while hovering over a web browser. Confirm custom-CSS cursors (e.g. `cursor: grab`, `cursor: crosshair`) appear as their actual shapes. +- [ ] Export to MP4. Confirm the cursor renders correctly in the exported video. +- [ ] Export to GIF. Same check. + +### P1 — affordance substitution (requires Accessibility) + +- [ ] Grant Accessibility permission and restart the app. +- [ ] Record hovering over a text input field. Confirm the text I-beam uses the bundled SVG version (prettier than the system bitmap). +- [ ] Record hovering over a link/button. Confirm the pointer hand uses the bundled SVG. + +### P1 — hotspot alignment (Retina) + +- [ ] On a Retina display, record a precise click on a small button. In the editor, confirm the cursor tip aligns with the actual click point. The helper reports `scaleFactor: 2.0`; the renderer divides pixel dimensions and hotspot by this value to recover point sizes. + +### P1 — click detection + +- [ ] Record several left-clicks. In the editor, confirm the click-bounce animation fires on each click. +- [ ] Confirm `interactionType: "click"` and `"mouseup"` events are present in the recording session sidecar (`cursorRecordingData` inside `.cursor.json`). + +### P2 — graceful degradation + +- [ ] Remove **both** build-output copies of the helper binary and start a recording. The session should succeed with `provider: "none"` (position-only telemetry, default arrow rendered). Restore both binaries afterward. + ```bash + ARCH=$([ "$(uname -m)" = "arm64" ] && echo darwin-arm64 || echo darwin-x64) + mv electron/native/screencapturekit/build/openscreen-macos-cursor-helper /tmp/cursor-helper-build + mv electron/native/bin/$ARCH/openscreen-macos-cursor-helper /tmp/cursor-helper-bin + # ... start recording, then restore: + mv /tmp/cursor-helper-bin electron/native/bin/$ARCH/openscreen-macos-cursor-helper + mv /tmp/cursor-helper-build electron/native/screencapturekit/build/openscreen-macos-cursor-helper + ``` +- [ ] Revoke Accessibility. Confirm recording still works and cursors render from bitmaps (no SVG substitution). + +### P2 — multi-display + +- [ ] Move the cursor to a secondary display during recording. Confirm the cursor clips to the canvas edge rather than snapping invisible on fast swipes. Confirm it hides after ≈100 ms of sustained out-of-bounds movement. + +### P2 — long recording memory + +- [ ] Record for 3–5 minutes while switching between many apps (browser, terminal, editor). The helper should not grow in memory because each iteration drains Cocoa objects via `autoreleasepool`. Check `Activity Monitor` → `openscreen-macos-cursor-helper` RSS stays flat after the first few seconds. + +## What a healthy recording looks like + +Inspect the cursor sidecar file written alongside the recorded video (`.cursor.json`). For a recording saved to `/tmp/rec.mp4`, the sidecar is `/tmp/rec.mp4.cursor.json`: + +```json +{ + "version": 2, + "provider": "native", + "assets": [ + { "id": "a7472...", "platform": "darwin", "imageDataUrl": "data:image/png;base64,...", "width": 64, "height": 64, "hotspotX": 26.0, "hotspotY": 16.0, "scaleFactor": 2.0 } + ], + "samples": [ + { "timeMs": 0, "cx": 0.42, "cy": 0.38, "visible": true, "assetId": "a7472...", "interactionType": "move" }, + ... + ] +} +``` + +`provider: "native"` and a non-empty `assets` array confirm bitmap capture is active. If you see `provider: "none"` and `assets: []`, the helper was not found or exited before `ready`. + +## Native macOS capture backend + +The app routes macOS recordings through the ScreenCaptureKit helper (`openscreen-screencapturekit-helper`) when it is available, so the real system cursor is excluded from the video frame. The cursor position and bitmap are captured separately by the cursor helper and composited in the editor and export pipeline. + +Current native availability rules: + +- macOS 13 (Ventura) or newer +- `openscreen-screencapturekit-helper` binary is present +- Screen Recording permission is granted + +Build both helpers locally: + +```bash +npm run build:native:mac +``` + +For local diagnostics with a custom helper binary, use the environment override: + +```bash +export OPENSCREEN_MAC_CURSOR_HELPER_EXE=/path/to/openscreen-macos-cursor-helper +npm run dev +``` + +## Known limitations + +- **Intel (x86\_64) Macs**: the distributed helper is built for `darwin-arm64`. On Intel Macs, you need to build from source with `npm run build:native:mac` on the target machine. +- **Accessibility permission in unsigned/dev builds**: `getMediaAccessStatus("accessibility")` may not reflect the toggle state for unsigned Electron in dev mode. The helper will always probe and report `accessibilityTrusted` in its `ready` event — use that as the authoritative signal. +- **App-defined custom cursors (CGS layer)**: `NSCursor.currentSystem` captures the active AppKit cursor. Cursors set at the CoreGraphics/CGS layer by some games or GPU-accelerated apps may not be visible here. This is a known macOS API limitation. diff --git a/electron-builder.json5 b/electron-builder.json5 index 8ad4a80eb..16a7f2c60 100644 --- a/electron-builder.json5 +++ b/electron-builder.json5 @@ -8,8 +8,13 @@ "**/*.node" ], "productName": "Openscreen", + // Fetch the auto-caption model + ORT wasm into caption-assets/ before packaging (idempotent). + "beforePack": "scripts/before-pack.cjs", "npmRebuild": true, - "buildDependenciesFromSource": true, + // sharp ships ABI-stable (napi) prebuilt binaries with bundled libvips. Building it from source + // needs a system libvips we don't provide and breaks on CI/local ("vips-cpp.42 not found"), so we + // let electron-builder use the prebuilt instead of recompiling. + "buildDependenciesFromSource": false, "compression": "normal", "directories": { "output": "release/${version}" @@ -24,12 +29,20 @@ "!CONTRIBUTING.md", "!LICENSE" ], - // Asset layout contract: "wallpapers/" under resourcesPath must align with - // assetBaseDir in electron/preload.ts (packaged branch). + // Asset layout contract: "wallpapers/" and "cursors/" under resourcesPath must + // align with assetBaseDir in electron/preload.ts (packaged branch). "extraResources": [ { "from": "public/wallpapers", "to": "wallpapers" + }, + { + "from": "public/cursors", + "to": "cursors" + }, + { + "from": "caption-assets", + "to": "caption-assets" } ], diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index ac29d45af..18d44f1ae 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -229,7 +229,7 @@ interface Window { canceled?: boolean; error?: string; }>; - loadProjectFile: () => Promise<{ + loadProjectFile: (projectFolder?: string) => Promise<{ success: boolean; path?: string; project?: unknown; @@ -275,6 +275,7 @@ interface Window { hudOverlayClose: () => void; setHudOverlayIgnoreMouseEvents: (ignore: boolean) => void; moveHudOverlayBy: (deltaX: number, deltaY: number) => void; + setHudOverlaySize: (width: number, height: number) => void; showCountdownOverlay: (value: number, runId: number) => Promise; setCountdownOverlayValue: (value: number, runId: number) => Promise; hideCountdownOverlay: (runId: number) => Promise; diff --git a/electron/globalShortcut.ts b/electron/globalShortcut.ts index f1b046a34..2765bad22 100644 --- a/electron/globalShortcut.ts +++ b/electron/globalShortcut.ts @@ -41,16 +41,14 @@ let currentAccelerator: string | null = null; export function registerOpenAppShortcut(binding: ShortcutBinding, onTrigger: () => void): boolean { const accelerator = bindingToAccelerator(binding); - // Same shortcut already registered, nothing to do if (accelerator === currentAccelerator) { return true; } - // Try to register new shortcut first (before unregistering old one) + // Register the new shortcut before unregistering the old, so a failure leaves the old binding intact const success = globalShortcut.register(accelerator, onTrigger); if (success) { - // Only unregister old shortcut after new one succeeds if (currentAccelerator) { globalShortcut.unregister(currentAccelerator); } diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 15a6539a7..1eb569efe 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -62,10 +62,7 @@ const ALLOWED_IMPORT_VIDEO_EXTENSIONS = new Set([ const PREVIEW_AUDIO_DIR = path.join(app.getPath("userData"), "preview-audio"); const nativeMacCaptureEvents = new EventEmitter(); -/** - * Paths explicitly approved by the user via file picker dialogs or project loads. - * These are added at runtime when the user selects files from outside the default directories. - */ +// Paths the user approved via file picker or project load (i.e. outside the default dirs). const approvedPaths = new Set(); function approveFilePath(filePath: string): void { @@ -101,10 +98,7 @@ function resolveApprovedVideoPath(videoPath?: string | null): string | null { return normalizedPath; } -/** - * Helper function to build dialog options with a parent window only when it's valid. - * This prevents passing stale or destroyed BrowserWindow references to dialog calls. - */ +// Attach the parent window only when valid, to avoid passing a destroyed BrowserWindow to dialogs. function buildDialogOptions( baseOptions: T, parentWindow: BrowserWindow | null, @@ -233,9 +227,8 @@ async function approveReadableVideoPath( return null; } - // When called with trustedDirs (e.g. from project load), only auto-approve - // paths within those directories. This prevents malicious project files from - // approving reads to arbitrary filesystem locations. + // With trustedDirs (e.g. project load), only auto-approve paths inside them so a + // malicious project file can't approve reads to arbitrary locations. if (trustedDirs) { const resolved = path.resolve(normalizedPath); const withinTrusted = trustedDirs.some((dir) => isPathWithinDir(resolved, dir)); @@ -282,11 +275,9 @@ function isValidDurationMs(value: number | undefined): value is number { } /** - * Finalize a single recording file: if it was streamed to disk, flush and close - * the stream; otherwise (a short recording, or the stream failed to open and the - * renderer fell back to in-memory buffering) write the buffered bytes. Returns - * whether the file was streamed, which the caller uses to decide whether the - * WebM duration needs patching on disk. + * Finalize one recording file: flush/close the stream if it was streamed, else write + * the buffered bytes (short recording or stream failed to open). Returns whether it was + * streamed, so the caller knows if the WebM duration needs patching on disk. */ async function finalizeRecordingFile( registry: RecordingStreamRegistry, @@ -322,8 +313,8 @@ async function getApprovedProjectSession( return null; } - // Only auto-approve media paths within the project's directory or RECORDINGS_DIR. - // This prevents crafted project files from approving reads to arbitrary locations. + // Only auto-approve media within the project's dir or RECORDINGS_DIR, so a crafted + // project file can't approve reads to arbitrary locations. const trustedDirs = [RECORDINGS_DIR]; if (projectFilePath) { trustedDirs.push(path.dirname(path.resolve(projectFilePath))); @@ -366,10 +357,7 @@ let lastEnumeratedSources = new Map(); let currentProjectPath: string | null = null; let currentRecordingSession: RecordingSession | null = null; -/** - * Returns the cached DesktopCapturerSource set when the user picked a source. - * Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. - */ +// Cached source from the user's pick. Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture. export function getSelectedDesktopSource(): DesktopCapturerSource | null { return selectedDesktopSource; } @@ -1290,7 +1278,7 @@ export function registerIpcHandlers( return { success: true, granted: true, status }; } - // Screen recording has no askForMediaAccess equivalent. Trigger the + // Screen recording has no askForMediaAccess equivalent, so trigger the // TCC prompt without opening OpenScreen's source selector above it. if (status === "not-determined") { const mainWin = getMainWindow(); @@ -1396,7 +1384,36 @@ export function registerIpcHandlers( }); ipcMain.handle("request-native-mac-cursor-access", async () => { - return requestMacCursorAccessibilityAccess(); + const access = await requestMacCursorAccessibilityAccess(); + + // When the editable cursor can't get Accessibility trust, pop a native dialog + // that deep-links to the Accessibility pane (mirrors the Screen Recording flow). + if (process.platform === "darwin" && !access.granted) { + const mainWin = getMainWindow(); + const detail = + access.status === "missing-helper" + ? "The cursor helper couldn't be found in this build, so the editable cursor can't be enabled. Rebuild the native helper (npm run build:native:mac) or switch the HUD cursor mode to system." + : "Allow OpenScreen under System Settings → Privacy & Security → Accessibility, then press record again to start the countdown."; + const messageOptions = { + type: "warning", + buttons: ["Open Accessibility Settings", "Cancel"], + defaultId: 0, + cancelId: 1, + message: "Accessibility access is required for the editable cursor", + detail, + } satisfies Electron.MessageBoxOptions; + const result = + mainWin && !mainWin.isDestroyed() + ? await dialog.showMessageBox(mainWin, messageOptions) + : await dialog.showMessageBox(messageOptions); + if (result.response === 0) { + await shell.openExternal( + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility", + ); + } + } + + return access; }); ipcMain.handle("open-source-selector", async () => { @@ -1440,10 +1457,9 @@ export function registerIpcHandlers( }); ipcMain.handle("switch-to-editor", () => { - // createEditorWindow is createEditorWindowWrapper — it already closes - // the current mainWindow (the HUD) before opening the editor. Closing - // it here too causes a double-close which leaves ghost transparent - // windows and makes the HUD shadow compound on each cycle. + // createEditorWindow already closes the current mainWindow (the HUD) before + // opening the editor. Closing it here too double-closes, leaving ghost + // transparent windows and compounding the HUD shadow each cycle. createEditorWindow(); }); @@ -1463,9 +1479,8 @@ export function registerIpcHandlers( return; } - // Wait for the first frame to be painted before showing the window. - // Showing before ready-to-show produces a black rectangle flash because - // Chromium hasn't rendered any pixels yet. + // Wait for the first frame before showing, else Chromium flashes a black + // rectangle because it hasn't rendered any pixels yet. if (overlayWindow.webContents.isLoading()) { await new Promise((resolve) => { overlayWindow.once("ready-to-show", resolve); @@ -2181,8 +2196,7 @@ export function registerIpcHandlers( ); // On-disk write streams for in-progress recordings, keyed by output file name. - // Chunks are appended as they arrive from ondataavailable so the renderer - // never buffers the full video in memory (the #616 fix). + // Chunks append as they arrive so the renderer never buffers the full video (#616). const recordingStreams = new RecordingStreamRegistry(); registerRecordingStreamHandlers(ipcMain, recordingStreams, resolveRecordingOutputPath); @@ -2225,9 +2239,9 @@ export function registerIpcHandlers( ); } - // Streamed files lack the WebM Duration header (the renderer no longer holds - // the blob to patch). Patch on disk so the editor's seek bar and timeline - // work. Best-effort and independent per file, so the patches run together. + // Streamed files lack the WebM Duration header (renderer no longer holds the + // blob), so patch on disk for the editor's seek bar and timeline. Best-effort, + // independent per file, so they run together. if (isValidDurationMs(payload.durationMs)) { const patches: Promise[] = []; if (screenStreamed) { @@ -2358,9 +2372,8 @@ export function registerIpcHandlers( ? [{ name: mainT("dialogs", "fileDialogs.gifImage"), extensions: ["gif"] }] : [{ name: mainT("dialogs", "fileDialogs.mp4Video"), extensions: ["mp4"] }]; - // Prefer the user's last export folder if it still exists, otherwise fall - // back to ~/Downloads. Validation must happen here because the renderer - // can't stat the filesystem. + // Prefer the user's last export folder if it still exists, else ~/Downloads. + // Validate here because the renderer can't stat the filesystem. let defaultDir = app.getPath("downloads"); if (exportFolder) { try { @@ -2405,8 +2418,8 @@ export function registerIpcHandlers( ipcMain.handle("write-export-to-path", async (_, videoData: ArrayBuffer, filePath: string) => { try { - // Sanity-check the path. The renderer is trusted (contextIsolation is on), - // but a stale state bug shouldn't be able to clobber arbitrary files. + // Sanity-check the path: the renderer is trusted (contextIsolation on), but a + // stale-state bug shouldn't be able to clobber arbitrary files. if (typeof filePath !== "string" || !path.isAbsolute(filePath)) { return { success: false, message: "Invalid path" }; } @@ -2482,14 +2495,13 @@ export function registerIpcHandlers( ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { - // shell.showItemInFolder doesn't return a value, it throws on error + // showItemInFolder returns nothing, it throws on error shell.showItemInFolder(filePath); return { success: true }; } catch (error) { console.error(`Error revealing item in folder: ${filePath}`, error); - // Fallback to open the directory if revealing the item fails - // This might happen if the file was moved or deleted after export, - // or if the path is somehow invalid for showItemInFolder + // Fall back to opening the directory if revealing fails (file moved/deleted + // after export, or a path showItemInFolder rejects). try { const openPathResult = await shell.openPath(path.dirname(filePath)); if (openPathResult) { @@ -2622,16 +2634,34 @@ export function registerIpcHandlers( } } - ipcMain.handle("load-project-file", async () => { - return loadProjectFile(); + ipcMain.handle("load-project-file", async (_, projectFolder?: string) => { + return loadProjectFile(projectFolder); }); - async function loadProjectFile(): Promise { + async function loadProjectFile(projectFolder?: string): Promise { try { + // Prefer the user's last opened-project folder if it still exists, else + // RECORDINGS_DIR. Validate here because the renderer can't stat the filesystem. + let defaultDir = RECORDINGS_DIR; + if (projectFolder) { + try { + const stats = await fs.stat(projectFolder); + if (stats.isDirectory()) { + defaultDir = projectFolder; + } + } catch (err) { + // Stat can fail if the folder was moved/deleted (expected) or on a + // permission error (worth surfacing). We fall back either way, but log it. + console.warn( + `Could not access remembered project folder "${projectFolder}", falling back to RECORDINGS_DIR:`, + err, + ); + } + } const dialogOptions = buildDialogOptions( { title: mainT("dialogs", "fileDialogs.openProject"), - defaultPath: RECORDINGS_DIR, + defaultPath: defaultDir, filters: [ { name: mainT("dialogs", "fileDialogs.openscreenProject"), @@ -2692,9 +2722,8 @@ export function registerIpcHandlers( const project = JSON.parse(content); currentProjectPath = filePath; - // Approve session paths; tolerate failures (e.g. video moved outside - // trusted dirs) so the project still loads and the renderer can surface - // a "video not found" error rather than a generic load failure. + // Approve session paths but tolerate failures (e.g. video moved outside trusted + // dirs) so the project still loads and the renderer can show "video not found". let session: import("../../src/lib/recordingSession").RecordingSession | null = null; try { session = await getApprovedProjectSession(project, filePath); diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 425f93e1a..2669300a9 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -25,7 +25,7 @@ export interface NativeBridgeContext { suggestedName?: string, existingProjectPath?: string, ) => Promise; - loadProjectFile: () => Promise; + loadProjectFile: (projectFolder?: string) => Promise; loadCurrentProjectFile: () => Promise; loadProjectFileFromPath: (path: string) => Promise; setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; @@ -164,7 +164,10 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { ), ); case "loadProjectFile": - return createSuccessResponse(requestId, await projectService.loadProjectFile()); + return createSuccessResponse( + requestId, + await projectService.loadProjectFile(request.payload?.projectFolder), + ); case "loadCurrentProjectFile": return createSuccessResponse( requestId, diff --git a/electron/ipc/recordingStream.ts b/electron/ipc/recordingStream.ts index 3dce5b955..665ea4194 100644 --- a/electron/ipc/recordingStream.ts +++ b/electron/ipc/recordingStream.ts @@ -3,23 +3,18 @@ import { unlink } from "node:fs/promises"; import type { IpcMain } from "electron"; /** - * Owns the lifecycle of on-disk write streams for in-progress recordings, keyed - * by the recording's output file name. Browser MediaRecorder chunks are appended - * here as they arrive so a long recording never buffers the whole video in the - * renderer (the #616 fix). - * - * The file name is the key because it is the one value the renderer and main - * process already exchange and it is globally unique per recording, so there is - * no derived/offset key to keep in sync across the IPC boundary. + * Owns write streams for in-progress recordings, keyed by output file name. + * MediaRecorder chunks are appended as they arrive so a long recording never + * buffers the whole video in the renderer (#616 fix). File name is the key + * because it's already exchanged across IPC and is unique per recording. */ export class RecordingStreamRegistry { private readonly streams = new Map(); /** - * Open a write stream and resolve only once the OS confirms it is writable. - * Resolving on the `open` event (rather than on `createWriteStream` returning) - * means a bad path or permission error rejects here instead of surfacing as a - * silent chunk drop later, so the renderer's fallback can take over. + * Open a write stream, resolving only on the `open` event so a bad path or + * permission error rejects here instead of becoming a silent chunk drop later, + * letting the renderer's fallback take over. */ async open(fileName: string, filePath: string): Promise { await this.endStream(fileName); @@ -33,9 +28,8 @@ export class RecordingStreamRegistry { resolve(); }); }); - // Keep a listener for the stream's lifetime so a late error logs rather - // than crashing the main process with an unhandled 'error' event. Per-write - // failures still surface through the `append` callback below. + // Keep a lifetime listener so a late error logs instead of crashing the main + // process with an unhandled 'error'. Per-write failures still surface in `append`. ws.on("error", (error) => { console.error(`[recording-stream] ${fileName}:`, error); }); @@ -59,9 +53,8 @@ export class RecordingStreamRegistry { } /** - * Flush and close the stream, keeping the file. Returns whether a stream was - * open — i.e. whether the recording was streamed to disk (true) or needs its - * in-memory buffer written by the caller (false). + * Flush and close the stream, keeping the file. Returns true if a stream was + * open (streamed to disk) or false if the caller still needs to write its buffer. */ async finalize(fileName: string): Promise { const ws = this.streams.get(fileName); @@ -76,9 +69,8 @@ export class RecordingStreamRegistry { } /** - * Close the stream (if any) and delete the partial file. Used when a streamed - * recording is discarded or fails before a successful save, so cancelled runs - * don't leak file descriptors or orphan partial recordings on disk. + * Close the stream (if any) and delete the partial file, so a discarded or + * failed recording doesn't leak descriptors or orphan partial files on disk. */ async discard(fileName: string, filePath: string): Promise { await this.endStream(fileName); diff --git a/electron/main.ts b/electron/main.ts index 14255d5b3..bfb949ed7 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -28,15 +28,14 @@ import { const __dirname = path.dirname(fileURLToPath(import.meta.url)); -// Use Screen & System Audio Recording permissions instead of CoreAudio Tap API on macOS. -// CoreAudio Tap requires NSAudioCaptureUsageDescription in the parent app's Info.plist, -// which doesn't work when running from a terminal/IDE during development, makes my life easier +// Use Screen & System Audio Recording permissions instead of the CoreAudio Tap API on macOS. +// Tap needs NSAudioCaptureUsageDescription in the parent app's Info.plist, which breaks when +// running from a terminal/IDE during dev. if (process.platform === "darwin") { app.commandLine.appendSwitch("disable-features", "MacCatapLoopbackAudioForScreenShare"); } -// Enable Wayland support for proper screen capture and window management -// on Wayland compositors (Hyprland, GNOME, KDE, etc.) +// Wayland support for screen capture and window management on Wayland compositors. if (process.platform === "linux") { const isWayland = process.env.XDG_SESSION_TYPE === "wayland" || process.env.WAYLAND_DISPLAY !== undefined; @@ -384,7 +383,7 @@ function createEditorWindowWrapper() { const windowToClose = mainWindow; if (!windowToClose || windowToClose.isDestroyed()) return; - // Ask renderer to show the custom in-app dialog + // Ask renderer to show the in-app close dialog. windowToClose.webContents.send("request-close-confirm"); ipcMain.once("close-confirm-response", (event, choice: "save" | "discard" | "cancel") => { @@ -393,7 +392,7 @@ function createEditorWindowWrapper() { if (!windowToClose || windowToClose.isDestroyed()) return; if (choice === "save") { - // Tell renderer to save the project, then close when done + // Save first, then close when the renderer reports done. windowToClose.webContents.send("request-save-before-close"); ipcMain.once("save-before-close-done", (event, shouldClose: boolean) => { if (event.sender.id !== windowToClose?.webContents.id) return; @@ -428,16 +427,14 @@ function createCountdownOverlayWindowWrapper() { return countdownOverlayWindow; } -// Closing every window quits the app entirely (tray icon goes too). -// The in-app "Return to Recorder" button covers the editor → HUD round-trip, -// so closing the last window is an explicit "I'm done" signal. +// Closing every window quits the app (tray goes too). The in-app "Return to Recorder" +// button covers the editor-to-HUD round-trip, so closing the last window means "I'm done". app.on("window-all-closed", () => { app.quit(); }); app.on("activate", () => { - // On OS X it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. + // On macOS, re-open a window when the dock icon is clicked and none are open. const hasVisibleWindow = BrowserWindow.getAllWindows().some((window) => { if (window.isDestroyed() || !window.isVisible()) { return false; @@ -456,16 +453,14 @@ app.on("will-quit", () => { unregisterAllGlobalShortcuts(); }); -// Register all IPC handlers when app is ready app.whenReady().then(async () => { - // Force the app into "regular" activation policy so the Dock icon appears. - // The HUD overlay (transparent + frameless + skipTaskbar) is the first - // window we open, and AppKit otherwise classifies us as an accessory app. + // Force "regular" activation policy so the Dock icon appears. The HUD overlay + // (transparent, frameless, skipTaskbar) is the first window, and AppKit would + // otherwise classify us as an accessory app. if (process.platform === "darwin") { app.dock?.show(); } - // Allow microphone/media/screen permission checks session.defaultSession.setPermissionCheckHandler((_webContents, permission) => { const allowed = [ "media", @@ -508,9 +503,8 @@ app.whenReady().then(async () => { { useSystemPicker: false }, ); - // Request microphone permission from macOS. Screen Recording is requested - // lazily from the source-picker action so the system prompt is not hidden - // behind OpenScreen's source selector window. + // Request mic permission now. Screen Recording is requested lazily from the + // source-picker action so its prompt isn't hidden behind the selector window. if (process.platform === "darwin") { const micStatus = systemPreferences.getMediaAccessStatus("microphone"); if (micStatus !== "granted") { @@ -518,7 +512,6 @@ app.whenReady().then(async () => { } } - // Listen for HUD overlay quit event (macOS only) ipcMain.on("hud-overlay-close", () => { app.quit(); }); @@ -536,7 +529,6 @@ app.whenReady().then(async () => { createTray(); updateTrayMenu(); setupApplicationMenu(); - // Ensure recordings directory exists await ensureRecordingsDir(); function switchToHudWrapper() { diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts index 0ba307788..9a82ffd55 100644 --- a/electron/native-bridge/cursor/recording/factory.ts +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -36,7 +36,7 @@ export function createCursorRecordingSession( } // Linux: capture cursor positions via Electron's `screen` API on an interval. - // No cursor sprites/assets and no clicks — just position telemetry. + // No cursor sprites/assets and no clicks, just position telemetry. return new TelemetryRecordingSession({ getDisplayBounds: options.getDisplayBounds, maxSamples: options.maxSamples, diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index 5e09e9298..27a8a870c 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -193,7 +193,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { private readyTimer: NodeJS.Timeout | null = null; private previousLeftButtonDown = false; private consecutiveOutsideSamples = 0; - // Only hide after this many consecutive out-of-bounds samples (≈100ms at 33ms interval). + // Hide only after this many consecutive out-of-bounds samples (~100ms at 33ms interval). // Fast swipes that briefly exit the display are clipped by clip-path instead of disappearing. private static readonly OUTSIDE_HIDE_THRESHOLD = 3; @@ -211,7 +211,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { systemPreferences.isTrustedAccessibilityClient(true); } catch { // Without Accessibility, text/pointer affordance detection is unavailable; - // cursor bitmaps are still captured natively via NSCursor. + // bitmaps are still captured natively via NSCursor. } const helperPath = findMacCursorHelperPath(); @@ -370,10 +370,9 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { const normalizedY = (cursor.y - bounds.y) / height; const isOutsideDisplay = normalizedX < 0 || normalizedX > 1 || normalizedY < 0 || normalizedY > 1; - // Fast swipes that briefly exit the display (=THRESHOLD, ~100ms) mark visible=false to + // avoid ghost cursors and motion trails from multi-display movement. if (isOutsideDisplay) { this.consecutiveOutsideSamples++; } else { diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts index 5c318f0f2..ea037b3fa 100644 --- a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -22,10 +22,14 @@ function getCursorSamplerCandidates(): string[] { const p = join(app.getAppPath(), ...segs); return app.isPackaged ? p.replace(/\.asar([/\\])/, ".asar.unpacked$1") : p; }; + const resolvePackaged = (...segs: string[]) => { + return app.isPackaged ? join(process.resourcesPath, ...segs) : null; + }; return [ envPath, resolve("electron", "native", "wgc-capture", "build", "cursor-sampler.exe"), resolve("electron", "native", "bin", archTag, "cursor-sampler.exe"), + resolvePackaged("electron", "native", "bin", archTag, "cursor-sampler.exe"), ].filter((c): c is string => Boolean(c)); } diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts index 9e96aa22d..0c363cc13 100644 --- a/electron/native-bridge/services/projectService.ts +++ b/electron/native-bridge/services/projectService.ts @@ -14,7 +14,7 @@ interface ProjectServiceOptions { suggestedName?: string, existingProjectPath?: string, ) => Promise; - loadProjectFile: () => Promise; + loadProjectFile: (projectFolder?: string) => Promise; loadCurrentProjectFile: () => Promise; loadProjectFileFromPath: (path: string) => Promise; setCurrentVideoPath: (path: string) => ProjectPathResult | Promise; @@ -49,8 +49,8 @@ export class ProjectService { return result; } - async loadProjectFile() { - const result = await this.options.loadProjectFile(); + async loadProjectFile(projectFolder?: string) { + const result = await this.options.loadProjectFile(projectFolder); this.getCurrentContext(); return result; } diff --git a/electron/native/wgc-capture/CMakeLists.txt b/electron/native/wgc-capture/CMakeLists.txt index 32c5d6ef5..4b9a24a4e 100644 --- a/electron/native/wgc-capture/CMakeLists.txt +++ b/electron/native/wgc-capture/CMakeLists.txt @@ -37,7 +37,7 @@ target_compile_definitions(wgc-capture PRIVATE _WIN32_WINNT=0x0A00 ) -target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8) +target_compile_options(wgc-capture PRIVATE /EHsc /W4 /utf-8 /await) target_link_libraries(wgc-capture PRIVATE d3d11 diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index bb741d33e..9ea4e59df 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -632,8 +632,8 @@ int main(int argc, char* argv[]) { (webcamOutputFrameIndex * 10'000'000ULL) / std::max(1, webcamCapture.fps())); if (!webcamEncoder.writeBgraFrame(webcamFrame, webcamTimestampHns)) { encodeFailed = true; - stopRequested = true; - cv.notify_all(); + control.stopRequested = true; + control.cv.notify_all(); return; } lastWrittenWebcamSequence = latestWebcamSequence; diff --git a/electron/preload.ts b/electron/preload.ts index a89d296ee..96cc831fd 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -29,6 +29,9 @@ contextBridge.exposeInMainWorld("electronAPI", { moveHudOverlayBy: (deltaX: number, deltaY: number) => { ipcRenderer.send("hud-overlay-move-by", deltaX, deltaY); }, + setHudOverlaySize: (width: number, height: number) => { + ipcRenderer.send("hud-overlay-set-size", width, height); + }, getSources: async (opts: Electron.SourcesOptions) => { return await ipcRenderer.invoke("get-sources", opts); }, @@ -170,8 +173,8 @@ contextBridge.exposeInMainWorld("electronAPI", { saveProjectFile: (projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { return ipcRenderer.invoke("save-project-file", projectData, suggestedName, existingProjectPath); }, - loadProjectFile: () => { - return ipcRenderer.invoke("load-project-file"); + loadProjectFile: (projectFolder?: string) => { + return ipcRenderer.invoke("load-project-file", projectFolder); }, loadProjectFileFromPath: (filePath: string) => { return ipcRenderer.invoke("load-project-file-from-path", filePath); diff --git a/electron/recording/webm-duration.ts b/electron/recording/webm-duration.ts index 5b2c197c9..a93451f0a 100644 --- a/electron/recording/webm-duration.ts +++ b/electron/recording/webm-duration.ts @@ -9,23 +9,16 @@ export type DurationPatchResult = /** * Patch the WebM Duration header on a finalized recording file. * - * Browser MediaRecorder writes WebM with no Duration EBML element. With the - * streaming-to-disk path the renderer never holds the blob, so the historical - * `fixWebmDuration(blob, durationMs)` call can't run. Patching on disk after - * `WriteStream.end()` produces an equivalent result: the editor's seek bar and - * timeline read a real duration instead of `N/A`. + * MediaRecorder writes WebM with no Duration EBML element, and the streaming-to-disk + * path never holds the blob so the old `fixWebmDuration(blob, durationMs)` can't run. + * Patching on disk after `WriteStream.end()` gives the editor a real duration instead of `N/A`. * - * Atomic by design: writes the patched bytes to `.duration-patch.tmp` - * and renames in place. If the process crashes mid-rewrite, the original file - * survives intact, so the user never loses their recording to a partial write. + * Atomic: writes to `.duration-patch.tmp` and renames in place, so a mid-rewrite + * crash leaves the original intact. Best-effort: any read/parse/write failure logs and returns + * a non-`patched` result rather than throwing; the file still plays without the patch (decoders + * walk frames sequentially), only the seek bar and timeline break. * - * Best-effort by intent: any failure (read, parse, write) logs and returns a - * non-`patched` result rather than throwing. The file is still playable without - * the patch (decoders walk frames sequentially); the only cost is that the - * editor's seek bar and timeline break until it is patched. - * - * Memory: reads the whole file into a main-process Buffer, the same footprint - * as the pre-streaming renderer path, just on the side without V8's heap cap. + * Reads the whole file into a main-process Buffer, off the renderer so it dodges V8's heap cap. */ export async function patchWebmDurationOnDisk( filePath: string, @@ -37,9 +30,8 @@ export async function patchWebmDurationOnDisk( const patched = fixParsedWebmDuration(webm, durationMs, { logger: false }); if (!patched) { - // fixParsedWebmDuration returns false for: missing Segment, missing - // Info, or a Duration that is already valid. The first two mean a - // malformed (most likely truncated) file; the third is a no-op. + // false means missing Segment, missing Info, or an already-valid Duration. + // The first two mean a malformed (likely truncated) file; the third is a no-op. const reason = inferUnpatchedReason(webm); if (reason === "no-section") { console.warn( @@ -66,8 +58,7 @@ export async function patchWebmDurationOnDisk( return { patched: true }; } catch (writeError) { console.error(`[webm-duration] failed to write patched ${filePath}:`, writeError); - // Best-effort cleanup of the temp file; if unlink also fails, leave it. - // The original recording is untouched because the rename never ran. + // Clean up the temp file; the original is untouched since the rename never ran. await fs.unlink(tmpPath).catch(() => undefined); return { patched: false, reason: "io-error" }; } @@ -78,14 +69,12 @@ export async function patchWebmDurationOnDisk( } /** - * Distinguish "no Segment/Info section" (malformed/truncated file) from "Info - * present but Duration already valid" (patch unnecessary). + * Distinguish "no Segment/Info section" (malformed/truncated file) from "Info present + * but Duration already valid" (patch unnecessary). * - * The IDs are the length-descriptor-stripped form that @fix-webm-duration/parser - * uses as its lookup keys (Segment `0x8538067`, Info `0x549a966`), verified - * against the parser's `src/lib/sections.js` — not the canonical 4-byte EBML - * IDs (`0x18538067` / `0x1549A966`), which this parser's `getSectionById` would - * never match. + * The IDs are the length-descriptor-stripped form @fix-webm-duration/parser uses as lookup + * keys (Segment `0x8538067`, Info `0x549a966`), per the parser's `src/lib/sections.js`, not + * the canonical 4-byte EBML IDs (`0x18538067` / `0x1549A966`) that `getSectionById` never matches. */ function inferUnpatchedReason(webm: WebmFile): "no-section" | "already-valid" { const segment = webm.getSectionById?.(0x8538067); diff --git a/electron/windows.ts b/electron/windows.ts index 5d34fe80c..c3830157f 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -10,7 +10,7 @@ const RENDERER_DIST = path.join(APP_ROOT, "dist"); const HEADLESS = process.env["HEADLESS"] === "true"; // Asset base URL for renderer (wallpapers, etc.). Packaged: extraResources copies -// public/wallpapers -> resources/wallpapers. Unpackaged: /public/. +// public/wallpapers to resources/wallpapers. Unpackaged: /public/. const ASSET_BASE_DIR = process.defaultApp ? path.join(__dirname, "..", "public") : process.resourcesPath; @@ -44,10 +44,45 @@ ipcMain.on("hud-overlay-move-by", (_event, deltaX: number, deltaY: number) => { hudOverlayWindow.setPosition(Math.round(x + deltaX), Math.round(y + deltaY), false); }); +// Resize the HUD to fit its rendered content. Anchored by its bottom-centre so it +// stays where the user dragged it while only growing/shrinking, which lets the +// vertical tray layout grow tall instead of scrolling inside a fixed window. +ipcMain.on("hud-overlay-set-size", (_event, width: number, height: number) => { + if ( + !hudOverlayWindow || + hudOverlayWindow.isDestroyed() || + !Number.isFinite(width) || + !Number.isFinite(height) + ) { + return; + } + + const bounds = hudOverlayWindow.getBounds(); + + // Clamp to the work area of the display the HUD sits on; on a short screen the + // vertical layout can exceed the display, where the bar's own overflow scroll takes over. + const { workArea } = screen.getDisplayMatching(bounds); + const nextWidth = Math.min(workArea.width, Math.max(1, Math.round(width))); + const nextHeight = Math.min(workArea.height, Math.max(1, Math.round(height))); + + if (bounds.width === nextWidth && bounds.height === nextHeight) { + return; + } + + const centerX = bounds.x + bounds.width / 2; + const bottomY = bounds.y + bounds.height; + + hudOverlayWindow.setBounds({ + x: Math.round(centerX - nextWidth / 2), + y: Math.round(bottomY - nextHeight), + width: nextWidth, + height: nextHeight, + }); +}); + /** - * Creates the always-on-top HUD overlay window centred at the bottom of the - * primary display. The window is frameless, transparent, and follows the user - * across macOS Spaces so it is never lost when switching virtual desktops. + * Frameless transparent HUD overlay, always-on-top, centred at the bottom of the + * primary display. Follows the user across macOS Spaces so it isn't lost on switch. */ export function createHudOverlayWindow(): BrowserWindow { const primaryDisplay = screen.getPrimaryDisplay(); @@ -62,14 +97,20 @@ export function createHudOverlayWindow(): BrowserWindow { const win = new BrowserWindow({ width: windowWidth, height: windowHeight, - minWidth: 600, - maxWidth: 600, - minHeight: 160, - maxHeight: 160, + // Min/max are intentionally loose: the renderer resizes to fit content via + // "hud-overlay-set-size" (above), needed for the vertical tray to grow taller. + minWidth: 120, + minHeight: 80, x: x, y: y, frame: false, transparent: true, + // Fully-transparent ARGB backing. Without this macOS draws the window as a + // rounded glass panel with a border around the HUD content. + backgroundColor: "#00000000", + // Don't let macOS mask the window into a rounded rect; the HUD bar provides + // its own rounding and the window itself must be invisible. + roundedCorners: false, resizable: false, alwaysOnTop: true, skipTaskbar: true, @@ -85,14 +126,14 @@ export function createHudOverlayWindow(): BrowserWindow { }); win.setIgnoreMouseEvents(true, { forward: true }); - // Follow the user across macOS Spaces (virtual desktops). - // Without this the HUD stays pinned to the Space it was first opened on. + // Follow the user across macOS Spaces, else the HUD stays pinned to the Space + // it was first opened on. if (process.platform === "darwin") { win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); } - // Show only once content is painted — prevents the black rectangle flash - // that appears when a transparent window is shown before its first paint. + // Show only once painted to avoid the black rectangle flash when a transparent + // window is shown before its first paint. win.once("ready-to-show", () => { if (!HEADLESS) win.show(); }); @@ -121,8 +162,8 @@ export function createHudOverlayWindow(): BrowserWindow { } /** - * Creates the main editor window. Starts maximised with a hidden title bar on - * macOS. This window is not always-on-top and appears in the taskbar/dock. + * Main editor window. Starts maximised with a hidden title bar on macOS; not + * always-on-top and appears in the taskbar/dock. */ export function createEditorWindow(): BrowserWindow { const isMac = process.platform === "darwin"; @@ -153,16 +194,15 @@ export function createEditorWindow(): BrowserWindow { }, }); - // Maximize the window by default win.maximize(); - // Show only once content is painted — prevents white flash on cold Vite start. + // Show only once painted to avoid a white flash on cold Vite start. win.once("ready-to-show", () => { if (!HEADLESS) win.show(); }); - // Inject dark background before any React paint so the sub-titlebar area - // never flashes white even on the very first cold Vite load. + // Inject dark background before any React paint so the sub-titlebar area never + // flashes white on a cold Vite load. win.webContents.on("dom-ready", () => { win.webContents.insertCSS("html, body, #root { background: #09090b !important; }").catch(() => { // Best-effort cosmetic; ignore if the page is mid-teardown. @@ -185,8 +225,8 @@ export function createEditorWindow(): BrowserWindow { } /** - * Creates the floating source-selector window used to pick a screen or window - * to record. Frameless, transparent, and follows the user across macOS Spaces. + * Floating source-selector window for picking a screen or window to record. + * Frameless, transparent, and follows the user across macOS Spaces. */ export function createSourceSelectorWindow(): BrowserWindow { const { width, height } = screen.getPrimaryDisplay().workAreaSize; @@ -211,8 +251,8 @@ export function createSourceSelectorWindow(): BrowserWindow { }, }); - // Follow the user across macOS Spaces so the selector appears on the - // active desktop regardless of where the HUD was originally opened. + // Follow the user across macOS Spaces so the selector appears on the active + // desktop regardless of where the HUD was opened. if (process.platform === "darwin") { win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); } @@ -229,8 +269,8 @@ export function createSourceSelectorWindow(): BrowserWindow { } /** - * Creates a centered transparent countdown overlay window that sits above the - * HUD while recording pre-roll is running. + * Centered transparent countdown overlay that sits above the HUD during + * recording pre-roll. */ export function createCountdownOverlayWindow(): BrowserWindow { const { workArea } = screen.getPrimaryDisplay(); diff --git a/nix/package.nix b/nix/package.nix index 33dc4f735..bca77c91b 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -11,7 +11,7 @@ buildNpmPackage { nodejs = nodejs_22; pname = "openscreen"; - version = "1.4.0"; + version = "1.5.0"; src = let @@ -33,7 +33,7 @@ buildNpmPackage { ); }; - npmDepsHash = "sha256-tOpoJPzaZDK3HJijGHpZ0+jWsbrYyQUuw1pO0Uxcifg="; + npmDepsHash = "sha256-lx38H0qG5IrjQRekLG2N+x90Zq/emPfbxOo/qDSn7iE="; env.ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; diff --git a/package-lock.json b/package-lock.json index 50ecc9d88..f3f21713b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "openscreen", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openscreen", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@fix-webm-duration/fix": "^1.0.1", "@pixi/filter-drop-shadow": "^5.2.0", @@ -26,6 +26,7 @@ "@uiw/color-convert": "^2.10.1", "@uiw/react-color-block": "^2.10.1", "@uiw/react-color-colorful": "^2.9.2", + "@xenova/transformers": "^2.17.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "dnd-timeline": "^2.4.0", @@ -1772,6 +1773,15 @@ "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", "license": "MIT" }, + "node_modules/@huggingface/jinja": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.2.2.tgz", + "integrity": "sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -2104,6 +2114,70 @@ "dev": true, "license": "MIT" }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.1.tgz", + "integrity": "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, "node_modules/@radix-ui/number": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.1.tgz", @@ -3822,6 +3896,12 @@ "@types/node": "*" } }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT" + }, "node_modules/@types/ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", @@ -3833,7 +3913,6 @@ "version": "22.19.17", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -4293,6 +4372,20 @@ "integrity": "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==", "license": "BSD-3-Clause" }, + "node_modules/@xenova/transformers": { + "version": "2.17.2", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.17.2.tgz", + "integrity": "sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.2.2", + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, "node_modules/@xmldom/xmldom": { "version": "0.8.13", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.13.tgz", @@ -4763,11 +4856,101 @@ "node": "18 || 20 || >=22" } }, + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.1.tgz", + "integrity": "sha512-WDRsyVN52eAx/lBamKD6uyw8H4228h/x0sGGGegOamM2cd7Pag88GfMQalobXI+HaEUxpCkbKQUDOQqt9wawRw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.9.1.tgz", + "integrity": "sha512-6M5XjcnsygQNPMCMPXSK379xrJFiZ/AEMNBmFEmQW8d/789VQATvriyi5r0HYTL9TkQ26rn3kgdTG3aisbrXkQ==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.1.tgz", + "integrity": "sha512-Vp0cnjYyrEC4whYTymQ+YZi6pBpfiICZO3cfRG8sy67ZNWe951urv1x4eW1BKNngw3U+3fPYb5JQvHbCtxH7Ow==", + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.3.tgz", + "integrity": "sha512-Kccpc7ACfXaxfeInfqKcZtW4pT5YBn1mesc4sCsun6sRwtbJ4h+sNOaksUpYEJUKfN65YWC6Bw2OJEFiKxq8nQ==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -4819,6 +5002,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/boolean": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", @@ -4891,7 +5085,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -4907,7 +5100,6 @@ } ], "license": "MIT", - "optional": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -5306,11 +5498,23 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -5323,9 +5527,18 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -5529,7 +5742,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, "license": "MIT", "dependencies": { "mimic-response": "^3.1.0" @@ -5545,7 +5757,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -5554,6 +5765,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/defer-to-connect": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", @@ -5622,6 +5842,15 @@ "node": ">=6" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-node": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", @@ -6096,7 +6325,6 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -6289,6 +6517,24 @@ "license": "MIT", "peer": true }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -6368,6 +6614,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -6503,6 +6755,12 @@ "integrity": "sha512-IKlE+pNvL2R+kVL1kEhUYqRxVqeFnjiIvHWDMLFXNaqyUdFXQM2wte44EfMYJNHkW16X991t2Zg8apKkhv7OBA==", "license": "MIT" }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==", + "license": "SEE LICENSE IN LICENSE.txt" + }, "node_modules/form-data": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", @@ -6561,6 +6819,12 @@ } } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, "node_modules/fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -6716,6 +6980,12 @@ "js-binary-schema-parser": "^2.0.3" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -6883,6 +7153,12 @@ "integrity": "sha512-dMW4CWBTUK1AEEDeZc1g4xpPGIrSf9fJF960qbTZmN/QwZIWY5wgliS6JWl9/25fpTGJrMRtSjGtOmPnfjZB+A==", "license": "Standard 'no charge' license: https://gsap.com/standard-license." }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -7093,7 +7369,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -7108,8 +7383,7 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause", - "optional": true + "license": "BSD-3-Clause" }, "node_modules/indent-string": { "version": "4.0.0", @@ -7137,9 +7411,20 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", + "license": "MIT" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -7652,6 +7937,12 @@ "url": "https://github.com/chalk/slice-ansi?sponsor=1" } }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -7884,7 +8175,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7927,6 +8217,12 @@ "mkdirp": "bin/cmd.js" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/motion": { "version": "12.38.0", "resolved": "https://registry.npmjs.org/motion/-/motion-12.38.0.tgz", @@ -8023,6 +8319,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/node-abi": { "version": "4.28.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-4.28.0.tgz", @@ -8256,7 +8558,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -8278,6 +8579,50 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/onnx-proto": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz", + "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==", + "license": "MIT", + "dependencies": { + "protobufjs": "^6.8.8" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz", + "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.14.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, "node_modules/p-cancelable": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", @@ -8470,6 +8815,12 @@ "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/playwright": { "version": "1.59.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", @@ -8713,6 +9064,91 @@ "node": "^12.20.0 || >=14" } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/prebuild-install/node_modules/node-abi": { + "version": "3.92.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.92.0.tgz", + "integrity": "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/prebuild-install/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", @@ -8806,11 +9242,36 @@ "signal-exit": "^3.0.2" } }, + "node_modules/protobufjs": { + "version": "6.11.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.6.tgz", + "integrity": "sha512-k8BHqgPBOtrlougZZqF2uUk5Z7bN8f0wj+3e8M3hvtSv0NBAz4VBy5f6R5Nxq/l+i7mRFTgNZb2trxqTpHNY/A==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", - "dev": true, "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", @@ -8893,6 +9354,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, "node_modules/re-resizable": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/re-resizable/-/re-resizable-6.11.2.tgz", @@ -9091,6 +9567,20 @@ "pify": "^2.3.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -9373,6 +9863,26 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -9457,6 +9967,47 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT" + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -9570,6 +10121,60 @@ "dev": true, "license": "ISC" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", @@ -9711,6 +10316,26 @@ "dev": true, "license": "MIT" }, + "node_modules/streamx": { + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-argv": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz", @@ -9791,6 +10416,15 @@ "node": ">=8" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/sucrase": { "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", @@ -9949,6 +10583,46 @@ "node": ">=18" } }, + "node_modules/tar-fs": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/tar-stream/node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/tar/node_modules/yallist": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", @@ -9959,6 +10633,15 @@ "node": ">=18" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, "node_modules/temp": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/temp/-/temp-0.9.4.tgz", @@ -10049,6 +10732,29 @@ "dev": true, "license": "MIT" }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/text-decoder/node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -10252,6 +10958,18 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-fest": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", @@ -10294,7 +11012,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/universalify": { @@ -10806,7 +11523,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/ws": { diff --git a/package.json b/package.json index fd0c4cf3d..54e0c833d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "openscreen", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "packageManager": "npm@10.9.4", "engines": { @@ -63,6 +63,7 @@ "@uiw/color-convert": "^2.10.1", "@uiw/react-color-block": "^2.10.1", "@uiw/react-color-colorful": "^2.9.2", + "@xenova/transformers": "^2.17.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "dnd-timeline": "^2.4.0", diff --git a/public/cursors/among-us-sus-knife-and-red-animated/arrow.png b/public/cursors/among-us-sus-knife-and-red-animated/arrow.png new file mode 100644 index 000000000..84e9a3b34 Binary files /dev/null and b/public/cursors/among-us-sus-knife-and-red-animated/arrow.png differ diff --git a/public/cursors/among-us-sus-knife-and-red-animated/pointer.png b/public/cursors/among-us-sus-knife-and-red-animated/pointer.png new file mode 100644 index 000000000..b4212f732 Binary files /dev/null and b/public/cursors/among-us-sus-knife-and-red-animated/pointer.png differ diff --git a/public/cursors/black-and-rainbow-stroke-gradient-animated/arrow.png b/public/cursors/black-and-rainbow-stroke-gradient-animated/arrow.png new file mode 100644 index 000000000..adfcec8dc Binary files /dev/null and b/public/cursors/black-and-rainbow-stroke-gradient-animated/arrow.png differ diff --git a/public/cursors/black-and-rainbow-stroke-gradient-animated/pointer.png b/public/cursors/black-and-rainbow-stroke-gradient-animated/pointer.png new file mode 100644 index 000000000..2a0ebf08f Binary files /dev/null and b/public/cursors/black-and-rainbow-stroke-gradient-animated/pointer.png differ diff --git a/public/cursors/black-pixel/arrow.png b/public/cursors/black-pixel/arrow.png new file mode 100644 index 000000000..ec2d6a315 Binary files /dev/null and b/public/cursors/black-pixel/arrow.png differ diff --git a/public/cursors/black-pixel/pointer.png b/public/cursors/black-pixel/pointer.png new file mode 100644 index 000000000..ffd561641 Binary files /dev/null and b/public/cursors/black-pixel/pointer.png differ diff --git a/public/cursors/christmas-miles-morales/arrow.png b/public/cursors/christmas-miles-morales/arrow.png new file mode 100644 index 000000000..6931394ec Binary files /dev/null and b/public/cursors/christmas-miles-morales/arrow.png differ diff --git a/public/cursors/christmas-miles-morales/pointer.png b/public/cursors/christmas-miles-morales/pointer.png new file mode 100644 index 000000000..049a68e2e Binary files /dev/null and b/public/cursors/christmas-miles-morales/pointer.png differ diff --git a/public/cursors/hello-kitty-watermelon/arrow.png b/public/cursors/hello-kitty-watermelon/arrow.png new file mode 100644 index 000000000..9e6a20319 Binary files /dev/null and b/public/cursors/hello-kitty-watermelon/arrow.png differ diff --git a/public/cursors/hello-kitty-watermelon/pointer.png b/public/cursors/hello-kitty-watermelon/pointer.png new file mode 100644 index 000000000..f45fdfd06 Binary files /dev/null and b/public/cursors/hello-kitty-watermelon/pointer.png differ diff --git a/public/cursors/hollow-knight-and-game-arrow/arrow.png b/public/cursors/hollow-knight-and-game-arrow/arrow.png new file mode 100644 index 000000000..efb4b7708 Binary files /dev/null and b/public/cursors/hollow-knight-and-game-arrow/arrow.png differ diff --git a/public/cursors/hollow-knight-and-game-arrow/pointer.png b/public/cursors/hollow-knight-and-game-arrow/pointer.png new file mode 100644 index 000000000..dcea8e91d Binary files /dev/null and b/public/cursors/hollow-knight-and-game-arrow/pointer.png differ diff --git a/public/cursors/hollow-knight-nail-sword-and-mask/arrow.png b/public/cursors/hollow-knight-nail-sword-and-mask/arrow.png new file mode 100644 index 000000000..dbd4f52a2 Binary files /dev/null and b/public/cursors/hollow-knight-nail-sword-and-mask/arrow.png differ diff --git a/public/cursors/hollow-knight-nail-sword-and-mask/pointer.png b/public/cursors/hollow-knight-nail-sword-and-mask/pointer.png new file mode 100644 index 000000000..ac77b710c Binary files /dev/null and b/public/cursors/hollow-knight-nail-sword-and-mask/pointer.png differ diff --git a/public/cursors/mickey-mouse-black-hand-inflated-glove/arrow.png b/public/cursors/mickey-mouse-black-hand-inflated-glove/arrow.png new file mode 100644 index 000000000..143076372 Binary files /dev/null and b/public/cursors/mickey-mouse-black-hand-inflated-glove/arrow.png differ diff --git a/public/cursors/mickey-mouse-black-hand-inflated-glove/pointer.png b/public/cursors/mickey-mouse-black-hand-inflated-glove/pointer.png new file mode 100644 index 000000000..f52a4e650 Binary files /dev/null and b/public/cursors/mickey-mouse-black-hand-inflated-glove/pointer.png differ diff --git a/public/cursors/naruto-akatsuki-cloud-arrow/arrow.png b/public/cursors/naruto-akatsuki-cloud-arrow/arrow.png new file mode 100644 index 000000000..d6361dff2 Binary files /dev/null and b/public/cursors/naruto-akatsuki-cloud-arrow/arrow.png differ diff --git a/public/cursors/naruto-akatsuki-cloud-arrow/pointer.png b/public/cursors/naruto-akatsuki-cloud-arrow/pointer.png new file mode 100644 index 000000000..fa10e9d2d Binary files /dev/null and b/public/cursors/naruto-akatsuki-cloud-arrow/pointer.png differ diff --git a/public/cursors/old-roblox/arrow.png b/public/cursors/old-roblox/arrow.png new file mode 100644 index 000000000..a55875df9 Binary files /dev/null and b/public/cursors/old-roblox/arrow.png differ diff --git a/public/cursors/old-roblox/pointer.png b/public/cursors/old-roblox/pointer.png new file mode 100644 index 000000000..d490a65fd Binary files /dev/null and b/public/cursors/old-roblox/pointer.png differ diff --git a/public/cursors/pink-glossy-arrow-and-hand-3d/arrow.png b/public/cursors/pink-glossy-arrow-and-hand-3d/arrow.png new file mode 100644 index 000000000..69ecd5b19 Binary files /dev/null and b/public/cursors/pink-glossy-arrow-and-hand-3d/arrow.png differ diff --git a/public/cursors/pink-glossy-arrow-and-hand-3d/pointer.png b/public/cursors/pink-glossy-arrow-and-hand-3d/pointer.png new file mode 100644 index 000000000..f8d5938a7 Binary files /dev/null and b/public/cursors/pink-glossy-arrow-and-hand-3d/pointer.png differ diff --git a/public/cursors/pinky-pixel/arrow.png b/public/cursors/pinky-pixel/arrow.png new file mode 100644 index 000000000..4bcd3dcaa Binary files /dev/null and b/public/cursors/pinky-pixel/arrow.png differ diff --git a/public/cursors/pinky-pixel/pointer.png b/public/cursors/pinky-pixel/pointer.png new file mode 100644 index 000000000..7eb897a74 Binary files /dev/null and b/public/cursors/pinky-pixel/pointer.png differ diff --git a/public/cursors/pokemon-neon-gengar/arrow.png b/public/cursors/pokemon-neon-gengar/arrow.png new file mode 100644 index 000000000..4dcccb360 Binary files /dev/null and b/public/cursors/pokemon-neon-gengar/arrow.png differ diff --git a/public/cursors/pokemon-neon-gengar/pointer.png b/public/cursors/pokemon-neon-gengar/pointer.png new file mode 100644 index 000000000..e7b4437f0 Binary files /dev/null and b/public/cursors/pokemon-neon-gengar/pointer.png differ diff --git a/public/cursors/sanrio-gudetama-and-arrow-kawaii/arrow.png b/public/cursors/sanrio-gudetama-and-arrow-kawaii/arrow.png new file mode 100644 index 000000000..123117b05 Binary files /dev/null and b/public/cursors/sanrio-gudetama-and-arrow-kawaii/arrow.png differ diff --git a/public/cursors/sanrio-gudetama-and-arrow-kawaii/pointer.png b/public/cursors/sanrio-gudetama-and-arrow-kawaii/pointer.png new file mode 100644 index 000000000..37afa7bd1 Binary files /dev/null and b/public/cursors/sanrio-gudetama-and-arrow-kawaii/pointer.png differ diff --git a/public/cursors/sanrio-kuromi-skull-arrow/arrow.png b/public/cursors/sanrio-kuromi-skull-arrow/arrow.png new file mode 100644 index 000000000..6fe7e89d6 Binary files /dev/null and b/public/cursors/sanrio-kuromi-skull-arrow/arrow.png differ diff --git a/public/cursors/sanrio-kuromi-skull-arrow/pointer.png b/public/cursors/sanrio-kuromi-skull-arrow/pointer.png new file mode 100644 index 000000000..2971a7f16 Binary files /dev/null and b/public/cursors/sanrio-kuromi-skull-arrow/pointer.png differ diff --git a/public/cursors/solo-leveling-sung-jinwoo-dark-flames/arrow.png b/public/cursors/solo-leveling-sung-jinwoo-dark-flames/arrow.png new file mode 100644 index 000000000..2155e0fd9 Binary files /dev/null and b/public/cursors/solo-leveling-sung-jinwoo-dark-flames/arrow.png differ diff --git a/public/cursors/solo-leveling-sung-jinwoo-dark-flames/pointer.png b/public/cursors/solo-leveling-sung-jinwoo-dark-flames/pointer.png new file mode 100644 index 000000000..0e0b1a0b7 Binary files /dev/null and b/public/cursors/solo-leveling-sung-jinwoo-dark-flames/pointer.png differ diff --git a/public/cursors/spring-gradient/arrow.png b/public/cursors/spring-gradient/arrow.png new file mode 100644 index 000000000..51889625d Binary files /dev/null and b/public/cursors/spring-gradient/arrow.png differ diff --git a/public/cursors/spring-gradient/pointer.png b/public/cursors/spring-gradient/pointer.png new file mode 100644 index 000000000..9ccdc2fb6 Binary files /dev/null and b/public/cursors/spring-gradient/pointer.png differ diff --git a/public/demo.png b/public/demo.png new file mode 100644 index 000000000..a0f008fe4 Binary files /dev/null and b/public/demo.png differ diff --git a/public/sample.png b/public/sample.png new file mode 100644 index 000000000..0138b39ac Binary files /dev/null and b/public/sample.png differ diff --git a/public/wallpapers/wallpaper1.jpg b/public/wallpapers/wallpaper1.jpg index 98c93cb14..dbd8afb8b 100644 Binary files a/public/wallpapers/wallpaper1.jpg and b/public/wallpapers/wallpaper1.jpg differ diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs new file mode 100644 index 000000000..934756a4d --- /dev/null +++ b/scripts/before-pack.cjs @@ -0,0 +1,14 @@ +// electron-builder beforePack hook: ensure the auto-caption assets (Whisper model + ORT wasm) exist +// before packaging, so the `caption-assets` extraResources entry has something to copy. Runs on +// every package invocation (local `npm run build:*` and CI's bare `electron-builder`). The fetch +// script is idempotent, so it's a no-op once the assets are present. + +const { execFileSync } = require("node:child_process"); +const path = require("node:path"); + +exports.default = async function beforePack() { + execFileSync("node", [path.join(__dirname, "fetch-caption-model.mjs")], { + stdio: "inherit", + cwd: path.join(__dirname, ".."), + }); +}; diff --git a/scripts/build-macos-screencapturekit-helper.mjs b/scripts/build-macos-screencapturekit-helper.mjs index 8e7c97396..b94836cff 100644 --- a/scripts/build-macos-screencapturekit-helper.mjs +++ b/scripts/build-macos-screencapturekit-helper.mjs @@ -18,14 +18,24 @@ const cursorHelperName = "openscreen-macos-cursor-helper"; const packageDir = path.join(root, "electron", "native", "screencapturekit"); const buildDir = path.join(packageDir, "build"); const swiftBuildDir = path.join(buildDir, "swiftpm"); -const builtHelperPath = path.join(swiftBuildDir, "release", helperName); const localHelperPath = path.join(buildDir, helperName); -const builtCursorHelperPath = path.join(swiftBuildDir, "release", cursorHelperName); const localCursorHelperPath = path.join(buildDir, cursorHelperName); -const archTag = process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; -const distributableDir = path.join(root, "electron", "native", "bin", archTag); -const distributablePath = path.join(distributableDir, helperName); -const distributableCursorHelperPath = path.join(distributableDir, cursorHelperName); + +// Build a separate single-arch binary per requested arch and place each in its own +// electron/native/bin/darwin- folder (the runtime resolves that folder by the running app's +// arch). No universal/fat binary. Defaults to the host arch for local builds; CI sets +// OPENSCREEN_MAC_HELPER_ARCHS per matrix entry (accepts arm64, x64, or x86_64). +function normalizeArch(value) { + return value === "x64" || value === "x86_64" + ? { swift: "x86_64", tag: "darwin-x64" } + : { swift: "arm64", tag: "darwin-arm64" }; +} +const hostArch = process.arch === "arm64" ? "arm64" : "x86_64"; +const archs = (process.env.OPENSCREEN_MAC_HELPER_ARCHS ?? hostArch) + .split(",") + .map((a) => a.trim()) + .filter(Boolean) + .map(normalizeArch); const xcodebuildVersion = spawnSync("xcodebuild", ["-version"], { cwd: root, @@ -50,42 +60,85 @@ if (xcodebuildVersion.status !== 0) { process.exit(1); } -const result = spawnSync( - "swift", - ["build", "-c", "release", "--package-path", packageDir, "--build-path", swiftBuildDir], - { - cwd: root, - stdio: "inherit", - }, -); +// SwiftPM writes a single-arch release build to /-apple-macosx/release/. +// Fall back to a search that skips the identically-named file inside the .dSYM debug bundle (matching +// that file and feeding it forward is what produced an unrunnable "exec format error" binary before). +function findExecutable(dir, swiftArch, name) { + const expected = path.join(dir, `${swiftArch}-apple-macosx`, "release", name); + if (fs.existsSync(expected)) return expected; -if (result.error) { - console.error(`Failed to start Swift build: ${result.error.message}`); - process.exit(1); -} - -if (result.status !== 0) { - process.exit(result.status ?? 1); + const stack = [dir]; + const matches = []; + while (stack.length > 0) { + const current = stack.pop(); + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + if (entry.name.endsWith(".dSYM")) continue; + const full = path.join(current, entry.name); + if (entry.isDirectory()) stack.push(full); + else if (entry.isFile() && entry.name === name && /[/\\]release[/\\]/i.test(full)) { + matches.push(full); + } + } + } + return matches[0] ?? null; } fs.mkdirSync(buildDir, { recursive: true }); -fs.mkdirSync(distributableDir, { recursive: true }); -for (const artifactPath of [builtHelperPath, builtCursorHelperPath]) { - if (!fs.existsSync(artifactPath)) { - console.error(`Swift build completed but expected artifact was not found: ${artifactPath}`); + +for (const { swift, tag } of archs) { + const archBuildDir = path.join(swiftBuildDir, swift); + const result = spawnSync( + "swift", + [ + "build", + "-c", + "release", + "--arch", + swift, + "--package-path", + packageDir, + "--build-path", + archBuildDir, + ], + { + cwd: root, + stdio: "inherit", + }, + ); + if (result.error) { + console.error(`Failed to start Swift build (${swift}): ${result.error.message}`); process.exit(1); } -} -fs.copyFileSync(builtHelperPath, localHelperPath); -fs.copyFileSync(builtHelperPath, distributablePath); -fs.copyFileSync(builtCursorHelperPath, localCursorHelperPath); -fs.copyFileSync(builtCursorHelperPath, distributableCursorHelperPath); -fs.chmodSync(localHelperPath, 0o755); -fs.chmodSync(distributablePath, 0o755); -fs.chmodSync(localCursorHelperPath, 0o755); -fs.chmodSync(distributableCursorHelperPath, 0o755); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } -console.log(`Built macOS ScreenCaptureKit helper: ${localHelperPath}`); -console.log(`Copied redistributable helper: ${distributablePath}`); -console.log(`Built macOS cursor helper: ${localCursorHelperPath}`); -console.log(`Copied redistributable cursor helper: ${distributableCursorHelperPath}`); + const targetDir = path.join(root, "electron", "native", "bin", tag); + fs.mkdirSync(targetDir, { recursive: true }); + + for (const [name, localPath] of [ + [helperName, localHelperPath], + [cursorHelperName, localCursorHelperPath], + ]) { + const exe = findExecutable(archBuildDir, swift, name); + if (!exe) { + console.error(`Swift build (${swift}) completed but executable was not found: ${name}`); + process.exit(1); + } + // Always place it in the arch's bin folder; mirror the host-arch build into the dev build + // dir so `npm run dev` (candidate path #2) can spawn it. + const dests = [path.join(targetDir, name)]; + if (swift === hostArch) dests.push(localPath); + for (const dest of dests) { + fs.copyFileSync(exe, dest); + fs.chmodSync(dest, 0o755); + } + } + console.log(`Built ${tag} helpers (${swift})`); +} diff --git a/scripts/fetch-caption-model.mjs b/scripts/fetch-caption-model.mjs new file mode 100644 index 000000000..f1d0a1f50 --- /dev/null +++ b/scripts/fetch-caption-model.mjs @@ -0,0 +1,154 @@ +// Populates `caption-assets/` so the packaged app can transcribe offline (under file://) +// instead of fetching the Whisper model from HuggingFace and the onnxruntime wasm from a CDN. +// +// caption-assets/ +// models/Xenova/whisper-tiny/... ← downloaded from HuggingFace (config + quantized ONNX) +// ort/ort-wasm*.wasm ← copied from @xenova/transformers/dist +// +// Idempotent: existing non-empty files are left alone, so re-runs and CI cache hits are no-ops. +// `caption-assets/` is gitignored and shipped via electron-builder `extraResources`. + +import { createWriteStream } from "node:fs"; +import { copyFile, mkdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const OUT = path.join(ROOT, "caption-assets"); +const MODEL_ID = "Xenova/whisper-tiny"; +const HF_BASE = `https://huggingface.co/${MODEL_ID}/resolve/main`; + +// Small config/tokenizer/preprocessor files plus the quantized ONNX the ASR pipeline loads by +// default (encoder + merged decoder). Grab every metadata file so transformers never requests +// one we forgot to bundle. +const MODEL_FILES = [ + "config.json", + "generation_config.json", + "preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "added_tokens.json", + "special_tokens_map.json", + "normalizer.json", + "merges.txt", + "vocab.json", + "quantize_config.json", + "onnx/encoder_model_quantized.onnx", + "onnx/decoder_model_merged_quantized.onnx", +]; + +async function exists(filePath) { + try { + const s = await stat(filePath); + return s.isFile() && s.size > 0; + } catch { + return false; + } +} + +const MAX_ATTEMPTS = 6; +// HuggingFace rate-limits (429) when the parallel CI matrix builds all hit it at once; also retry the +// usual transient server errors. +const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function backoffMs(attempt, retryAfter) { + // Honor Retry-After when the server sends it (seconds or an HTTP date). + if (retryAfter) { + const secs = Number(retryAfter); + if (Number.isFinite(secs)) return Math.min(60_000, secs * 1000); + const at = Date.parse(retryAfter); + if (!Number.isNaN(at)) return Math.min(60_000, Math.max(0, at - Date.now())); + } + // Exponential backoff with jitter: ~2s, 4s, 8s, 16s, 32s, capped at 60s. + return Math.min(60_000, 2000 * 2 ** (attempt - 1)) + Math.floor(Math.random() * 1000); +} + +async function fetchWithRetry(url) { + let lastErr; + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + const res = await fetch(url, { headers: { "user-agent": "openscreen-build" } }); + if (res.ok && res.body) return res; + if (RETRYABLE_STATUS.has(res.status) && attempt < MAX_ATTEMPTS) { + const wait = backoffMs(attempt, res.headers.get("retry-after")); + console.log( + ` … HTTP ${res.status}, retry ${attempt}/${MAX_ATTEMPTS - 1} in ${Math.round(wait / 1000)}s`, + ); + await sleep(wait); + continue; + } + throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`); + } catch (err) { + lastErr = err; + const isHttp = err instanceof Error && err.message.startsWith("Failed to download"); + if (isHttp || attempt >= MAX_ATTEMPTS) throw err; + // Network/DNS error: back off and retry. + const wait = backoffMs(attempt, null); + console.log( + ` … ${err.message}, retry ${attempt}/${MAX_ATTEMPTS - 1} in ${Math.round(wait / 1000)}s`, + ); + await sleep(wait); + } + } + throw lastErr; +} + +async function download(url, dest) { + if (await exists(dest)) { + console.log(` ✓ cached ${path.relative(OUT, dest)}`); + return; + } + await mkdir(path.dirname(dest), { recursive: true }); + const res = await fetchWithRetry(url); + const tmp = `${dest}.partial`; + await pipeline(Readable.fromWeb(res.body), createWriteStream(tmp)); + const { rename } = await import("node:fs/promises"); + await rename(tmp, dest); + const mb = ((await stat(dest)).size / 1_000_000).toFixed(1); + console.log(` ↓ ${path.relative(OUT, dest)} (${mb} MB)`); +} + +async function copyOrtWasm() { + const distDir = path.join(ROOT, "node_modules", "@xenova", "transformers", "dist"); + // Non-threaded variants only: the worker runs ORT with numThreads=1 (no SharedArrayBuffer + // under file://), so the threaded wasm is never loaded. Saves ~20MB. + const wasm = ["ort-wasm.wasm", "ort-wasm-simd.wasm"]; + const ortOut = path.join(OUT, "ort"); + await mkdir(ortOut, { recursive: true }); + for (const name of wasm) { + const src = path.join(distDir, name); + const dest = path.join(ortOut, name); + if (!(await exists(src))) { + throw new Error(`Missing ${src} — is @xenova/transformers installed? Run npm ci first.`); + } + if (await exists(dest)) { + console.log(` ✓ cached ort/${name}`); + continue; + } + await copyFile(src, dest); + console.log(` + copied ort/${name}`); + } +} + +async function main() { + console.log(`Fetching caption assets → ${path.relative(ROOT, OUT)}/`); + console.log("ONNX Runtime wasm:"); + await copyOrtWasm(); + console.log(`Whisper model (${MODEL_ID}):`); + const modelDir = path.join(OUT, "models", ...MODEL_ID.split("/")); + for (const rel of MODEL_FILES) { + await download(`${HF_BASE}/${rel}`, path.join(modelDir, rel)); + } + console.log("Caption assets ready."); +} + +main().catch((err) => { + console.error(`\nfetch-caption-model failed: ${err.message}`); + process.exit(1); +}); diff --git a/src/App.tsx b/src/App.tsx index 6c36aa8c5..0c8875d04 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -112,7 +112,7 @@ export default function App() { return ( {content} - + ); } diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index bba5f494c..a93514c5a 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -21,13 +21,13 @@ import { import { RxDragHandleDots2 } from "react-icons/rx"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; +import { loadUserPreferences, saveUserPreferences } from "@/lib/userPreferences"; import { nativeBridgeClient } from "@/native"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useCameraDevices } from "../../hooks/useCameraDevices"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; import { requestCameraAccess } from "../../lib/requestCameraAccess"; -import { loadUserPreferences, saveUserPreferences } from "../../lib/userPreferences"; import { formatTimePadded } from "../../utils/timeUtils"; import { AudioLevelMeter } from "../ui/audio-level-meter"; import { Button } from "../ui/button"; @@ -37,6 +37,11 @@ import { openSourceSelectorWithPermissionRetry } from "./openSourceSelectorFlow" const ICON_SIZE = 20; +// Vertical tray gap (px): bar's `bottom-5` (20px) plus an 8px gap. +const HUD_DEVICE_POPUP_GAP = 28; +// Horizontal layout: mirrors the `bottom-[68px]` class on the popup element. +const HUD_DEVICE_POPUP_HORIZONTAL_BOTTOM = 68; + const ICON_CONFIG = { drag: { icon: RxDragHandleDots2, size: ICON_SIZE }, monitor: { icon: MdMonitor, size: ICON_SIZE }, @@ -140,6 +145,10 @@ export function LaunchWindow() { const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); const languageTriggerRef = useRef(null); const languageMenuPanelRef = useRef(null); + const hudBarRef = useRef(null); + const deviceSelectorRef = useRef(null); + // Measured bar height, anchors the popups above the tall vertical tray so they don't overlap it. + const [hudBarHeight, setHudBarHeight] = useState(0); const [languageMenuStyle, setLanguageMenuStyle] = useState<{ right: number; top: number; @@ -291,6 +300,106 @@ export function LaunchWindow() { return () => cancelAnimationFrame(id); }, [isLanguageMenuOpen]); + // Resize the overlay window to fit content, else the taller vertical tray gets clipped + // and scrolls. Measure from the window's bottom-centre (the anchor the main process + // preserves) so fixed bottom/centre offsets keep this stable and it doesn't oscillate. + const lastHudSizeRef = useRef({ width: 0, height: 0 }); + const measureHudSize = useCallback(() => { + const barEl = hudBarRef.current; + if (!barEl || !window.electronAPI?.setHudOverlaySize) return; + + // Breathing room so the drop shadow isn't clipped. TOP_MARGIN must also exceed the + // slack in the bar's `max-h: calc(100vh - 2.5rem)` cap (40px reserved - 20px bottom + // gap = 20px) so the window stays tall enough that the cap never engages and adds a scrollbar. + const SIDE_MARGIN = 24; + const TOP_MARGIN = 24; + // Wide enough that the language menu (11rem) never clips, even when the bar is narrow. + const MIN_WIDTH = 220; + + const viewportHeight = window.innerHeight; + const centerX = window.innerWidth / 2; + + // Use natural (scroll) size, not the clipped box: vertical mode's max-h cap is a + // small-screen fallback, and reading clipped height would pin the window to it. + // scrollHeight gives full content height; the cap only engages when the main process clamps to screen. + let topFromBottom = viewportHeight - barEl.getBoundingClientRect().bottom + barEl.scrollHeight; + let halfWidth = barEl.scrollWidth / 2; + + // Popups drive both dimensions too. Their vertical anchor depends on bar height, + // which is fed back through React state and lags by a frame, so derive their top + // edge from the bar's natural height instead of the stale rendered position. Keeps + // one measurement pass authoritative and avoids a feedback re-measure. + if (deviceSelectorRef.current) { + const rect = deviceSelectorRef.current.getBoundingClientRect(); + if (rect.width !== 0 || rect.height !== 0) { + const popupBottomOffset = + trayLayout === "vertical" + ? barEl.scrollHeight + HUD_DEVICE_POPUP_GAP + : HUD_DEVICE_POPUP_HORIZONTAL_BOTTOM; + topFromBottom = Math.max(topFromBottom, popupBottomOffset + rect.height); + halfWidth = Math.max(halfWidth, rect.width / 2); + } + } + + // The language menu scrolls within available height, so it only influences width. + // Its presence in the DOM means it's open. + if (languageMenuPanelRef.current) { + const rect = languageMenuPanelRef.current.getBoundingClientRect(); + halfWidth = Math.max(halfWidth, centerX - rect.left, rect.right - centerX); + } + + setHudBarHeight((prev) => { + const next = Math.round(barEl.scrollHeight); + return Math.abs(prev - next) > 1 ? next : prev; + }); + + const width = Math.max(MIN_WIDTH, Math.ceil(halfWidth * 2) + SIDE_MARGIN); + const height = Math.ceil(topFromBottom) + TOP_MARGIN; + if (width === lastHudSizeRef.current.width && height === lastHudSizeRef.current.height) { + return; + } + lastHudSizeRef.current = { width, height }; + window.electronAPI.setHudOverlaySize(width, height); + }, [trayLayout]); + + // One persistent observer; elements wire themselves up via callback refs as they + // mount/unmount so measurement re-runs without recreating it or threading mount state through deps. + const hudResizeObserverRef = useRef(null); + useEffect(() => { + const observer = new ResizeObserver(() => measureHudSize()); + hudResizeObserverRef.current = observer; + if (hudBarRef.current) observer.observe(hudBarRef.current); + if (deviceSelectorRef.current) observer.observe(deviceSelectorRef.current); + measureHudSize(); + return () => { + observer.disconnect(); + hudResizeObserverRef.current = null; + }; + }, [measureHudSize]); + + const observeHudElement = useCallback( + (el: T | null, ref: React.MutableRefObject) => { + const observer = hudResizeObserverRef.current; + if (ref.current && observer) observer.unobserve(ref.current); + ref.current = el; + if (el && observer) observer.observe(el); + measureHudSize(); + }, + [measureHudSize], + ); + const setHudBarEl = useCallback( + (el: HTMLDivElement | null) => observeHudElement(el, hudBarRef), + [observeHudElement], + ); + const setDeviceSelectorEl = useCallback( + (el: HTMLDivElement | null) => observeHudElement(el, deviceSelectorRef), + [observeHudElement], + ); + const setLanguageMenuPanelEl = useCallback( + (el: HTMLDivElement | null) => observeHudElement(el, languageMenuPanelRef), + [observeHudElement], + ); + const hudMouseEventsEnabledRef = useRef(undefined); const setHudMouseEventsEnabled = useCallback((enabled: boolean) => { if (hudMouseEventsEnabledRef.current === enabled) { @@ -391,10 +500,8 @@ export function LaunchWindow() { }; return ( - // Root fills the HUD window only. Avoid w-screen/h-screen (100vw/100vh): - // 100vw can exceed the inner layout width when scrollbars affect the - // viewport (notably on Windows), causing a horizontal scrollbar once the - // recording toolbar widened (issue #305). + // Avoid w-screen/h-screen: 100vw can exceed the inner layout width when scrollbars + // affect the viewport (Windows), causing a horizontal scrollbar (issue #305).
{ @@ -446,11 +553,19 @@ export function LaunchWindow() {
)} - {/* Device selectors — fixed above HUD bar, viewport-relative, never clipped */} + {/* Device selectors, fixed above HUD bar, viewport-relative, never clipped */} {(showMicControls || showWebcamControls) && (
{/* Mic selector */} {showMicControls && ( @@ -581,8 +696,9 @@ export function LaunchWindow() {
)} - {/* HUD bar — fixed at bottom center, viewport-relative, never moves */} + {/* HUD bar, fixed at bottom center, viewport-relative, never moves */}
{ const isSelected = selectedSource?.id === source.id; + const sourceKind = source.id.startsWith("screen:") ? "screen" : "window"; return (
handleSourceSelect(source)} > diff --git a/src/components/ui/color-picker.tsx b/src/components/ui/color-picker.tsx index d8ec2b33c..72f0135e2 100644 --- a/src/components/ui/color-picker.tsx +++ b/src/components/ui/color-picker.tsx @@ -46,8 +46,7 @@ export default function ColorPicker(props: ColorPickerProps) { return "#ffffff"; }; - // Normalize the hex input. - // Adds a # at the beginning of the input if it's not there. + // Prefix a # when the user typed a bare hex value. const normalizeHexDraft = (raw: string) => { const trimmed = raw.trim(); if (trimmed === "") return ""; @@ -58,8 +57,7 @@ export default function ColorPicker(props: ColorPickerProps) { const handleColorInputChange = (e: React.ChangeEvent) => { const normalized = normalizeHexDraft(e.target.value); setHexInput(normalized); - // Check if the normalized hex is a valid hex color. - // It should follow the format #RRGGBB or #RGB. + // Only push when it's a complete #RGB or #RRGGBB value. const isValidHexColor = /^#[0-9A-Fa-f]{3}$/.test(normalized) || /^#[0-9A-Fa-f]{6}$/.test(normalized); if (isValidHexColor) { diff --git a/src/components/ui/select.tsx b/src/components/ui/select.tsx index d151d164e..bdbf64e9a 100644 --- a/src/components/ui/select.tsx +++ b/src/components/ui/select.tsx @@ -82,7 +82,8 @@ const SelectContent = React.forwardRef< ; -const Toaster = ({ ...props }: ToasterProps) => { +const Toaster = ({ className, ...props }: ToasterProps) => { return ( diff --git a/src/components/video-editor/AddCustomFontDialog.tsx b/src/components/video-editor/AddCustomFontDialog.tsx index 9ab9ce3da..872559d28 100644 --- a/src/components/video-editor/AddCustomFontDialog.tsx +++ b/src/components/video-editor/AddCustomFontDialog.tsx @@ -36,7 +36,6 @@ export function AddCustomFontDialog({ onFontAdded }: AddCustomFontDialogProps) { const handleImportUrlChange = (url: string) => { setImportUrl(url); - // Auto-extract font name if valid Google Fonts URL if (isValidGoogleFontsUrl(url)) { const extracted = parseFontFamilyFromImport(url); if (extracted && !fontName) { @@ -46,7 +45,6 @@ export function AddCustomFontDialog({ onFontAdded }: AddCustomFontDialogProps) { }; const handleAdd = async () => { - // Validate inputs if (!importUrl.trim()) { toast.error(t("customFont.errorEmptyUrl")); return; @@ -65,7 +63,6 @@ export function AddCustomFontDialog({ onFontAdded }: AddCustomFontDialogProps) { setLoading(true); try { - // Extract font family from URL const fontFamily = parseFontFamilyFromImport(importUrl); if (!fontFamily) { toast.error(t("customFont.errorExtractFailed")); @@ -73,7 +70,6 @@ export function AddCustomFontDialog({ onFontAdded }: AddCustomFontDialogProps) { return; } - // Create custom font object const newFont: CustomFont = { id: generateFontId(fontName), name: fontName.trim(), @@ -81,17 +77,15 @@ export function AddCustomFontDialog({ onFontAdded }: AddCustomFontDialogProps) { importUrl: importUrl.trim(), }; - // Add font (this will load and verify it) - throws if it fails + // Loads and verifies the font; throws on failure await addCustomFont(newFont); - // Notify parent if (onFontAdded) { onFontAdded(newFont); } toast.success(t("customFont.successMessage", { fontName })); - // Reset and close setImportUrl(""); setFontName(""); setOpen(false); diff --git a/src/components/video-editor/AnnotationOverlay.tsx b/src/components/video-editor/AnnotationOverlay.tsx index 13d245b8a..09669d4e7 100644 --- a/src/components/video-editor/AnnotationOverlay.tsx +++ b/src/components/video-editor/AnnotationOverlay.tsx @@ -48,7 +48,7 @@ interface AnnotationOverlayProps { onBlurDataCommit?: () => void; onClick: (id: string) => void; zIndex: number; - isSelectedBoost: boolean; // Boost z-index when selected for easy editing + isSelectedBoost: boolean; // raise z-index when selected, for easier editing previewSourceCanvas?: PreviewCanvasSource | null; previewFrameVersion?: number; currentTimeMs: number; @@ -537,7 +537,7 @@ export function AnnotationOverlay({ const yPercent = (d.y / containerHeight) * 100; onPositionChange(annotation.id, { x: xPercent, y: yPercent }); - // Reset dragging flag after a short delay to prevent click event + // Delay clearing so the trailing click doesn't fire onClick setTimeout(() => { isDraggingRef.current = false; }, 100); @@ -576,7 +576,7 @@ export function AnnotationOverlay({ "ring-2 ring-[#34B27B] ring-offset-2 ring-offset-transparent", )} style={{ - zIndex: isSelectedBoost ? zIndex + 1000 : zIndex, // Boost selected annotation to ensure it's on top + zIndex: isSelectedBoost ? zIndex + 1000 : zIndex, // keep the selected annotation on top pointerEvents: isSelected ? "auto" : "none", border: isSelected && annotation.type !== "blur" ? "2px solid rgba(52, 178, 123, 0.8)" : "none", diff --git a/src/components/video-editor/AnnotationSettingsPanel.tsx b/src/components/video-editor/AnnotationSettingsPanel.tsx index 4fe3f505e..f98c4cc11 100644 --- a/src/components/video-editor/AnnotationSettingsPanel.tsx +++ b/src/components/video-editor/AnnotationSettingsPanel.tsx @@ -108,7 +108,6 @@ export function AnnotationSettingsPanel({ const getFontLabel = (font: (typeof FONT_FAMILIES)[number]) => font.labelKey ? fontStyleLabels[font.labelKey] : font.name; - // Load custom fonts on mount useEffect(() => { setCustomFonts(getCustomFonts()); }, []); @@ -138,7 +137,6 @@ export function AnnotationSettingsPanel({ const file = files[0]; - // Validate file type const validTypes = ["image/jpeg", "image/jpg", "image/png", "image/gif", "image/webp"]; if (!validTypes.includes(file.type)) { toast.error(t("annotation.invalidImageType"), { diff --git a/src/components/video-editor/ArrowSvgs.tsx b/src/components/video-editor/ArrowSvgs.tsx index 99c542ff7..5078b844e 100644 --- a/src/components/video-editor/ArrowSvgs.tsx +++ b/src/components/video-editor/ArrowSvgs.tsx @@ -7,9 +7,7 @@ interface ArrowSvgProps { } /** - * Inline SVG arrow components for 8 directions. - * These match the visual style of the previous icon-based arrows but use - * pure SVG paths for easy replication in export. + * Inline SVG arrows for 8 directions. Pure paths (not icon fonts) so export can replicate them. */ export function ArrowUp({ color, strokeWidth, className }: ArrowSvgProps) { diff --git a/src/components/video-editor/EditorEmptyState.tsx b/src/components/video-editor/EditorEmptyState.tsx index 511323abe..2a5c49547 100644 --- a/src/components/video-editor/EditorEmptyState.tsx +++ b/src/components/video-editor/EditorEmptyState.tsx @@ -2,11 +2,12 @@ import { AlertCircle, Film, FolderOpen, Upload, X } from "lucide-react"; import { useCallback, useRef, useState } from "react"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { useScopedT } from "@/contexts/I18nContext"; +import { getProjectFolder, parentDirectoryOf, saveUserPreferences } from "@/lib/userPreferences"; import { nativeBridgeClient } from "@/native"; interface EditorEmptyStateProps { onVideoImported: (videoPath: string) => void; - /** Called with the loaded project data — handles both button click and drag-drop */ + /** Called with the loaded project data; handles both button click and drag-drop */ onProjectOpened: (project: unknown, path: string | null) => void; } @@ -17,8 +18,8 @@ export function EditorEmptyState({ onVideoImported, onProjectOpened }: EditorEmp const tc = useScopedT("common"); const [isDraggingOver, setIsDraggingOver] = useState(false); const [dropError, setDropError] = useState(null); - // Freeze the last non-null error type so dialog content doesn't snap to the - // else-branch during the closing animation (same pattern as UnsavedChangesDialog). + // Freeze the last non-null error type so dialog content doesn't snap to the else-branch + // during the closing animation (same pattern as UnsavedChangesDialog). const lastDropErrorRef = useRef>("unsupported-format"); if (dropError !== null) { lastDropErrorRef.current = dropError; @@ -35,8 +36,14 @@ export function EditorEmptyState({ onVideoImported, onProjectOpened }: EditorEmp }, [onVideoImported]); const handleLoadProject = useCallback(async () => { - const result = await nativeBridgeClient.project.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(getProjectFolder()); if (result.canceled || !result.success || !result.project) return; + if (result.path) { + const folder = parentDirectoryOf(result.path); + if (folder) { + saveUserPreferences({ projectFolder: folder }); + } + } onProjectOpened(result.project, result.path ?? null); }, [onProjectOpened]); @@ -67,7 +74,7 @@ export function EditorEmptyState({ onVideoImported, onProjectOpened }: EditorEmp return; } - // Use Electron's webUtils.getPathForFile — File.path was removed in Electron 32+ + // Use Electron's webUtils.getPathForFile; File.path was removed in Electron 32+ let filePath: string; try { filePath = window.electronAPI.getPathForFile(projectFile); diff --git a/src/components/video-editor/ExportDialog.tsx b/src/components/video-editor/ExportDialog.tsx index 203228909..44eae5d2e 100644 --- a/src/components/video-editor/ExportDialog.tsx +++ b/src/components/video-editor/ExportDialog.tsx @@ -30,14 +30,13 @@ export function ExportDialog({ const t = useScopedT("dialogs"); const [showSuccess, setShowSuccess] = useState(false); - // Reset showSuccess when a new export starts or dialog reopens useEffect(() => { if (isExporting) { setShowSuccess(false); } }, [isExporting]); - // Reset showSuccess when dialog opens fresh + // Reset when the dialog opens fresh (not mid-export). useEffect(() => { if (isOpen && !isExporting && !progress) { setShowSuccess(false); @@ -59,13 +58,12 @@ export function ExportDialog({ const formatLabel = exportFormat === "gif" ? "GIF" : "Video"; - // Determine if we're in the compiling phase (frames done but still exporting) + // Compiling phase: frames are done but the export is still finishing. const isCompiling = isExporting && progress && progress.percentage >= 100 && exportFormat === "gif"; const isFinalizing = progress?.phase === "finalizing"; const renderProgress = progress?.renderProgress; - // Get status message based on phase const getStatusMessage = () => { if (error) return t("export.tryAgain"); if (isCompiling || isFinalizing) { @@ -80,7 +78,6 @@ export function ExportDialog({ return t("export.takeMoment"); }; - // Get title based on phase const getTitle = () => { if (error) return t("export.failed"); if (isFinalizing && exportFormat === "mp4") return t("export.finalizingVideoTitle"); @@ -194,7 +191,7 @@ export function ExportDialog({
{isCompiling || isFinalizing ? ( - // Show render progress if available, otherwise animated indeterminate bar + // Real progress if we have it, otherwise an indeterminate bar. renderProgress !== undefined && renderProgress > 0 ? (
- {SHORTCUT_ACTIONS.map((action) => ( -
- {t(`actions.${action}`)} - - {formatBinding(shortcuts[action], isMac)} - -
- ))} + {SHORTCUT_ACTIONS.filter((action) => BLUR_REGIONS_ENABLED || action !== "addBlur").map( + (action) => ( +
+ {t(`actions.${action}`)} + + {formatBinding(shortcuts[action], isMac)} + +
+ ), + )}
{FIXED_SHORTCUTS.map((fixed) => ( diff --git a/src/components/video-editor/PlaybackControls.tsx b/src/components/video-editor/PlaybackControls.tsx index 061ae5c5e..bdec37cc7 100644 --- a/src/components/video-editor/PlaybackControls.tsx +++ b/src/components/video-editor/PlaybackControls.tsx @@ -96,7 +96,8 @@ export default function PlaybackControls({
); - // If an annotation is selected, show annotation settings instead + // Annotation selected: show its settings panel instead. if ( selectedAnnotation && onAnnotationContentChange && @@ -772,7 +811,7 @@ export function SettingsPanel({ ); } - if (selectedBlur && onBlurDataChange && onBlurDelete) { + if (BLUR_REGIONS_ENABLED && selectedBlur && onBlurDataChange && onBlurDelete) { return (
@@ -937,32 +976,42 @@ export function SettingsPanel({
)} {zoomEnabled && hasCursorTelemetry && ( -
- - {t("zoom.focusMode.title")} - -
- {(["manual", "auto"] as const).map((mode) => { - const isActive = selectedZoomFocusMode === mode; - return ( - - ); - })} +
+
+ + {t("zoom.focusMode.title")} + +
+ {(["manual", "auto"] as const).map((mode) => { + const isActive = selectedZoomFocusMode === mode; + return ( + + ); + })} +
+ {focusModeLocked && ( +
+ + {t("zoom.focusMode.lockedDisclaimer")} +
+ )}
)} {zoomEnabled && onZoomPreviewStart && onZoomPreviewEnd && ( @@ -1234,6 +1283,44 @@ export function SettingsPanel({
+ {webcamLayoutPreset !== "no-webcam" && ( +
+
+ {t("layout.mirrorWebcam")} +
+ +
+ )} + {webcamLayoutPreset === "picture-in-picture" && ( +
+
+ {t("layout.reactiveWebcam")} + + + +
+ +
+ )} {webcamLayoutPreset === "picture-in-picture" && (
@@ -1469,8 +1556,20 @@ export function SettingsPanel({ {showCursor && ( <>
-
- {t("cursor.clipToBounds")} +
+ {t("cursor.clipToBounds")} + + +
+ {cursorThemeOptions.length > 1 && ( +
+
+ {t("cursor.theme")} +
+
+ {cursorThemeOptions.map((option) => { + const isSelected = cursorTheme === option.id; + return ( + + ); + })} +
+
+ )}
diff --git a/src/components/video-editor/ShortcutsConfigDialog.tsx b/src/components/video-editor/ShortcutsConfigDialog.tsx index c5e950397..dcd073513 100644 --- a/src/components/video-editor/ShortcutsConfigDialog.tsx +++ b/src/components/video-editor/ShortcutsConfigDialog.tsx @@ -22,6 +22,7 @@ import { type ShortcutConflict, type ShortcutsConfig, } from "@/lib/shortcuts"; +import { BLUR_REGIONS_ENABLED } from "./featureFlags"; const MODIFIER_KEYS = new Set(["Control", "Shift", "Alt", "Meta"]); @@ -143,61 +144,63 @@ export function ShortcutsConfigDialog() {

{t("configurable")}

- {SHORTCUT_ACTIONS.map((action) => { - const isCapturing = captureFor === action; - const hasConflict = conflict?.forAction === action; - return ( -
-
- {t(`actions.${action}`)} - -
- {hasConflict && conflict?.conflictWith.type === "configurable" && ( -
- - ⚠{" "} - {t("alreadyUsedBy", { - action: t(`actions.${conflict.conflictWith.action}`), - })} - -
- - -
+ {SHORTCUT_ACTIONS.filter((action) => BLUR_REGIONS_ENABLED || action !== "addBlur").map( + (action) => { + const isCapturing = captureFor === action; + const hasConflict = conflict?.forAction === action; + return ( +
+
+ {t(`actions.${action}`)} +
- )} -
- ); - })} + {hasConflict && conflict?.conflictWith.type === "configurable" && ( +
+ + ⚠{" "} + {t("alreadyUsedBy", { + action: t(`actions.${conflict.conflictWith.action}`), + })} + +
+ + +
+
+ )} +
+ ); + }, + )}
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 05034632e..445a09060 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -1,8 +1,9 @@ import type { Span } from "dnd-timeline"; import { FolderOpen, Languages, Save, Video } from "lucide-react"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { type CSSProperties, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels"; import { toast } from "sonner"; +import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, @@ -11,11 +12,28 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { useShortcuts } from "@/contexts/ShortcutsContext"; import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory"; import { type Locale } from "@/i18n/config"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; +import { + captionSegmentsToAnnotationRegions, + extractMono16kFromVideoUrl, + MAX_CAPTION_AUDIO_SEC, + reconcileAutoCaptionTimelineGaps, + shiftTrimRegionsMsForCaptionBuffer, + transcribeMono16kToSegments, + trimLeadingSilenceMono16k, +} from "@/lib/captioning"; import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; import { calculateEffectiveSourceDimensions, @@ -36,6 +54,7 @@ import type { CursorCaptureMode, ProjectMedia } from "@/lib/recordingSession"; import { matchesShortcut } from "@/lib/shortcuts"; import { getExportFolder, + getProjectFolder, loadUserPreferences, parentDirectoryOf, saveUserPreferences, @@ -70,6 +89,7 @@ import { } from "./projectPersistence"; import { SettingsPanel } from "./SettingsPanel"; import TimelineEditor from "./timeline/TimelineEditor"; +import { buildAutoZoomSuggestions } from "./timeline/zoomSuggestionUtils"; import { type AnnotationRegion, type BlurData, @@ -95,6 +115,9 @@ import { import { UnsavedChangesDialog } from "./UnsavedChangesDialog"; import VideoPlayback, { VideoPlaybackRef } from "./VideoPlayback"; +/** Single Sonner slot so auto-caption phases update in place instead of stacking. */ +const AUTO_CAPTION_PROGRESS_TOAST_ID = "auto-caption-progress"; + function isClickInteractionType(interactionType: string | null | undefined) { return ( interactionType === "click" || @@ -151,6 +174,8 @@ function buildSaveDiagnosticMessage(formatLabel: "GIF" | "Video", reason?: strin return `${formatLabel} export save failed${reason ? `\nReason: ${reason}` : ""}`; } +const CAPTION_WORD_CHOICES = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] as const; + export default function VideoEditor() { const { state: editorState, @@ -164,6 +189,8 @@ export default function VideoEditor() { const { zoomRegions, + autoZoomEnabled, + autoFocusAll, trimRegions, speedRegions, annotationRegions, @@ -178,11 +205,13 @@ export default function VideoEditor() { aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, webcamSizePreset, webcamPosition, } = editorState; - // ── Non-undoable state + // Non-undoable state const [videoPath, setVideoPath] = useState(null); const [videoSourcePath, setVideoSourcePath] = useState(null); const [webcamVideoPath, setWebcamVideoPath] = useState(null); @@ -226,8 +255,8 @@ export default function VideoEditor() { } | null>(null); const [isFullscreen, setIsFullscreen] = useState(false); const [showCloseConfirmDialog, setShowCloseConfirmDialog] = useState(false); - // Unsaved-changes confirmation for New Project / Load Project actions. - // (The window-close flow uses showCloseConfirmDialog above.) + // Unsaved-changes confirmation for New Project / Load Project. + // The window-close flow uses showCloseConfirmDialog above. const [confirmDialogVariant, setConfirmDialogVariant] = useState< "newProject" | "loadProject" | null >(null); @@ -260,6 +289,7 @@ export default function VideoEditor() { const [cursorClipToBounds, setCursorClipToBounds] = useState( DEFAULT_CURSOR_SETTINGS.clipToBounds, ); + const [cursorTheme, setCursorTheme] = useState(DEFAULT_CURSOR_SETTINGS.theme); const [nativePlatform, setNativePlatform] = useState(null); const [recordingCursorCaptureMode, setRecordingCursorCaptureMode] = useState(null); @@ -271,9 +301,9 @@ export default function VideoEditor() { const nextSpeedIdRef = useRef(1); const { shortcuts, isMac } = useShortcuts(); - // Native Windows recordings include captured cursor assets. Native macOS - // recordings hide the system cursor in ScreenCaptureKit and use telemetry - // samples with OpenScreen's default arrow asset for the editable overlay. + // Windows recordings include captured cursor assets. macOS hides the system + // cursor in ScreenCaptureKit and renders telemetry samples with OpenScreen's + // default arrow asset for the editable overlay. const hasEditableCursorRecording = recordingCursorCaptureMode === "editable-overlay" && (nativePlatform === "win32" || nativePlatform === "darwin") && @@ -287,6 +317,11 @@ export default function VideoEditor() { const nextAnnotationIdRef = useRef(1); const nextAnnotationZIndexRef = useRef(1); + const isAutoCaptioningRef = useRef(false); + const [isAutoCaptioning, setIsAutoCaptioning] = useState(false); + const [showAutoCaptionsDialog, setShowAutoCaptionsDialog] = useState(false); + const [captionWordsMin, setCaptionWordsMin] = useState(2); + const [captionWordsMax, setCaptionWordsMax] = useState(7); const exporterRef = useRef(null); const annotationOnlyRegions = useMemo( @@ -359,6 +394,10 @@ export default function VideoEditor() { setRecordingCursorCaptureMode(projectCursorCaptureMode); setCurrentProjectPath(path ?? null); + // A loaded project keeps its zooms exactly as saved, so never auto-suggest + // over it (even if it has zero zooms because the user deleted them all). + autoProcessedSourceRef.current = sourcePath; + pushState({ wallpaper: normalizedEditor.wallpaper, shadowIntensity: normalizedEditor.shadowIntensity, @@ -369,12 +408,16 @@ export default function VideoEditor() { padding: normalizedEditor.padding, cropRegion: normalizedEditor.cropRegion, zoomRegions: normalizedEditor.zoomRegions, + autoZoomEnabled: normalizedEditor.autoZoomEnabled, + autoFocusAll: normalizedEditor.autoFocusAll, trimRegions: normalizedEditor.trimRegions, speedRegions: normalizedEditor.speedRegions, annotationRegions: normalizedEditor.annotationRegions, aspectRatio: normalizedEditor.aspectRatio, webcamLayoutPreset: normalizedEditor.webcamLayoutPreset, webcamMaskShape: normalizedEditor.webcamMaskShape, + webcamMirrored: normalizedEditor.webcamMirrored, + webcamReactiveZoom: normalizedEditor.webcamReactiveZoom, webcamSizePreset: normalizedEditor.webcamSizePreset, webcamPosition: normalizedEditor.webcamPosition, }); @@ -383,6 +426,7 @@ export default function VideoEditor() { setGifFrameRate(normalizedEditor.gifFrameRate); setGifLoop(normalizedEditor.gifLoop); setGifSizePreset(normalizedEditor.gifSizePreset); + setCursorTheme(normalizedEditor.cursorTheme); setSelectedZoomId(null); setSelectedTrimId(null); @@ -441,21 +485,28 @@ export default function VideoEditor() { padding, cropRegion, zoomRegions, + autoZoomEnabled, + autoFocusAll, trimRegions, speedRegions, annotationRegions, aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, + webcamSizePreset, webcamPosition, exportQuality, exportFormat, gifFrameRate, gifLoop, gifSizePreset, + cursorTheme, }); }, [ currentProjectMedia, + cursorTheme, wallpaper, shadowIntensity, showBlur, @@ -465,12 +516,17 @@ export default function VideoEditor() { padding, cropRegion, zoomRegions, + autoZoomEnabled, + autoFocusAll, trimRegions, speedRegions, annotationRegions, aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, + webcamSizePreset, webcamPosition, exportQuality, exportFormat, @@ -533,8 +589,8 @@ export default function VideoEditor() { createProjectSnapshot({ screenVideoPath: result.path }, INITIAL_EDITOR_STATE), ); } - // No video/project/session — leave videoPath null so the - // EditorEmptyState dashboard renders instead of an error screen. + // No video/project/session, so leave videoPath null and let the + // EditorEmptyState dashboard render instead of an error screen. } catch (err) { setError("Error loading video: " + String(err)); } finally { @@ -545,8 +601,7 @@ export default function VideoEditor() { loadInitialData(); }, [applyLoadedProject]); - // Track whether user preferences have been loaded to avoid - // overwriting saved prefs with defaults on the first render + // Avoid overwriting saved prefs with defaults before they've loaded. const [prefsHydrated, setPrefsHydrated] = useState(false); // Load persisted user preferences on mount (intentionally runs once) @@ -589,12 +644,16 @@ export default function VideoEditor() { padding, cropRegion, zoomRegions, + autoZoomEnabled, + autoFocusAll, trimRegions, speedRegions, annotationRegions, aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, webcamSizePreset, webcamPosition, exportQuality, @@ -602,6 +661,7 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + cursorTheme, }; const projectData = createProjectData(currentProjectMedia, editorState); @@ -610,8 +670,8 @@ export default function VideoEditor() { .split(/[\\/]/) .pop() ?.replace(/\.[^.]+$/, "") || `project-${Date.now()}`; - // Match the normalization path used by `currentProjectSnapshot` so the - // post-save baseline compares equal and `hasUnsavedChanges` clears. + // Normalize the same way as currentProjectSnapshot so the post-save + // baseline compares equal and hasUnsavedChanges clears. const projectSnapshot = createProjectSnapshot(currentProjectMedia, editorState); const result = await nativeBridgeClient.project.saveProjectFile( projectData, @@ -649,21 +709,26 @@ export default function VideoEditor() { padding, cropRegion, zoomRegions, + autoZoomEnabled, + autoFocusAll, trimRegions, speedRegions, annotationRegions, aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, + webcamSizePreset, webcamPosition, exportQuality, exportFormat, gifFrameRate, gifLoop, gifSizePreset, + cursorTheme, videoPath, t, - webcamSizePreset, ], ); @@ -719,7 +784,7 @@ export default function VideoEditor() { }, []); const doLoadProject = useCallback(async () => { - const result = await nativeBridgeClient.project.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(getProjectFolder()); if (result.canceled) { return; @@ -736,6 +801,13 @@ export default function VideoEditor() { return; } + if (result.path) { + const folder = parentDirectoryOf(result.path); + if (folder) { + saveUserPreferences({ projectFolder: folder }); + } + } + toast.success(t("project.loadedFrom", { path: result.path ?? "" })); }, [applyLoadedProject, t]); @@ -788,6 +860,7 @@ export default function VideoEditor() { setCursorMotionBlur(DEFAULT_CURSOR_SETTINGS.motionBlur); setCursorClickBounce(DEFAULT_CURSOR_SETTINGS.clickBounce); setCursorClipToBounds(DEFAULT_CURSOR_SETTINGS.clipToBounds); + setCursorTheme(DEFAULT_CURSOR_SETTINGS.theme); // Reset region ID counters. nextZoomIdRef.current = 1; nextTrimIdRef.current = 1; @@ -947,6 +1020,9 @@ export default function VideoEditor() { depth: DEFAULT_ZOOM_DEPTH, customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], focus: { cx: 0.5, cy: 0.5 }, + // Auto-Focus on means new zooms follow the cursor too. + focusMode: autoFocusAll ? "auto" : undefined, + source: "manual", }; pushState((prev) => ({ zoomRegions: [...prev.zoomRegions, newRegion] })); setSelectedZoomId(id); @@ -955,24 +1031,91 @@ export default function VideoEditor() { setSelectedAnnotationId(null); setSelectedBlurId(null); }, - [pushState], + [pushState, autoFocusAll], ); - const handleZoomSuggested = useCallback( - (span: Span, focus: ZoomFocus) => { - const id = `zoom-${nextZoomIdRef.current++}`; - const newRegion: ZoomRegion = { - id, - startMs: Math.round(span.start), - endMs: Math.round(span.end), + // Builds fresh "auto" zoom regions from cursor telemetry without overlapping + // existing ones. Used by both the on-load auto-suggest pass and the wand toggle. + const buildAutoZoomRegions = useCallback( + (existingRegions: ZoomRegion[]): ZoomRegion[] => { + const totalMs = Math.round(duration * 1000); + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry, + totalMs, + existingRegions, + defaultDurationMs: Math.max(1000, Math.round(totalMs * 0.05)), + }); + return suggestions.map((suggestion) => ({ + id: `zoom-${nextZoomIdRef.current++}`, + startMs: Math.round(suggestion.span.start), + endMs: Math.round(suggestion.span.end), depth: DEFAULT_ZOOM_DEPTH, customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], - focus: clampFocusToDepth(focus, DEFAULT_ZOOM_DEPTH), - }; - // Bulk suggest must not steal selection — keeping a zoom selected hides - // the export panel (SettingsPanel gates it on !hasTimelineSelection), - // trapping users who just want to export after auto-zoom. - pushState((prev) => ({ zoomRegions: [...prev.zoomRegions, newRegion] })); + focus: clampFocusToDepth(suggestion.focus, DEFAULT_ZOOM_DEPTH), + focusMode: autoFocusAll ? ("auto" as const) : undefined, + source: "auto" as const, + })); + }, + [cursorTelemetry, duration, autoFocusAll], + ); + + // Auto-suggest zooms once per fresh recording (no existing zooms, telemetry + // available, wand enabled). Loaded projects are marked processed elsewhere so + // they're never touched. The ref guard runs this once per source and survives undo. + const autoProcessedSourceRef = useRef(null); + useEffect(() => { + if (!autoZoomEnabled || !cursorTelemetrySourcePath) return; + if (autoProcessedSourceRef.current === cursorTelemetrySourcePath) return; + if (cursorTelemetry.length < 2 || duration <= 0) return; + // Only auto-suggest for a fresh recording; don't disturb existing zooms. + if (zoomRegions.length > 0) { + autoProcessedSourceRef.current = cursorTelemetrySourcePath; + return; + } + const newRegions = buildAutoZoomRegions([]); + autoProcessedSourceRef.current = cursorTelemetrySourcePath; + if (newRegions.length === 0) return; + pushState((prev) => ({ zoomRegions: [...prev.zoomRegions, ...newRegions] })); + }, [ + autoZoomEnabled, + cursorTelemetrySourcePath, + cursorTelemetry, + duration, + zoomRegions, + buildAutoZoomRegions, + pushState, + ]); + + // Wand toggle: ON regenerates suggestions around existing zooms; OFF removes + // only untouched auto zooms (manual and edited-to-manual survive). + const handleToggleAutoZoom = useCallback( + (enabled: boolean) => { + if (enabled) { + autoProcessedSourceRef.current = cursorTelemetrySourcePath; + pushState((prev) => ({ + autoZoomEnabled: true, + zoomRegions: [...prev.zoomRegions, ...buildAutoZoomRegions(prev.zoomRegions)], + })); + } else { + pushState((prev) => ({ + autoZoomEnabled: false, + zoomRegions: prev.zoomRegions.filter((region) => region.source !== "auto"), + })); + } + }, + [pushState, buildAutoZoomRegions, cursorTelemetrySourcePath], + ); + + // Flip every zoom between auto (cursor-follow) and manual at once. + const handleToggleAutoFocusAll = useCallback( + (on: boolean) => { + pushState((prev) => ({ + autoFocusAll: on, + zoomRegions: prev.zoomRegions.map((region) => ({ + ...region, + focusMode: on ? "auto" : "manual", + })), + })); }, [pushState], ); @@ -1004,6 +1147,7 @@ export default function VideoEditor() { ...region, startMs: Math.round(span.start), endMs: Math.round(span.end), + source: "manual", } : region, ), @@ -1029,12 +1173,14 @@ export default function VideoEditor() { [pushState], ); - // Focus drag: updateState for live preview, commitState on pointer-up + // Focus drag: updateState for live preview, commitState on pointer-up. const handleZoomFocusChange = useCallback( (id: string, focus: ZoomFocus) => { updateState((prev) => ({ zoomRegions: prev.zoomRegions.map((region) => - region.id === id ? { ...region, focus: clampFocusToDepth(focus, region.depth) } : region, + region.id === id + ? { ...region, focus: clampFocusToDepth(focus, region.depth), source: "manual" } + : region, ), })); }, @@ -1052,6 +1198,7 @@ export default function VideoEditor() { depth, customScale: ZOOM_DEPTH_SCALES[depth], focus: clampFocusToDepth(region.focus, depth), + source: "manual", } : region, ), @@ -1067,7 +1214,9 @@ export default function VideoEditor() { if (!Number.isFinite(rounded)) return; updateState((prev) => ({ zoomRegions: prev.zoomRegions.map((region) => - region.id === selectedZoomId ? { ...region, customScale: rounded } : region, + region.id === selectedZoomId + ? { ...region, customScale: rounded, source: "manual" } + : region, ), })); }, @@ -1083,7 +1232,7 @@ export default function VideoEditor() { if (!selectedZoomId) return; pushState((prev) => ({ zoomRegions: prev.zoomRegions.map((region) => - region.id === selectedZoomId ? { ...region, focusMode } : region, + region.id === selectedZoomId ? { ...region, focusMode, source: "manual" } : region, ), })); }, @@ -1110,9 +1259,9 @@ export default function VideoEditor() { if (region.id !== selectedZoomId) return region; if (preset === null) { const { rotationPreset: _p, ...rest } = region; - return rest; + return { ...rest, source: "manual" }; } - return { ...region, rotationPreset: preset }; + return { ...region, rotationPreset: preset, source: "manual" }; }), })); }, @@ -1260,8 +1409,11 @@ export default function VideoEditor() { const handleAnnotationSpanChange = useCallback( (id: string, span: Span) => { - pushState((prev) => ({ - annotationRegions: prev.annotationRegions.map((region) => + pushState((prev) => { + const editedAutoCaption = + prev.annotationRegions.find((region) => region.id === id)?.annotationSource === + "auto-caption"; + const next = prev.annotationRegions.map((region) => region.id === id ? { ...region, @@ -1269,8 +1421,11 @@ export default function VideoEditor() { endMs: Math.round(span.end), } : region, - ), - })); + ); + return { + annotationRegions: editedAutoCaption ? reconcileAutoCaptionTimelineGaps(next) : next, + }; + }); }, [pushState], ); @@ -1283,8 +1438,10 @@ export default function VideoEditor() { const source = prev.annotationRegions.find((region) => region.id === id); if (!source) return {}; + const { annotationSource: _stripCaptionLink, ...sourceWithoutCaptionLink } = source; + const duplicate: AnnotationRegion = { - ...source, + ...sourceWithoutCaptionLink, id: duplicateId, zIndex: duplicateZIndex, position: { x: source.position.x + 4, y: source.position.y + 4 }, @@ -1375,11 +1532,18 @@ export default function VideoEditor() { const handleAnnotationStyleChange = useCallback( (id: string, style: Partial) => { - pushState((prev) => ({ - annotationRegions: prev.annotationRegions.map((region) => - region.id === id ? { ...region, style: { ...region.style, ...style } } : region, - ), - })); + pushState((prev) => { + const touched = prev.annotationRegions.find((r) => r.id === id); + const syncAutoCaptions = touched?.annotationSource === "auto-caption"; + return { + annotationRegions: prev.annotationRegions.map((region) => { + if (syncAutoCaptions && region.annotationSource === "auto-caption") { + return { ...region, style: { ...region.style, ...style } }; + } + return region.id === id ? { ...region, style: { ...region.style, ...style } } : region; + }), + }; + }); }, [pushState], ); @@ -1442,22 +1606,36 @@ export default function VideoEditor() { const handleAnnotationPositionChange = useCallback( (id: string, position: { x: number; y: number }) => { - pushState((prev) => ({ - annotationRegions: prev.annotationRegions.map((region) => - region.id === id ? { ...region, position } : region, - ), - })); + pushState((prev) => { + const moved = prev.annotationRegions.find((r) => r.id === id); + const syncAutoCaptions = moved?.annotationSource === "auto-caption"; + return { + annotationRegions: prev.annotationRegions.map((region) => { + if (syncAutoCaptions && region.annotationSource === "auto-caption") { + return { ...region, position }; + } + return region.id === id ? { ...region, position } : region; + }), + }; + }); }, [pushState], ); const handleAnnotationSizeChange = useCallback( (id: string, size: { width: number; height: number }) => { - pushState((prev) => ({ - annotationRegions: prev.annotationRegions.map((region) => - region.id === id ? { ...region, size } : region, - ), - })); + pushState((prev) => { + const resized = prev.annotationRegions.find((r) => r.id === id); + const syncAutoCaptions = resized?.annotationSource === "auto-caption"; + return { + annotationRegions: prev.annotationRegions.map((region) => { + if (syncAutoCaptions && region.annotationSource === "auto-caption") { + return { ...region, size }; + } + return region.id === id ? { ...region, size } : region; + }), + }; + }); }, [pushState], ); @@ -1522,7 +1700,7 @@ export default function VideoEditor() { } if (matchesShortcut(e, shortcuts.playPause, isMac)) { - // Allow space only in inputs/textareas + // Let space pass through inside inputs/textareas. if (isInput) { return; } @@ -1658,9 +1836,8 @@ export default function VideoEditor() { return; } - // Ask the user where to save BEFORE starting the export. This avoids the - // post-export save dialog getting hidden behind other windows after a - // long-running export. + // Pick the save path before exporting, otherwise the save dialog can end up + // hidden behind other windows after a long-running export. const isGifFormat = settings.format === "gif"; const targetFileName = `export-${Date.now()}.${isGifFormat ? "gif" : "mp4"}`; const pickResult = await window.electronAPI.pickExportSavePath( @@ -1696,7 +1873,7 @@ export default function VideoEditor() { ? getNativeAspectRatioValue(sourceWidth, sourceHeight, cropRegion) : getAspectRatioValue(aspectRatio); - // Get preview CONTAINER dimensions for scaling + // Preview container dimensions, used for scaling. const playbackRef = videoPlaybackRef.current; const containerElement = playbackRef?.containerRef?.current; const previewWidth = containerElement?.clientWidth || DEFAULT_SOURCE_DIMENSIONS.width; @@ -1730,9 +1907,12 @@ export default function VideoEditor() { cursorMotionBlur, cursorClickBounce, cursorClipToBounds, + cursorTheme, annotationRegions, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, webcamSizePreset, webcamPosition, previewWidth, @@ -1821,9 +2001,12 @@ export default function VideoEditor() { cursorMotionBlur, cursorClickBounce, cursorClipToBounds, + cursorTheme, annotationRegions, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, webcamSizePreset, webcamPosition, previewWidth, @@ -1899,8 +2082,8 @@ export default function VideoEditor() { } finally { setIsExporting(false); exporterRef.current = null; - // Reset dialog state to ensure it can be opened again on next export - // This fixes the bug where second export doesn't show save dialog + // Reset so the next export can reopen the dialog (second export + // otherwise wouldn't show the save dialog). setShowExportDialog(false); setExportProgress(null); } @@ -1925,6 +2108,8 @@ export default function VideoEditor() { aspectRatio, webcamLayoutPreset, webcamMaskShape, + webcamMirrored, + webcamReactiveZoom, webcamSizePreset, webcamPosition, exportQuality, @@ -1937,6 +2122,7 @@ export default function VideoEditor() { cursorMotionBlur, cursorClickBounce, cursorClipToBounds, + cursorTheme, t, ], ); @@ -2018,6 +2204,138 @@ export default function VideoEditor() { } }, []); + const generateAutoCaptions = useCallback( + async (minWords: number, maxWords: number) => { + if (!videoPath) { + toast.error(t("errors.noVideoLoaded")); + return; + } + if (isAutoCaptioningRef.current) { + toast.error(t("autoCaptions.busy")); + return; + } + const minW = Math.max(1, Math.min(minWords, maxWords)); + const maxW = Math.max(minW, maxWords); + + isAutoCaptioningRef.current = true; + setIsAutoCaptioning(true); + toast.loading(t("autoCaptions.generating"), { id: AUTO_CAPTION_PROGRESS_TOAST_ID }); + try { + const { samples, truncated, durationSec } = await extractMono16kFromVideoUrl(videoPath); + if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) { + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + toast.error(t("autoCaptions.noAudio")); + return; + } + + const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples); + if (speechSamples.length < 800) { + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + toast.error(t("autoCaptions.noAudio")); + return; + } + + const trimMs = Math.round(trimSec * 1000); + const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs); + + const transcribeOptions = { + onStatus: (phase: "model" | "transcribe") => { + if (phase === "model") { + toast.loading(t("autoCaptions.loadingModel"), { + id: AUTO_CAPTION_PROGRESS_TOAST_ID, + }); + } else { + toast.loading(t("autoCaptions.transcribing"), { + id: AUTO_CAPTION_PROGRESS_TOAST_ID, + }); + } + }, + }; + + let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments( + speechSamples, + { + trimRegions: trimRegionsForTranscribe, + ...transcribeOptions, + }, + ); + let transcribedFromTrimmedBuffer = true; + + // Leading-silence trimming can return empty even when the full source has + // speech. Retry once against the untrimmed buffer before giving up. + if (segmentsRaw.length === 0 && trimSec > 0) { + ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, { + trimRegions, + ...transcribeOptions, + })); + transcribedFromTrimmedBuffer = false; + } + + const segments = + transcribedFromTrimmedBuffer && trimSec > 0 + ? segmentsRaw.map((s) => ({ + ...s, + startSec: s.startSec + trimSec, + endSec: s.endSec + trimSec, + })) + : segmentsRaw; + + let { regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( + segments, + nextAnnotationIdRef.current, + nextAnnotationZIndexRef.current, + { + minWordsPerCaption: minW, + maxWordsPerCaption: maxW, + timestampGranularity: granularity, + }, + ); + + if (regions.length === 0 && segments.length > 0) { + ({ regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( + segments, + nextAnnotationIdRef.current, + nextAnnotationZIndexRef.current, + { + minWordsPerCaption: 1, + maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + timestampGranularity: granularity, + }, + )); + } + + if (regions.length === 0) { + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + toast.info(t("autoCaptions.noneHeard")); + return; + } + + pushState((prev) => ({ annotationRegions: [...prev.annotationRegions, ...regions] })); + nextAnnotationIdRef.current = nextNumericId; + nextAnnotationZIndexRef.current = nextZIndex; + + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + const minutesTrunc = String(Math.round(MAX_CAPTION_AUDIO_SEC / 60)); + if (truncated) { + toast.success(t("autoCaptions.done", { count: String(regions.length) }), { + description: t("autoCaptions.truncated", { minutes: minutesTrunc }), + }); + } else { + toast.success(t("autoCaptions.done", { count: String(regions.length) })); + } + } catch (e) { + console.error(e); + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + const detail = e instanceof Error ? e.message : String(e); + toast.error(t("autoCaptions.failed"), { description: detail }); + } finally { + isAutoCaptioningRef.current = false; + setIsAutoCaptioning(false); + } + }, + [videoPath, trimRegions, pushState, t], + ); + const handleSaveDiagnostic = useCallback(async () => { const result = await window.electronAPI.saveDiagnostic({ error: exportError ?? "Manual diagnostic export", @@ -2060,7 +2378,7 @@ export default function VideoEditor() { {t("newRecording.title")} @@ -2085,13 +2403,92 @@ export default function VideoEditor() { + + + + {t("autoCaptions.dialogTitle")} + {t("autoCaptions.dialogDescription")} + +
+
+ + +
+
+ + +
+
+ + + + +
+
+
- {/* Empty state — shown when no video is loaded */} + {/* Empty state shown when no video is loaded */} {!videoPath && (
updateState({ webcamPosition: pos })} @@ -2243,6 +2642,7 @@ export default function VideoEditor() { cursorMotionBlur={cursorMotionBlur} cursorClickBounce={cursorClickBounce} cursorClipToBounds={cursorClipToBounds} + cursorTheme={cursorTheme} isPreviewingZoom={isPreviewingZoom} />
@@ -2291,6 +2691,7 @@ export default function VideoEditor() { onZoomFocusModeChange={(mode) => selectedZoomId && handleZoomFocusModeChange(mode) } + focusModeLocked={autoFocusAll} selectedZoomFocus={ selectedZoomId ? (zoomRegions.find((z) => z.id === selectedZoomId)?.focus ?? null) @@ -2340,6 +2741,12 @@ export default function VideoEditor() { } webcamMaskShape={webcamMaskShape} onWebcamMaskShapeChange={(shape) => pushState({ webcamMaskShape: shape })} + webcamMirrored={webcamMirrored} + webcamReactiveZoom={webcamReactiveZoom} + onWebcamMirroredChange={(mirrored) => pushState({ webcamMirrored: mirrored })} + onWebcamReactiveZoomChange={(reactive) => + pushState({ webcamReactiveZoom: reactive }) + } webcamSizePreset={webcamSizePreset} onWebcamSizePresetChange={(v) => updateState({ webcamSizePreset: v })} onWebcamSizePresetCommit={commitState} @@ -2423,6 +2830,8 @@ export default function VideoEditor() { onCursorClickBounceChange={setCursorClickBounce} cursorClipToBounds={cursorClipToBounds} onCursorClipToBoundsChange={setCursorClipToBounds} + cursorTheme={cursorTheme} + onCursorThemeChange={setCursorTheme} hasCursorData={ cursorTelemetry.length > 0 || hasNativeCursorRecordingData(cursorRecordingData) @@ -2444,10 +2853,12 @@ export default function VideoEditor() { videoDuration={duration} currentTime={currentTime} onSeek={handleSeek} - cursorTelemetry={cursorTelemetry} zoomRegions={zoomRegions} onZoomAdded={handleZoomAdded} - onZoomSuggested={handleZoomSuggested} + autoZoomEnabled={autoZoomEnabled} + onToggleAutoZoom={handleToggleAutoZoom} + autoFocusAll={autoFocusAll} + onToggleAutoFocusAll={handleToggleAutoFocusAll} onZoomSpanChange={handleZoomSpanChange} onZoomDelete={handleZoomDelete} selectedZoomId={selectedZoomId} @@ -2489,6 +2900,19 @@ export default function VideoEditor() { } videoUrl={videoPath ?? undefined} showTrimWaveform={showTrimWaveform} + captionsLabel={t("autoCaptions.button")} + isGeneratingCaptions={isAutoCaptioning} + onGenerateCaptions={() => { + if (!videoPath) { + toast.error(t("errors.noVideoLoaded")); + return; + } + if (isAutoCaptioningRef.current) { + toast.error(t("autoCaptions.busy")); + return; + } + setShowAutoCaptionsDialog(true); + }} />
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 9f7d8a17d..1b4ad5263 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -20,14 +20,15 @@ import { } from "react"; import { getWebcamLayoutCssBoxShadow, + reactiveWebcamScale, type Size, type StyledRenderRect, type WebcamLayoutPreset, type WebcamSizePreset, } from "@/lib/compositeLayout"; +import { getSmoothedCursorPath } from "@/lib/cursor/cursorPathSmoothing"; import { createNativeCursorMotionBlurState, - createNativeCursorSmoothingState, getNativeCursorClickBounceProgress, getNativeCursorClickBounceScale, getNativeCursorMotionBlurPx, @@ -35,10 +36,8 @@ import { projectNativeCursorToLocal, projectNativeCursorToStage, resetNativeCursorMotionBlurState, - resetNativeCursorSmoothingState, resolveInterpolatedNativeCursorFrame, resolveNativeCursorRenderAsset, - smoothNativeCursorSample, } from "@/lib/cursor/nativeCursor"; import { classifyWallpaper, DEFAULT_WALLPAPER, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; @@ -66,19 +65,11 @@ import { rotation3DPerspective, type SpeedRegion, type TrimRegion, - ZOOM_DEPTH_SCALES, type ZoomFocus, type ZoomRegion, } from "./types"; -import { - AUTO_FOLLOW_RAMP_DISTANCE, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, - DEFAULT_FOCUS, - ZOOM_SCALE_DEADZONE, - ZOOM_TRANSLATION_DEADZONE_PX, -} from "./videoPlayback/constants"; -import { adaptiveSmoothFactor, smoothCursorFocus } from "./videoPlayback/cursorFollowUtils"; +import { AUTO_FOLLOW_PARAMS, DEFAULT_FOCUS } from "./videoPlayback/constants"; +import { advanceFollowFocus } from "./videoPlayback/cursorFollowUtils"; import { DEFAULT_CURSOR_CONFIG, PixiCursorOverlay, @@ -90,6 +81,7 @@ import { clamp01 } from "./videoPlayback/mathUtils"; import { updateOverlayIndicator } from "./videoPlayback/overlayUtils"; import { createVideoEventHandlers } from "./videoPlayback/videoEventHandlers"; import { findDominantRegion } from "./videoPlayback/zoomRegionUtils"; +import { createZoomSpringState, resetZoomSpring, stepZoomSpring } from "./videoPlayback/zoomSpring"; import { applyZoomTransform, computeFocusFromTransform, @@ -103,6 +95,8 @@ interface VideoPlaybackProps { webcamVideoPath?: string; webcamLayoutPreset: WebcamLayoutPreset; webcamMaskShape?: import("./types").WebcamMaskShape; + webcamMirrored?: boolean; + webcamReactiveZoom?: boolean; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; onWebcamPositionChange?: (position: { cx: number; cy: number }) => void; @@ -150,8 +144,9 @@ interface VideoPlaybackProps { cursorMotionBlur?: number; cursorClickBounce?: number; cursorClipToBounds?: boolean; - // When true, render the selected zoom at the playhead even while paused — - // lets the editor preview the zoom effect without leaving the focus-edit view. + cursorTheme?: string; + // Render the selected zoom at the playhead even while paused, so the editor can + // preview the effect without leaving the focus-edit view. isPreviewingZoom?: boolean; } @@ -227,6 +222,8 @@ const VideoPlayback = forwardRef( webcamVideoPath, webcamLayoutPreset, webcamMaskShape, + webcamMirrored = false, + webcamReactiveZoom = false, webcamSizePreset, webcamPosition, onWebcamPositionChange, @@ -274,6 +271,7 @@ const VideoPlayback = forwardRef( cursorMotionBlur = DEFAULT_CURSOR_SETTINGS.motionBlur, cursorClickBounce = DEFAULT_CURSOR_SETTINGS.clickBounce, cursorClipToBounds = DEFAULT_CURSOR_SETTINGS.clipToBounds, + cursorTheme = DEFAULT_CURSOR_SETTINGS.theme, isPreviewingZoom = false, }, ref, @@ -281,6 +279,10 @@ const VideoPlayback = forwardRef( const videoRef = useRef(null); const supplementalAudioRef = useRef(null); const webcamVideoRef = useRef(null); + const webcamWrapperRef = useRef(null); + const webcamReactiveZoomRef = useRef(webcamReactiveZoom); + const webcamLayoutPresetRef = useRef(webcamLayoutPreset); + const webcamPositionRef = useRef(webcamPosition); const containerRef = useRef(null); const appRef = useRef(null); const videoSpriteRef = useRef(null); @@ -313,6 +315,9 @@ const VideoPlayback = forwardRef( y: 0, appliedScale: 1, }); + // Spring that chases the eased zoom target so the camera glides instead of jerking. + const zoomSpringRef = useRef(createZoomSpringState()); + const prevZoomTimeMsRef = useRef(null); const blurFilterRef = useRef(null); const motionBlurFilterRef = useRef(null); const isDraggingFocusRef = useRef(false); @@ -346,6 +351,7 @@ const VideoPlayback = forwardRef( const cursorMotionBlurRef = useRef(cursorMotionBlur); const cursorClickBounceRef = useRef(cursorClickBounce); const cursorClipToBoundsRef = useRef(cursorClipToBounds); + const cursorThemeRef = useRef(cursorTheme); const isPreviewingZoomRef = useRef(isPreviewingZoom); const motionBlurStateRef = useRef(createMotionBlurState()); const onTimeUpdateRef = useRef(onTimeUpdate); @@ -363,7 +369,6 @@ const VideoPlayback = forwardRef( const nativeCursorTextureIdRef = useRef(null); const nativeCursorImageRef = useRef(null); const nativeCursorImageIdRef = useRef(null); - const nativeCursorSmoothingStateRef = useRef(createNativeCursorSmoothingState()); const nativeCursorMotionBlurStateRef = useRef(createNativeCursorMotionBlurState()); const nativeCursorClipRef = useRef(null); const borderRadiusRef = useRef(0); @@ -477,17 +482,8 @@ const VideoPlayback = forwardRef( [onDurationChange, syncResolvedDuration], ); - // IMPORTANT: must use clampFocusToScale(focus, getZoomScale(region)) here, - // NOT clampFocusToStage(focus, region.depth). - // - // region.depth is the preset slot (1×/2×/4×) and ignores customScale entirely. - // getZoomScale(region) returns customScale when set, falling back to the preset - // depth scale — so drag-to-reposition respects the actual zoom level the user - // configured, not the preset bucket it sits in. - // - // This was previously broken (invisible drag boundaries near canvas edges) and - // has been fixed twice. If you're refactoring this drag handler, keep this call - // as clampFocusForRegion(focus, region) — do not switch it back to region.depth. + // Clamp against getZoomScale(region), not region.depth: depth is just the preset + // slot (1x/2x/4x) and ignores customScale, which gives wrong drag bounds near the edges. const clampFocusForRegion = useCallback((focus: ZoomFocus, region: ZoomRegion) => { return clampFocusToScale(focus, getZoomScale(region)); }, []); @@ -501,7 +497,6 @@ const VideoPlayback = forwardRef( return; } - // Update stage size from overlay dimensions const stageWidth = overlayEl.clientWidth; const stageHeight = overlayEl.clientHeight; if (stageWidth && stageHeight) { @@ -821,7 +816,6 @@ const VideoPlayback = forwardRef( useEffect(() => { cursorRecordingDataRef.current = cursorRecordingData; - resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); }, [cursorRecordingData]); @@ -849,6 +843,24 @@ const VideoPlayback = forwardRef( cursorClipToBoundsRef.current = cursorClipToBounds; }, [cursorClipToBounds]); + useEffect(() => { + cursorThemeRef.current = cursorTheme; + }, [cursorTheme]); + + useEffect(() => { + webcamReactiveZoomRef.current = webcamReactiveZoom; + webcamLayoutPresetRef.current = webcamLayoutPreset; + webcamPositionRef.current = webcamPosition; + // Clear any reactive transform when the effect is turned off or layout changes, + // so a stale shrink doesn't linger while the ticker isn't updating it. + if ( + webcamWrapperRef.current && + (!webcamReactiveZoom || webcamLayoutPreset !== "picture-in-picture") + ) { + webcamWrapperRef.current.style.transform = ""; + } + }, [webcamReactiveZoom, webcamLayoutPreset, webcamPosition]); + useEffect(() => { isPreviewingZoomRef.current = isPreviewingZoom; }, [isPreviewingZoom]); @@ -912,10 +924,9 @@ const VideoPlayback = forwardRef( }; }, [pixiReady, videoReady, layoutVideoContent]); - // Drop the PIXI canvas resolution to 1.0 while scrubbing (the user is - // navigating, not previewing) and restore native DPR on play/idle so the - // preview stays faithful. Mutating renderer.resolution per-frame would - // thrash texture uploads; we only do it on scrub-state transitions. + // Drop canvas resolution to 1.0 while scrubbing and restore native DPR on play/idle. + // Only on scrub-state transitions; mutating renderer.resolution per-frame thrashes + // texture uploads. useEffect(() => { if (!pixiReady) return; const app = appRef.current; @@ -1301,12 +1312,32 @@ const VideoPlayback = forwardRef( motionBlurAmount: motionBlurAmountRef.current, transformOverride: transform, motionBlurState: motionBlurStateRef.current, - frameTimeMs: performance.now(), + // Content time, not wall-clock, so motion-blur velocity matches export and stays + // correct under speed regions (frameRenderer passes the same content timeMs). + frameTimeMs: currentTimeRef.current, }); state.x = appliedTransform.x; state.y = appliedTransform.y; state.appliedScale = appliedTransform.scale; + + // Scale the PiP webcam inversely with the (eased) zoom, anchored to the docked + // corner (bottom-right by default) so it stays flush instead of drifting to center. + const webcamWrapper = webcamWrapperRef.current; + if (webcamWrapper) { + const reactive = + webcamReactiveZoomRef.current && webcamLayoutPresetRef.current === "picture-in-picture"; + const factor = reactive ? reactiveWebcamScale(state.appliedScale) : 1; + if (factor < 1) { + const pos = webcamPositionRef.current; + const originX = (pos ? pos.cx >= 0.5 : true) ? "100%" : "0%"; + const originY = (pos ? pos.cy >= 0.5 : true) ? "100%" : "0%"; + webcamWrapper.style.transformOrigin = `${originX} ${originY}`; + webcamWrapper.style.transform = `scale(${factor})`; + } else { + webcamWrapper.style.transform = ""; + } + } }; let lastMotionBlurActive: boolean | null = null; @@ -1327,53 +1358,55 @@ const VideoPlayback = forwardRef( let targetFocus = defaultFocus; let targetProgress = 0; - // If a zoom is selected but video is not playing, show default unzoomed view + // If a zoom is selected but not playing, show the default unzoomed view. const selectedId = selectedZoomIdRef.current; const hasSelectedZoom = selectedId !== null; const shouldShowUnzoomedView = hasSelectedZoom && !isPlayingRef.current && !isPreviewingZoomRef.current; if (region && strength > 0 && !shouldShowUnzoomedView) { - const zoomScale = blendedScale ?? ZOOM_DEPTH_SCALES[region.depth]; + // Use getZoomScale (customScale-aware) to match export and the magnification + // findDominantRegion resolved focus at. Falling back to the depth preset would + // zoom/pan to a different level than export. + const zoomScale = blendedScale ?? getZoomScale(region); const regionFocus = region.focus; targetScaleFactor = zoomScale; targetFocus = regionFocus; targetProgress = strength; - // Apply adaptive smoothing for auto-follow mode + // Adaptive smoothing for auto-follow mode. if (region.focusMode === "auto" && !transition) { const raw = targetFocus; const isZoomingIn = targetProgress < 0.999 && targetProgress >= prevTargetProgressRef.current; + // Follow the cursor in content time (frame-rate independent) so the camera pans + // at the same speed in preview and export. Snap to target when not actively + // playing (paused/seek/scrub), matching the zoom spring's snap. + const focusAnimating = + isPlayingRef.current && !isSeekingRef.current && !isScrubbingRef.current; + const focusDtMs = + prevZoomTimeMsRef.current === null + ? 0 + : currentTimeRef.current - prevZoomTimeMsRef.current; if (targetProgress >= 0.999) { - // Full zoom: adaptive smoothing — moves faster when far, decelerates when close + // Full zoom: adaptive smoothing, faster when far, decelerating when close. const prev = smoothedAutoFocusRef.current ?? raw; - const factor = adaptiveSmoothFactor( - raw, - prev, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, - AUTO_FOLLOW_RAMP_DISTANCE, - ); - const smoothed = smoothCursorFocus(raw, prev, factor); + const smoothed = focusAnimating + ? advanceFollowFocus(prev, raw, focusDtMs, AUTO_FOLLOW_PARAMS) + : raw; smoothedAutoFocusRef.current = smoothed; targetFocus = smoothed; } else if (isZoomingIn) { - // Zoom-in: track cursor directly so zoom always aims at current cursor - // position; keep ref in sync to avoid snap when full-zoom begins + // Zoom-in: track cursor directly so zoom always aims at the current position; + // keep ref in sync to avoid a snap when full-zoom begins. smoothedAutoFocusRef.current = raw; } else { - // Zoom-out: keep smoothing for continuity — avoids snap at zoom-out start + // Zoom-out: keep smoothing for continuity to avoid a snap at zoom-out start. const prev = smoothedAutoFocusRef.current ?? raw; - const factor = adaptiveSmoothFactor( - raw, - prev, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, - AUTO_FOLLOW_RAMP_DISTANCE, - ); - const smoothed = smoothCursorFocus(raw, prev, factor); + const smoothed = focusAnimating + ? advanceFollowFocus(prev, raw, focusDtMs, AUTO_FOLLOW_PARAMS) + : raw; smoothedAutoFocusRef.current = smoothed; targetFocus = smoothed; } @@ -1382,7 +1415,7 @@ const VideoPlayback = forwardRef( } prevTargetProgressRef.current = targetProgress; - // Handle connected zoom transitions (pan between adjacent zoom regions) + // Connected zoom transitions: pan between adjacent regions. if (transition) { const startTransform = computeZoomTransform({ stageSize: stageSizeRef.current, @@ -1440,18 +1473,28 @@ const VideoPlayback = forwardRef( focusY: state.focusY, }); - const appliedScale = - Math.abs(projectedTransform.scale - prevScale) < ZOOM_SCALE_DEADZONE - ? projectedTransform.scale - : projectedTransform.scale; - const appliedX = - Math.abs(projectedTransform.x - prevX) < ZOOM_TRANSLATION_DEADZONE_PX - ? projectedTransform.x - : projectedTransform.x; - const appliedY = - Math.abs(projectedTransform.y - prevY) < ZOOM_TRANSLATION_DEADZONE_PX - ? projectedTransform.y - : projectedTransform.y; + // Chase the eased target with a spring so the camera glides (no jerk at the steep + // start of the ease, no snap at close-region seams). Step by content time while + // playing; snap to the exact target when paused/seeking/scrubbing for crisp frames. + const nowMs = currentTimeRef.current; + const prevMs = prevZoomTimeMsRef.current; + const animating = isPlayingRef.current && !isSeekingRef.current && !isScrubbingRef.current; + const dtMs = prevMs === null ? 0 : nowMs - prevMs; + let appliedScale: number; + let appliedX: number; + let appliedY: number; + if (!animating || prevMs === null || dtMs <= 0 || dtMs > 80) { + resetZoomSpring(zoomSpringRef.current, projectedTransform); + appliedScale = projectedTransform.scale; + appliedX = projectedTransform.x; + appliedY = projectedTransform.y; + } else { + const sprung = stepZoomSpring(zoomSpringRef.current, projectedTransform, dtMs); + appliedScale = sprung.scale; + appliedX = sprung.x; + appliedY = sprung.y; + } + prevZoomTimeMsRef.current = nowMs; const motionIntensity = Math.max( Math.abs(appliedScale - prevScale), @@ -1489,7 +1532,6 @@ const VideoPlayback = forwardRef( } } - // Update cursor overlay const cursorOverlay = cursorOverlayRef.current; if (cursorOverlay) { const timeMs = currentTimeRef.current; // already in ms @@ -1516,7 +1558,6 @@ const VideoPlayback = forwardRef( if (nativeCursorClipRef.current) { nativeCursorClipRef.current.style.clipPath = ""; } - resetNativeCursorSmoothingState(nativeCursorSmoothingStateRef.current); resetNativeCursorMotionBlurState(nativeCursorMotionBlurStateRef.current); }; if (nativeCursorImage) { @@ -1527,13 +1568,15 @@ const VideoPlayback = forwardRef( timeMs, ); if (frame) { - const displaySample = smoothNativeCursorSample({ - forceSnap: !isPlayingRef.current || isSeekingRef.current, - sample: frame.sample, - smoothing: cursorSmoothingRef.current, - state: nativeCursorSmoothingStateRef.current, - timeMs, - }); + // Position comes from the precomputed offline-smoothed path; the frame still + // supplies the cursor image, type, and click timing. + const smoothedPos = getSmoothedCursorPath( + cursorRecordingDataRef.current, + cursorSmoothingRef.current, + )?.sampleAt(timeMs); + const displaySample = smoothedPos + ? { ...frame.sample, cx: smoothedPos.cx, cy: smoothedPos.cy } + : frame.sample; const cameraContainer = cameraContainerRef.current; const videoContainer = videoContainerRef.current; const cropRegionValue = cropRegionRef.current ?? { x: 0, y: 0, width: 1, height: 1 }; @@ -1556,9 +1599,14 @@ const VideoPlayback = forwardRef( }) : null; if (projectedLocalPoint && projectedStagePoint) { - // Pass deviceScaleFactor=1 — asset.scaleFactor already encodes DPR. + // Pass deviceScaleFactor=1 since asset.scaleFactor already encodes DPR. // Size is normalized below so preview matches export proportionally. - const renderAsset = resolveNativeCursorRenderAsset(frame.asset, 1, displaySample); + const renderAsset = resolveNativeCursorRenderAsset( + frame.asset, + 1, + displaySample, + cursorThemeRef.current, + ); const bounceProgress = getNativeCursorClickBounceProgress( cursorRecordingDataRef.current, timeMs, @@ -1587,9 +1635,8 @@ const VideoPlayback = forwardRef( nativeCursorImageIdRef.current = renderAsset.id; } nativeCursorImage.style.display = "block"; - // Update clip-path on nativeCursorClipRef to the camera-aware video boundary. - // clip-path works correctly here because nativeCursorClipRef is outside preserve-3d. - // When cursorClipToBounds is off, allow the cursor to overflow the canvas. + // Clip to the camera-aware video boundary. Works here because nativeCursorClipRef + // sits outside preserve-3d. When cursorClipToBounds is off, let the cursor overflow. if (nativeCursorClipRef.current) { if (!cursorClipToBoundsRef.current) { nativeCursorClipRef.current.style.clipPath = "none"; @@ -1612,7 +1659,7 @@ const VideoPlayback = forwardRef( nativeCursorImage.style.filter = blurPx > 0 ? `blur(${blurPx.toFixed(2)}px)` : "none"; // translate3d is relative to nativeCursorClipRef (absolute inset-0 = stage origin). - // projectedStagePoint.x is the stage-space cursor position — no offset needed. + // projectedStagePoint.x is the stage-space cursor position, so no offset is needed. nativeCursorImage.style.transform = `translate3d(${ projectedStagePoint.x - renderAsset.hotspotX * transformedScale }px, ${projectedStagePoint.y - renderAsset.hotspotY * transformedScale}px, 0)`; @@ -1854,7 +1901,7 @@ const VideoPlayback = forwardRef( ), }} > - {/* Background layer - always render as DOM element with blur */} + {/* Background always renders as a DOM element so it can be blurred. */}
( const useClipPath = !!clipPath; return (
( clipPath: clipPath ?? undefined, boxShadow: useClipPath ? "none" : webcamCssBoxShadow, backgroundColor: "#000", + transform: webcamMirrored ? "scaleX(-1)" : undefined, }} onPointerDown={handleWebcamPointerDown} onPointerMove={handleWebcamPointerMove} @@ -1921,7 +1970,7 @@ const VideoPlayback = forwardRef(
); })()} - {/* Only render overlay after PIXI and video are fully initialized */} + {/* Render the overlay only once PIXI and video are ready. */} {pixiReady && videoReady && (
( })() : null; - // Handle click-through cycling: when clicking same annotation, cycle to next + // Re-clicking a selected annotation cycles through any overlapping ones. const handleAnnotationClick = (clickedId: string) => { if (!onSelectAnnotation) return; - // If clicking on already selected annotation and there are multiple overlapping if (clickedId === selectedAnnotationId && filteredAnnotations.length > 1) { - // Find current index and cycle to next const currentIndex = filteredAnnotations.findIndex((a) => a.id === clickedId); const nextIndex = (currentIndex + 1) % filteredAnnotations.length; onSelectAnnotation(filteredAnnotations[nextIndex].id); } else { - // First click or clicking different annotation onSelectAnnotation(clickedId); } }; @@ -2062,10 +2108,8 @@ const VideoPlayback = forwardRef(
)}
- {/* Clip the native cursor overlay to the exact video canvas boundary. - Placed OUTSIDE composite3DRef (preserve-3d) so clip-path works - correctly even during 3D zoom rotation regions. - clip-path is set dynamically to the camera-aware video bounds. */} + {/* Native cursor clip. Lives outside composite3DRef (preserve-3d) so clip-path + keeps working during 3D zoom rotations; bounds are set dynamically. */}
{ aspectRatio: "16:9", webcamLayoutPreset: "picture-in-picture", webcamMaskShape: "circle", + webcamMirrored: true, + webcamSizePreset: 25, webcamPosition: null, exportQuality: "good", exportFormat: "mp4", @@ -68,6 +70,12 @@ describe("projectPersistence media compatibility", () => { ).toBe("rectangle"); }); + it("normalizes webcam mirroring safely", () => { + expect(normalizeProjectEditor({ webcamMirrored: true }).webcamMirrored).toBe(true); + expect(normalizeProjectEditor({ webcamMirrored: false }).webcamMirrored).toBe(false); + expect(normalizeProjectEditor({ webcamMirrored: "yes" as never }).webcamMirrored).toBe(false); + }); + it("normalizes blur region type and mosaic block size safely", () => { const editor = normalizeProjectEditor({ annotationRegions: [ diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index ff59427f2..258efdf6a 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -1,5 +1,6 @@ import { normalizeTextAnimation } from "@/lib/annotationTextAnimation"; import { normalizeBlurColor, normalizeBlurType } from "@/lib/blurEffects"; +import { normalizeCursorThemeId } from "@/lib/cursor/cursorThemes"; import type { ExportFormat, ExportQuality, GifFrameRate, GifSizePreset } from "@/lib/exporter"; import type { ProjectMedia } from "@/lib/recordingSession"; import { normalizeProjectMedia } from "@/lib/recordingSession"; @@ -25,6 +26,8 @@ import { DEFAULT_BLUR_INTENSITY, DEFAULT_FIGURE_DATA, DEFAULT_PLAYBACK_SPEED, + DEFAULT_WEBCAM_MIRRORED, + DEFAULT_WEBCAM_REACTIVE_ZOOM, DEFAULT_ZOOM_DEPTH, DEFAULT_ZOOM_MOTION_BLUR, MAX_BLUR_BLOCK_SIZE, @@ -44,11 +47,10 @@ import { const VALID_BLUR_SHAPES = new Set(["rectangle", "oval", "freehand"] as const); -// Pre-fix projects could persist resolved file:// URLs (machine-specific) for -// bundled wallpapers. Rewrite only paths that match a known install layout -// (resources/[assets/]wallpapers for packaged, public/wallpapers for dev) so -// a legitimate user file that happens to live in a folder named "wallpapers" -// elsewhere is never silently replaced. +// Old projects persisted machine-specific file:// URLs for bundled wallpapers. +// Match only the known install layouts (packaged resources/[assets/]wallpapers, +// dev public/wallpapers) so a user's own file under some "wallpapers" folder isn't +// silently replaced. const LEGACY_FILE_WALLPAPER_RE = /^file:\/\/.*?\/(?:resources\/(?:assets\/)?|public\/)wallpapers\/(wallpaper\d+\.jpg)$/i; const CANONICAL_WALLPAPERS = new Set(WALLPAPER_PATHS); @@ -72,12 +74,16 @@ export interface ProjectEditorState { padding: number; cropRegion: CropRegion; zoomRegions: ZoomRegion[]; + autoZoomEnabled: boolean; + autoFocusAll: boolean; trimRegions: TrimRegion[]; speedRegions: SpeedRegion[]; annotationRegions: AnnotationRegion[]; aspectRatio: AspectRatio; webcamLayoutPreset: WebcamLayoutPreset; webcamMaskShape: WebcamMaskShape; + webcamMirrored: boolean; + webcamReactiveZoom: boolean; webcamSizePreset: WebcamSizePreset; webcamPosition: WebcamPosition | null; exportQuality: ExportQuality; @@ -85,6 +91,7 @@ export interface ProjectEditorState { gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; + cursorTheme: string; } export interface EditorProjectData { @@ -258,6 +265,7 @@ export function normalizeProjectEditor(editor: Partial): Pro cy: clamp(isFiniteNumber(region.focus?.cy) ? region.focus.cy : 0.5, 0, 1), }, focusMode: region.focusMode === "auto" ? "auto" : "manual", + source: region.source === "auto" ? "auto" : "manual", ...(validPreset ? { rotationPreset: validPreset } : {}), }; }) @@ -333,6 +341,8 @@ export function normalizeProjectEditor(editor: Partial): Pro content: typeof region.content === "string" ? region.content : "", textContent: typeof region.textContent === "string" ? region.textContent : undefined, imageContent: typeof region.imageContent === "string" ? region.imageContent : undefined, + annotationSource: + region.annotationSource === "auto-caption" ? ("auto-caption" as const) : undefined, position: { x: clamp( isFiniteNumber(region.position?.x) @@ -436,6 +446,7 @@ export function normalizeProjectEditor(editor: Partial): Pro const cropHeight = clamp(rawCropHeight, 0.01, 1 - cropY); return { + cursorTheme: normalizeCursorThemeId(editor.cursorTheme), wallpaper: typeof editor.wallpaper === "string" ? normalizeWallpaperValue(editor.wallpaper) @@ -473,6 +484,10 @@ export function normalizeProjectEditor(editor: Partial): Pro height: cropHeight, }, zoomRegions: normalizedZoomRegions, + // Default on for legacy projects so re-opens match the new default. The + // on-load auto-suggest pass is gated separately, so this won't add zooms. + autoZoomEnabled: typeof editor.autoZoomEnabled === "boolean" ? editor.autoZoomEnabled : true, + autoFocusAll: typeof editor.autoFocusAll === "boolean" ? editor.autoFocusAll : false, trimRegions: normalizedTrimRegions, speedRegions: normalizedSpeedRegions, annotationRegions: normalizedAnnotationRegions, @@ -485,6 +500,12 @@ export function normalizeProjectEditor(editor: Partial): Pro editor.webcamMaskShape === "rounded" ? editor.webcamMaskShape : DEFAULT_WEBCAM_SETTINGS.maskShape, + webcamMirrored: + typeof editor.webcamMirrored === "boolean" ? editor.webcamMirrored : DEFAULT_WEBCAM_MIRRORED, + webcamReactiveZoom: + typeof editor.webcamReactiveZoom === "boolean" + ? editor.webcamReactiveZoom + : DEFAULT_WEBCAM_REACTIVE_ZOOM, webcamSizePreset: typeof editor.webcamSizePreset === "number" && isFiniteNumber(editor.webcamSizePreset) ? Math.max(10, Math.min(50, editor.webcamSizePreset)) diff --git a/src/components/video-editor/timeline/BackgroundWaveform.tsx b/src/components/video-editor/timeline/BackgroundWaveform.tsx index 815b472d1..d89a14c40 100644 --- a/src/components/video-editor/timeline/BackgroundWaveform.tsx +++ b/src/components/video-editor/timeline/BackgroundWaveform.tsx @@ -1,36 +1,28 @@ import { useTimelineContext } from "dnd-timeline"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; + +// Perceptual curve on normalized amplitude; exponent < 1 lifts quiet passages so +// one loud spike doesn't flatten the rest. +const WAVEFORM_GAMMA = 0.6; export interface BackgroundWaveformProps { - /** Pre-computed peaks array: pairs of [min, max] per block (length = 2 * N). */ + /** Pre-computed peaks: pairs of [min, max] per block (length = 2 * N). */ peaks: Float32Array | null; videoDurationMs: number; - /** - * Pixels to inset the drawn waveform from the top of the canvas row, - * so it aligns with the item content top edge. Defaults to 0. - */ + /** Inset from canvas top so the waveform aligns with item content top. Defaults to 0. */ topInset?: number; - /** - * Pixels to inset the drawn waveform from the bottom of the canvas row, - * so it aligns with the item content bottom edge. Defaults to 0. - */ + /** Inset from canvas bottom so the waveform aligns with item content bottom. Defaults to 0. */ bottomInset?: number; } /** - * Renders a rectified (half-wave) audio waveform on a `` that fills - * its containing block. Designed to be passed as the `background` prop of - * ``, which already provides `relative overflow-hidden` — no wrapper - * element needed. - * - * The canvas always uses `inset-0` (full row height). Vertical alignment with - * the item content is achieved via `topInset`/`bottomInset` in the draw calls - * rather than CSS positioning, so the result is immune to sub-pixel CSS layout - * differences. + * Renders a rectified (half-wave) audio waveform on a canvas filling its block. + * Pass as the `background` prop of ``, which already provides + * `relative overflow-hidden`. * - * - Accepts pre-computed `peaks` from the caller (see `useAudioPeaks`). - * - Redraws whenever the timeline zoom/pan range changes. - * - `pointer-events: none` — never blocks drag-to-create interactions. + * Canvas is always `inset-0` (full row height); vertical alignment comes from + * `topInset`/`bottomInset` in the draw calls, not CSS, so it's immune to + * sub-pixel layout rounding. `pointer-events: none` keeps drag-to-create working. */ export default function BackgroundWaveform({ peaks, @@ -42,8 +34,21 @@ export default function BackgroundWaveform({ const canvasRef = useRef(null); const [canvasSize, setCanvasSize] = useState({ w: 0, h: 0 }); - // Observe the canvas itself — Row's `relative overflow-hidden` parent - // makes it fill the row exactly, so no wrapper div is needed. + // Normalize against the track's own loudest peak so quiet recordings (mic/system + // audio rarely hit full scale) still fill the row. Recomputed only on peaks change, + // not zoom/pan, so height stays stable while scrolling. + const normFactor = useMemo(() => { + if (!peaks || peaks.length === 0) return 0; + let globalMax = 0; + for (let i = 0; i < peaks.length; i++) { + const a = Math.abs(peaks[i]); + if (a > globalMax) globalMax = a; + } + return globalMax > 0 ? 1 / globalMax : 0; + }, [peaks]); + + // Observe the canvas directly; Row's `relative overflow-hidden` parent makes + // it fill the row exactly, so no wrapper div is needed. useEffect(() => { const canvas = canvasRef.current; if (!canvas) return; @@ -55,7 +60,6 @@ export default function BackgroundWaveform({ return () => ro.disconnect(); }, []); - // Redraw whenever peaks, range, or canvas size changes. useEffect(() => { const canvas = canvasRef.current; if (!canvas || canvasSize.w <= 0 || canvasSize.h <= 0) return; @@ -70,7 +74,7 @@ export default function BackgroundWaveform({ ctx.scale(dpr, dpr); ctx.clearRect(0, 0, canvasSize.w, canvasSize.h); - if (!peaks || peaks.length === 0) return; + if (!peaks || peaks.length === 0 || normFactor === 0) return; const W = canvasSize.w; const H = canvasSize.h; @@ -78,7 +82,7 @@ export default function BackgroundWaveform({ if (rangeMs <= 0 || videoDurationMs <= 0) return; // Draw within [topY, bottomY] so the waveform aligns with item bounds - // regardless of CSS sub-pixel rounding on the canvas element itself. + // regardless of sub-pixel rounding on the canvas element. const topY = topInset; const bottomY = H - bottomInset; const drawHeight = bottomY - topY; @@ -87,8 +91,9 @@ export default function BackgroundWaveform({ const N = peaks.length / 2; const amp = drawHeight * 0.9; - // Rectified (half-wave): amplitude = max(|min|, |max|), drawn upward from bottomY. - const colAmp = new Float32Array(W); + // Rectified: amplitude = max(|min|, |max|), normalized to the loudest peak + // and gamma-curved, drawn upward from bottomY. + const colY = new Float32Array(W); for (let x = 0; x < W; x++) { const startMs = range.start + (x / W) * rangeMs; const endMs = range.start + ((x + 1) / W) * rangeMs; @@ -102,30 +107,32 @@ export default function BackgroundWaveform({ if (a > absMax) absMax = a; if (b > absMax) absMax = b; } - colAmp[x] = absMax; + const normalized = Math.min(1, absMax * normFactor); + const display = normalized > 0 ? normalized ** WAVEFORM_GAMMA : 0; + colY[x] = bottomY - display * amp; } - // Filled polygon: bottom-left → top silhouette → bottom-right. + // Filled polygon: bottom-left, up over the silhouette, down to bottom-right. ctx.beginPath(); ctx.moveTo(0, bottomY); for (let x = 0; x < W; x++) { - ctx.lineTo(x, bottomY - colAmp[x] * amp); + ctx.lineTo(x, colY[x]); } ctx.lineTo(W, bottomY); ctx.closePath(); ctx.fillStyle = "rgba(74, 222, 128, 0.55)"; ctx.fill(); - // Crisp top-edge stroke for the sharp silhouette. + // Crisp top-edge stroke. ctx.beginPath(); - ctx.moveTo(0, bottomY - colAmp[0] * amp); + ctx.moveTo(0, colY[0]); for (let x = 1; x < W; x++) { - ctx.lineTo(x, bottomY - colAmp[x] * amp); + ctx.lineTo(x, colY[x]); } ctx.strokeStyle = "rgba(74, 222, 128, 0.85)"; ctx.lineWidth = 1; ctx.stroke(); - }, [peaks, range, canvasSize, videoDurationMs, topInset, bottomInset]); + }, [peaks, normFactor, range, canvasSize, videoDurationMs, topInset, bottomInset]); return ; } diff --git a/src/components/video-editor/timeline/Item.tsx b/src/components/video-editor/timeline/Item.tsx index 7251af6de..254c4f94b 100644 --- a/src/components/video-editor/timeline/Item.tsx +++ b/src/components/video-editor/timeline/Item.tsx @@ -79,9 +79,8 @@ export default function Item({ [span.start, span.end], ); - // Minimum clickable width on the outer wrapper. - // Kept small (6px) so items visually distinguish their real positions; - // users should zoom in to interact with sub-second items precisely. + // Minimum clickable width on the outer wrapper. Kept small so items keep their real + // positions; zoom in to interact with sub-second items precisely. const MIN_ITEM_PX = 6; const safeItemStyle = { ...itemStyle, minWidth: MIN_ITEM_PX }; diff --git a/src/components/video-editor/timeline/Row.tsx b/src/components/video-editor/timeline/Row.tsx index 17a59e83c..a0b00c9a5 100644 --- a/src/components/video-editor/timeline/Row.tsx +++ b/src/components/video-editor/timeline/Row.tsx @@ -9,9 +9,8 @@ interface RowProps extends RowDefinition { } /** - * A single horizontal lane in the timeline. Wraps the dnd-timeline `useRow` - * hook and adds an optional `background` layer (e.g. `BackgroundWaveform`), - * an empty-state hint label, and a minimum height. + * A horizontal timeline lane. Wraps dnd-timeline's `useRow` and adds an optional + * `background` layer, an empty-state hint label, and a minimum height. */ export default function Row({ id, children, hint, isEmpty, background }: RowProps) { const { setNodeRef, rowWrapperStyle, rowStyle } = useRow({ id }); diff --git a/src/components/video-editor/timeline/TimelineEditor.tsx b/src/components/video-editor/timeline/TimelineEditor.tsx index f84d038a9..17894ad1c 100644 --- a/src/components/video-editor/timeline/TimelineEditor.tsx +++ b/src/components/video-editor/timeline/TimelineEditor.tsx @@ -1,11 +1,13 @@ import type { Range, Span } from "dnd-timeline"; import { useTimelineContext } from "dnd-timeline"; import { + Captions, Check, ChevronDown, Gauge, MessageSquare, Plus, + ScanEye, Scissors, WandSparkles, ZoomIn, @@ -27,20 +29,13 @@ import { matchesShortcut } from "@/lib/shortcuts"; import { cn } from "@/lib/utils"; import { ASPECT_RATIOS, type AspectRatio, getAspectRatioLabel } from "@/utils/aspectRatioUtils"; import { formatShortcut } from "@/utils/platformUtils"; -import type { - AnnotationRegion, - CursorTelemetryPoint, - SpeedRegion, - TrimRegion, - ZoomFocus, - ZoomRegion, -} from "../types"; +import { BLUR_REGIONS_ENABLED } from "../featureFlags"; +import type { AnnotationRegion, SpeedRegion, TrimRegion, ZoomRegion } from "../types"; import BackgroundWaveform from "./BackgroundWaveform"; import Item from "./Item"; import KeyframeMarkers from "./KeyframeMarkers"; import Row from "./Row"; import TimelineWrapper from "./TimelineWrapper"; -import { detectZoomDwellCandidates, normalizeCursorTelemetry } from "./zoomSuggestionUtils"; const ZOOM_ROW_ID = "row-zoom"; const TRIM_ROW_ID = "row-trim"; @@ -49,17 +44,20 @@ const BLUR_ROW_ID = "row-blur"; const SPEED_ROW_ID = "row-speed"; const FALLBACK_RANGE_MS = 1000; const TARGET_MARKER_COUNT = 12; -const SUGGESTION_SPACING_MS = 1800; interface TimelineEditorProps { videoDuration: number; hasVideoSource?: boolean; currentTime: number; onSeek?: (time: number) => void; - cursorTelemetry?: CursorTelemetryPoint[]; zoomRegions: ZoomRegion[]; onZoomAdded: (span: Span) => void; - onZoomSuggested?: (span: Span, focus: ZoomFocus) => void; + /** Magic-wand auto-zoom toggle state + handler. */ + autoZoomEnabled?: boolean; + onToggleAutoZoom?: (enabled: boolean) => void; + /** Global Auto-Focus toggle state + handler. */ + autoFocusAll?: boolean; + onToggleAutoFocusAll?: (on: boolean) => void; onZoomSpanChange: (id: string, span: Span) => void; onZoomDelete: (id: string) => void; selectedZoomId: string | null; @@ -92,6 +90,11 @@ interface TimelineEditorProps { onAspectRatioChange: (aspectRatio: AspectRatio) => void; videoUrl?: string; showTrimWaveform?: boolean; + /** Opens the auto-captions flow. When omitted, the captions button is hidden. */ + onGenerateCaptions?: () => void; + isGeneratingCaptions?: boolean; + /** Localized label for the auto-captions button (lives in the `editor` namespace). */ + captionsLabel?: string; } interface TimelineScaleConfig { @@ -133,9 +136,8 @@ const SCALE_CANDIDATES = [ ]; /** - * Picks the best axis interval for the currently visible time range. - * Called dynamically — re-runs on every zoom change so the axis always - * shows a meaningful density of markers regardless of video length. + * Picks the best axis interval for the currently visible time range, so marker + * density stays meaningful regardless of video length. */ function calculateAxisScale(visibleRangeMs: number): { intervalMs: number; gridMs: number } { const visibleSeconds = visibleRangeMs / 1000; @@ -153,19 +155,17 @@ function calculateAxisScale(visibleRangeMs: number): { intervalMs: number; gridM function calculateTimelineScale(durationSeconds: number): TimelineScaleConfig { const totalMs = Math.max(0, Math.round(durationSeconds * 1000)); - // Minimum item duration: fixed at 100ms (0.1s). - // Allows precise cuts while remaining interactive. + // 100ms, precise enough to cut but still grabbable. const minItemDurationMs = 100; - // Default placement size: 5% of video duration, clamped between 1s and 30s. + // 5% of duration, clamped to 1-30s. const defaultItemDurationMs = totalMs > 0 ? Math.max(minItemDurationMs, Math.min(Math.round(totalMs * 0.05), 30000)) : Math.max(minItemDurationMs, 1000); - // Minimum visible range: 300ms — allows comfortably viewing 0.1s items. - // Axis markers adapt dynamically via calculateAxisScale, so there is no - // upper constraint on how far the user can zoom in. + // 300ms, enough to view 0.1s items comfortably. Axis markers adapt via + // calculateAxisScale, so there's no cap on zoom-in. const minVisibleRangeMs = 300; return { @@ -296,11 +296,11 @@ function PlaybackCursor({ const clickX = e.clientX - rect.left - sidebarWidth; const contentWidth = Math.max(rect.width - sidebarWidth, 1); - // Allow dragging outside to 0 or max, but clamp the value + // Allow dragging past the edges, but clamp the value const relativeMs = pixelsToValue(clickX); let absoluteMs = Math.max(0, Math.min(range.start + relativeMs, videoDurationMs)); - // Snap to nearby keyframe if within threshold (150ms) + // Snap to a keyframe within 150ms const snapThresholdMs = 150; const nearbyKeyframe = keyframes.find( (kf) => @@ -403,7 +403,7 @@ function PlaybackCursor({ className="absolute top-0 bottom-0 z-50 group/cursor" style={{ [sideProperty === "right" ? "marginRight" : "marginLeft"]: `${sidebarWidth - 1}px`, - pointerEvents: "none", // Allow clicks to pass through to timeline, but we'll enable pointer events on the handle + pointerEvents: "none", // pass clicks through to the timeline; the handle re-enables them }} >
calculateAxisScale(range.end - range.start), [range.end, range.start], @@ -479,13 +478,12 @@ function TimelineAxis({ .filter((time) => time <= maxTime) .sort((a, b) => a - b); - // Generate minor ticks (4 ticks between major intervals) + // 4 minor ticks between major intervals const minorTicks = []; const minorInterval = intervalMs / 5; for (let time = firstMarker; time <= maxTime; time += minorInterval) { if (time >= visibleStart && time <= visibleEnd) { - // Skip if it's close to a major marker const isMajor = Math.abs(time % intervalMs) < 1; if (!isMajor) { minorTicks.push(time); @@ -636,8 +634,7 @@ function Timeline({ const handleTimelineClick = useCallback( (e: React.MouseEvent) => { - // Only clear selection if clicking on empty space (not on items) - // This is handled by event propagation - items stop propagation + // Items stop propagation, so this only fires on empty space clearTimelineSelection(); seekTimelineAtClientX(e.currentTarget, e.clientX); }, @@ -847,21 +844,23 @@ function Timeline({ ))} - - {blurItems.map((item) => ( - onSelectBlur?.(item.id)} - variant={item.variant} - > - {item.label} - - ))} - + {BLUR_REGIONS_ENABLED && ( + + {blurItems.map((item) => ( + onSelectBlur?.(item.id)} + variant={item.variant} + > + {item.label} + + ))} + + )} {speedItems.map((item) => ( @@ -888,10 +887,12 @@ export default function TimelineEditor({ hasVideoSource = false, currentTime, onSeek, - cursorTelemetry = [], zoomRegions, onZoomAdded, - onZoomSuggested, + autoZoomEnabled = true, + onToggleAutoZoom, + autoFocusAll = false, + onToggleAutoFocusAll, onZoomSpanChange, onZoomDelete, selectedZoomId, @@ -924,6 +925,9 @@ export default function TimelineEditor({ onAspectRatioChange, videoUrl, showTrimWaveform = false, + onGenerateCaptions, + isGeneratingCaptions = false, + captionsLabel, }: TimelineEditorProps) { const t = useScopedT("timeline"); const totalMs = useMemo(() => Math.max(0, Math.round(videoDuration * 1000)), [videoDuration]); @@ -953,7 +957,6 @@ export default function TimelineEditor({ }); }, []); - // Add keyframe at current playhead position const addKeyframe = useCallback(() => { if (totalMs === 0) return; const time = Math.max(0, Math.min(currentTimeMs, totalMs)); @@ -961,14 +964,12 @@ export default function TimelineEditor({ setKeyframes((prev) => [...prev, { id: uuidv4(), time }]); }, [currentTimeMs, totalMs, keyframes]); - // Delete selected keyframe const deleteSelectedKeyframe = useCallback(() => { if (!selectedKeyframeId) return; setKeyframes((prev) => prev.filter((kf) => kf.id !== selectedKeyframeId)); setSelectedKeyframeId(null); }, [selectedKeyframeId]); - // Move keyframe to new time position const handleKeyframeMove = useCallback( (id: string, newTime: number) => { setKeyframes((prev) => @@ -980,14 +981,12 @@ export default function TimelineEditor({ [totalMs], ); - // Delete selected zoom item const deleteSelectedZoom = useCallback(() => { if (!selectedZoomId) return; onZoomDelete(selectedZoomId); onSelectZoom(null); }, [selectedZoomId, onZoomDelete, onSelectZoom]); - // Delete selected trim item const deleteSelectedTrim = useCallback(() => { if (!selectedTrimId || !onTrimDelete || !onSelectTrim) return; onTrimDelete(selectedTrimId); @@ -1016,9 +1015,8 @@ export default function TimelineEditor({ setRange(createInitialRange(totalMs)); }, [totalMs]); - // Normalize regions only when timeline bounds change (not on every region edit). - // Using refs to read current regions avoids a dependency-loop that re-fires - // this effect on every drag/resize and races with dnd-timeline's internal state. + // Normalize regions only when timeline bounds change. Reading via refs avoids a + // dependency loop that would re-fire on every drag and race dnd-timeline's state. const zoomRegionsRef = useRef(zoomRegions); const trimRegionsRef = useRef(trimRegions); const speedRegionsRef = useRef(speedRegions); @@ -1066,12 +1064,10 @@ export default function TimelineEditor({ onSpeedSpanChange?.(region.id, { start: normalizedStart, end: normalizedEnd }); } }); - // Only re-run when the timeline scale changes, not on every region edit }, [totalMs, safeMinDurationMs, onZoomSpanChange, onTrimSpanChange, onSpeedSpanChange]); const hasOverlap = useCallback( (newSpan: Span, excludeId?: string): boolean => { - // Determine which row the item belongs to const isZoomItem = zoomRegions.some((r) => r.id === excludeId); const isTrimItem = trimRegions.some((r) => r.id === excludeId); const isAnnotationItem = annotationRegions.some((r) => r.id === excludeId); @@ -1082,11 +1078,10 @@ export default function TimelineEditor({ return false; } - // Helper to check overlap against a specific set of regions const checkOverlap = (regions: (ZoomRegion | TrimRegion | SpeedRegion)[]) => { return regions.some((region) => { if (region.id === excludeId) return false; - // True overlap: regions actually intersect (not just adjacent) + // True intersection, adjacency is allowed return newSpan.end > region.startMs && newSpan.start < region.endMs; }); }; @@ -1108,8 +1103,7 @@ export default function TimelineEditor({ [zoomRegions, trimRegions, annotationRegions, blurRegions, speedRegions], ); - // At least 5% of the timeline or 1000ms, whichever is larger, so the region - // is always wide enough to grab and resize comfortably. + // 5% of the timeline or 1000ms, whichever is larger, so it's wide enough to grab. const defaultRegionDurationMs = useMemo( () => Math.max(1000, Math.round(totalMs * 0.05)), [totalMs], @@ -1125,14 +1119,11 @@ export default function TimelineEditor({ return; } - // Always place zoom at playhead const startPos = Math.max(0, Math.min(currentTimeMs, totalMs)); - // Find the next zoom region after the playhead const sorted = [...zoomRegions].sort((a, b) => a.startMs - b.startMs); const nextRegion = sorted.find((region) => region.startMs > startPos); const gapToNext = nextRegion ? nextRegion.startMs - startPos : totalMs - startPos; - // Check if playhead is inside any zoom region const isOverlapping = sorted.some( (region) => startPos >= region.startMs && startPos < region.endMs, ); @@ -1147,103 +1138,6 @@ export default function TimelineEditor({ onZoomAdded({ start: startPos, end: startPos + actualDuration }); }, [videoDuration, totalMs, currentTimeMs, zoomRegions, onZoomAdded, defaultRegionDurationMs, t]); - const handleSuggestZooms = useCallback(() => { - if (!videoDuration || videoDuration === 0 || totalMs === 0) { - return; - } - - if (!onZoomSuggested) { - toast.error(t("errors.zoomSuggestionUnavailable")); - return; - } - - if (cursorTelemetry.length < 2) { - toast.info(t("errors.noCursorTelemetry"), { - description: t("errors.noCursorTelemetryDescription"), - }); - return; - } - - const defaultDuration = Math.min(defaultRegionDurationMs, totalMs); - if (defaultDuration <= 0) { - return; - } - - const reservedSpans = [...zoomRegions] - .map((region) => ({ start: region.startMs, end: region.endMs })) - .sort((a, b) => a.start - b.start); - - const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs); - - if (normalizedSamples.length < 2) { - toast.info(t("errors.noUsableTelemetry"), { - description: t("errors.noUsableTelemetryDescription"), - }); - return; - } - - const dwellCandidates = detectZoomDwellCandidates(normalizedSamples); - - if (dwellCandidates.length === 0) { - toast.info(t("errors.noDwellMoments"), { - description: t("errors.noDwellMomentsDescription"), - }); - return; - } - - const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength); - const acceptedCenters: number[] = []; - - let addedCount = 0; - - sortedCandidates.forEach((candidate) => { - const tooCloseToAccepted = acceptedCenters.some( - (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS, - ); - - if (tooCloseToAccepted) { - return; - } - - const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); - const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); - const candidateEnd = candidateStart + defaultDuration; - const hasOverlap = reservedSpans.some( - (span) => candidateEnd > span.start && candidateStart < span.end, - ); - - if (hasOverlap) { - return; - } - - reservedSpans.push({ start: candidateStart, end: candidateEnd }); - acceptedCenters.push(candidate.centerTimeMs); - onZoomSuggested({ start: candidateStart, end: candidateEnd }, candidate.focus); - addedCount += 1; - }); - - if (addedCount === 0) { - toast.info(t("errors.noAutoZoomSlots"), { - description: t("errors.noAutoZoomSlotsDescription"), - }); - return; - } - - toast.success( - addedCount === 1 - ? t("success.addedZoomSuggestions", { count: String(addedCount) }) - : t("success.addedZoomSuggestionsPlural", { count: String(addedCount) }), - ); - }, [ - videoDuration, - totalMs, - defaultRegionDurationMs, - zoomRegions, - onZoomSuggested, - cursorTelemetry, - t, - ]); - const handleAddTrim = useCallback(() => { if (!videoDuration || videoDuration === 0 || totalMs === 0 || !onTrimAdded) { return; @@ -1254,14 +1148,11 @@ export default function TimelineEditor({ return; } - // Always place trim at playhead const startPos = Math.max(0, Math.min(currentTimeMs, totalMs)); - // Find the next trim region after the playhead const sorted = [...trimRegions].sort((a, b) => a.startMs - b.startMs); const nextRegion = sorted.find((region) => region.startMs > startPos); const gapToNext = nextRegion ? nextRegion.startMs - startPos : totalMs - startPos; - // Check if playhead is inside any trim region const isOverlapping = sorted.some( (region) => startPos >= region.startMs && startPos < region.endMs, ); @@ -1286,14 +1177,11 @@ export default function TimelineEditor({ return; } - // Always place speed region at playhead const startPos = Math.max(0, Math.min(currentTimeMs, totalMs)); - // Find the next speed region after the playhead const sorted = [...speedRegions].sort((a, b) => a.startMs - b.startMs); const nextRegion = sorted.find((region) => region.startMs > startPos); const gapToNext = nextRegion ? nextRegion.startMs - startPos : totalMs - startPos; - // Check if playhead is inside any speed region const isOverlapping = sorted.some( (region) => startPos >= region.startMs && startPos < region.endMs, ); @@ -1366,19 +1254,19 @@ export default function TimelineEditor({ if (matchesShortcut(e, keyShortcuts.addAnnotation, isMac)) { handleAddAnnotation(); } - if (matchesShortcut(e, keyShortcuts.addBlur, isMac)) { + if (BLUR_REGIONS_ENABLED && matchesShortcut(e, keyShortcuts.addBlur, isMac)) { handleAddBlur(); } if (matchesShortcut(e, keyShortcuts.addSpeed, isMac)) { handleAddSpeed(); } - // Tab: Cycle through overlapping annotations at current time + // Tab cycles through overlapping annotations at the current time if (e.key === "Tab" && annotationRegions.length > 0) { const currentTimeMs = Math.round(currentTime * 1000); const overlapping = annotationRegions .filter((a) => currentTimeMs >= a.startMs && currentTimeMs <= a.endMs) - .sort((a, b) => a.zIndex - b.zIndex); // Sort by z-index + .sort((a, b) => a.zIndex - b.zIndex); if (overlapping.length > 0) { e.preventDefault(); @@ -1386,11 +1274,10 @@ export default function TimelineEditor({ if (!selectedAnnotationId || !overlapping.some((a) => a.id === selectedAnnotationId)) { onSelectAnnotation?.(overlapping[0].id); } else { - // Cycle to next annotation const currentIndex = overlapping.findIndex((a) => a.id === selectedAnnotationId); const nextIndex = e.shiftKey - ? (currentIndex - 1 + overlapping.length) % overlapping.length // Shift+Tab = backward - : (currentIndex + 1) % overlapping.length; // Tab = forward + ? (currentIndex - 1 + overlapping.length) % overlapping.length // Shift+Tab steps backward + : (currentIndex + 1) % overlapping.length; onSelectAnnotation?.(overlapping[nextIndex].id); } } @@ -1479,7 +1366,6 @@ export default function TimelineEditor({ let label: string; if (region.type === "text") { - // Show text preview const preview = region.content.trim() || t("labels.emptyText"); label = preview.length > 20 ? `${preview.substring(0, 20)}...` : preview; } else if (region.type === "image") { @@ -1517,10 +1403,8 @@ export default function TimelineEditor({ return [...zooms, ...trims, ...annotations, ...blurs, ...speeds]; }, [zoomRegions, trimRegions, annotationRegions, blurRegions, speedRegions, t]); - // Spans that participate in overlap resolution (clampToNeighbours). - // Excludes annotation/blur deliberately — those are allowed to overlap and - // must NOT act as hard constraints when a zoom/trim/speed drag is being - // resolved. + // Spans that participate in overlap resolution (clampToNeighbours). Annotation + // and blur are excluded since they may overlap and shouldn't constrain a drag. const allRegionSpans = useMemo(() => { const zooms = zoomRegions.map((r) => ({ id: r.id, start: r.startMs, end: r.endMs })); const trims = trimRegions.map((r) => ({ id: r.id, start: r.startMs, end: r.endMs })); @@ -1528,8 +1412,7 @@ export default function TimelineEditor({ return [...zooms, ...trims, ...speeds]; }, [zoomRegions, trimRegions, speedRegions]); - // Additional snap targets that are NOT clamping constraints. Their edges - // pull during snap, but they don't push anyone away. + // Snap targets whose edges pull during a snap but don't push anyone away. const softSnapSpans = useMemo(() => { const annotations = annotationRegions.map((r) => ({ id: r.id, @@ -1544,7 +1427,6 @@ export default function TimelineEditor({ const handleItemSpanChange = useCallback( (id: string, span: Span) => { - // Check if it's a zoom, trim, speed, or annotation item if (zoomRegions.some((r) => r.id === id)) { onZoomSpanChange(id, span); } else if (trimRegions.some((r) => r.id === id)) { @@ -1605,14 +1487,31 @@ export default function TimelineEditor({ + - + + + + + + + )} + {onGenerateCaptions && ( + + )}
diff --git a/src/components/video-editor/timeline/TimelineWrapper.tsx b/src/components/video-editor/timeline/TimelineWrapper.tsx index 2a44262af..9cd71cb05 100644 --- a/src/components/video-editor/timeline/TimelineWrapper.tsx +++ b/src/components/video-editor/timeline/TimelineWrapper.tsx @@ -21,11 +21,9 @@ interface TimelineWrapperProps { minVisibleRangeMs: number; gridSizeMs?: number; onItemSpanChange: (id: string, span: Span) => void; - // Spans that act as hard overlap constraints (zoom/trim/speed). Used by - // clampToNeighbours AND as snap targets. + // Hard overlap constraints (zoom/trim/speed), used by clampToNeighbours and as snap targets. allRegionSpans?: { id: string; start: number; end: number }[]; - // Spans that act ONLY as snap targets (annotation/blur). They never push - // other items away during overlap resolution. + // Snap targets only (annotation/blur); never push other items during overlap resolution. softSnapSpans?: { id: string; start: number; end: number }[]; currentTimeMs?: number; keyframeTimesMs?: number[]; @@ -36,9 +34,8 @@ interface SnapGuideHandle { hide: () => void; } -// Lives inside TimelineContext so it can read valueToPixels. Updates DOM -// directly via an imperative handle — same pattern as the drag tooltip — to -// avoid re-rendering the timeline on every pointer move. +// Lives inside TimelineContext to read valueToPixels. Updates the DOM directly via +// an imperative handle (like the drag tooltip) to avoid re-rendering on every pointer move. const SnapGuide = forwardRef((_, ref) => { const { sidebarWidth, direction, range, valueToPixels } = useTimelineContext(); const elRef = useRef(null); @@ -198,10 +195,9 @@ export default function TimelineWrapper({ const snapGuideRef = useRef(null); - // Pull the active span's edges to nearby region boundaries, timeline bounds, - // the playhead, and keyframes. Threshold scales with zoom (~1% of visible - // range, min 50ms) so snap feels right at any zoom level. - // Returns the snapped span plus the actual snap target used (for guide rendering). + // Pull the active span's edges to nearby region boundaries, timeline bounds, playhead, + // and keyframes. Threshold scales with zoom (~1% of visible range, min 50ms). Returns + // the snapped span plus the snap target used (for guide rendering). const snapSpanToTargets = useCallback( ( span: Span, @@ -296,12 +292,10 @@ export default function TimelineWrapper({ ], ); - // dnd-timeline's resize event doesn't expose direction. Compare the live - // span to the committed one (committed spans only update on commit, so - // during a single resize they still reflect the pre-resize state). - // Returns null when the deltas are equal — including the common clamped - // case where both are 0 — because we can't tell which handle the user - // grabbed, and guessing wrong would snap the other edge. + // dnd-timeline's resize event doesn't expose direction, so compare the live span to + // the committed one (committed only updates on commit, so it's the pre-resize state). + // Returns null when deltas are equal (including the common clamped both-0 case): we + // can't tell which handle was grabbed, and guessing wrong snaps the other edge. const inferResizeMode = useCallback( (activeItemId: string, span: Span): "resize-left" | "resize-right" | null => { const old = diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index 9f807d32c..f083a7f7c 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -3,6 +3,8 @@ import type { CursorTelemetryPoint, ZoomFocus } from "../types"; export const MIN_DWELL_DURATION_MS = 450; export const MAX_DWELL_DURATION_MS = 2600; export const DWELL_MOVE_THRESHOLD = 0.02; +/** Minimum spacing between two accepted suggestion centres. */ +export const SUGGESTION_SPACING_MS = 1800; export interface ZoomDwellCandidate { centerTimeMs: number; @@ -79,3 +81,76 @@ export function detectZoomDwellCandidates(samples: CursorTelemetryPoint[]): Zoom return dwellCandidates; } + +export interface AutoZoomSuggestion { + span: { start: number; end: number }; + focus: ZoomFocus; +} + +/** + * Build non-overlapping zoom suggestions from cursor telemetry: detect dwell moments, + * rank by duration, space by SUGGESTION_SPACING_MS, drop any overlapping an existing + * region. Pure, shared by the magic-wand toggle and the on-load auto-suggest pass. + */ +export function buildAutoZoomSuggestions(options: { + cursorTelemetry: CursorTelemetryPoint[]; + totalMs: number; + existingRegions: { startMs: number; endMs: number }[]; + defaultDurationMs: number; +}): AutoZoomSuggestion[] { + const { cursorTelemetry, totalMs, existingRegions, defaultDurationMs } = options; + if (totalMs <= 0 || cursorTelemetry.length < 2) { + return []; + } + + const defaultDuration = Math.min(defaultDurationMs, totalMs); + if (defaultDuration <= 0) { + return []; + } + + const normalizedSamples = normalizeCursorTelemetry(cursorTelemetry, totalMs); + if (normalizedSamples.length < 2) { + return []; + } + + const dwellCandidates = detectZoomDwellCandidates(normalizedSamples); + if (dwellCandidates.length === 0) { + return []; + } + + const reservedSpans = existingRegions + .map((region) => ({ start: region.startMs, end: region.endMs })) + .sort((a, b) => a.start - b.start); + + const sortedCandidates = [...dwellCandidates].sort((a, b) => b.strength - a.strength); + const acceptedCenters: number[] = []; + const suggestions: AutoZoomSuggestion[] = []; + + for (const candidate of sortedCandidates) { + const tooCloseToAccepted = acceptedCenters.some( + (center) => Math.abs(center - candidate.centerTimeMs) < SUGGESTION_SPACING_MS, + ); + if (tooCloseToAccepted) { + continue; + } + + const centeredStart = Math.round(candidate.centerTimeMs - defaultDuration / 2); + const candidateStart = Math.max(0, Math.min(centeredStart, totalMs - defaultDuration)); + const candidateEnd = candidateStart + defaultDuration; + const hasOverlap = reservedSpans.some( + (span) => candidateEnd > span.start && candidateStart < span.end, + ); + if (hasOverlap) { + continue; + } + + reservedSpans.push({ start: candidateStart, end: candidateEnd }); + acceptedCenters.push(candidate.centerTimeMs); + suggestions.push({ + span: { start: candidateStart, end: candidateEnd }, + focus: candidate.focus, + }); + } + + return suggestions; +} diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 0f2267cca..93ae00237 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -3,7 +3,7 @@ import type { WebcamLayoutPreset } from "@/lib/compositeLayout"; export type ZoomDepth = 1 | 2 | 3 | 4 | 5 | 6; export type ZoomFocusMode = "manual" | "auto"; export type { WebcamLayoutPreset }; -/** Webcam size as a percentage of the canvas reference dimension (10–50). */ +/** Webcam size as a percentage of the canvas reference dimension (10-50). */ export type WebcamSizePreset = number; export const DEFAULT_WEBCAM_SIZE_PRESET: WebcamSizePreset = 25; @@ -14,6 +14,11 @@ export type WebcamMaskShape = "rectangle" | "circle" | "square" | "rounded"; export const DEFAULT_WEBCAM_MASK_SHAPE: WebcamMaskShape = "rectangle"; +export const DEFAULT_WEBCAM_MIRRORED = false; + +/** When true, the picture-in-picture webcam scales inversely with zoom (shrinks as you zoom in). */ +export const DEFAULT_WEBCAM_REACTIVE_ZOOM = true; + export interface WebcamPosition { cx: number; // normalized horizontal center (0-1) cy: number; // normalized vertical center (0-1) @@ -48,15 +53,21 @@ export const ROTATION_3D_PRESETS: Record = { export const ROTATION_3D_PRESET_ORDER: Rotation3DPreset[] = ["iso", "left", "right"]; -/** Perspective distance in CSS px is computed at render-time as this factor times - * min(viewport width, viewport height). Same factor used in preview and export so - * the visual look is identical regardless of canvas resolution. */ +/** Perspective distance in CSS px is this factor times min(viewport w, h). Same + * factor in preview and export so the look matches at any canvas resolution. */ export const ROTATION_3D_PERSPECTIVE_FACTOR = 2.6; export function rotation3DPerspective(width: number, height: number): number { return Math.min(width, height) * ROTATION_3D_PERSPECTIVE_FACTOR; } +/** + * Origin of a zoom region. "auto" marks zooms from the magic-wand suggest pass; + * toggling the wand off removes only these. Editing an auto zoom promotes it to + * "manual" so it survives. Undefined is treated as "manual" for back-compat. + */ +export type ZoomRegionSource = "auto" | "manual"; + export interface ZoomRegion { id: string; startMs: number; @@ -65,8 +76,9 @@ export interface ZoomRegion { focus: ZoomFocus; focusMode?: ZoomFocusMode; rotationPreset?: Rotation3DPreset; - /** Custom scale overriding the preset depth (1.0–5.0, two decimal precision). */ + /** Custom scale overriding the preset depth (1.0-5.0, two decimal precision). */ customScale?: number; + source?: ZoomRegionSource; } export function getRotation3D(region: Pick): Rotation3D { @@ -87,13 +99,10 @@ export function lerpRotation3D(a: Rotation3D, b: Rotation3D, t: number): Rotatio } /** - * Compute the maximum uniform scale that, when applied alongside `rot` and a perspective - * of `perspective` CSS px, keeps the projected bounding box of a `width × height` element - * inside its original `width × height` rectangle. Returns 1 when no scaling is needed. - * - * Math: project each rotated corner onto the screen via x' = x·P/(P−z); take the worst-case - * |x'|/|y'| against the half-extents and return the limiting ratio. This makes the rotated - * recording sit *inside* the zoom window instead of bleeding past it. + * Max uniform scale that, with `rot` and a perspective of `perspective` CSS px, keeps + * the projected bounding box of a width x height element inside its original rectangle. + * Returns 1 when no scaling is needed. Projects each rotated corner (x' = x*P/(P-z)) and + * returns the limiting half-extent ratio so the rotated recording stays inside the zoom window. */ export function computeRotation3DContainScale( rot: Rotation3D, @@ -123,7 +132,7 @@ export function computeRotation3DContainScale( let maxAbsY = 0; for (const [x0, y0] of corners) { - // CSS "rotateX(α) rotateY(β) rotateZ(γ)" reads right-to-left: Z first, then Y, then X. + // CSS "rotateX rotateY rotateZ" applies right-to-left: Z first, then Y, then X. let px = x0; let py = y0; let pz = 0; @@ -146,11 +155,11 @@ export function computeRotation3DContainScale( py = xy; pz = xz; - // Perspective projection: viewer at (0, 0, P), looking toward −z. A point at z=pz - // is scaled by P / (P − pz). When perspective ≤ 0 we treat as orthographic. + // Viewer at (0, 0, P) looking toward -z; a point at z=pz scales by P/(P-pz). + // perspective <= 0 means orthographic. if (perspective > 0) { const denom = perspective - pz; - if (denom <= 0) return 1; // pathological — skip scaling rather than crash + if (denom <= 0) return 1; // pathological, skip scaling rather than crash const f = perspective / denom; px *= f; py *= f; @@ -195,8 +204,7 @@ export const DEFAULT_CURSOR_SIZE = 3.0; export const DEFAULT_CURSOR_SMOOTHING = 0.67; export const DEFAULT_CURSOR_MOTION_BLUR = 0.35; export const DEFAULT_CURSOR_CLICK_BOUNCE = 2.5; -// false = allow the cursor to overflow into the background by default. -// true = clip the native cursor to the video canvas bounds. +// false lets the cursor overflow into the background; true clips it to the canvas bounds. export const DEFAULT_CURSOR_CLIP_TO_BOUNDS = false; export const DEFAULT_ZOOM_MOTION_BLUR = 0.35; @@ -288,6 +296,8 @@ export interface AnnotationRegion { size: AnnotationSize; style: AnnotationTextStyle; zIndex: number; + /** When set, layout/style edits on one region can sync to all auto-caption siblings. */ + annotationSource?: "auto-caption"; figureData?: FigureData; blurData?: BlurData; } @@ -358,8 +368,7 @@ export const DEFAULT_CROP_REGION: CropRegion = { export type PlaybackSpeed = number; export const MIN_PLAYBACK_SPEED = 0.1; -// Anything above 16x causes the playhead to stall during preview -// due to the video decoder not being able to keep up. +// Above 16x the decoder can't keep up and the playhead stalls during preview. export const MAX_PLAYBACK_SPEED = 16; export function clampPlaybackSpeed(speed: number): PlaybackSpeed { diff --git a/src/components/video-editor/videoPlayback/constants.ts b/src/components/video-editor/videoPlayback/constants.ts index b5b4bd1e0..0cfa17c6b 100644 --- a/src/components/video-editor/videoPlayback/constants.ts +++ b/src/components/video-editor/videoPlayback/constants.ts @@ -11,3 +11,14 @@ export const ZOOM_SCALE_DEADZONE = 0.002; export const AUTO_FOLLOW_SMOOTHING_FACTOR = 0.1; export const AUTO_FOLLOW_SMOOTHING_FACTOR_MAX = 0.25; export const AUTO_FOLLOW_RAMP_DISTANCE = 0.15; +// Reference frame interval so preview and export normalize their per-frame +// smoothing identically regardless of render fps. Lower fps = floatier follow +// (tuned to the live-preview feel). +export const AUTO_FOLLOW_REFERENCE_MS = 1000 / 40; +// Shared by preview and export so the camera follows the cursor identically. +export const AUTO_FOLLOW_PARAMS = { + minFactor: AUTO_FOLLOW_SMOOTHING_FACTOR, + maxFactor: AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, + rampDistance: AUTO_FOLLOW_RAMP_DISTANCE, + referenceMs: AUTO_FOLLOW_REFERENCE_MS, +} as const; diff --git a/src/components/video-editor/videoPlayback/cursorFollowUtils.ts b/src/components/video-editor/videoPlayback/cursorFollowUtils.ts index 14dad24f1..12113970c 100644 --- a/src/components/video-editor/videoPlayback/cursorFollowUtils.ts +++ b/src/components/video-editor/videoPlayback/cursorFollowUtils.ts @@ -1,9 +1,6 @@ import type { CursorTelemetryPoint, ZoomFocus } from "../types"; -/** - * Binary-search the sorted telemetry array and linearly interpolate - * the cursor position at the given playback time. - */ +/** Binary-search the sorted telemetry and lerp the cursor position at the given playback time. */ export function interpolateCursorAt( telemetry: CursorTelemetryPoint[], timeMs: number, @@ -44,7 +41,7 @@ export function interpolateCursorAt( /** * Exponential smoothing to reduce jitter from high-frequency cursor data. - * Lower factor = smoother / more lag, higher = more responsive. + * Lower factor = smoother/more lag, higher = more responsive. */ export function smoothCursorFocus(raw: ZoomFocus, prev: ZoomFocus, factor: number): ZoomFocus { return { @@ -53,10 +50,54 @@ export function smoothCursorFocus(raw: ZoomFocus, prev: ZoomFocus, factor: numbe }; } +export interface FollowParams { + minFactor: number; + maxFactor: number; + rampDistance: number; + referenceMs: number; +} + +/** + * Advance the auto-follow focus from `prev` toward target `raw` over `dtMs` of content time. The + * distance-adaptive factor is reframed against `referenceMs` so convergence is content-time based and + * matches between preview and export. Returns `prev` unchanged when paused so the camera holds still. + */ +export function advanceFollowFocus( + prev: ZoomFocus, + raw: ZoomFocus, + dtMs: number, + params: FollowParams, +): ZoomFocus { + if (!(dtMs > 0)) return prev; + const base = adaptiveSmoothFactor( + raw, + prev, + params.minFactor, + params.maxFactor, + params.rampDistance, + ); + const factor = timeCorrectedFollowFactor(base, dtMs, params.referenceMs); + return smoothCursorFocus(raw, prev, factor); +} + +/** + * Make a per-frame smoothing `baseFactor` frame-rate independent by reframing it in content time. + * The camera converges as `(1 - baseFactor)^(dtMs / referenceMs)` regardless of frame chunking, so + * preview (variable fps) and export (fixed fps) follow at the same speed. Larger `referenceMs` = + * floatier. Returns 0 when paused so the camera holds still. + */ +export function timeCorrectedFollowFactor( + baseFactor: number, + dtMs: number, + referenceMs: number, +): number { + if (!(dtMs > 0) || !(referenceMs > 0)) return 0; + return 1 - (1 - baseFactor) ** (dtMs / referenceMs); +} + /** - * Compute an adaptive smoothing factor that scales with distance: - * far from target → faster (maxFactor), close → slower (minFactor). - * This replaces the hard deadzone with a natural deceleration curve. + * Adaptive smoothing factor that scales with distance: far from target = faster (maxFactor), close = + * slower (minFactor). Replaces a hard deadzone with a natural deceleration curve. */ export function adaptiveSmoothFactor( raw: ZoomFocus, diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts index dd3087bd2..2edcca97d 100644 --- a/src/components/video-editor/videoPlayback/cursorRenderer.ts +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -43,11 +43,11 @@ export interface CursorRenderConfig { dotRadius: number; /** Cursor fill color (hex number for PixiJS) */ dotColor: number; - /** Cursor opacity (0–1) */ + /** Cursor opacity (0-1) */ dotAlpha: number; /** Unused, kept for interface compatibility */ trailLength: number; - /** Smoothing factor for cursor interpolation (0–1, lower = smoother/slower) */ + /** Smoothing factor for cursor interpolation (0-1, lower = smoother/slower) */ smoothingFactor: number; /** Directional cursor motion blur amount. */ motionBlur: number; @@ -122,10 +122,9 @@ function getNormalizedAnchor( } /** - * Loads an SVG at `sampleSize × sampleSize`, crops the trim region out of it, - * and returns a PNG data-URL of the cropped result. This is required because - * SVG files have their own natural pixel size (e.g. 32×32) which does not - * match the 1024-sample coordinate space used by the trim measurements. + * Loads an SVG at `sampleSize × sampleSize`, crops the trim region, and returns + * a PNG data-URL. Needed because an SVG's natural size (e.g. 32×32) doesn't match + * the 1024-sample coordinate space the trim measurements use. */ async function rasterizeAndCropSvg( url: string, @@ -137,14 +136,12 @@ async function rasterizeAndCropSvg( ): Promise<{ dataUrl: string; width: number; height: number }> { const img = await loadImage(url); - // Draw at full sample size const srcCanvas = document.createElement("canvas"); srcCanvas.width = sampleSize; srcCanvas.height = sampleSize; const srcCtx = srcCanvas.getContext("2d")!; srcCtx.drawImage(img, 0, 0, sampleSize, sampleSize); - // Crop to trim bounds const dstCanvas = document.createElement("canvas"); dstCanvas.width = trimWidth; dstCanvas.height = trimHeight; @@ -349,7 +346,7 @@ function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: nu } function findLatestStableCursorType(samples: CursorTelemetryPoint[], timeMs: number) { - // Binary search to find position at timeMs, then scan backwards + // Binary search to position at timeMs, then scan backwards let lo = 0; let hi = samples.length - 1; while (lo < hi) { @@ -361,8 +358,8 @@ function findLatestStableCursorType(samples: CursorTelemetryPoint[], timeMs: num } } - // Scan backwards from the position to find a sample with cursorType - // Skip click events only (not mouseup) to avoid transient re-type during clicks + // Scan back for a sample with cursorType. Skip click events (not mouseup) to + // avoid a transient re-type during clicks. for (let index = lo; index >= 0; index -= 1) { const sample = samples[index]; if (sample.timeMs > timeMs) { diff --git a/src/components/video-editor/videoPlayback/layoutUtils.ts b/src/components/video-editor/videoPlayback/layoutUtils.ts index 6bf094634..bdedf6a2f 100644 --- a/src/components/video-editor/videoPlayback/layoutUtils.ts +++ b/src/components/video-editor/videoPlayback/layoutUtils.ts @@ -73,10 +73,8 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null { app.canvas.style.width = "100%"; app.canvas.style.height = "100%"; - // Apply crop region const crop = cropRegion || { x: 0, y: 0, width: 1, height: 1 }; - // Calculate the cropped dimensions const croppedVideoWidth = videoWidth * crop.width; const croppedVideoHeight = videoHeight * crop.height; @@ -85,9 +83,8 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null { const cropEndX = cropStartX + croppedVideoWidth; const cropEndY = cropStartY + croppedVideoHeight; - // Calculate scale to fit the cropped area in the viewport - // Padding is a percentage (0-100), where 50 matches the original VIEWPORT_SCALE of 0.8 - // Vertical stack ignores padding — it's full-bleed + // Padding is a percent (0-100); 50 matches the original VIEWPORT_SCALE of 0.8. + // Vertical stack is full-bleed, so it ignores padding. const effectivePadding = webcamLayoutPreset === "vertical-stack" ? 0 : padding; const paddingScale = 1.0 - (effectivePadding / 100) * 0.4; const maxDisplayWidth = width * paddingScale; @@ -120,11 +117,10 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null { videoSprite.scale.set(scale); - // Calculate display size of the full video at this scale const fullVideoDisplayWidth = videoWidth * scale; const fullVideoDisplayHeight = videoHeight * scale; - // Position the video so the cropped region is centered within the screenRect + // Position the video so the cropped region is centered within screenRect. const croppedDisplayWidth = croppedVideoWidth * scale; const croppedDisplayHeight = croppedVideoHeight * scale; const offsetX = screenRect.x + (screenRect.width - croppedDisplayWidth) / 2; @@ -134,7 +130,7 @@ export function layoutVideoContent(params: LayoutParams): LayoutResult | null { videoSprite.position.set(spriteX, spriteY); - // Apply border radius — mask clips the video to the screenRect + // Mask clips the video to screenRect, with border radius. maskGraphics.clear(); maskGraphics.roundRect( screenRect.x, diff --git a/src/components/video-editor/videoPlayback/mathUtils.ts b/src/components/video-editor/videoPlayback/mathUtils.ts index 78c9414f3..6553c622c 100644 --- a/src/components/video-editor/videoPlayback/mathUtils.ts +++ b/src/components/video-editor/videoPlayback/mathUtils.ts @@ -68,8 +68,7 @@ export function smoothStep(t: number) { } /** - * Gentle ease-in-out cubic — slow start, smooth middle, gentle landing. - * Used for zoom-in transitions. + * Ease-in-out cubic. Used for zoom-in transitions. */ export function easeInOutCubic(t: number) { const x = clamp01(t); @@ -77,8 +76,7 @@ export function easeInOutCubic(t: number) { } /** - * Ease-out cubic — starts at speed, then decelerates to a gentle stop. - * Used for zoom-out transitions so strength eases smoothly to zero. + * Ease-out cubic. Used for zoom-out transitions so strength eases to zero. */ export function easeOutCubic(t: number) { const x = clamp01(t); diff --git a/src/components/video-editor/videoPlayback/videoEventHandlers.ts b/src/components/video-editor/videoPlayback/videoEventHandlers.ts index a26107daa..3b26aa812 100644 --- a/src/components/video-editor/videoPlayback/videoEventHandlers.ts +++ b/src/components/video-editor/videoPlayback/videoEventHandlers.ts @@ -1,9 +1,8 @@ import type React from "react"; import type { SpeedRegion, TrimRegion } from "../types"; -// Keep "scrub mode" on for a brief tail after `seeked` — rapid drag-scrubbing -// fires `seeking`/`seeked` dozens of times per second, and toggling effects -// each time would flicker. +// Keep "scrub mode" on for a brief tail after `seeked`: rapid drag-scrubbing fires +// `seeking`/`seeked` dozens of times a second and toggling effects each time would flicker. const SCRUB_END_DEBOUNCE_MS = 150; interface VideoEventHandlersParams { @@ -51,7 +50,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { onTimeUpdate(timeValue); }; - // Helper function to check if current time is within a trim region const findActiveTrimRegion = (currentTimeMs: number): TrimRegion | null => { const trimRegions = trimRegionsRef.current; return ( @@ -61,7 +59,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { ); }; - // Helper function to find the active speed region at the current time const findActiveSpeedRegion = (currentTimeMs: number): SpeedRegion | null => { return ( speedRegionsRef.current.find( @@ -76,11 +73,11 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { const currentTimeMs = video.currentTime * 1000; const activeTrimRegion = findActiveTrimRegion(currentTimeMs); - // If we're in a trim region during playback, skip to the end of it + // In a trim region during playback: skip to its end if (activeTrimRegion && !video.paused && !video.ended) { const skipToTime = activeTrimRegion.endMs / 1000; - // If the skip would take us past the video duration, pause instead + // Pause if the skip would run past the end if (skipToTime >= video.duration) { video.pause(); } else { @@ -88,7 +85,6 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { emitTime(skipToTime); } } else { - // Apply playback speed from active speed region const activeSpeedRegion = findActiveSpeedRegion(currentTimeMs); video.playbackRate = activeSpeedRegion ? activeSpeedRegion.speed : 1; emitTime(video.currentTime); @@ -143,7 +139,7 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { const currentTimeMs = video.currentTime * 1000; const activeTrimRegion = findActiveTrimRegion(currentTimeMs); - // If we seeked into a trim region while playing, skip to the end + // Seeked into a trim region while playing: skip to the end if (activeTrimRegion && isPlayingRef.current && !video.paused) { const skipToTime = activeTrimRegion.endMs / 1000; diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 1bfa4655b..34fceffe7 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -271,8 +271,8 @@ type DominantRegionResult = { }; // Single-slot cache: the ticker calls findDominantRegion at 60fps with mostly -// unchanged inputs (especially while paused). Reusing the previous result when -// inputs match avoids the per-frame O(N) region scan + allocations. +// unchanged inputs (especially while paused), so reusing the last result skips +// the per-frame O(N) scan and allocations. let dominantRegionCache: { regions: ZoomRegion[]; timeMsKey: number; diff --git a/src/components/video-editor/videoPlayback/zoomSpring.test.ts b/src/components/video-editor/videoPlayback/zoomSpring.test.ts new file mode 100644 index 000000000..c21134262 --- /dev/null +++ b/src/components/video-editor/videoPlayback/zoomSpring.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { createZoomSpringState, resetZoomSpring, stepZoomSpring } from "./zoomSpring"; + +const DT = 1000 / 60; + +describe("zoom spring chase", () => { + it("resetZoomSpring snaps every axis exactly to the target", () => { + const state = createZoomSpringState(); + resetZoomSpring(state, { scale: 1.8, x: -120, y: 40 }); + expect(stepZoomSpring(state, { scale: 1.8, x: -120, y: 40 }, DT)).toEqual({ + scale: 1.8, + x: -120, + y: 40, + }); + }); + + it("eases into a jumped target instead of snapping (velocity continuity)", () => { + const state = createZoomSpringState(); + resetZoomSpring(state, { scale: 1, x: 0, y: 0 }); + // Target jumps from 1 to 2; a single step must NOT teleport there. + const first = stepZoomSpring(state, { scale: 2, x: 0, y: 0 }, DT); + expect(first.scale).toBeGreaterThan(1); + expect(first.scale).toBeLessThan(2); + }); + + it("converges to a static target without overshooting it", () => { + const state = createZoomSpringState(); + resetZoomSpring(state, { scale: 1, x: 0, y: 0 }); + const target = { scale: 2.2, x: 0, y: 0 }; + let maxScale = 1; + let last = 1; + for (let i = 0; i < 200; i++) { + last = stepZoomSpring(state, target, DT).scale; + maxScale = Math.max(maxScale, last); + } + expect(last).toBeCloseTo(2.2, 2); // settled onto the target + expect(maxScale).toBeLessThanOrEqual(2.2 + 1e-6); // never overshot past it + }); + + it("does not overshoot when the target reverses mid-motion", () => { + const state = createZoomSpringState(); + resetZoomSpring(state, { scale: 1, x: 0, y: 0 }); + // Build upward momentum chasing a high target... + for (let i = 0; i < 8; i++) stepZoomSpring(state, { scale: 3, x: 0, y: 0 }, DT); + // ...then reverse the target below the current value; momentum must not carry it past. + const reverseTarget = { scale: 1.5, x: 0, y: 0 }; + let min = Number.POSITIVE_INFINITY; + for (let i = 0; i < 200; i++) { + min = Math.min(min, stepZoomSpring(state, reverseTarget, DT).scale); + } + expect(min).toBeGreaterThanOrEqual(1.5 - 1e-6); // never dipped below the reversed target + }); + + it("steps each axis independently", () => { + const state = createZoomSpringState(); + resetZoomSpring(state, { scale: 1, x: 0, y: 0 }); + const out = stepZoomSpring(state, { scale: 1, x: 100, y: 0 }, DT); + expect(out.scale).toBe(1); // already at target → unchanged + expect(out.x).toBeGreaterThan(0); + expect(out.x).toBeLessThan(100); + expect(out.y).toBe(0); + }); +}); diff --git a/src/components/video-editor/videoPlayback/zoomSpring.ts b/src/components/video-editor/videoPlayback/zoomSpring.ts new file mode 100644 index 000000000..94e19250e --- /dev/null +++ b/src/components/video-editor/videoPlayback/zoomSpring.ts @@ -0,0 +1,85 @@ +import { + createSpringState, + getZoomSpringConfig, + type SpringState, + stepSpringValue, +} from "./motionSmoothing"; + +/** + * Spring-chase for the camera zoom transform. + * + * computeZoomTransform is a time-driven target shaped by an ease curve. Applying it + * straight to the camera reproduces every velocity discontinuity (the ease-in launch, + * seams between close regions), which reads as a jerk. Instead we chase the target with + * a per-axis spring: the target keeps the authored timing, the spring keeps the rendered + * motion velocity-continuous. + */ + +export interface ZoomTransform { + scale: number; + x: number; + y: number; +} + +export interface ZoomSpringState { + scale: SpringState; + x: SpringState; + y: SpringState; +} + +export function createZoomSpringState(): ZoomSpringState { + return { + scale: createSpringState(1), + x: createSpringState(0), + y: createSpringState(0), + }; +} + +/** Snap every axis straight to the target (used on seek / pause / first frame). */ +export function resetZoomSpring(state: ZoomSpringState, target: ZoomTransform): void { + for (const [axis, value] of [ + [state.scale, target.scale], + [state.x, target.x], + [state.y, target.y], + ] as const) { + axis.value = value; + axis.velocity = 0; + axis.initialized = true; + } +} + +/** + * Step one axis toward target with a moving-target overshoot clamp. The target moves + * every frame, so a fast spring can carry velocity past it on a reversal and wobble. If + * the step crosses the target, snap to it and zero the velocity to stay quick without jelly. + */ +function stepAxis( + axis: SpringState, + target: number, + deltaMs: number, + config: ReturnType, +): number { + const before = axis.initialized ? axis.value : target; + const after = stepSpringValue(axis, target, deltaMs, config); + const crossed = (before <= target && after > target) || (before >= target && after < target); + if (crossed) { + axis.value = target; + axis.velocity = 0; + return target; + } + return after; +} + +/** Advance the spring toward target by deltaMs (content time); returns the smoothed transform. */ +export function stepZoomSpring( + state: ZoomSpringState, + target: ZoomTransform, + deltaMs: number, +): ZoomTransform { + const config = getZoomSpringConfig(); + return { + scale: stepAxis(state.scale, target.scale, deltaMs, config), + x: stepAxis(state.x, target.x, deltaMs, config), + y: stepAxis(state.y, target.y, deltaMs, config), + }; +} diff --git a/src/components/video-editor/videoPlayback/zoomTransform.ts b/src/components/video-editor/videoPlayback/zoomTransform.ts index 800949f08..ea96077e2 100644 --- a/src/components/video-editor/videoPlayback/zoomTransform.ts +++ b/src/components/video-editor/videoPlayback/zoomTransform.ts @@ -8,7 +8,7 @@ const MAX_AMOUNT_BOOST = 2.2; function getMotionBlurAmountResponse(motionBlurAmount: number) { const clampedAmount = Math.min(1, Math.max(0, motionBlurAmount)); - // Keep the low end usable while giving the top of the slider substantially more headroom. + // Keep the low end usable while giving the top of the slider more headroom. return clampedAmount * (1 + (MAX_AMOUNT_BOOST - 1) * clampedAmount); } @@ -90,8 +90,7 @@ export function computeZoomTransform({ } const progress = Math.min(1, Math.max(0, zoomProgress)); - // Focus coordinates are stage-normalized (0-1 of full canvas), - // so map directly to stage pixels, not through baseMask. + // Focus coords are stage-normalized (0-1 of full canvas), so map directly to stage pixels, not via baseMask. const focusStagePxX = focusX * stageSize.width; const focusStagePxY = focusY * stageSize.height; const stageCenterX = stageSize.width / 2; @@ -173,7 +172,6 @@ export function applyZoomTransform({ focusY, }); - // Apply position & scale to camera container cameraContainer.scale.set(transform.scale); cameraContainer.position.set(transform.x, transform.y); diff --git a/src/contexts/I18nContext.tsx b/src/contexts/I18nContext.tsx index 5d7534b34..894ba56d5 100644 --- a/src/contexts/I18nContext.tsx +++ b/src/contexts/I18nContext.tsx @@ -106,7 +106,6 @@ export function I18nProvider({ children }: { children: ReactNode }) { // localStorage may be unavailable } document.documentElement.lang = newLocale; - // Notify Electron main process window.electronAPI?.setLocale?.(newLocale); }, []); diff --git a/src/hooks/audioPeaksWorker.ts b/src/hooks/audioPeaksWorker.ts index 27812fd96..8a487193d 100644 --- a/src/hooks/audioPeaksWorker.ts +++ b/src/hooks/audioPeaksWorker.ts @@ -1,11 +1,8 @@ /** * Web Worker: computes min/max peak pairs from raw audio channel data. - * - * Input message: { channels: Float32Array[]; duration: number } - * Output message: Float32Array of length 2*N — [min0, max0, min1, max1, …] - * - * Channel buffers are transferred (zero-copy) from the caller. - * The peaks buffer is transferred back. + * In: { channels: Float32Array[]; duration: number }. + * Out: Float32Array of length 2*N, [min0, max0, min1, max1, ...]. + * Channel buffers and the peaks buffer are transferred (zero-copy). */ self.onmessage = (event: MessageEvent<{ channels: Float32Array[]; duration: number }>) => { const { channels, duration } = event.data; @@ -18,7 +15,7 @@ self.onmessage = (event: MessageEvent<{ channels: Float32Array[]; duration: numb const totalSamples = channels[0].length; const N = Math.min(24000, Math.ceil(duration * 200)); const blockSize = totalSamples / N; - const peaks = new Float32Array(N * 2); // [min0, max0, min1, max1, …] + const peaks = new Float32Array(N * 2); // [min0, max0, min1, max1, ...] for (let i = 0; i < N; i++) { const start = Math.floor(i * blockSize); diff --git a/src/hooks/recorderHandle.ts b/src/hooks/recorderHandle.ts index e98000e6e..6264b9be5 100644 --- a/src/hooks/recorderHandle.ts +++ b/src/hooks/recorderHandle.ts @@ -3,23 +3,19 @@ const RECORDER_TIMESLICE_MS = 1000; export type RecorderHandle = { recorder: MediaRecorder; /** - * Resolves once the recording has fully drained. For a streamed recording the - * blob is empty (the bytes are already on disk); for an in-memory recording it - * holds the full WebM. Rejects if a chunk failed to write to disk mid-stream, - * so a truncated recording surfaces as an error instead of a silent partial save. + * Resolves once the recording drains. Empty blob when streamed (bytes already on + * disk), full WebM when in-memory. Rejects on a mid-stream write failure so a + * truncated recording surfaces as an error instead of a silent partial save. */ recordedBlobPromise: Promise; /** - * Whether the recording's bytes went to disk via the streaming path. Computed - * at finalize time rather than construction, so a stream that fails to open is - * correctly reported as not-streamed and its in-memory fallback is used. + * Whether bytes went to disk via streaming. Computed at finalize, not construction, + * so a stream that fails to open reports as not-streamed and uses its memory fallback. */ isStreaming: () => boolean; /** - * Close the disk stream (if one opened) and delete its partial file. Called - * when a recording is discarded or fails before a successful save, so cancelled - * runs don't leak the stream or orphan a partial file. No-op for in-memory - * recorders. + * Close the disk stream (if any) and delete its partial file. Called when a recording + * is discarded or fails before save, so cancelled runs don't leak. No-op in-memory. */ discard: () => Promise; }; @@ -27,13 +23,10 @@ export type RecorderHandle = { /** * Wrap a MediaRecorder, optionally streaming its chunks to disk. * - * When `fileName` is given, chunks are written to disk in arrival order through - * the main process as they arrive, so a long recording never buffers the whole - * video in the renderer (the #616 fix). Until the disk stream confirms it is - * open, chunks are held in memory; if the open fails, that buffer becomes a - * complete in-memory fallback so nothing is lost. Native-capture webcam sidecars - * omit `fileName` and always buffer in memory, since their finalize path reads - * the blob directly to attach the webcam track. + * With `fileName`, chunks stream to disk through the main process so a long recording + * never buffers the whole video in the renderer (#616). Chunks held in memory until the + * stream confirms open; if the open fails, that buffer is the complete fallback. Webcam + * sidecars omit `fileName` and buffer in memory, since finalize reads the blob directly. */ export function createRecorderHandle( stream: MediaStream, @@ -44,26 +37,23 @@ export function createRecorderHandle( const mimeType = options.mimeType || "video/webm"; const api = window.electronAPI; - // Chunks held in memory: everything before the stream opens, plus everything - // when not streaming at all. On a successful open these flush to disk and are - // dropped; on open failure they remain as the complete fallback recording. + // Chunks held in memory before the stream opens, or for the whole recording when not + // streaming. On open they flush to disk and drop; on open failure they're the fallback. const memoryChunks: Blob[] = []; let mode: "pending" | "streaming" | "buffering" = fileName ? "pending" : "buffering"; let streamOpened = false; let appendError: Error | null = null; - // Serialize chunk writes so they land on disk in arrival order, and so stop - // can await every in-flight write before the main process closes the stream - // (otherwise a late chunk arrives after close and truncates the recording). + // Serialize writes so chunks land in arrival order and stop can await every in-flight + // write before the stream closes (a late chunk after close truncates the recording). let writeChain: Promise = Promise.resolve(); const enqueueWrite = (chunk: Blob) => { writeChain = writeChain.then(async () => { if (appendError || !fileName || !api?.appendRecordingChunk) { return; } - // Capture both outcomes — a `{ success: false }` result and an outright - // rejection (channel/handler error) — into appendError, so writeChain - // never rejects and isStreaming() stays consistent after a failure. + // Capture both a `{ success: false }` result and an outright rejection into + // appendError, so writeChain never rejects and isStreaming() stays consistent. try { const buffer = await chunk.arrayBuffer(); const result = await api.appendRecordingChunk(fileName, buffer); @@ -76,10 +66,9 @@ export function createRecorderHandle( }); }; - // Require BOTH stream IPC methods before attempting to stream. If only - // openRecordingStream exists (renderer/main version skew), streaming would - // open but every append would silently no-op, saving an empty file — so in - // that case fall through to in-memory buffering instead. + // Require both stream IPC methods before streaming. With only openRecordingStream + // (renderer/main version skew) the open succeeds but appends no-op, saving an empty + // file, so fall through to in-memory buffering instead. const openPromise: Promise<{ success: boolean; error?: string }> = fileName !== undefined && typeof api?.openRecordingStream === "function" && @@ -101,8 +90,7 @@ export function createRecorderHandle( } }, () => { - // The IPC call itself rejected (channel or handler error). Treat it the - // same as a failed open: keep buffering in memory so nothing is lost. + // IPC call rejected. Treat like a failed open: keep buffering in memory. mode = "buffering"; }, ); @@ -115,7 +103,7 @@ export function createRecorderHandle( if (mode === "streaming") { enqueueWrite(event.data); } else { - // "pending" (stream not open yet) or "buffering" (not streaming). + // pending (stream not open yet) or buffering (not streaming). memoryChunks.push(event.data); } }; @@ -130,9 +118,9 @@ export function createRecorderHandle( }); async function finalizeBlob(): Promise { - // Wait for the open attempt to settle so its flush (or fallback switch) has - // been applied, then for every queued write to land, so we never resolve - // while chunks are still in flight to the about-to-close disk stream. + // Wait for the open to settle (flush or fallback applied) then for every queued + // write to land, so we don't resolve while chunks are still in flight to the + // about-to-close stream. await openPromise.catch(() => undefined); await writeChain; if (appendError) { diff --git a/src/hooks/useAudioPeaks.ts b/src/hooks/useAudioPeaks.ts index 3be6ac613..337f50344 100644 --- a/src/hooks/useAudioPeaks.ts +++ b/src/hooks/useAudioPeaks.ts @@ -10,8 +10,7 @@ function getAudioCtx(): AudioContext { /** * Offloads peak computation to a Web Worker (zero-copy via Transferable). - * Accepts an optional AbortSignal — if aborted, the worker is terminated - * immediately and the promise rejects with an AbortError. + * On abort, the worker is terminated and the promise rejects with AbortError. */ function computePeaksInWorker( audioBuffer: AudioBuffer, @@ -60,15 +59,10 @@ function computePeaksInWorker( } /** - * Decodes audio from `videoUrl` and returns a Float32Array of paired - * [min, max] peak values (length = 2 * N blocks). Returns `null` while - * decoding is in progress, and stays `null` when the file has no audio - * track or decoding fails (silent degradation). - * - * - File loading uses the Electron IPC bridge for local paths (same as the exporter). - * - Peak computation runs in a Web Worker to avoid blocking the main thread. - * - Results are cached in a ref scoped to the hook instance (survives re-renders - * and waveform toggle off/on, but not component unmount). + * Decodes audio from `videoUrl` into paired [min, max] peaks (length = 2 * N + * blocks). Returns `null` while decoding, and stays `null` on no audio track or + * decode failure (silent degradation). Results are cached in a ref scoped to the + * hook instance, so they survive re-renders and waveform toggles but not unmount. */ export function useAudioPeaks(videoUrl?: string): Float32Array | null { const cacheRef = useRef>(new Map()); @@ -103,9 +97,11 @@ export function useAudioPeaks(videoUrl?: string): Float32Array | null { cacheRef.current.set(videoUrl, p); setPeaks(p); } catch (err) { - // AbortError means the effect cleaned up — no state update needed. + // AbortError means the effect cleaned up, so no state update needed. if (err instanceof DOMException && err.name === "AbortError") return; - // No audio track or unsupported format — clear stale data silently. + // No audio track or unsupported format: degrade to no waveform, but log + // so an unexpectedly-missing waveform is diagnosable. + console.warn("useAudioPeaks: could not decode audio for waveform:", err); if (!cancelled) setPeaks(null); } })(); diff --git a/src/hooks/useCameraDevices.test.ts b/src/hooks/useCameraDevices.test.ts index 5ca21bc85..e6ec8a9ed 100644 --- a/src/hooks/useCameraDevices.test.ts +++ b/src/hooks/useCameraDevices.test.ts @@ -91,7 +91,7 @@ describe("useCameraDevices", () => { expect(result.current.selectedDeviceId).toBe("cam1"); }); - // Simulate cam1 being unplugged — only cam2 remains + // Simulate cam1 being unplugged, only cam2 remains const cam2Only = [ { kind: "videoinput", deviceId: "cam2", label: "Camera 2", groupId: "group1" }, ]; diff --git a/src/hooks/useCameraDevices.ts b/src/hooks/useCameraDevices.ts index a02e65aa2..bf402c7f0 100644 --- a/src/hooks/useCameraDevices.ts +++ b/src/hooks/useCameraDevices.ts @@ -23,8 +23,8 @@ export function useCameraDevices(enabled: boolean = false) { setIsLoading(true); setError(null); - // Enumerate without requesting a second stream — the recorder handles - // the real acquisition; unlabeled devices fall back to their device ID. + // Enumerate without requesting a second stream; the recorder handles + // the real acquisition. Unlabeled devices fall back to their device ID. const allDevices = await navigator.mediaDevices.enumerateDevices(); const videoInputs = allDevices .filter((device) => device.kind === "videoinput") diff --git a/src/hooks/useEditorHistory.ts b/src/hooks/useEditorHistory.ts index 25b6e21a1..9ca00440b 100644 --- a/src/hooks/useEditorHistory.ts +++ b/src/hooks/useEditorHistory.ts @@ -15,13 +15,22 @@ import type { WebcamSizePreset, ZoomRegion, } from "@/components/video-editor/types"; -import { DEFAULT_CROP_REGION } from "@/components/video-editor/types"; +import { + DEFAULT_CROP_REGION, + DEFAULT_WEBCAM_MIRRORED, + DEFAULT_WEBCAM_REACTIVE_ZOOM, +} from "@/components/video-editor/types"; import type { AspectRatio } from "@/utils/aspectRatioUtils"; -// Undoable state — selection IDs are intentionally excluded (undoing a -// selection change would feel surprising to the user). +// Undoable state. Selection IDs are excluded, since undoing a selection change +// would feel surprising. export interface EditorState { zoomRegions: ZoomRegion[]; + /** Magic-wand auto-zoom toggle. When on, fresh recordings get suggested zooms. */ + autoZoomEnabled: boolean; + /** Global Auto-Focus toggle: when on, all zooms follow the cursor and the + * per-zoom Focus Mode selector is locked. */ + autoFocusAll: boolean; trimRegions: TrimRegion[]; speedRegions: SpeedRegion[]; annotationRegions: AnnotationRegion[]; @@ -36,12 +45,16 @@ export interface EditorState { aspectRatio: AspectRatio; webcamLayoutPreset: WebcamLayoutPreset; webcamMaskShape: WebcamMaskShape; + webcamMirrored: boolean; + webcamReactiveZoom: boolean; webcamSizePreset: WebcamSizePreset; webcamPosition: WebcamPosition | null; } export const INITIAL_EDITOR_STATE: EditorState = { zoomRegions: [], + autoZoomEnabled: true, + autoFocusAll: false, trimRegions: [], speedRegions: [], annotationRegions: [], @@ -56,6 +69,8 @@ export const INITIAL_EDITOR_STATE: EditorState = { aspectRatio: DEFAULT_EDITOR_LAYOUT_SETTINGS.aspectRatio, webcamLayoutPreset: DEFAULT_WEBCAM_SETTINGS.layoutPreset, webcamMaskShape: DEFAULT_WEBCAM_SETTINGS.maskShape, + webcamMirrored: DEFAULT_WEBCAM_MIRRORED, + webcamReactiveZoom: DEFAULT_WEBCAM_REACTIVE_ZOOM, webcamSizePreset: DEFAULT_WEBCAM_SETTINGS.sizePreset, webcamPosition: DEFAULT_WEBCAM_SETTINGS.position, }; @@ -86,8 +101,8 @@ function withCheckpoint(history: History, newPresent: EditorState): History { export function useEditorHistory(initial: EditorState = INITIAL_EDITOR_STATE) { const [history, setHistory] = useState({ past: [], present: initial, future: [] }); - // Tracks whether a live-update series (e.g. slider drag) is in progress. - // The first updateState call saves the pre-interaction state as a checkpoint. + // True while a live-update series (e.g. slider drag) is in progress. The first + // updateState call checkpoints the pre-interaction state. const dirtyRef = useRef(false); const pushState = useCallback((update: StateUpdate) => { diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index f5fb92032..ce0e0b77f 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -135,10 +135,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }, []); const selectMimeType = () => { - // H.264 first: hardware-accelerated on all modern devices, gives sharp - // real-time output. AV1/VP9 are great for distribution but too - // CPU-intensive for live 60 fps capture — they produce blurry frames - // when the software encoder can't keep up. + // H.264 first: hardware-accelerated, so sharp real-time output. AV1/VP9 are + // better for distribution but too CPU-heavy for live 60 fps capture (software + // encoder falls behind and produces blurry frames). const preferred = [ "video/webm;codecs=h264", "video/webm;codecs=vp8", @@ -339,7 +338,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { window.electronAPI?.discardCursorTelemetry(activeRecordingId); return; } - // When streaming succeeded the blob is empty — the data is already on disk. + // When streaming succeeded the blob is empty; the data is already on disk. if (!activeScreenRecorder.isStreaming() && screenBlob.size === 0) { return; } @@ -398,10 +397,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } catch (error) { console.error("Error saving recording:", error); } finally { - // Discard any recorder whose data was not part of a successful save - // — a discarded run, a failed save, or a webcam whose disk write - // failed (so it was omitted while the screen still saved) — so no - // stream or partial file is left open or orphaned. + // Discard any recorder whose data wasn't part of a successful save (discarded + // run, failed save, or a webcam whose disk write failed while the screen still + // saved) so no stream or partial file is left open or orphaned. if (!storeSucceeded) { await activeScreenRecorder.discard().catch(() => undefined); } @@ -1069,9 +1067,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { try { const platform = await window.electronAPI.getPlatform(); if (platform === "darwin" && cursorCaptureMode === "editable-overlay") { + // The main process shows a native dialog that deep-links to the + // Accessibility settings pane when access is missing, so we just stop + // here and let the user grant it and press record again. const access = await window.electronAPI.requestNativeMacCursorAccess(); if (!access.granted) { - toast.info(t("recording.accessibilityAllowAndRetry")); return; } } diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index b3e122280..39750e5eb 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -44,6 +44,25 @@ "permissionDenied": "تم رفض إذن التسجيل. يرجى السماح بتسجيل الشاشة.", "accessibilityAllowAndRetry": "اسمح بوصول تسهيلات الاستخدام لـ OpenScreen، ثم اضغط على التسجيل مرة أخرى لبدء العد التنازلي." }, + "autoCaptions": { + "button": "التسميات التوضيحية التلقائية", + "dialogTitle": "التسميات التوضيحية التلقائية", + "dialogDescription": "اختر تقريبا كم عدد الكلمات التي تظهر في كل تسمية توضيحية. يتم توزيع التوقيت عبر الكلمات في تلك العبارة.", + "minWords": "الحد الأدنى من الكلمات لكل تسمية", + "maxWords": "الحد الأقصى من الكلمات لكل تسمية", + "wordsCount": "{{count}} كلمة", + "generate": "توليد", + "dialogCancel": "إلغاء", + "generating": "جارٍ توليد التسميات من الصوت…", + "loadingModel": "جارٍ تحميل نموذج الكلام (سيتم تنزيل ~75 ميغابايت عند الاستخدام الأول)…", + "transcribing": "جارٍ نسخ الكلام إلى نص…", + "busy": "توليد التسميات قيد التنفيذ بالفعل.", + "done": "تمت إضافة {{count}} تسمية.", + "noneHeard": "لم يتم الكشف عن أي كلام.", + "noAudio": "لا يحتوي هذا الفيديو على صوت صالح للنسخ.", + "failed": "تعذّر توليد التسميات.", + "truncated": "تم نسخ الدقائق الأولى فقط: {{minutes}} دقيقة." + }, "emptyState": { "title": "لا يوجد مشروع مفتوح", "description": "استورد مقطع فيديو للبدء في التحرير، أو حمّل مشروع OpenScreen موجود.", diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index 9ddc2ba7d..c91928890 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -45,7 +45,10 @@ "dualFrame": "إطار مزدوج", "webcamShape": "شكل الكاميرا", "webcamSize": "حجم كاميرا الويب", - "noWebcam": "بدون كاميرا" + "noWebcam": "بدون كاميرا", + "mirrorWebcam": "عكس كاميرا الويب", + "reactiveWebcam": "تصغير عند التكبير", + "reactiveWebcamDescription": "تتقلص الكاميرا بسلاسة أثناء تكبير الفيديو حتى لا تعيق الرؤية." }, "effects": { "title": "تأثيرات الفيديو", @@ -197,6 +200,8 @@ "errorLoadFailed": "تعذر تحميل الخط. يرجى التحقق من صحة رابط خطوط Google." }, "cursor": { + "theme": "نمط المؤشر", + "themeDefault": "افتراضي", "show": "إظهار المؤشر", "size": "الحجم", "smoothing": "التنعيم", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index ebd9a5d5f..d6a56f033 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -44,6 +44,25 @@ "permissionDenied": "Recording permission denied. Please allow screen recording.", "accessibilityAllowAndRetry": "Allow Accessibility access for OpenScreen, then press record again to start the countdown." }, + "autoCaptions": { + "button": "Auto captions", + "dialogTitle": "Auto captions", + "dialogDescription": "Choose roughly how many words each caption shows at once. Timing is spread across the words in that phrase.", + "minWords": "Minimum words per caption", + "maxWords": "Maximum words per caption", + "wordsCount": "{{count}} words", + "generate": "Generate", + "dialogCancel": "Cancel", + "generating": "Generating captions from audio…", + "loadingModel": "Loading speech model (first use downloads ~75 MB)…", + "transcribing": "Transcribing speech…", + "busy": "Caption generation is already in progress.", + "done": "Added {{count}} captions.", + "noneHeard": "No speech was detected.", + "noAudio": "This video has no usable audio to transcribe.", + "failed": "Could not generate captions.", + "truncated": "Only the first {{minutes}} minutes were transcribed." + }, "emptyState": { "title": "No project open", "description": "Import a video to start editing, or load an existing OpenScreen project.", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 02d44c658..3594ac51e 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -9,7 +9,8 @@ "title": "Focus Mode", "manual": "Manual", "auto": "Auto", - "autoDescription": "Camera follows the recorded cursor position" + "autoDescription": "Camera follows the recorded cursor position", + "lockedDisclaimer": "Controlled by the global Auto-Focus toggle in the timeline. Turn it off to set focus mode per zoom." }, "threeD": { "title": "3D Rotation", @@ -45,7 +46,10 @@ "dualFrame": "Dual Frame", "noWebcam": "No Webcam", "webcamShape": "Camera Shape", - "webcamSize": "Webcam Size" + "webcamSize": "Webcam Size", + "mirrorWebcam": "Mirror Webcam", + "reactiveWebcam": "Shrink on Zoom", + "reactiveWebcamDescription": "Camera smoothly shrinks while the video is zoomed in, so it stays out of the way." }, "effects": { "title": "Video Effects", @@ -206,12 +210,15 @@ "errorLoadFailed": "The font could not be loaded. Please verify the Google Fonts URL is correct." }, "cursor": { + "theme": "Cursor Style", + "themeDefault": "Default", "show": "Show Cursor", "size": "Size", "smoothing": "Smoothing", "motionBlur": "Motion Blur", "clickBounce": "Click Bounce", - "clipToBounds": "Clip to Canvas" + "clipToBounds": "Clip to Canvas", + "clipToBoundsDescription": "Keeps the cursor inside the video frame. Turn off to let the cursor extend past the edges - useful when zoomed in or panned." }, "language": { "title": "Language" diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index 389184b2b..4fc250828 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -2,6 +2,10 @@ "buttons": { "addZoom": "Add Zoom (Z)", "suggestZooms": "Suggest Zooms from Cursor", + "autoZoomOn": "Auto zoom suggestions on — click to remove suggested zooms", + "autoZoomOff": "Auto zoom suggestions off — click to suggest zooms from cursor", + "autoFocusAllOn": "Auto-Focus on for all zooms — click to switch all to manual", + "autoFocusAllOff": "Auto-Focus all zooms (camera follows the cursor)", "addTrim": "Add Trim (T)", "addAnnotation": "Add Annotation (A)", "addBlur": "Add Blur (B)", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 16a2c8547..277ce40ff 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -44,6 +44,25 @@ "cancel": "Cancelar", "confirm": "Confirmar" }, + "autoCaptions": { + "button": "Subtítulos automáticos", + "dialogTitle": "Subtítulos automáticos", + "dialogDescription": "Elige aproximadamente cuántas palabras muestra cada subtítulo a la vez. El tiempo se reparte entre las palabras de esa frase.", + "minWords": "Número mínimo de palabras por subtítulo", + "maxWords": "Número máximo de palabras por subtítulo", + "wordsCount": "{{count}} palabras", + "generate": "Generar", + "dialogCancel": "Cancelar", + "generating": "Generando subtítulos a partir del audio…", + "loadingModel": "Cargando el modelo de voz (el primer uso descarga ~75 MB)…", + "transcribing": "Transcribiendo el habla…", + "busy": "La generación de subtítulos ya está en curso.", + "done": "Se añadieron {{count}} subtítulos.", + "noneHeard": "No se detectó voz.", + "noAudio": "Este video no tiene audio utilizable para transcribir.", + "failed": "No se pudieron generar los subtítulos.", + "truncated": "Solo se transcribieron los primeros {{minutes}} minutos." + }, "emptyState": { "title": "No hay proyecto abierto", "description": "Importa un video para empezar a editar o carga un proyecto de OpenScreen existente.", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 21295bf80..fa046e794 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -45,7 +45,10 @@ "dualFrame": "Marco dual", "webcamShape": "Forma de cámara", "webcamSize": "Tamaño de cámara", - "noWebcam": "Sin cámara" + "noWebcam": "Sin cámara", + "mirrorWebcam": "Reflejar cámara", + "reactiveWebcam": "Reducir al ampliar", + "reactiveWebcamDescription": "La cámara se reduce suavemente mientras el vídeo está ampliado, para no estorbar." }, "effects": { "title": "Efectos de video", @@ -206,6 +209,8 @@ "errorLoadFailed": "No se pudo cargar la fuente. Por favor verifica que la URL de Google Fonts sea correcta." }, "cursor": { + "theme": "Estilo del cursor", + "themeDefault": "Predeterminado", "show": "Mostrar cursor", "size": "Tamaño", "smoothing": "Suavizado", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 4eb57a9cc..40dc24fd7 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -44,6 +44,25 @@ }, "loadingVideo": "Chargement de la vidéo...", "loadingEditor": "Chargement de l'éditeur...", + "autoCaptions": { + "button": "Sous-titres automatiques", + "dialogTitle": "Sous-titres automatiques", + "dialogDescription": "Choisissez approximativement combien de mots chaque sous-titre affiche à la fois. Le timing est réparti entre les mots de cette phrase.", + "minWords": "Nombre minimum de mots par sous-titre", + "maxWords": "Nombre maximum de mots par sous-titre", + "wordsCount": "{{count}} mots", + "generate": "Générer", + "dialogCancel": "Annuler", + "generating": "Génération des sous-titres à partir de l'audio…", + "loadingModel": "Chargement du modèle vocal (le premier usage télécharge ~75 MB)…", + "transcribing": "Transcription de la parole…", + "busy": "La génération des sous-titres est déjà en cours.", + "done": "{{count}} sous-titres ajoutés.", + "noneHeard": "Aucune parole n'a été détectée.", + "noAudio": "Cette vidéo ne contient pas d'audio exploitable pour la transcription.", + "failed": "Impossible de générer les sous-titres.", + "truncated": "Seules les {{minutes}} premières minutes ont été transcrites." + }, "emptyState": { "title": "Aucun projet ouvert", "description": "Importez une vidéo pour commencer à éditer, ou chargez un projet OpenScreen existant.", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index f5224afdd..06d51e2da 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -45,7 +45,10 @@ "dualFrame": "Double cadre", "webcamShape": "Forme de la caméra", "webcamSize": "Taille de la caméra", - "noWebcam": "Sans webcam" + "noWebcam": "Sans webcam", + "mirrorWebcam": "Inverser la webcam", + "reactiveWebcam": "Réduire au zoom", + "reactiveWebcamDescription": "La caméra rétrécit doucement pendant le zoom, pour ne pas gêner." }, "effects": { "title": "Effets vidéo", @@ -206,6 +209,8 @@ "errorLoadFailed": "La police n'a pas pu être chargée. Vérifiez que l'URL Google Fonts est correcte." }, "cursor": { + "theme": "Style du curseur", + "themeDefault": "Par défaut", "show": "Afficher le curseur", "size": "Taille", "smoothing": "Lissage", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 336d3e6ba..0e94b9a9f 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -42,5 +42,24 @@ "cameraNotFound": "Fotocamera non trovata.", "permissionDenied": "Autorizzazione di registrazione negata. Consenti la registrazione dello schermo.", "accessibilityAllowAndRetry": "Consenti l'accesso all'accessibilità per OpenScreen, poi premi di nuovo registra per avviare il conto alla rovescia." + }, + "autoCaptions": { + "button": "Sottotitoli automatici", + "dialogTitle": "Sottotitoli automatici", + "dialogDescription": "Scegli all'incirca quante parole mostrare per ogni sottotitolo. La temporizzazione viene distribuita tra le parole della frase.", + "minWords": "Numero minimo di parole per sottotitolo", + "maxWords": "Numero massimo di parole per sottotitolo", + "wordsCount": "{{count}} parole", + "generate": "Genera", + "dialogCancel": "Annulla", + "generating": "Generazione dei sottotitoli dall'audio…", + "loadingModel": "Caricamento del modello vocale (al primo utilizzo vengono scaricati ~75 MB)…", + "transcribing": "Trascrizione del parlato…", + "busy": "La generazione dei sottotitoli è già in corso.", + "done": "Aggiunti {{count}} sottotitoli.", + "noneHeard": "Nessun parlato rilevato.", + "noAudio": "Questo video non contiene audio utilizzabile per la trascrizione.", + "failed": "Impossibile generare i sottotitoli.", + "truncated": "Sono stati trascritti solo i primi {{minutes}} minuti." } } diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index a1c8c5647..9345f95ee 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -45,7 +45,10 @@ "dualFrame": "Doppio frame", "noWebcam": "Nessuna webcam", "webcamShape": "Forma fotocamera", - "webcamSize": "Dimensione webcam" + "webcamSize": "Dimensione webcam", + "mirrorWebcam": "Specchia webcam", + "reactiveWebcam": "Riduci con lo zoom", + "reactiveWebcamDescription": "La webcam si riduce dolcemente durante lo zoom, così non intralcia." }, "effects": { "title": "Effetti video", @@ -205,6 +208,8 @@ "errorLoadFailed": "Impossibile caricare il font. Verifica che l'URL di Google Fonts sia corretto." }, "cursor": { + "theme": "Stile del cursore", + "themeDefault": "Predefinito", "show": "Mostra cursore", "size": "Dimensione", "smoothing": "Smussatura", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 5151d1054..8e0da42e1 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -44,6 +44,25 @@ "cameraNotFound": "カメラが見つかりません。", "accessibilityAllowAndRetry": "OpenScreenにアクセシビリティアクセスを許可してから、もう一度録画を押してカウントダウンを開始してください。" }, + "autoCaptions": { + "button": "自動キャプション", + "dialogTitle": "自動キャプション", + "dialogDescription": "各キャプションに一度に表示する語数の目安を選びます。タイミングはそのフレーズ内の語に分配されます。", + "minWords": "キャプションあたりの最小語数", + "maxWords": "キャプションあたりの最大語数", + "wordsCount": "{{count}} 語", + "generate": "生成", + "dialogCancel": "キャンセル", + "generating": "音声からキャプションを生成しています…", + "loadingModel": "音声モデルを読み込んでいます(初回利用時は約 75 MB をダウンロードします)…", + "transcribing": "音声を文字起こししています…", + "busy": "キャプションの生成はすでに実行中です。", + "done": "{{count}} 件のキャプションを追加しました。", + "noneHeard": "音声が検出されませんでした。", + "noAudio": "この動画には書き起こしに使える音声がありません。", + "failed": "キャプションを生成できませんでした。", + "truncated": "最初の {{minutes}} 分のみが書き起こされました。" + }, "emptyState": { "title": "プロジェクトが開かれていません", "description": "動画をインポートして編集を開始するか、既存の OpenScreen プロジェクトを読み込んでください。", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index efecf27e2..2950deb03 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -45,7 +45,10 @@ "dualFrame": "デュアルフレーム", "webcamShape": "カメラの形状", "webcamSize": "カメラのサイズ", - "noWebcam": "Webカメラなし" + "noWebcam": "Webカメラなし", + "mirrorWebcam": "Webカメラを反転", + "reactiveWebcam": "ズーム時に縮小", + "reactiveWebcamDescription": "ズーム中はカメラがスムーズに縮小し、邪魔になりません。" }, "effects": { "title": "動画効果", @@ -206,6 +209,8 @@ "errorLoadFailed": "フォントを読み込めませんでした。GoogleフォントのURLが正しいことを確認してください。" }, "cursor": { + "theme": "カーソルのスタイル", + "themeDefault": "デフォルト", "show": "カーソルを表示", "size": "サイズ", "smoothing": "スムージング", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 23990c386..a63a22a57 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -44,6 +44,25 @@ "cameraNotFound": "카메라를 찾을 수 없습니다.", "accessibilityAllowAndRetry": "OpenScreen의 손쉬운 사용 접근을 허용한 다음, 카운트다운을 시작하려면 다시 녹화를 누르세요." }, + "autoCaptions": { + "button": "자동 자막", + "dialogTitle": "자동 자막", + "dialogDescription": "각 자막에 한 번에 표시할 단어 수의 대략적인 값을 선택하세요. 타이밍은 해당 구문의 단어들에 나뉩니다.", + "minWords": "자막당 최소 단어 수", + "maxWords": "자막당 최대 단어 수", + "wordsCount": "{{count}}개 단어", + "generate": "생성", + "dialogCancel": "취소", + "generating": "오디오에서 자막을 생성하는 중…", + "loadingModel": "음성 모델을 불러오는 중(첫 사용 시 약 75MB 다운로드)…", + "transcribing": "음성을 전사하는 중…", + "busy": "자막 생성이 이미 진행 중입니다.", + "done": "자막 {{count}}개를 추가했습니다.", + "noneHeard": "음성이 감지되지 않았습니다.", + "noAudio": "이 동영상에는 전사에 사용할 수 있는 음성이 없습니다.", + "failed": "자막을 생성할 수 없습니다.", + "truncated": "처음 {{minutes}}분만 전사되었습니다." + }, "emptyState": { "title": "열린 프로젝트 없음", "description": "동영상을 가져와 편집을 시작하거나 기존 OpenScreen 프로젝트를 불러오세요.", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index 5921ca3e2..d593014ed 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -45,7 +45,10 @@ "webcamShape": "카메라 모양", "webcamSize": "웹캠 크기", "dualFrame": "듀얼 프레임", - "noWebcam": "웹캠 없음" + "noWebcam": "웹캠 없음", + "mirrorWebcam": "웹캠 미러링", + "reactiveWebcam": "확대 시 축소", + "reactiveWebcamDescription": "확대하는 동안 카메라가 부드럽게 작아져 방해되지 않습니다." }, "effects": { "title": "비디오 효과", @@ -206,6 +209,8 @@ "errorLoadFailed": "폰트를 불러올 수 없습니다. Google Fonts URL이 올바른지 확인해 주세요." }, "cursor": { + "theme": "커서 스타일", + "themeDefault": "기본", "show": "커서 표시", "size": "크기", "smoothing": "부드러움", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 7e3f69531..b0e9ab8c9 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -41,5 +41,24 @@ "cameraDisconnected": "Webcam desconectada.", "cameraNotFound": "Câmera não encontrada.", "permissionDenied": "Permissão de gravação negada. Por favor, permita a gravação de tela." + }, + "autoCaptions": { + "button": "Legendas automáticas", + "dialogTitle": "Legendas automáticas", + "dialogDescription": "Escolha aproximadamente quantas palavras cada legenda mostra de cada vez. O tempo é distribuído entre as palavras da frase.", + "minWords": "Mínimo de palavras por legenda", + "maxWords": "Máximo de palavras por legenda", + "wordsCount": "{{count}} palavras", + "generate": "Gerar", + "dialogCancel": "Cancelar", + "generating": "Gerando legendas a partir do áudio…", + "loadingModel": "Carregando o modelo de fala (o primeiro uso baixa ~75 MB)…", + "transcribing": "Transcrevendo a fala…", + "busy": "A geração de legendas já está em andamento.", + "done": "{{count}} legendas adicionadas.", + "noneHeard": "Nenhuma fala foi detectada.", + "noAudio": "Este vídeo não tem áudio utilizável para transcrição.", + "failed": "Não foi possível gerar as legendas.", + "truncated": "Apenas os primeiros {{minutes}} minutos foram transcritos." } } diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 6788bdb2e..357db548a 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -44,7 +44,9 @@ "dualFrame": "Quadro Duplo", "noWebcam": "Sem Webcam", "webcamShape": "Formato da Câmera", - "webcamSize": "Tamanho da Webcam" + "webcamSize": "Tamanho da Webcam", + "reactiveWebcam": "Encolher ao ampliar", + "reactiveWebcamDescription": "A câmera diminui suavemente enquanto o vídeo está ampliado, para não atrapalhar." }, "effects": { "title": "Efeitos de Vídeo", @@ -191,6 +193,17 @@ "errorTimeout": "A fonte demorou muito para carregar. Por favor, verifique a URL e tente novamente.", "errorLoadFailed": "A fonte não pôde ser carregada. Por favor, verifique se a URL do Google Fonts está correta." }, + "cursor": { + "theme": "Estilo do cursor", + "themeDefault": "Padrão", + "show": "Mostrar cursor", + "size": "Tamanho", + "smoothing": "Suavização", + "motionBlur": "Desfoque de movimento", + "clickBounce": "Rebote ao clicar", + "clipToBounds": "Recortar à tela", + "clipToBoundsDescription": "Mantém o cursor dentro do quadro do vídeo. Desative para permitir que o cursor ultrapasse as bordas — útil ao aplicar zoom ou ao deslocar." + }, "language": { "title": "Idioma" } diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index ff0c80b8b..78fa129a1 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -44,6 +44,25 @@ "permissionDenied": "Разрешение на запись запрещено. Пожалуйста, разрешите запись экрана.", "accessibilityAllowAndRetry": "Разрешите OpenScreen доступ к Универсальному доступу, затем снова нажмите запись, чтобы начать обратный отсчет." }, + "autoCaptions": { + "button": "Автосубтитры", + "dialogTitle": "Автосубтитры", + "dialogDescription": "Выберите, сколько примерно слов показывать в одном субтитре. Время распределяется между словами фразы.", + "minWords": "Минимум слов в субтитре", + "maxWords": "Максимум слов в субтитре", + "wordsCount": "{{count}} слов", + "generate": "Создать", + "dialogCancel": "Отмена", + "generating": "Создание субтитров из звука…", + "loadingModel": "Загрузка речевой модели (при первом запуске скачивается ~75 МБ)…", + "transcribing": "Распознавание речи…", + "busy": "Создание субтитров уже выполняется.", + "done": "Добавлено субтитров: {{count}}.", + "noneHeard": "Речь не обнаружена.", + "noAudio": "В этом видео нет звука, пригодного для расшифровки.", + "failed": "Не удалось создать субтитры.", + "truncated": "Расшифрованы только первые {{minutes}} мин." + }, "emptyState": { "title": "Нет открытых проектов", "description": "Импортируйте видео для начала редактирования или загрузите существующий проект OpenScreen.", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index e08684490..17205a099 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -45,7 +45,10 @@ "dualFrame": "Двойной кадр", "webcamShape": "Форма камеры", "webcamSize": "Размер веб-камеры", - "noWebcam": "Без веб-камеры" + "noWebcam": "Без веб-камеры", + "mirrorWebcam": "Зеркалить веб-камеру", + "reactiveWebcam": "Уменьшать при зуме", + "reactiveWebcamDescription": "Камера плавно уменьшается во время приближения, чтобы не мешать." }, "effects": { "title": "Видеоэффекты", @@ -206,6 +209,8 @@ "errorLoadFailed": "Не удалось загрузить шрифт. Пожалуйста, проверьте правильность URL Google Fonts." }, "cursor": { + "theme": "Стиль курсора", + "themeDefault": "По умолчанию", "show": "Показывать курсор", "size": "Размер", "smoothing": "Сглаживание", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index de45a180f..89203e719 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -44,6 +44,25 @@ "cancel": "İptal", "confirm": "Onayla" }, + "autoCaptions": { + "button": "Otomatik altyazılar", + "dialogTitle": "Otomatik altyazılar", + "dialogDescription": "Her altyazının aynı anda yaklaşık kaç kelime göstermesini istediğinizi seçin. Zamanlama, o ifadedeki kelimelere dağıtılır.", + "minWords": "Altyazı başına en az kelime", + "maxWords": "Altyazı başına en fazla kelime", + "wordsCount": "{{count}} kelime", + "generate": "Oluştur", + "dialogCancel": "İptal", + "generating": "Sesten altyazılar oluşturuluyor…", + "loadingModel": "Konuşma modeli yükleniyor (ilk kullanımda ~75 MB indirilir)…", + "transcribing": "Konuşma yazıya dökülüyor…", + "busy": "Altyazı oluşturma zaten devam ediyor.", + "done": "{{count}} altyazı eklendi.", + "noneHeard": "Konuşma algılanmadı.", + "noAudio": "Bu videoda yazıya dökülebilecek kullanılabilir bir ses yok.", + "failed": "Altyazılar oluşturulamadı.", + "truncated": "Yalnızca ilk {{minutes}} dakika yazıya döküldü." + }, "emptyState": { "title": "Açık proje yok", "description": "Düzenlemeye başlamak için bir video içe aktarın veya mevcut bir OpenScreen projesi yükleyin.", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 7155a0842..209cddd09 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -45,7 +45,10 @@ "webcamShape": "Kamera Şekli", "dualFrame": "Çift Kare", "webcamSize": "Webcam Boyutu", - "noWebcam": "Web kamerası yok" + "noWebcam": "Web kamerası yok", + "mirrorWebcam": "Web kamerasını aynala", + "reactiveWebcam": "Yakınlaştırınca küçült", + "reactiveWebcamDescription": "Video yakınlaştırıldığında kamera yumuşakça küçülür, böylece yoldan çekilir." }, "effects": { "title": "Video Efektleri", @@ -197,6 +200,8 @@ "errorLoadFailed": "Yazı tipi yüklenemedi. Lütfen Google Fonts URL'sinin doğruluğunu kontrol edin." }, "cursor": { + "theme": "İmleç Stili", + "themeDefault": "Varsayılan", "show": "İmleci Göster", "size": "Boyut", "smoothing": "Yumuşatma", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 1875bb559..90004091e 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -44,6 +44,25 @@ "permissionDenied": "Quyền ghi hình bị từ chối. Vui lòng cho phép ghi màn hình.", "accessibilityAllowAndRetry": "Cho phép OpenScreen truy cập Trợ năng, sau đó nhấn ghi lại để bắt đầu đếm ngược." }, + "autoCaptions": { + "button": "Phụ đề tự động", + "dialogTitle": "Phụ đề tự động", + "dialogDescription": "Chọn khoảng bao nhiêu từ mỗi phụ đề hiển thị cùng lúc. Thời gian được phân bổ cho các từ trong cụm từ đó.", + "minWords": "Số từ tối thiểu mỗi phụ đề", + "maxWords": "Số từ tối đa mỗi phụ đề", + "wordsCount": "{{count}} từ", + "generate": "Tạo", + "dialogCancel": "Hủy", + "generating": "Đang tạo phụ đề từ âm thanh…", + "loadingModel": "Đang tải mô hình giọng nói (lần đầu sử dụng sẽ tải ~75 MB)…", + "transcribing": "Đang chuyển lời nói thành văn bản…", + "busy": "Việc tạo phụ đề đang được tiến hành.", + "done": "Đã thêm {{count}} phụ đề.", + "noneHeard": "Không phát hiện thấy lời nói.", + "noAudio": "Video này không có âm thanh dùng được để chuyển thành văn bản.", + "failed": "Không thể tạo phụ đề.", + "truncated": "Chỉ {{minutes}} phút đầu tiên được chuyển thành văn bản." + }, "emptyState": { "title": "Không có dự án nào được mở", "description": "Nhập video để bắt đầu chỉnh sửa hoặc tải một dự án OpenScreen hiện có.", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index 60139ccb6..f86e4851c 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -45,7 +45,10 @@ "dualFrame": "Khung kép", "webcamShape": "Hình dạng máy ảnh", "webcamSize": "Kích thước Webcam", - "noWebcam": "Không có webcam" + "noWebcam": "Không có webcam", + "mirrorWebcam": "Lật webcam", + "reactiveWebcam": "Thu nhỏ khi phóng to", + "reactiveWebcamDescription": "Camera thu nhỏ mượt mà khi video được phóng to, để không che khuất." }, "effects": { "title": "Hiệu ứng video", @@ -197,6 +200,8 @@ "errorLoadFailed": "Không thể tải phông chữ. Vui lòng xác minh URL Google Fonts là chính xác." }, "cursor": { + "theme": "Kiểu con trỏ", + "themeDefault": "Mặc định", "show": "Hiện con trỏ", "size": "Kích thước", "smoothing": "Làm mượt", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index d11f1dd95..58f6ae27b 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -44,6 +44,25 @@ "permissionDenied": "录屏权限被拒绝。请允许屏幕录制。", "accessibilityAllowAndRetry": "允许 OpenScreen 使用辅助功能权限,然后再次按录制以开始倒计时。" }, + "autoCaptions": { + "button": "自动字幕", + "dialogTitle": "自动字幕", + "dialogDescription": "大致选择每条字幕一次显示多少个字词。时间会在该语句内的字词之间分配。", + "minWords": "每条字幕的最少字数", + "maxWords": "每条字幕的最多字数", + "wordsCount": "{{count}} 个词", + "generate": "生成", + "dialogCancel": "取消", + "generating": "正在从音频生成字幕…", + "loadingModel": "正在加载语音模型(首次使用将下载约 75 MB)…", + "transcribing": "正在转写语音…", + "busy": "字幕生成已在进行中。", + "done": "已添加 {{count}} 条字幕。", + "noneHeard": "未检测到语音。", + "noAudio": "此视频没有可用于转写的音频。", + "failed": "无法生成字幕。", + "truncated": "仅转写了最前 {{minutes}} 分钟。" + }, "emptyState": { "title": "未打开任何项目", "description": "导入视频开始编辑,或加载已有的 OpenScreen 项目。", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 9455bf581..dab854077 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -45,7 +45,10 @@ "dualFrame": "双画框", "webcamShape": "摄像头形状", "webcamSize": "摄像头大小", - "noWebcam": "无摄像头" + "noWebcam": "无摄像头", + "mirrorWebcam": "镜像摄像头", + "reactiveWebcam": "缩放时缩小", + "reactiveWebcamDescription": "放大视频时摄像头会平滑缩小,以免遮挡内容。" }, "effects": { "title": "视频效果", @@ -197,6 +200,8 @@ "errorLoadFailed": "无法加载该字体。请确认 Google Fonts URL 是否正确。" }, "cursor": { + "theme": "光标样式", + "themeDefault": "默认", "show": "显示光标", "size": "大小", "smoothing": "平滑", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index 131518713..8a6485409 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -44,6 +44,25 @@ "cameraNotFound": "找不到攝影機。", "accessibilityAllowAndRetry": "允許 OpenScreen 使用輔助使用權限,然後再次按下錄製以開始倒數。" }, + "autoCaptions": { + "button": "自動字幕", + "dialogTitle": "自動字幕", + "dialogDescription": "大致選擇每條字幕一次顯示多少字詞。時間會在該語句內的字詞之間分配。", + "minWords": "每條字幕的最少字數", + "maxWords": "每條字幕的最多字數", + "wordsCount": "{{count}} 個詞", + "generate": "產生", + "dialogCancel": "取消", + "generating": "正在從音訊產生字幕…", + "loadingModel": "正在載入語音模型(首次使用將下載約 75 MB)…", + "transcribing": "正在轉錄語音…", + "busy": "字幕產生已在進行中。", + "done": "已新增 {{count}} 條字幕。", + "noneHeard": "未偵測到語音。", + "noAudio": "此影片沒有可用於轉寫的音訊。", + "failed": "無法產生字幕。", + "truncated": "僅轉寫了最前 {{minutes}} 分鐘。" + }, "emptyState": { "title": "未開啟任何專案", "description": "匯入影片以開始編輯,或載入現有的 OpenScreen 專案。", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 44058e7b3..b5fbcda8c 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -46,7 +46,10 @@ "dualFrame": "雙畫框", "webcamShape": "攝影機形狀", "webcamSize": "攝影機大小", - "noWebcam": "無網路攝影機" + "noWebcam": "無網路攝影機", + "mirrorWebcam": "鏡像攝影機", + "reactiveWebcam": "縮放時縮小", + "reactiveWebcamDescription": "放大影片時鏡頭會平滑縮小,以免遮擋內容。" }, "effects": { "title": "影片效果", @@ -198,6 +201,8 @@ "errorLoadFailed": "無法載入該字體。請確認 Google Fonts URL 是否正確。" }, "cursor": { + "theme": "游標樣式", + "themeDefault": "預設", "show": "顯示游標", "size": "大小", "smoothing": "平滑", diff --git a/src/lib/captioning/annotationsFromCaptions.test.ts b/src/lib/captioning/annotationsFromCaptions.test.ts new file mode 100644 index 000000000..bbf26fed2 --- /dev/null +++ b/src/lib/captioning/annotationsFromCaptions.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from "vitest"; + +import { + captionSegmentsToAnnotationRegions, + groupPhraseCaptionSegmentsIntoLines, + groupTimedCaptionWordsIntoLines, + reconcileAutoCaptionTimelineGaps, +} from "./annotationsFromCaptions"; + +describe("groupPhraseCaptionSegmentsIntoLines", () => { + it("preserves phrase boundaries when formatting phrase-timestamp captions", () => { + const lines = groupPhraseCaptionSegmentsIntoLines( + [ + { startSec: 0, endSec: 0.5, text: "alpha beta" }, + { startSec: 0.62, endSec: 1.6, text: "gamma delta" }, + ], + 2, + 2, + ); + + expect(lines).toHaveLength(2); + expect(lines[0]).toMatchObject({ text: "alpha beta", startSec: 0 }); + expect(lines[1]).toMatchObject({ text: "gamma delta", startSec: 0.62 }); + expect(lines[0]!.endSec).toBeLessThanOrEqual(0.62); + }); + + it("slices a single merged phrase into timed caption lines by word bounds", () => { + const lines = groupPhraseCaptionSegmentsIntoLines( + [{ startSec: 0, endSec: 1, text: "alpha beta gamma delta" }], + 2, + 2, + ); + + expect(lines).toHaveLength(2); + expect(lines[0]).toMatchObject({ + startSec: 0, + endSec: 0.5, + text: "alpha beta", + }); + expect(lines[1]).toMatchObject({ + startSec: 0.5, + endSec: 1, + text: "gamma delta", + }); + }); +}); + +describe("captionSegmentsToAnnotationRegions", () => { + it("uses raw phrase timing instead of shifting caption boundaries", () => { + const { regions } = captionSegmentsToAnnotationRegions( + [ + { startSec: 0, endSec: 0.5, text: "first second" }, + { startSec: 0.62, endSec: 1.2, text: "third fourth" }, + ], + 1, + 1, + { minWordsPerCaption: 2, maxWordsPerCaption: 2, timestampGranularity: "phrase" }, + ); + + expect(regions).toHaveLength(2); + expect(regions[0]).toMatchObject({ startMs: 0, endMs: 500 }); + expect(regions[1]).toMatchObject({ startMs: 620, endMs: 1200 }); + }); + + it("preserves empty timeline space when word timestamps contain a real pause", () => { + const lines = groupTimedCaptionWordsIntoLines( + [ + { startSec: 0, endSec: 0.12, text: "first" }, + { startSec: 0.13, endSec: 0.28, text: "caption" }, + { startSec: 0.7, endSec: 0.83, text: "second" }, + { startSec: 0.84, endSec: 0.98, text: "caption" }, + ], + 2, + 2, + ); + + expect(lines).toHaveLength(2); + expect(lines[0]).toMatchObject({ startSec: 0, endSec: 0.28, text: "first caption" }); + expect(lines[1]).toMatchObject({ startSec: 0.7, endSec: 0.98, text: "second caption" }); + }); + + it("preserves repeated words before grouping in word mode", () => { + const { regions } = captionSegmentsToAnnotationRegions( + [ + { startSec: 0, endSec: 0.12, text: "I" }, + { startSec: 0.13, endSec: 0.25, text: "I" }, + ], + 1, + 1, + { minWordsPerCaption: 2, maxWordsPerCaption: 2, timestampGranularity: "word" }, + ); + + expect(regions).toHaveLength(1); + expect(regions[0]).toMatchObject({ content: "I I" }); + }); +}); + +describe("reconcileAutoCaptionTimelineGaps", () => { + it("does not change regions when the minimum enforced gap is zero", () => { + const regions = reconcileAutoCaptionTimelineGaps([ + { + id: "annotation-1", + startMs: 0, + endMs: 120, + type: "text", + content: "one", + annotationSource: "auto-caption", + position: { x: 0, y: 0 }, + size: { width: 10, height: 10 }, + style: { + color: "#fff", + backgroundColor: "transparent", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + }, + zIndex: 1, + }, + { + id: "manual-1", + startMs: 50, + endMs: 1000, + type: "text", + content: "manual", + position: { x: 10, y: 10 }, + size: { width: 10, height: 10 }, + style: { + color: "#fff", + backgroundColor: "transparent", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + }, + zIndex: 2, + }, + { + id: "annotation-2", + startMs: 130, + endMs: 300, + type: "text", + content: "two", + annotationSource: "auto-caption", + position: { x: 0, y: 0 }, + size: { width: 10, height: 10 }, + style: { + color: "#fff", + backgroundColor: "transparent", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + }, + zIndex: 3, + }, + ]); + + expect(regions.find((r) => r.id === "manual-1")).toMatchObject({ + startMs: 50, + endMs: 1000, + }); + expect(regions.find((r) => r.id === "annotation-1")).toMatchObject({ + startMs: 0, + endMs: 120, + }); + expect(regions.find((r) => r.id === "annotation-2")).toMatchObject({ + startMs: 130, + endMs: 300, + }); + }); +}); diff --git a/src/lib/captioning/annotationsFromCaptions.ts b/src/lib/captioning/annotationsFromCaptions.ts new file mode 100644 index 000000000..c3e22b21d --- /dev/null +++ b/src/lib/captioning/annotationsFromCaptions.ts @@ -0,0 +1,604 @@ +import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types"; + +import type { CaptionSegment } from "./transcribe"; + +/** Wide lower-third bar; `position.x` is top-left as % of container, so center with (100 - width) / 2. */ +const CAPTION_WIDTH = 92; +const CAPTION_HEIGHT = 12; +const CAPTION_BOTTOM_MARGIN = 2; + +const CAPTION_POSITION = { + x: (100 - CAPTION_WIDTH) / 2, + y: 100 - CAPTION_HEIGHT - CAPTION_BOTTOM_MARGIN, +}; + +const CAPTION_SIZE = { width: CAPTION_WIDTH, height: CAPTION_HEIGHT }; + +const CAPTION_STYLE: AnnotationTextStyle = { + color: "#ffffff", + backgroundColor: "rgba(255, 255, 255, 0)", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", +}; + +/** Nudge caption starts earlier (seconds); Whisper onsets run slightly late. Do not offset ends too, that pulls lines off-screen early. */ +const AUTO_CAPTION_START_BIAS_SEC = 0; + +/** Extra hold after Whisper's segment end (seconds); model end times run early vs trailing vowels. Separate from the start bias. */ +const AUTO_CAPTION_END_HOLD_SEC = 0; + +/** Inside one Whisper phrase, sub-lines can be shorter (do not steal time from neighbors). */ +const WORD_SPLIT_MIN_SPAN_SEC = 0.02; + +/** Brief linger after the last word in a line (seconds); trimmed if it would overlap the next line. */ +const CAPTION_LINE_END_TAIL_SEC = 0; + +/** A real silence between word-level timestamps should start a new caption run. */ +const WORD_RUN_BREAK_GAP_SEC = 0.24; + +/** Min time between consecutive caption regions (seconds); keeps a visible gap so blocks don't read as one clip. Small so short pauses survive. */ +const MIN_CAPTION_TIMELINE_GAP_SEC = 0; + +/** Same text again with almost no gap or overlap; common Whisper/chunk artifact. */ +const DEDUPE_SAME_TEXT_MAX_GAP_SEC = 0.55; + +export const SAME_CONTENT_ECHO_MAX_GAP_SEC = 1.15; + +function normalizeCaptionKey(text: string): string { + return text + .trim() + .replace(/\s+/g, " ") + .replace(/[\u2018\u2019]/g, "'") + .replace(/[\u201C\u201D]/g, '"') + .toLowerCase() + .replace(/[.!?,;:]+$/g, ""); +} + +/** Legacy echo-collapse helper kept for reference while phrase timing uses raw model spans. */ +export function collapseSameContentEchoes(segments: CaptionSegment[]): CaptionSegment[] { + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + const out: CaptionSegment[] = []; + const lastIndexByKey = new Map(); + + for (const seg of sorted) { + const key = normalizeCaptionKey(seg.text); + const hit = lastIndexByKey.get(key); + if (hit !== undefined) { + const prev = out[hit]!; + if (seg.startSec < prev.endSec + SAME_CONTENT_ECHO_MAX_GAP_SEC) { + prev.startSec = Math.min(prev.startSec, seg.startSec); + prev.endSec = Math.max(prev.endSec, seg.endSec); + continue; + } + } + out.push({ + startSec: seg.startSec, + endSec: seg.endSec, + text: seg.text.trim(), + }); + lastIndexByKey.set(key, out.length - 1); + } + return out; +} + +/** + * Collapse adjacent duplicate lines (overlapping or tiny gap). Does not merge the same phrase + * repeated later in the video when separated by real silence. + */ +function dedupeAdjacentCaptionRepeats(segments: CaptionSegment[]): CaptionSegment[] { + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + const out: CaptionSegment[] = []; + for (const seg of sorted) { + const t = seg.text.trim(); + const prev = out[out.length - 1]; + if (prev && normalizeCaptionKey(prev.text) === normalizeCaptionKey(t)) { + const overlap = prev.endSec - seg.startSec; + const gap = seg.startSec - prev.endSec; + if (overlap > 0.015 || gap < DEDUPE_SAME_TEXT_MAX_GAP_SEC) { + prev.startSec = Math.min(prev.startSec, seg.startSec); + prev.endSec = Math.max(prev.endSec, seg.endSec); + continue; + } + } + out.push({ startSec: seg.startSec, endSec: seg.endSec, text: t }); + } + return out; +} + +/** Trim only real overlaps. Avoid synthetic lead/lag so caption timing matches model output. */ +function finalizeCaptionSegmentsForPlayback(segments: CaptionSegment[]): CaptionSegment[] { + const OVERLAP_TRIM_SEC = 0.002; + + const sortedRaw = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + + const a = sortedRaw.map((seg) => { + let s = seg.startSec + AUTO_CAPTION_START_BIAS_SEC; + let e = seg.endSec + AUTO_CAPTION_END_HOLD_SEC; + s = Math.max(0, s); + if (e <= s) e = s + 0.02; + return { startSec: s, endSec: e, text: seg.text.trim() }; + }); + + for (let i = 1; i < a.length; i++) { + if (a[i].startSec < a[i - 1].endSec - OVERLAP_TRIM_SEC) { + a[i - 1].endSec = Math.max(a[i - 1].startSec + 1e-4, a[i].startSec); + } + } + + return a; +} + +/** Default min gap between auto-caption blocks on the timeline (ms); matches `MIN_CAPTION_TIMELINE_GAP_SEC`. */ +export const DEFAULT_AUTO_CAPTION_MIN_GAP_MS = Math.round(MIN_CAPTION_TIMELINE_GAP_SEC * 1000); + +/** + * Enforce a min gap between consecutive `auto-caption` regions (by start time). Shortens the previous + * region's end when possible, else shifts the following region later so blocks can't sit completely flush. + */ +export function reconcileAutoCaptionTimelineGaps( + regions: AnnotationRegion[], + minGapMs: number = DEFAULT_AUTO_CAPTION_MIN_GAP_MS, +): AnnotationRegion[] { + const gap = Math.max(0, Math.round(minGapMs)); + if (regions.length === 0 || gap === 0) return regions; + + const autoCandidates = regions.filter((r) => r.annotationSource === "auto-caption"); + if (autoCandidates.length <= 1) return regions; + + const sorted = [...autoCandidates].sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + const fixed: AnnotationRegion[] = []; + let prev = { ...sorted[0]! }; + fixed.push(prev); + + for (let i = 1; i < sorted.length; i++) { + let cur = { ...sorted[i]! }; + const minStart = prev.endMs + gap; + + if (cur.startMs < minStart) { + const newPrevEnd = cur.startMs - gap; + if (newPrevEnd >= prev.startMs + 1) { + prev = { ...prev, endMs: newPrevEnd }; + fixed[fixed.length - 1] = prev; + } else { + const dur = Math.max(1, cur.endMs - cur.startMs); + cur = { ...cur, startMs: minStart, endMs: minStart + dur }; + } + } + + fixed.push(cur); + prev = cur; + } + + const fixedById = new Map(fixed.map((r) => [r.id, r])); + return regions.map((r) => fixedById.get(r.id) ?? r); +} + +/** Join phrases that are close in time so the editor does not create dozens of separate overlays. */ +export function mergeAdjacentCaptionSegments( + segments: CaptionSegment[], + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[] { + const maxGapSec = options?.maxGapSec ?? 1.35; + const maxChars = options?.maxChars ?? 320; + const maxBlockDurationSec = options?.maxBlockDurationSec ?? 12; + + const sorted = [...segments].sort((a, b) => a.startSec - b.startSec); + const out: CaptionSegment[] = []; + + for (const seg of sorted) { + const text = seg.text.trim(); + if (!text) continue; + + const prev = out[out.length - 1]; + if (!prev) { + out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); + continue; + } + + const gap = seg.startSec - prev.endSec; + const mergedText = `${prev.text} ${text}`.trim(); + const mergedEnd = Math.max(prev.endSec, seg.endSec); + const wouldSpan = mergedEnd - prev.startSec; + if (gap <= maxGapSec && mergedText.length <= maxChars && wouldSpan <= maxBlockDurationSec) { + prev.endSec = mergedEnd; + prev.text = mergedText; + } else { + out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); + } + } + + return out; +} + +function partitionPhraseCaptionSegments( + segments: CaptionSegment[], + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[][] { + const maxGapSec = options?.maxGapSec ?? 0; + const maxChars = options?.maxChars ?? Number.POSITIVE_INFINITY; + const maxBlockDurationSec = options?.maxBlockDurationSec ?? Number.POSITIVE_INFINITY; + + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + if (sorted.length === 0) return []; + + const groups: CaptionSegment[][] = []; + let current: CaptionSegment[] = []; + + for (const seg of sorted) { + const text = seg.text.trim(); + if (!text) continue; + + if (current.length === 0) { + current.push({ ...seg, text }); + continue; + } + + const prev = current[current.length - 1]!; + const groupStart = current[0]!.startSec; + const gap = seg.startSec - prev.endSec; + const currentChars = current.reduce((sum, item) => sum + item.text.length, 0); + const wouldChars = currentChars + 1 + text.length; + const wouldSpan = Math.max(prev.endSec, seg.endSec) - groupStart; + + if (gap <= maxGapSec && wouldChars <= maxChars && wouldSpan <= maxBlockDurationSec) { + current.push({ ...seg, text }); + continue; + } + + groups.push(current); + current = [{ ...seg, text }]; + } + + if (current.length > 0) { + groups.push(current); + } + + return groups; +} + +export interface CaptionSegmentLayoutOptions { + /** Lower bound on words per on-screen caption (default 2). */ + minWordsPerCaption?: number; + /** Upper bound on words per on-screen caption (default 7). */ + maxWordsPerCaption?: number; + /** + * `word`: each `CaptionSegment` is a single token with Whisper word timestamps (default). + * `phrase`: merged phrase spans; use proportional line splitting inside each span. + */ + timestampGranularity?: "word" | "phrase"; +} + +function computeCaptionLineIndexRanges( + wordCount: number, + minWords: number, + maxWords: number, +): Array<{ from: number; to: number }> { + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const sliceRanges: Array<{ from: number; to: number }> = []; + let i = 0; + while (i < wordCount) { + const remaining = wordCount - i; + if (remaining <= maxW) { + if (sliceRanges.length > 0 && remaining < minW) { + sliceRanges[sliceRanges.length - 1]!.to = wordCount; + } else { + sliceRanges.push({ from: i, to: wordCount }); + } + break; + } + + let take = maxW; + const after = remaining - take; + if (after > 0 && after < minW) { + take = remaining - minW; + if (take < minW) { + sliceRanges.push({ from: i, to: wordCount }); + break; + } + if (take > maxW) { + take = maxW; + } + } + sliceRanges.push({ from: i, to: i + take }); + i += take; + } + return sliceRanges; +} + +/** + * Groups per-word segments into on-screen lines using each token's Whisper timestamps + * (no proportional stretching across a long phrase span). + */ +export function groupTimedCaptionWordsIntoLines( + segments: CaptionSegment[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const words = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + if (words.length === 0) return []; + + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const out: CaptionSegment[] = []; + + let runStart = 0; + const flushRun = (runEndExclusive: number) => { + const run = words.slice(runStart, runEndExclusive); + if (run.length === 0) return; + const ranges = computeCaptionLineIndexRanges(run.length, minW, maxW); + for (const { from, to } of ranges) { + const slice = run.slice(from, to); + const s = slice[0]!.startSec; + const rawEnd = slice[slice.length - 1]!.endSec; + const e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, rawEnd + CAPTION_LINE_END_TAIL_SEC); + out.push({ + startSec: s, + endSec: e, + text: slice.map((w) => w.text.trim()).join(" "), + }); + } + }; + + for (let i = 1; i < words.length; i++) { + const prev = words[i - 1]!; + const cur = words[i]!; + const gap = cur.startSec - prev.endSec; + if (gap >= WORD_RUN_BREAK_GAP_SEC) { + flushRun(i); + runStart = i; + } + } + flushRun(words.length); + + for (let i = 0; i < out.length - 1; i++) { + if (out[i]!.endSec > out[i + 1]!.startSec + 1e-3) { + out[i]!.endSec = Math.max( + out[i]!.startSec + WORD_SPLIT_MIN_SPAN_SEC, + out[i + 1]!.startSec - 1e-4, + ); + } + } + return out; +} + +/** + * Splits each merged transcription span into shorter captions with about + * `minWords`-`maxWords` words. Times are interpolated by character weight inside the span. + */ +export function splitMergedCaptionsByWordBounds( + merged: CaptionSegment[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const out: CaptionSegment[] = []; + + for (const seg of merged) { + const words = seg.text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) continue; + + if (words.length <= maxW) { + out.push({ + startSec: seg.startSec, + endSec: seg.endSec, + text: words.join(" "), + }); + continue; + } + + out.push(...splitOneSegmentByWordBounds(seg.startSec, seg.endSec, words, minW, maxW)); + } + + return out; +} + +function wrapCaptionTextByWordBounds(text: string, minWords: number, maxWords: number): string { + const words = text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) return ""; + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const ranges = computeCaptionLineIndexRanges(words.length, minW, maxW); + return ranges.map(({ from, to }) => words.slice(from, to).join(" ")).join("\n"); +} + +function expandPhraseSegmentToPseudoWords(segment: CaptionSegment): CaptionSegment[] { + const words = segment.text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) return []; + if (words.length === 1) { + return [ + { + startSec: segment.startSec, + endSec: segment.endSec, + text: words[0]!, + }, + ]; + } + + return splitOneSegmentByWordBounds(segment.startSec, segment.endSec, words, 1, 1); +} + +export function groupPhraseCaptionSegmentsIntoLines( + segments: CaptionSegment[], + minWords: number, + maxWords: number, + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[] { + const groups = partitionPhraseCaptionSegments(segments, options); + const out: CaptionSegment[] = []; + + for (const group of groups) { + if (group.length === 1) { + const only = group[0]!; + const wrapped = wrapCaptionTextByWordBounds(only.text, minWords, maxWords).trim(); + if (!wrapped) continue; + const lineTexts = wrapped + .split("\n") + .map((t) => t.trim()) + .filter(Boolean); + const n = lineTexts.length; + const rawDur = only.endSec - only.startSec; + if (n > 1 && rawDur < n * WORD_SPLIT_MIN_SPAN_SEC) { + out.push({ + startSec: only.startSec, + endSec: only.endSec, + text: lineTexts.join(" "), + }); + continue; + } + const dur = Math.max(rawDur, WORD_SPLIT_MIN_SPAN_SEC * n); + if (n <= 1) { + out.push({ + startSec: only.startSec, + endSec: only.endSec, + text: lineTexts[0] ?? wrapped, + }); + continue; + } + for (let i = 0; i < n; i++) { + const startSec = only.startSec + (dur * i) / n; + const boundary = only.startSec + (dur * (i + 1)) / n; + const endSec = + i === n - 1 ? only.endSec : Math.max(startSec + WORD_SPLIT_MIN_SPAN_SEC, boundary); + out.push({ + startSec, + endSec, + text: lineTexts[i]!, + }); + } + continue; + } + + const pseudoWords = group.flatMap(expandPhraseSegmentToPseudoWords); + out.push(...groupTimedCaptionWordsIntoLines(pseudoWords, minWords, maxWords)); + } + + return out; +} + +function splitOneSegmentByWordBounds( + startSec: number, + endSec: number, + words: string[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const sliceRanges = computeCaptionLineIndexRanges(words.length, minWords, maxWords); + + const dur = Math.max(endSec - startSec, 0.05); + const weights = words.map((w) => Math.max(1, w.length)); + const totalW = weights.reduce((a, b) => a + b, 0); + + const weightSum = (from: number, to: number) => { + let s = 0; + for (let k = from; k < to; k++) s += weights[k] ?? 0; + return s; + }; + + const result: CaptionSegment[] = []; + let prevEnd = startSec; + for (const { from, to } of sliceRanges) { + const wb = weightSum(0, from); + const ws = weightSum(from, to); + let s = startSec + (wb / totalW) * dur; + let e = startSec + ((wb + ws) / totalW) * dur; + s = Math.max(s, prevEnd); + e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, e); + e = Math.min(e, endSec); + if (e <= s) { + e = Math.min(endSec, s + WORD_SPLIT_MIN_SPAN_SEC); + } + prevEnd = e; + result.push({ + startSec: s, + endSec: e, + text: words.slice(from, to).join(" "), + }); + } + if (result.length > 0) { + result[result.length - 1].endSec = endSec; + for (let i = 0; i < result.length - 1; i++) { + if (result[i].endSec > result[i + 1].startSec + 0.002) { + result[i].endSec = Math.max(result[i].startSec + 1e-4, result[i + 1].startSec); + } + } + } + return result; +} + +export function captionSegmentsToAnnotationRegions( + segments: CaptionSegment[], + startNumericId: number, + startZIndex: number, + layout?: CaptionSegmentLayoutOptions, +): { regions: AnnotationRegion[]; nextNumericId: number; nextZIndex: number } { + // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a + // normalized key and would merge spans while keeping only the first token's text. + const minW = layout?.minWordsPerCaption ?? 2; + const maxW = layout?.maxWordsPerCaption ?? 7; + const granularity = layout?.timestampGranularity ?? "word"; + + const grouped = + granularity === "phrase" + ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW) + : groupTimedCaptionWordsIntoLines(segments, minW, maxW); + + const dedupedOut = dedupeAdjacentCaptionRepeats(grouped); + const finalized = finalizeCaptionSegmentsForPlayback(dedupedOut); + + let nid = startNumericId; + let z = startZIndex; + const regions: AnnotationRegion[] = []; + + for (const seg of finalized) { + const startMs = Math.round(seg.startSec * 1000); + const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1); + regions.push({ + id: `annotation-${nid++}`, + startMs, + endMs, + type: "text", + content: seg.text, + annotationSource: "auto-caption", + position: { ...CAPTION_POSITION }, + size: { ...CAPTION_SIZE }, + style: { ...CAPTION_STYLE }, + zIndex: z++, + }); + } + + return { + regions: reconcileAutoCaptionTimelineGaps(regions), + nextNumericId: nid, + nextZIndex: z, + }; +} + +export function maxAnnotationNumericId(regions: AnnotationRegion[]): number { + let max = 0; + for (const r of regions) { + const m = /^annotation-(\d+)$/.exec(r.id); + if (m) max = Math.max(max, Number.parseInt(m[1], 10)); + } + return max; +} + +export function maxAnnotationZIndex(regions: AnnotationRegion[]): number { + if (regions.length === 0) return 0; + return Math.max(...regions.map((r) => r.zIndex)); +} diff --git a/src/lib/captioning/captionConstants.ts b/src/lib/captioning/captionConstants.ts new file mode 100644 index 000000000..1bacb7cc7 --- /dev/null +++ b/src/lib/captioning/captionConstants.ts @@ -0,0 +1,2 @@ +/** Max audio length for auto-captions (decode + transcribe); keep demuxer read aligned with this. */ +export const MAX_CAPTION_AUDIO_SEC = 4 * 60 * 60; diff --git a/src/lib/captioning/extractMono16k.ts b/src/lib/captioning/extractMono16k.ts new file mode 100644 index 000000000..bce9a9fd8 --- /dev/null +++ b/src/lib/captioning/extractMono16k.ts @@ -0,0 +1,159 @@ +import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; +import { extractMonoPcmViaWebDemuxer } from "./extractMono16kWebDemuxer"; + +export { MAX_CAPTION_AUDIO_SEC }; + +const FETCH_TIMEOUT_MS = 120_000; + +async function fetchWithTimeout(url: string, signal?: AbortSignal): Promise { + const ctrl = new AbortController(); + const timer = window.setTimeout(() => ctrl.abort(), FETCH_TIMEOUT_MS); + const onAbort = () => ctrl.abort(); + if (signal) { + if (signal.aborted) ctrl.abort(); + else signal.addEventListener("abort", onAbort, { once: true }); + } + try { + return await fetch(url, { signal: ctrl.signal }); + } finally { + window.clearTimeout(timer); + if (signal) signal.removeEventListener("abort", onAbort); + } +} + +/** + * Load the editor video like `StreamingVideoDecoder`: Electron `readBinaryFile` + * for local paths (fetch(file://) is unreliable in the renderer), otherwise + * HTTP/blob/data URLs via fetch. + */ +async function loadSourceVideoFile(videoUrl: string, signal?: AbortSignal): Promise { + const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); + + if (!isRemoteUrl && window.electronAPI?.readBinaryFile) { + const result = await window.electronAPI.readBinaryFile(videoUrl); + if (!result.success || !result.data) { + throw new Error(result.message || result.error || "Failed to read source video"); + } + const filename = (result.path || videoUrl).split(/[\\/]/).pop() || "video"; + return new File([result.data], filename, { type: "video/webm" }); + } + + const response = await fetchWithTimeout(videoUrl, signal); + if (!response.ok) { + throw new Error(`Failed to load video for captions: ${response.status} ${response.statusText}`); + } + const blob = await response.blob(); + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const filename = videoUrl.split("/").pop() || "video"; + return new File([blob], filename, { type: blob.type || "video/webm" }); +} + +function mixToMono(audioBuffer: AudioBuffer): Float32Array { + const { length, numberOfChannels } = audioBuffer; + const out = new Float32Array(length); + if (numberOfChannels === 0) return out; + for (let i = 0; i < length; i++) { + let sum = 0; + for (let c = 0; c < numberOfChannels; c++) { + sum += audioBuffer.getChannelData(c)[i]; + } + out[i] = sum / numberOfChannels; + } + return out; +} + +async function resampleMono( + mono: Float32Array, + fromRate: number, + toRate: number, + signal?: AbortSignal, +): Promise { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + if (fromRate === toRate) return mono; + const durationSec = mono.length / fromRate; + const outLength = Math.max(1, Math.ceil(durationSec * toRate)); + const offline = new OfflineAudioContext(1, outLength, toRate); + const buf = offline.createBuffer(1, mono.length, fromRate); + buf.copyToChannel(Float32Array.from(mono), 0); + const src = offline.createBufferSource(); + src.buffer = buf; + src.connect(offline.destination); + src.start(0); + const rendered = await offline.startRendering(); + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + return rendered.getChannelData(0).slice(); +} + +async function truncateAndResampleTo16k( + mono: Float32Array, + fromRate: number, + durationSec: number, + signal?: AbortSignal, +): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { + let truncated = false; + let work = mono; + if (durationSec > MAX_CAPTION_AUDIO_SEC) { + const maxSamples = Math.floor(MAX_CAPTION_AUDIO_SEC * fromRate); + work = mono.subarray(0, Math.min(mono.length, maxSamples)); + truncated = true; + } + + const samples = await resampleMono(work, fromRate, 16_000, signal); + return { samples, truncated, durationSec: samples.length / 16_000 }; +} + +/** + * Decode the video's audio track to mono 16 kHz float samples (Whisper input). + * Prefers `decodeAudioData` when the container is supported, else the same + * web-demuxer + AudioDecoder path as export. + */ +export async function extractMono16kFromVideoUrl( + videoUrl: string, + options?: { signal?: AbortSignal }, +): Promise<{ samples: Float32Array; truncated: boolean; durationSec: number }> { + const file = await loadSourceVideoFile(videoUrl, options?.signal); + + /** When this returns null, use web-demuxer + AudioDecoder (same as export). */ + const tryDecodeAudioDataPath = async (): Promise<{ + samples: Float32Array; + truncated: boolean; + durationSec: number; + } | null> => { + const audioContext = new AudioContext(); + try { + const ab = await file.arrayBuffer(); + if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const audioBuffer = await audioContext.decodeAudioData(ab.slice(0)); + if ( + audioBuffer.numberOfChannels === 0 || + audioBuffer.length === 0 || + !Number.isFinite(audioBuffer.duration) || + audioBuffer.duration <= 0 + ) { + return null; + } + const durationSec = audioBuffer.duration; + const mono = mixToMono(audioBuffer); + const fromRate = audioBuffer.sampleRate; + const out = await truncateAndResampleTo16k(mono, fromRate, durationSec, options?.signal); + // decodeAudioData can resolve for some WebM/Matroska inputs yet yield almost no usable + // PCM, and captions only fall back to the demuxer path on throw, so return null to recover. + if (out.samples.length < 800) { + return null; + } + return out; + } catch { + return null; + } finally { + await audioContext.close().catch(() => undefined); + } + }; + + const primary = await tryDecodeAudioDataPath(); + if (primary) { + return primary; + } + + const pcm = await extractMonoPcmViaWebDemuxer(file, options?.signal); + return truncateAndResampleTo16k(pcm.mono, pcm.sampleRate, pcm.durationSec, options?.signal); +} diff --git a/src/lib/captioning/extractMono16kWebDemuxer.ts b/src/lib/captioning/extractMono16kWebDemuxer.ts new file mode 100644 index 000000000..f86b6dc57 --- /dev/null +++ b/src/lib/captioning/extractMono16kWebDemuxer.ts @@ -0,0 +1,187 @@ +import { WebDemuxer } from "web-demuxer"; + +import { MAX_CAPTION_AUDIO_SEC } from "./captionConstants"; + +const DECODE_QUEUE_BACKPRESSURE = 20; +const SOURCE_LOAD_TIMEOUT_MS = 60_000; +const READ_END_PADDING_SEC = 0.5; + +function webDemuxerWasmUrl(): string { + return new URL("../exporter/wasm/web-demuxer.wasm", window.location.href).href; +} + +function audioDataFrameToMono(frame: AudioData): Float32Array { + const frames = frame.numberOfFrames; + const ch = frame.numberOfChannels; + const out = new Float32Array(frames); + const fmt = frame.format || ""; + const planar = fmt.includes("planar"); + + if (planar) { + const plane = new Float32Array(frames); + for (let c = 0; c < ch; c++) { + frame.copyTo(plane, { planeIndex: c }); + for (let i = 0; i < frames; i++) { + out[i] += plane[i]; + } + } + for (let i = 0; i < frames; i++) { + out[i] /= ch; + } + } else { + const interleaved = new Float32Array(frames * ch); + frame.copyTo(interleaved, { planeIndex: 0 }); + for (let i = 0; i < frames; i++) { + let sum = 0; + for (let c = 0; c < ch; c++) { + sum += interleaved[i * ch + c]; + } + out[i] = sum / ch; + } + } + return out; +} + +function mergeAndConsumeDecodedAudioToMonoLinear( + frames: AudioData[], + sampleRate: number, + durationSec: number, +): Float32Array { + const sorted = [...frames].sort((a, b) => a.timestamp - b.timestamp); + const totalSamples = Math.max(1, Math.ceil(durationSec * sampleRate)); + const acc = new Float32Array(totalSamples); + const weight = new Float32Array(totalSamples); + + for (const frame of sorted) { + const startSample = Math.round((frame.timestamp / 1e6) * sampleRate); + const slice = audioDataFrameToMono(frame); + for (let i = 0; i < slice.length; i++) { + const pos = startSample + i; + if (pos >= 0 && pos < totalSamples) { + acc[pos] += slice[i]; + weight[pos] += 1; + } + } + frame.close(); + } + + for (let i = 0; i < totalSamples; i++) { + if (weight[i] > 0) { + acc[i] /= weight[i]; + } + } + return acc; +} + +function withTimeout(promise: Promise, ms: number, message: string): Promise { + return new Promise((resolve, reject) => { + const id = window.setTimeout(() => reject(new Error(message)), ms); + promise + .then((v) => { + window.clearTimeout(id); + resolve(v); + }) + .catch((e) => { + window.clearTimeout(id); + reject(e instanceof Error ? e : new Error(String(e))); + }); + }); +} + +/** + * Demux + WebCodecs audio decode (same stack as export). Use when `decodeAudioData` + * can't handle the container (e.g. WebM with video). + */ +export async function extractMonoPcmViaWebDemuxer( + file: File, + signal?: AbortSignal, +): Promise<{ mono: Float32Array; sampleRate: number; durationSec: number }> { + const demuxer = new WebDemuxer({ wasmFilePath: webDemuxerWasmUrl() }); + await withTimeout( + demuxer.load(file), + SOURCE_LOAD_TIMEOUT_MS, + "Timed out while parsing the source video for captions.", + ); + + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + + const mediaInfo = await withTimeout( + demuxer.getMediaInfo(), + SOURCE_LOAD_TIMEOUT_MS, + "Timed out while reading media info for captions.", + ); + + const reportedDurationSec = + Number.isFinite(mediaInfo.duration) && mediaInfo.duration > 0 ? mediaInfo.duration : 0; + + let audioConfig: AudioDecoderConfig; + try { + audioConfig = await demuxer.getDecoderConfig("audio"); + } catch { + throw new Error("No audio track found in this video."); + } + + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); + if (!codecCheck.supported) { + throw new Error(`Audio codec not supported for captions: ${audioConfig.codec}`); + } + + const sampleRate = audioConfig.sampleRate || 48_000; + + // Many WebM/Matroska files report a too-short duration, so capping read at reported time stops + // demux early and clips everything past that. Read to the caption-decode ceiling instead; the + // demuxer stops when the track ends. + const readEndSec = MAX_CAPTION_AUDIO_SEC + READ_END_PADDING_SEC; + const decodedFrames: AudioData[] = []; + + const decoder = new AudioDecoder({ + output: (data: AudioData) => decodedFrames.push(data), + error: (e: DOMException) => console.error("[captioning] AudioDecoder error:", e), + }); + decoder.configure(audioConfig); + + const reader = demuxer.read("audio", 0, readEndSec).getReader(); + try { + while (!signal?.aborted) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; + decoder.decode(chunk); + while (decoder.decodeQueueSize > DECODE_QUEUE_BACKPRESSURE && !signal?.aborted) { + await new Promise((r) => setTimeout(r, 1)); + } + } + } finally { + try { + await reader.cancel(); + } catch { + /* already closed */ + } + } + + if (decoder.state === "configured") { + await decoder.flush(); + decoder.close(); + } + + if (signal?.aborted) { + for (const f of decodedFrames) f.close(); + throw new DOMException("Aborted", "AbortError"); + } + + if (decodedFrames.length === 0) { + throw new Error("Decoded zero audio frames from this video."); + } + + let maxEndUs = 0; + for (const f of decodedFrames) { + const end = f.timestamp + (f.duration ?? 0); + if (end > maxEndUs) maxEndUs = end; + } + const inferredDurationSec = maxEndUs / 1e6; + // Prefer extent implied by decoded frames (fixes bad container duration); fall back to reported + // metadata when frames lack duration. + const durationSec = inferredDurationSec > 0.02 ? inferredDurationSec : reportedDurationSec; + + const mono = mergeAndConsumeDecodedAudioToMonoLinear(decodedFrames, sampleRate, durationSec); + return { mono, sampleRate, durationSec }; +} diff --git a/src/lib/captioning/index.ts b/src/lib/captioning/index.ts new file mode 100644 index 000000000..cc2e2a3a6 --- /dev/null +++ b/src/lib/captioning/index.ts @@ -0,0 +1,17 @@ +export type { CaptionSegmentLayoutOptions } from "./annotationsFromCaptions"; +export { + captionSegmentsToAnnotationRegions, + DEFAULT_AUTO_CAPTION_MIN_GAP_MS, + groupTimedCaptionWordsIntoLines, + mergeAdjacentCaptionSegments, + reconcileAutoCaptionTimelineGaps, + splitMergedCaptionsByWordBounds, +} from "./annotationsFromCaptions"; +export { extractMono16kFromVideoUrl, MAX_CAPTION_AUDIO_SEC } from "./extractMono16k"; +export { shiftTrimRegionsMsForCaptionBuffer, trimLeadingSilenceMono16k } from "./leadingSilence"; +export type { + CaptionSegment, + CaptionTimestampGranularity, + TranscribeMono16kResult, +} from "./transcribe"; +export { transcribeMono16kToSegments } from "./transcribe"; diff --git a/src/lib/captioning/leadingSilence.ts b/src/lib/captioning/leadingSilence.ts new file mode 100644 index 000000000..4bd6a11aa --- /dev/null +++ b/src/lib/captioning/leadingSilence.ts @@ -0,0 +1,78 @@ +/** Caption path is always mono 16 kHz after `extractMono16kFromVideoUrl`. */ +import type { TrimRegion } from "@/components/video-editor/types"; + +const SAMPLE_RATE = 16_000; + +/** Window length for peak detection (~50 ms). */ +const WINDOW_SAMPLES = 800; + +/** Coarse hop so long intros scan quickly (~50 ms steps). */ +const HOP_SAMPLES = 800; + +/** Max |sample| in a window below this counts as silence (float PCM ~[-1, 1]). */ +const PEAK_THRESHOLD = 0.012; + +/** Keep a little audio before the first peak so word onsets are not clipped. */ +const PRE_ROLL_SEC = 0.12; + +/** Do not scan more than this much audio for leading silence (performance + pathological files). */ +const MAX_LEADING_SCAN_SEC = 15 * 60; + +/** + * Drops quiet audio at the beginning so Whisper is not fed a long silent prefix (which can skew + * the first phrase and wastes work). Returned `trimSec` must be added back to every segment time. + */ +export function trimLeadingSilenceMono16k(samples: Float32Array): { + samples: Float32Array; + trimSec: number; +} { + if (samples.length < WINDOW_SAMPLES) { + return { samples, trimSec: 0 }; + } + + const maxIndex = Math.min( + samples.length - WINDOW_SAMPLES, + Math.floor(MAX_LEADING_SCAN_SEC * SAMPLE_RATE), + ); + + let firstSpeechSample = -1; + for (let i = 0; i <= maxIndex; i += HOP_SAMPLES) { + let peak = 0; + for (let j = 0; j < WINDOW_SAMPLES; j++) { + peak = Math.max(peak, Math.abs(samples[i + j]!)); + } + if (peak > PEAK_THRESHOLD) { + firstSpeechSample = i; + break; + } + } + + if (firstSpeechSample <= 0) { + return { samples, trimSec: 0 }; + } + + const preRollSamples = Math.round(PRE_ROLL_SEC * SAMPLE_RATE); + const start = Math.max(0, firstSpeechSample - preRollSamples); + return { + samples: samples.subarray(start), + trimSec: start / SAMPLE_RATE, + }; +} + +/** + * When audio is trimmed from the front, Whisper times are relative to the shortened buffer. + * Shift trim regions by the same offset so `segmentOverlapsTrim` still uses consistent coordinates. + */ +export function shiftTrimRegionsMsForCaptionBuffer( + regions: TrimRegion[], + trimMs: number, +): TrimRegion[] { + if (trimMs <= 0) return regions; + return regions + .map((r) => ({ + ...r, + startMs: Math.max(0, r.startMs - trimMs), + endMs: Math.max(0, r.endMs - trimMs), + })) + .filter((r) => r.endMs > r.startMs); +} diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts new file mode 100644 index 000000000..a72a89673 --- /dev/null +++ b/src/lib/captioning/transcribe.ts @@ -0,0 +1,106 @@ +import type { TrimRegion } from "@/components/video-editor/types"; + +export interface CaptionSegment { + startSec: number; + endSec: number; + text: string; +} + +/** How caption layout should interpret `CaptionSegment` times from `transcribeMono16kToSegments`. */ +export type CaptionTimestampGranularity = "word" | "phrase"; + +export interface TranscribeMono16kResult { + segments: CaptionSegment[]; + granularity: CaptionTimestampGranularity; +} + +/** Request payload posted from the renderer to the transcription worker. */ +export interface TranscribeWorkerRequest { + samples: Float32Array; + trimRegions: TrimRegion[]; + /** + * Load the Whisper model + ORT wasm from bundled `caption-assets` instead of remote CDNs. + * Required in the packaged app (runs from `file://` where remote fetches fail). The worker + * can't read `window.electronAPI`, so the renderer resolves this here. + */ + useLocalModels: boolean; + /** Base URL of bundled resources (packaged: resourcesPath file:// URL); used when `useLocalModels`. */ + assetBaseUrl?: string; +} + +/** Messages the transcription worker posts back to the renderer. */ +export type TranscribeWorkerResponse = + | { type: "status"; phase: "model" | "transcribe" } + | { type: "result"; segments: CaptionSegment[]; granularity: CaptionTimestampGranularity } + | { type: "error"; message: string }; + +/** + * Transcribes mono 16 kHz audio into timed caption segments using in-browser Whisper. + * + * Runs in a Web Worker so the editor's main thread stays responsive (WASM inference + * doesn't yield). First run downloads model weights. Aborting via `options.signal` + * terminates the worker, since load/inference can't be cooperatively cancelled. + */ +export function transcribeMono16kToSegments( + samples: Float32Array, + options?: { + trimRegions?: TrimRegion[]; + onStatus?: (phase: "model" | "transcribe") => void; + signal?: AbortSignal; + }, +): Promise { + if (options?.signal?.aborted) { + return Promise.reject(new DOMException("Aborted", "AbortError")); + } + + return new Promise((resolve, reject) => { + const worker = new Worker(new URL("./transcribe.worker.ts", import.meta.url), { + type: "module", + }); + + let settled = false; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + options?.signal?.removeEventListener("abort", onAbort); + worker.terminate(); + fn(); + }; + + const onAbort = () => finish(() => reject(new DOMException("Aborted", "AbortError"))); + options?.signal?.addEventListener("abort", onAbort, { once: true }); + + worker.onmessage = (e: MessageEvent) => { + const msg = e.data; + if (msg.type === "status") { + options?.onStatus?.(msg.phase); + return; + } + if (msg.type === "result") { + finish(() => resolve({ segments: msg.segments, granularity: msg.granularity })); + return; + } + finish(() => reject(new Error(msg.message))); + }; + + worker.onerror = (e) => { + finish(() => reject(new Error(e.message || "Caption transcription worker failed"))); + }; + + // Packaged app runs from file:// (remote fetches fail), so load bundled assets. + // Dev runs from http://localhost where the remote path works. + const useLocalModels = typeof window !== "undefined" && window.location?.protocol === "file:"; + const assetBaseUrl = + typeof window !== "undefined" ? window.electronAPI?.assetBaseUrl : undefined; + + // Structured-clone copy, not a transfer: the caller may reuse `samples` for the + // full-buffer retry pass, so the buffer must stay valid here. + const request: TranscribeWorkerRequest = { + samples, + trimRegions: options?.trimRegions ?? [], + useLocalModels, + assetBaseUrl, + }; + worker.postMessage(request); + }); +} diff --git a/src/lib/captioning/transcribe.worker.ts b/src/lib/captioning/transcribe.worker.ts new file mode 100644 index 000000000..ab65b2eea --- /dev/null +++ b/src/lib/captioning/transcribe.worker.ts @@ -0,0 +1,93 @@ +/** + * Web Worker running in-browser Whisper transcription off the renderer's main + * thread so the editor UI never blocks during model load or transcription. + * + * Input: { samples: Float32Array; trimRegions: TrimRegion[] } + * Output (see `TranscribeWorkerResponse`): status / result / error messages. + * + * The caller terminates this worker to abort (model load and inference can't be + * cooperatively cancelled), so there is no in-worker abort handling. + */ + +import type { TranscribeWorkerRequest, TranscribeWorkerResponse } from "./transcribe"; +import { runTranscription, type TranscriberFn } from "./transcribeCore"; + +function post(message: TranscribeWorkerResponse): void { + (self as unknown as Worker).postMessage(message); +} + +/** + * ONNX Runtime's wasm bundle treats `process.versions.node` (which can leak into + * an Electron worker) as Node and tries `require("fs")`, which Vite doesn't + * support. Mask it only while Transformers/ORT run. No-op when `process` is + * undefined (the usual case in a Web Worker). + */ +function withoutNodeVersion(fn: () => Promise): Promise { + const versions = + typeof process !== "undefined" && process.versions && typeof process.versions === "object" + ? process.versions + : null; + const hadNode = versions !== null && "node" in versions; + const savedNode = hadNode ? (versions as { node?: string }).node : undefined; + if (hadNode && versions) { + try { + Reflect.deleteProperty(versions, "node"); + } catch { + (versions as { node?: string }).node = undefined; + } + } + return fn().finally(() => { + if (hadNode && versions && savedNode !== undefined) { + (versions as { node: string }).node = savedNode; + } + }); +} + +async function loadTranscriber(opts: { + useLocalModels: boolean; + assetBaseUrl?: string; +}): Promise { + return withoutNodeVersion(async () => { + const { pipeline, env } = await import("@xenova/transformers"); + if (opts.useLocalModels && opts.assetBaseUrl) { + // Packaged app: load the bundled model and ORT wasm from disk so transcription + // needs no network and works under file:// (remote HuggingFace/CDN fetches fail there). + const base = new URL("caption-assets/", opts.assetBaseUrl).href; + env.allowLocalModels = true; + env.allowRemoteModels = false; + env.localModelPath = new URL("models/", base).href; + env.backends.onnx.wasm.wasmPaths = new URL("ort/", base).href; + // Non-threaded wasm: SharedArrayBuffer isn't available under file:// (no cross-origin isolation). + env.backends.onnx.wasm.numThreads = 1; + } else { + // Dev (http://localhost): fetch from the remote CDN, which works there. + env.allowLocalModels = false; + } + // Default tiny weights only: the `output_attentions` revision regresses inference in + // some environments (empty chunks, thrown errors) while phrase mode works on this model. + const transcriber = (await pipeline( + "automatic-speech-recognition", + "Xenova/whisper-tiny", + )) as unknown as TranscriberFn; + return transcriber; + }); +} + +self.onmessage = async (event: MessageEvent) => { + const { samples, trimRegions, useLocalModels, assetBaseUrl } = event.data; + try { + post({ type: "status", phase: "model" }); + const transcriber = await loadTranscriber({ useLocalModels, assetBaseUrl }); + + post({ type: "status", phase: "transcribe" }); + const { segments, granularity } = await runTranscription( + transcriber, + samples, + trimRegions ?? [], + ); + + post({ type: "result", segments, granularity }); + } catch (e) { + post({ type: "error", message: e instanceof Error ? e.message : String(e) }); + } +}; diff --git a/src/lib/captioning/transcribeCore.ts b/src/lib/captioning/transcribeCore.ts new file mode 100644 index 000000000..9834e3654 --- /dev/null +++ b/src/lib/captioning/transcribeCore.ts @@ -0,0 +1,268 @@ +import type { TrimRegion } from "@/components/video-editor/types"; +import type { CaptionSegment, TranscribeMono16kResult } from "./transcribe"; + +/** + * Pure transcription algorithm for the captioning Web Worker: takes a built Whisper + * `transcriber` and turns mono 16 kHz audio into timed caption segments. No DOM or + * Transformers.js imports so it runs in a worker and unit-tests in isolation. + */ + +/** A Transformers.js automatic-speech-recognition pipeline call. */ +export type TranscriberFn = ( + audio: Float32Array, + opts: Record, +) => Promise; + +function segmentOverlapsTrim(startMs: number, endMs: number, trims: TrimRegion[]): boolean { + return trims.some((t) => startMs < t.endMs && endMs > t.startMs); +} + +/** Same trim-out rule as {@link segmentsFromTranscriberChunks}; for retry passes that used empty trims. */ +function dropSegmentsOverlappingTrimRegions( + segments: CaptionSegment[], + trimRegions: TrimRegion[], +): CaptionSegment[] { + if (trimRegions.length === 0) return segments; + return segments.filter((s) => { + const startMs = Math.round(s.startSec * 1000); + const endMs = Math.round(s.endSec * 1000); + return !segmentOverlapsTrim(startMs, endMs, trimRegions); + }); +} + +/** Whisper runs with internal 30s chunks; keep each forward pass bounded for WASM memory. */ +const TRANSCRIBE_SLICE_SAMPLES = 12 * 60 * 16_000; + +/** Very short slices are skipped in the multi-slice loop unless padded (see `padTailSliceForTranscribe`). */ +const MIN_TRANSCRIBE_SLICE_SAMPLES = 800; + +/** + * Pad a short tail slice so Whisper still runs; timestamps are clamped with `realDurationSec` so + * padding does not extend perceived audio on the timeline. + */ +function padTailSliceForTranscribe(samples: Float32Array): { + slice: Float32Array; + realDurationSec: number; +} { + const realDurationSec = samples.length / 16_000; + if (samples.length >= MIN_TRANSCRIBE_SLICE_SAMPLES) { + return { slice: samples, realDurationSec }; + } + const padded = new Float32Array(MIN_TRANSCRIBE_SLICE_SAMPLES); + padded.set(samples); + return { slice: padded, realDurationSec }; +} + +/** Converts raw Whisper chunk output into sorted, deduped, trim-filtered caption segments. */ +function segmentsFromTranscriberChunks( + chunks: Array<{ timestamp?: [number | null, number | null]; text?: unknown }>, + timeOffsetSec: number, + trims: TrimRegion[], + audioDurationSec: number, +): CaptionSegment[] { + const sorted = [...chunks].sort((x, y) => { + const ax = x.timestamp?.[0]; + const ay = y.timestamp?.[0]; + const na = typeof ax === "number" ? ax : -1; + const nb = typeof ay === "number" ? ay : -1; + return na - nb; + }); + + const segments: CaptionSegment[] = []; + + for (let idx = 0; idx < sorted.length; idx++) { + const c = sorted[idx]!; + const ts = c.timestamp as [number | null, number | null] | undefined; + if (!ts) continue; + let a = ts[0]; + let b = ts[1]; + if (a == null) a = 0; + a = Math.max(0, a); + if (b == null) { + let nextStart: number | null = null; + for (let j = idx + 1; j < sorted.length; j++) { + const na = sorted[j]?.timestamp?.[0]; + if (typeof na === "number") { + nextStart = na; + break; + } + } + b = nextStart ?? audioDurationSec; + } + if (b <= a) { + b = Math.min(a + 0.25, audioDurationSec); + } + b = Math.min(b, audioDurationSec); + + const text = String(c.text ?? "") + .replace(/\s+/g, " ") + .trim(); + if (!text) continue; + + const startSec = a + timeOffsetSec; + const sliceEnd = timeOffsetSec + audioDurationSec; + const endSec = Math.min(Math.max(startSec + 0.08, b + timeOffsetSec), sliceEnd); + const startMs = Math.round(startSec * 1000); + const endMs = Math.round(endSec * 1000); + if (segmentOverlapsTrim(startMs, endMs, trims)) continue; + + segments.push({ startSec, endSec, text }); + } + + segments.sort((u, v) => u.startSec - v.startSec || u.endSec - v.endSec); + const rawDeduped: CaptionSegment[] = []; + for (const seg of segments) { + const prev = rawDeduped[rawDeduped.length - 1]; + if (prev && prev.text === seg.text && seg.startSec <= prev.endSec) { + prev.endSec = Math.max(prev.endSec, seg.endSec); + prev.startSec = Math.min(prev.startSec, seg.startSec); + continue; + } + rawDeduped.push(seg); + } + return rawDeduped; +} + +/** Runs the transcriber on one audio slice, chunking only long clips. */ +async function runTranscriberOnSlice( + transcriber: TranscriberFn, + samples: Float32Array, + opts: { forceFullSequences: boolean; timestampMode: "word" | "phrase" }, +): Promise { + const durationSec = samples.length / 16_000; + // Only chunk long clips; short-audio chunking regressed some Whisper.js runs (empty chunks). + const chunking = durationSec > 30 ? { chunk_length_s: 30, stride_length_s: 5 } : {}; + return transcriber(samples, { + return_timestamps: opts.timestampMode === "word" ? "word" : true, + force_full_sequences: opts.forceFullSequences, + ...chunking, + }); +} + +/** Flattens the various shapes a Transformers.js ASR result can take into a chunk list. */ +function getChunksFromTranscriberResult(result: unknown): Array<{ + timestamp?: [number | null, number | null]; + text?: unknown; +}> { + if (result == null) return []; + if (Array.isArray(result)) { + const out: Array<{ timestamp?: [number | null, number | null]; text?: unknown }> = []; + for (const item of result) { + const chunks = (item as { chunks?: unknown })?.chunks; + if (Array.isArray(chunks)) out.push(...chunks); + } + return out; + } + const chunks = (result as { chunks?: unknown })?.chunks; + return Array.isArray(chunks) ? chunks : []; +} + +/** Prefer `chunks`; if the model only returned top-level `text`, synthesize one span for timing. */ +function extractChunksFromAsrResult(result: unknown): Array<{ + timestamp?: [number | null, number | null]; + text?: unknown; +}> { + const fromChunks = getChunksFromTranscriberResult(result); + if (fromChunks.length > 0) return fromChunks; + const single = Array.isArray(result) ? result[0] : result; + const text = + typeof (single as { text?: unknown })?.text === "string" + ? String((single as { text: string }).text).trim() + : ""; + if (text) { + return [{ timestamp: [0, null], text }]; + } + return []; +} + +/** + * Drives Whisper over (possibly sliced) mono 16 kHz audio and returns timed segments. + * Long audio is split so one pass doesn't exhaust WASM memory; timestamps are shifted + * back onto the full timeline. Tries word- then phrase-level timestamps, with a + * trim-ignoring retry, before giving up. + */ +export async function runTranscription( + transcriber: TranscriberFn, + samples: Float32Array, + trims: TrimRegion[], +): Promise { + const transcribeOne = async ( + ignoreTrims: boolean, + forceFullSequences: boolean, + timestampMode: "word" | "phrase", + ): Promise => { + try { + const activeTrims = ignoreTrims ? [] : trims; + if (samples.length <= TRANSCRIBE_SLICE_SAMPLES) { + const { slice, realDurationSec } = padTailSliceForTranscribe(samples); + const result = await runTranscriberOnSlice(transcriber, slice, { + forceFullSequences, + timestampMode, + }); + return segmentsFromTranscriberChunks( + extractChunksFromAsrResult(result), + 0, + activeTrims, + realDurationSec, + ); + } + + const all: CaptionSegment[] = []; + for (let offset = 0; offset < samples.length; offset += TRANSCRIBE_SLICE_SAMPLES) { + const end = Math.min(offset + TRANSCRIBE_SLICE_SAMPLES, samples.length); + const sliceRaw = samples.subarray(offset, end); + const isFinalSlice = end >= samples.length; + if (sliceRaw.length === 0) continue; + if (sliceRaw.length < MIN_TRANSCRIBE_SLICE_SAMPLES && !isFinalSlice) continue; + + const { slice, realDurationSec } = + sliceRaw.length < MIN_TRANSCRIBE_SLICE_SAMPLES && isFinalSlice + ? padTailSliceForTranscribe(sliceRaw) + : { slice: sliceRaw, realDurationSec: sliceRaw.length / 16_000 }; + + const result = await runTranscriberOnSlice(transcriber, slice, { + forceFullSequences, + timestampMode, + }); + const tOff = offset / 16_000; + all.push( + ...segmentsFromTranscriberChunks( + extractChunksFromAsrResult(result), + tOff, + activeTrims, + realDurationSec, + ), + ); + } + return all; + } catch (e) { + console.warn("[captioning] Whisper pass failed:", e); + return []; + } + }; + + const attemptModes: Array<"word" | "phrase"> = ["word", "phrase"]; + for (const timestampMode of attemptModes) { + let segments = await transcribeOne(false, true, timestampMode); + if (segments.length === 0) { + segments = await transcribeOne(false, false, timestampMode); + } + if (segments.length === 0 && trims.length > 0) { + segments = dropSegmentsOverlappingTrimRegions( + await transcribeOne(true, true, timestampMode), + trims, + ); + if (segments.length === 0) { + segments = dropSegmentsOverlappingTrimRegions( + await transcribeOne(true, false, timestampMode), + trims, + ); + } + } + if (segments.length > 0) { + return { segments, granularity: timestampMode }; + } + } + + return { segments: [], granularity: "phrase" }; +} diff --git a/src/lib/compositeLayout.test.ts b/src/lib/compositeLayout.test.ts index 65cdfd573..51c3fd0b7 100644 --- a/src/lib/compositeLayout.test.ts +++ b/src/lib/compositeLayout.test.ts @@ -69,7 +69,7 @@ describe("computeCompositeLayout", () => { expect(landscape).not.toBeNull(); expect(portrait).not.toBeNull(); - // Same total pixel count — webcam area should be comparable + // Same total pixel count, so webcam area should be comparable. const landscapeArea = landscape!.webcamRect!.width * landscape!.webcamRect!.height; const portraitArea = portrait!.webcamRect!.width * portrait!.webcamRect!.height; expect(landscapeArea).toBe(portraitArea); @@ -135,10 +135,8 @@ describe("computeCompositeLayout", () => { webcamSizePreset: 100, }); - // Values below 10 should clamp to 10 expect(belowMin!.webcamRect!.width).toBe(atMin!.webcamRect!.width); expect(belowMin!.webcamRect!.height).toBe(atMin!.webcamRect!.height); - // Values above 50 should clamp to 50 expect(aboveMax!.webcamRect!.width).toBe(atMax!.webcamRect!.width); expect(aboveMax!.webcamRect!.height).toBe(atMax!.webcamRect!.height); }); diff --git a/src/lib/compositeLayout.ts b/src/lib/compositeLayout.ts index abb6b0f7b..5eedb266d 100644 --- a/src/lib/compositeLayout.ts +++ b/src/lib/compositeLayout.ts @@ -5,6 +5,20 @@ export interface RenderRect { height: number; } +/** Floor for the reactive webcam multiplier so the camera never shrinks below ~35% at deep zoom. */ +export const WEBCAM_REACTIVE_ZOOM_MIN_SCALE = 0.35; + +/** + * Maps the live zoom scale to a webcam size multiplier, inversely (2x zoom, half size; 3x, a + * third) so the camera stays out of the way while zoomed and returns to full size as zoom eases + * back. Clamped to a floor so it never disappears. appliedScale is already eased per frame, so + * the camera animates in sync for free. + */ +export function reactiveWebcamScale(zoomScale: number): number { + const safe = Number.isFinite(zoomScale) && zoomScale > 0 ? zoomScale : 1; + return Math.max(WEBCAM_REACTIVE_ZOOM_MIN_SCALE, Math.min(1, 1 / safe)); +} + export interface StyledRenderRect extends RenderRect { borderRadius: number; maskShape?: import("@/components/video-editor/types").WebcamMaskShape; @@ -192,7 +206,7 @@ export function computeCompositeLayout(params: { const { width: canvasWidth, height: canvasHeight } = canvasSize; const { width: screenWidth, height: screenHeight } = screenSize; - // "no-webcam" preset: hide the webcam entirely, screen fills the canvas normally + // no-webcam: hide the webcam, screen fills the canvas normally. if (layoutPreset === "no-webcam") { const screenRect = centerRect({ canvasSize, @@ -214,7 +228,7 @@ export function computeCompositeLayout(params: { if (preset.transform.type === "stack") { if (!webcamWidth || !webcamHeight || webcamWidth <= 0 || webcamHeight <= 0) { - // No webcam — screen fills the entire canvas (cover mode) + // No webcam, so screen fills the whole canvas (cover mode). return { screenRect: { x: 0, y: 0, width: canvasWidth, height: canvasHeight }, webcamRect: null, @@ -222,12 +236,12 @@ export function computeCompositeLayout(params: { }; } - // Webcam: full width at the bottom, maintaining its aspect ratio + // Webcam: full width at the bottom, keeping aspect ratio. const webcamAspect = webcamWidth / webcamHeight; const resolvedWebcamWidth = canvasWidth; const resolvedWebcamHeight = Math.round(canvasWidth / webcamAspect); - // Screen: fills remaining space at the top (cover mode — may crop sides) + // Screen: fills remaining space at the top (cover mode, may crop sides). const screenRectHeight = canvasHeight - resolvedWebcamHeight; return { @@ -326,8 +340,7 @@ export function computeCompositeLayout(params: { transform.minMargin, Math.round(Math.min(canvasWidth, canvasHeight) * transform.marginFraction), ); - // Use geometric mean so the webcam occupies a consistent visual proportion - // regardless of whether the canvas is portrait or landscape. + // Geometric mean so the webcam keeps a consistent visual proportion in portrait or landscape. const referenceDim = Math.sqrt(canvasWidth * canvasHeight); const maxWidth = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION); const maxHeight = Math.max(transform.minSize, referenceDim * MAX_STAGE_FRACTION); @@ -346,10 +359,10 @@ export function computeCompositeLayout(params: { let webcamY: number; if (webcamPosition) { - // Custom position: cx/cy represent the center of the webcam as a fraction of the canvas + // cx/cy are the webcam center as a fraction of the canvas. webcamX = Math.round(webcamPosition.cx * canvasWidth - width / 2); webcamY = Math.round(webcamPosition.cy * canvasHeight - height / 2); - // Clamp to stay within canvas bounds + // Clamp inside canvas bounds. webcamX = Math.max(0, Math.min(canvasWidth - width, webcamX)); webcamY = Math.max(0, Math.min(canvasHeight - height, webcamY)); } else { diff --git a/src/lib/cursor/cursorPathSmoothing.test.ts b/src/lib/cursor/cursorPathSmoothing.test.ts new file mode 100644 index 000000000..e9bdd5e38 --- /dev/null +++ b/src/lib/cursor/cursorPathSmoothing.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import type { CursorRecordingData, CursorRecordingSample } from "@/native/contracts"; +import { getSmoothedCursorPath } from "./cursorPathSmoothing"; + +function makeRecording(samples: CursorRecordingSample[]): CursorRecordingData { + return { version: 2, provider: "native", assets: [], samples }; +} + +/** Roughness proxy: sum of squared second differences of x on a uniform grid. */ +function roughness( + sampleAt: (t: number) => { cx: number; cy: number } | null, + t0: number, + t1: number, +) { + const xs: number[] = []; + for (let t = t0; t <= t1; t += 5) { + const p = sampleAt(t); + if (p) xs.push(p.cx); + } + let acc = 0; + for (let i = 2; i < xs.length; i++) { + const d2 = xs[i] - 2 * xs[i - 1] + xs[i - 2]; + acc += d2 * d2; + } + return acc; +} + +describe("cursor path smoothing", () => { + it("removes high-frequency jitter while tracking the overall path", () => { + // A rightward drift with alternating zig-zag noise on cy, then a dwell at the end. + const samples: CursorRecordingSample[] = []; + for (let i = 0; i <= 40; i++) { + samples.push({ + timeMs: i * 33, + cx: 0.2 + (i / 40) * 0.6, + cy: 0.5 + (i % 2 === 0 ? 0.05 : -0.05), + visible: true, + }); + } + const driftEnd = samples[samples.length - 1].timeMs; + for (let i = 1; i <= 60; i++) { + samples.push({ timeMs: driftEnd + i * 33, cx: 0.8, cy: 0.5, visible: true }); + } + const data = makeRecording(samples); + const smoothed = getSmoothedCursorPath(data, 0.7)!; + const raw = getSmoothedCursorPath(makeRecording(samples), 0)!; + + // Compare jitter on the cy channel (where the zig-zag lives) over the moving portion. + const cyAt = (path: typeof smoothed) => (t: number) => { + const p = path.sampleAt(t); + return p ? { cx: p.cy, cy: p.cx } : null; + }; + const smoothRough = roughness(cyAt(smoothed), 0, driftEnd); + const rawRough = roughness(cyAt(raw), 0, driftEnd); + expect(smoothRough).toBeLessThan(rawRough * 0.25); + + // After the cursor rests, the spring settles onto the true target (click accuracy). + const end = samples[samples.length - 1].timeMs; + const last = smoothed.sampleAt(end)!; + expect(last.cx).toBeCloseTo(0.8, 2); + expect(last.cy).toBeCloseTo(0.5, 2); + }); + + it("is a passthrough at smoothing 0", () => { + const samples: CursorRecordingSample[] = [ + { timeMs: 0, cx: 0.1, cy: 0.1, visible: true }, + { timeMs: 100, cx: 0.9, cy: 0.4, visible: true }, + ]; + const path = getSmoothedCursorPath(makeRecording(samples), 0)!; + expect(path.sampleAt(0)).toEqual({ cx: 0.1, cy: 0.1 }); + expect(path.sampleAt(50)!.cx).toBeCloseTo(0.5, 5); + expect(path.sampleAt(100)).toEqual({ cx: 0.9, cy: 0.4 }); + }); + + it("respects visibility gaps and never smooths across them", () => { + const samples: CursorRecordingSample[] = [ + { timeMs: 0, cx: 0.2, cy: 0.2, visible: true }, + { timeMs: 100, cx: 0.3, cy: 0.3, visible: true }, + { timeMs: 150, cx: 0.3, cy: 0.3, visible: false }, + { timeMs: 200, cx: 0.8, cy: 0.8, visible: true }, + { timeMs: 300, cx: 0.9, cy: 0.9, visible: true }, + ]; + const path = getSmoothedCursorPath(makeRecording(samples), 0.6)!; + expect(path.sampleAt(50)).not.toBeNull(); + expect(path.sampleAt(160)).toBeNull(); // inside the hidden gap + expect(path.sampleAt(250)).not.toBeNull(); + }); + + it("is deterministic for identical inputs", () => { + const build = () => + getSmoothedCursorPath( + makeRecording([ + { timeMs: 0, cx: 0.1, cy: 0.5, visible: true }, + { timeMs: 50, cx: 0.4, cy: 0.55, visible: true }, + { timeMs: 120, cx: 0.7, cy: 0.45, visible: true }, + ]), + 0.65, + )!; + const a = build(); + const b = build(); + for (const t of [0, 25, 60, 90, 120]) { + expect(a.sampleAt(t)).toEqual(b.sampleAt(t)); + } + }); + + it("returns null when there is no cursor data", () => { + expect(getSmoothedCursorPath(null, 0.5)).toBeNull(); + expect(getSmoothedCursorPath(makeRecording([]), 0.5)).toBeNull(); + }); +}); diff --git a/src/lib/cursor/cursorPathSmoothing.ts b/src/lib/cursor/cursorPathSmoothing.ts new file mode 100644 index 000000000..e89b9575d --- /dev/null +++ b/src/lib/cursor/cursorPathSmoothing.ts @@ -0,0 +1,237 @@ +import { getCursorSpringConfig } from "@/components/video-editor/videoPlayback/motionSmoothing"; +import type { CursorRecordingData, CursorRecordingSample } from "@/native/contracts"; + +/** + * Offline cursor-path smoothing for native recordings. + * + * We have the whole path up front, so instead of a per-frame causal filter we precompute once: + * resample to a fixed high rate, then run a spring-damper over it. The spring gives the motion + * inertia (it trails the real cursor) and is deterministic, so preview and export match exactly. + */ + +export interface SmoothedCursorPosition { + cx: number; + cy: number; +} + +export interface SmoothedCursorPath { + /** Smoothed normalized position at a time, or null when the cursor is hidden there. */ + sampleAt(timeMs: number): SmoothedCursorPosition | null; +} + +/** 240 steps/sec keeps the spring stable and crisp at any playback fps. */ +const STEP_MS = 1000 / 240; +const STEP_S = STEP_MS / 1000; + +interface SmoothedRun { + start: number; + end: number; + times: Float32Array; + xs: Float32Array; + ys: Float32Array; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function binarySearchAtOrBefore( + times: Float32Array | number[], + timeMs: number, + hi: number, +): number { + let low = 0; + let high = hi; + let result = -1; + while (low <= high) { + const mid = low + ((high - low) >> 1); + if (times[mid] <= timeMs) { + result = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return result; +} + +/** Linear interpolation of a sample run's position at an arbitrary time. */ +function interpolateRun(samples: CursorRecordingSample[], timeMs: number): SmoothedCursorPosition { + const last = samples.length - 1; + if (timeMs <= samples[0].timeMs) return { cx: samples[0].cx, cy: samples[0].cy }; + if (timeMs >= samples[last].timeMs) return { cx: samples[last].cx, cy: samples[last].cy }; + const i = binarySearchAtOrBefore( + samples.map((s) => s.timeMs), + timeMs, + last, + ); + const a = samples[i]; + const b = samples[i + 1] ?? a; + const span = b.timeMs - a.timeMs; + if (span <= 0) return { cx: a.cx, cy: a.cy }; + const t = (timeMs - a.timeMs) / span; + return { cx: a.cx + (b.cx - a.cx) * t, cy: a.cy + (b.cy - a.cy) * t }; +} + +/** + * Drive a spring across `targets`, returning the smoothed series. Semi-implicit + * (symplectic) Euler, stable for these stiffness values at the 240Hz grid. + */ +function springSmooth( + targets: Float32Array, + stiffness: number, + damping: number, + mass: number, +): Float32Array { + const out = new Float32Array(targets.length); + if (targets.length === 0) return out; + let x = targets[0]; + let v = 0; + out[0] = x; + for (let i = 1; i < targets.length; i++) { + const accel = (-stiffness * (x - targets[i]) - damping * v) / mass; + v += accel * STEP_S; + x += v * STEP_S; + out[i] = x; + } + return out; +} + +/** Maximal runs of visible samples, so we never smooth across a hidden gap. */ +function splitVisibleRuns(samples: CursorRecordingSample[]): CursorRecordingSample[][] { + const runs: CursorRecordingSample[][] = []; + let current: CursorRecordingSample[] = []; + for (const sample of samples) { + if (sample.visible === false) { + if (current.length) runs.push(current); + current = []; + continue; + } + current.push(sample); + } + if (current.length) runs.push(current); + return runs; +} + +function buildSmoothedRun( + samples: CursorRecordingSample[], + stiffness: number, + damping: number, + mass: number, +): SmoothedRun { + const start = samples[0].timeMs; + const end = samples[samples.length - 1].timeMs; + const stepCount = Math.max(1, Math.round((end - start) / STEP_MS)); + const n = stepCount + 1; + const times = new Float32Array(n); + const rawX = new Float32Array(n); + const rawY = new Float32Array(n); + for (let i = 0; i < n; i++) { + const t = i === n - 1 ? end : start + i * STEP_MS; + times[i] = t; + const p = interpolateRun(samples, t); + rawX[i] = p.cx; + rawY[i] = p.cy; + } + // The spring is itself a strong low-pass (~3Hz cutoff), so it removes capture tremor without a + // separate denoise pass. Chasing the raw target keeps the cursor accurate near sharp stops (no + // acausal pull toward neighbouring samples that would offset clicks/dwells). + return { + start, + end, + times, + xs: springSmooth(rawX, stiffness, damping, mass), + ys: springSmooth(rawY, stiffness, damping, mass), + }; +} + +function sampleRun(run: SmoothedRun, timeMs: number): SmoothedCursorPosition { + const last = run.times.length - 1; + if (timeMs <= run.times[0]) return { cx: run.xs[0], cy: run.ys[0] }; + if (timeMs >= run.times[last]) return { cx: run.xs[last], cy: run.ys[last] }; + const i = binarySearchAtOrBefore(run.times, timeMs, last); + const span = run.times[i + 1] - run.times[i]; + if (span <= 0) return { cx: run.xs[i], cy: run.ys[i] }; + const t = (timeMs - run.times[i]) / span; + return { + cx: run.xs[i] + (run.xs[i + 1] - run.xs[i]) * t, + cy: run.ys[i] + (run.ys[i + 1] - run.ys[i]) * t, + }; +} + +/** Passthrough path (smoothing 0): raw linear interpolation, still respecting visibility gaps. */ +function buildRawPath(runs: CursorRecordingSample[][]): SmoothedCursorPath { + return { + sampleAt(timeMs) { + for (const run of runs) { + if (timeMs >= run[0].timeMs && timeMs <= run[run.length - 1].timeMs) { + return interpolateRun(run, timeMs); + } + } + return null; + }, + }; +} + +function buildSmoothedPath( + recordingData: CursorRecordingData, + smoothing01: number, +): SmoothedCursorPath { + const runs = splitVisibleRuns(recordingData.samples).filter((run) => run.length > 0); + if (runs.length === 0) { + return { sampleAt: () => null }; + } + if (smoothing01 <= 0) { + return buildRawPath(runs); + } + + // Use the slider value directly to match the live overlay's spring strength so both cursor + // systems lag identically (an extra multiplier here over-smoothed, causing a visible offset). + const config = getCursorSpringConfig(clamp(smoothing01, 0, 1)); + + const smoothedRuns = runs.map((run) => + run.length < 2 + ? { + start: run[0].timeMs, + end: run[0].timeMs, + times: new Float32Array([run[0].timeMs]), + xs: new Float32Array([run[0].cx]), + ys: new Float32Array([run[0].cy]), + } + : buildSmoothedRun(run, config.stiffness, config.damping, config.mass), + ); + + return { + sampleAt(timeMs) { + for (const run of smoothedRuns) { + if (timeMs >= run.start && timeMs <= run.end) return sampleRun(run, timeMs); + } + return null; + }, + }; +} + +const pathCache = new WeakMap>(); + +/** + * Returns the smoothed cursor path for a recording at a given strength, memoized per + * (recordingData, strength) so it's built once and shared by preview and export. + */ +export function getSmoothedCursorPath( + recordingData: CursorRecordingData | null | undefined, + smoothing01: number, +): SmoothedCursorPath | null { + if (!recordingData || recordingData.samples.length === 0) return null; + const key = (Number.isFinite(smoothing01) ? clamp(smoothing01, 0, 1) : 0).toFixed(2); + let byStrength = pathCache.get(recordingData); + if (!byStrength) { + byStrength = new Map(); + pathCache.set(recordingData, byStrength); + } + let path = byStrength.get(key); + if (!path) { + path = buildSmoothedPath(recordingData, Number.parseFloat(key)); + byStrength.set(key, path); + } + return path; +} diff --git a/src/lib/cursor/cursorThemes.ts b/src/lib/cursor/cursorThemes.ts new file mode 100644 index 000000000..dd6fa7cc0 --- /dev/null +++ b/src/lib/cursor/cursorThemes.ts @@ -0,0 +1,421 @@ +import type { NativeCursorType } from "@/native/contracts"; + +/** + * A single themed cursor image override for one {@link NativeCursorType}. + * + * width/height/hotspot are in the same ~32-logical-pixel reference as the built-in + * PRETTY_NATIVE_CURSOR_ASSETS, so a theme asset matches the default cursor's on-screen + * size regardless of source PNG resolution. The PNG can be higher-res (e.g. 128x128) + * and is downscaled at draw time for crisper retina output. + */ +export interface CursorThemeAsset { + /** Path relative to the public asset root, e.g. "cursors/hello-kitty-watermelon/arrow.png". */ + assetPath: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +export interface CursorTheme { + id: string; + /** Display label. Proper nouns, so not run through i18n. */ + name: string; + /** Attribution / origin for the artwork. */ + source?: string; + /** + * Per-cursor-type overrides. Missing types fall back to the built-in default art. + * Sweezy packs only ship "arrow" and "pointer". + */ + assets: Partial>; +} + +/** Sentinel id for the built-in cursor art (no theme override). */ +export const DEFAULT_CURSOR_THEME_ID = "default"; + +/** + * Bundled cursor themes. To add a pack: drop arrow.png/pointer.png into + * public/cursors// and add an entry here with hotspots normalized to the + * 32-logical reference (divide a 128px-pack hotspot by 4). No renderer changes needed. + */ +export const CURSOR_THEMES: readonly CursorTheme[] = [ + { + id: "hello-kitty-watermelon", + name: "Hello Kitty & Watermelon", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/hello-kitty-watermelon/arrow.png", + width: 32, + height: 32, + hotspotX: 1.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/hello-kitty-watermelon/pointer.png", + width: 32, + height: 32, + hotspotX: 4, + hotspotY: 2, + }, + }, + }, + { + id: "among-us-sus-knife-and-red-animated", + name: "Among Us Sus Knife & Red Animated", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/among-us-sus-knife-and-red-animated/arrow.png", + width: 32, + height: 32, + hotspotX: 1.6, + hotspotY: 0.96, + }, + pointer: { + assetPath: "cursors/among-us-sus-knife-and-red-animated/pointer.png", + width: 32, + height: 32, + hotspotX: 12, + hotspotY: 2, + }, + }, + }, + { + id: "black-and-rainbow-stroke-gradient-animated", + name: "Black & Rainbow Stroke Gradient Animated", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/black-and-rainbow-stroke-gradient-animated/arrow.png", + width: 32, + height: 32, + hotspotX: 1.6, + hotspotY: 0.96, + }, + pointer: { + assetPath: "cursors/black-and-rainbow-stroke-gradient-animated/pointer.png", + width: 32, + height: 32, + hotspotX: 8, + hotspotY: 1.5, + }, + }, + }, + { + id: "black-pixel", + name: "Black Pixel", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/black-pixel/arrow.png", + width: 32, + height: 32, + hotspotX: 2, + hotspotY: 3.5, + }, + pointer: { + assetPath: "cursors/black-pixel/pointer.png", + width: 32, + height: 32, + hotspotX: 8, + hotspotY: 1.5, + }, + }, + }, + { + id: "christmas-miles-morales", + name: "Christmas Miles Morales", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/christmas-miles-morales/arrow.png", + width: 32, + height: 32, + hotspotX: 1, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/christmas-miles-morales/pointer.png", + width: 32, + height: 32, + hotspotX: 5.5, + hotspotY: 3, + }, + }, + }, + { + id: "hollow-knight-and-game-arrow", + name: "Hollow Knight & Game Arrow", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/hollow-knight-and-game-arrow/arrow.png", + width: 32, + height: 32, + hotspotX: 0.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/hollow-knight-and-game-arrow/pointer.png", + width: 32, + height: 32, + hotspotX: 5, + hotspotY: 0.5, + }, + }, + }, + { + id: "hollow-knight-nail-sword-and-mask", + name: "Hollow Knight Nail Sword & Mask", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/hollow-knight-nail-sword-and-mask/arrow.png", + width: 32, + height: 32, + hotspotX: 0.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/hollow-knight-nail-sword-and-mask/pointer.png", + width: 32, + height: 32, + hotspotX: 3.5, + hotspotY: 2, + }, + }, + }, + { + id: "naruto-akatsuki-cloud-arrow", + name: "Naruto Akatsuki Cloud Arrow", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/naruto-akatsuki-cloud-arrow/arrow.png", + width: 32, + height: 32, + hotspotX: 0.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/naruto-akatsuki-cloud-arrow/pointer.png", + width: 32, + height: 32, + hotspotX: 1, + hotspotY: 1, + }, + }, + }, + { + id: "old-roblox", + name: "Old Roblox", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/old-roblox/arrow.png", + width: 32, + height: 32, + hotspotX: 2.5, + hotspotY: 1.5, + }, + pointer: { + assetPath: "cursors/old-roblox/pointer.png", + width: 32, + height: 32, + hotspotX: 3.5, + hotspotY: 1.5, + }, + }, + }, + { + id: "pink-glossy-arrow-and-hand-3d", + name: "Pink Glossy Arrow & Hand 3D", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/pink-glossy-arrow-and-hand-3d/arrow.png", + width: 32, + height: 32, + hotspotX: 1.5, + hotspotY: 1.5, + }, + pointer: { + assetPath: "cursors/pink-glossy-arrow-and-hand-3d/pointer.png", + width: 32, + height: 32, + hotspotX: 3, + hotspotY: 1, + }, + }, + }, + { + id: "pinky-pixel", + name: "Pinky Pixel", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/pinky-pixel/arrow.png", + width: 32, + height: 32, + hotspotX: 0.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/pinky-pixel/pointer.png", + width: 32, + height: 32, + hotspotX: 7, + hotspotY: 1, + }, + }, + }, + { + id: "pokemon-neon-gengar", + name: "Pokemon Neon Gengar", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/pokemon-neon-gengar/arrow.png", + width: 32, + height: 32, + hotspotX: 1, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/pokemon-neon-gengar/pointer.png", + width: 32, + height: 32, + hotspotX: 2, + hotspotY: 2.5, + }, + }, + }, + { + id: "sanrio-gudetama-and-arrow-kawaii", + name: "Sanrio Gudetama & Arrow Kawaii", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/sanrio-gudetama-and-arrow-kawaii/arrow.png", + width: 32, + height: 32, + hotspotX: 0.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/sanrio-gudetama-and-arrow-kawaii/pointer.png", + width: 32, + height: 32, + hotspotX: 8, + hotspotY: 4, + }, + }, + }, + { + id: "spring-gradient", + name: "Spring Gradient", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/spring-gradient/arrow.png", + width: 32, + height: 32, + hotspotX: 1.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/spring-gradient/pointer.png", + width: 32, + height: 32, + hotspotX: 8, + hotspotY: 0.5, + }, + }, + }, + { + id: "mickey-mouse-black-hand-inflated-glove", + name: "Mickey Mouse Black Hand Inflated Glove", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/mickey-mouse-black-hand-inflated-glove/arrow.png", + width: 32, + height: 32, + hotspotX: 2.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/mickey-mouse-black-hand-inflated-glove/pointer.png", + width: 32, + height: 32, + hotspotX: 10, + hotspotY: 0.5, + }, + }, + }, + { + id: "sanrio-kuromi-skull-arrow", + name: "Sanrio Kuromi Skull Arrow", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/sanrio-kuromi-skull-arrow/arrow.png", + width: 32, + height: 32, + hotspotX: 1.5, + hotspotY: 0.5, + }, + pointer: { + assetPath: "cursors/sanrio-kuromi-skull-arrow/pointer.png", + width: 32, + height: 32, + hotspotX: 9.5, + hotspotY: 1, + }, + }, + }, + { + id: "solo-leveling-sung-jinwoo-dark-flames", + name: "Solo Leveling Sung Jinwoo Dark Flames", + source: "sweezy-cursors.com", + assets: { + arrow: { + assetPath: "cursors/solo-leveling-sung-jinwoo-dark-flames/arrow.png", + width: 32, + height: 32, + hotspotX: 2, + hotspotY: 1, + }, + pointer: { + assetPath: "cursors/solo-leveling-sung-jinwoo-dark-flames/pointer.png", + width: 32, + height: 32, + hotspotX: 7, + hotspotY: 4.5, + }, + }, + }, +]; + +/** All selectable theme ids, including the built-in default. */ +export const CURSOR_THEME_IDS: ReadonlySet = new Set([ + DEFAULT_CURSOR_THEME_ID, + ...CURSOR_THEMES.map((theme) => theme.id), +]); + +/** Returns the theme for `id`, or null for the default / unknown ids. */ +export function getCursorTheme(id: string | null | undefined): CursorTheme | null { + if (!id || id === DEFAULT_CURSOR_THEME_ID) { + return null; + } + return CURSOR_THEMES.find((theme) => theme.id === id) ?? null; +} + +/** + * Normalizes a persisted/incoming theme id to a known value, falling back to the + * default for anything unrecognized. + */ +export function normalizeCursorThemeId(id: unknown): string { + return typeof id === "string" && CURSOR_THEME_IDS.has(id) ? id : DEFAULT_CURSOR_THEME_ID; +} diff --git a/src/lib/cursor/nativeCursor.test.ts b/src/lib/cursor/nativeCursor.test.ts index 3e7a6760f..eb1ef5305 100644 --- a/src/lib/cursor/nativeCursor.test.ts +++ b/src/lib/cursor/nativeCursor.test.ts @@ -101,3 +101,119 @@ describe("native cursor click bounce", () => { expect(getNativeCursorClickBounceProgress(recordingData, 133)).toBeGreaterThan(0); }); }); + +describe("custom cursor themes", () => { + const arrowAsset: NativeCursorAsset = { + id: "telemetry-arrow", + platform: "darwin", + imageDataUrl: "default-arrow", + width: 32, + height: 32, + hotspotX: 16, + hotspotY: 15, + cursorType: "arrow", + }; + + it("substitutes the themed art for an overridden cursor type", () => { + const rendered = resolveNativeCursorRenderAsset( + arrowAsset, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5, cursorType: "arrow" }, + "hello-kitty-watermelon", + ); + + expect(rendered.id).toBe("theme:hello-kitty-watermelon:arrow"); + expect(rendered.imageDataUrl).toContain("cursors/hello-kitty-watermelon/arrow.png"); + expect(rendered.width).toBe(32); + expect(rendered.hotspotX).toBeCloseTo(1.5); + }); + + it("classifies an untyped macOS arrow bitmap (top-left hotspot) as the themed arrow", () => { + const macArrow: NativeCursorAsset = { + id: "sha-arrow", + platform: "darwin", + imageDataUrl: "captured-bitmap", + width: 34, + height: 46, + hotspotX: 8, + hotspotY: 8, + scaleFactor: 2, + }; + const rendered = resolveNativeCursorRenderAsset( + macArrow, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5 }, + "hello-kitty-watermelon", + ); + + expect(rendered.id).toBe("theme:hello-kitty-watermelon:arrow"); + expect(rendered.imageDataUrl).toContain("cursors/hello-kitty-watermelon/arrow.png"); + }); + + it("classifies an untyped macOS hand bitmap (upper-center hotspot) as the themed pointer", () => { + const macHand: NativeCursorAsset = { + id: "sha-hand", + platform: "darwin", + imageDataUrl: "captured-bitmap", + width: 64, + height: 64, + hotspotX: 26, + hotspotY: 16, + scaleFactor: 2, + }; + const rendered = resolveNativeCursorRenderAsset( + macHand, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5 }, + "hello-kitty-watermelon", + ); + + expect(rendered.id).toBe("theme:hello-kitty-watermelon:pointer"); + expect(rendered.imageDataUrl).toContain("cursors/hello-kitty-watermelon/pointer.png"); + }); + + it("leaves an untyped text/crosshair bitmap (centered hotspot) as the real captured cursor", () => { + const macText: NativeCursorAsset = { + id: "sha-text", + platform: "darwin", + imageDataUrl: "captured-ibeam", + width: 18, + height: 36, + hotspotX: 8, + hotspotY: 18, + scaleFactor: 2, + }; + const rendered = resolveNativeCursorRenderAsset( + macText, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5 }, + "hello-kitty-watermelon", + ); + + expect(rendered.id).toBe("sha-text"); + expect(rendered.imageDataUrl).toBe("captured-ibeam"); + }); + + it("keeps the default art for the default theme id", () => { + const rendered = resolveNativeCursorRenderAsset( + arrowAsset, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5, cursorType: "arrow" }, + "default", + ); + + expect(rendered.id).toBe("pretty:arrow"); + expect(rendered.imageDataUrl).not.toContain("hello-kitty-watermelon"); + }); + + it("falls back to default art for a cursor type the theme does not override", () => { + const rendered = resolveNativeCursorRenderAsset( + { ...arrowAsset, cursorType: "text" }, + 1, + { timeMs: 0, cx: 0.5, cy: 0.5, cursorType: "text" }, + "hello-kitty-watermelon", + ); + + expect(rendered.id).toBe("pretty:text"); + }); +}); diff --git a/src/lib/cursor/nativeCursor.ts b/src/lib/cursor/nativeCursor.ts index f20fd4259..dcf55eea5 100644 --- a/src/lib/cursor/nativeCursor.ts +++ b/src/lib/cursor/nativeCursor.ts @@ -16,6 +16,8 @@ import textUrl from "@/assets/cursors/Cursor=Text-Cursor.svg"; import upArrowUrl from "@/assets/cursors/Cursor=Up-Arrow.svg"; import waitUrl from "@/assets/cursors/Cursor=Wait.svg"; import type { CropRegion } from "@/components/video-editor/types"; +import { getAssetPath } from "@/lib/assetPath"; +import { DEFAULT_CURSOR_THEME_ID, getCursorTheme } from "@/lib/cursor/cursorThemes"; import type { CursorRecordingData, CursorRecordingSample, @@ -28,13 +30,6 @@ export interface ActiveNativeCursorFrame { sample: CursorRecordingSample; } -export interface NativeCursorSmoothingState { - cx: number; - cy: number; - lastTimeMs: number | null; - initialized: boolean; -} - export interface NativeCursorMotionBlurState { x: number; y: number; @@ -276,22 +271,6 @@ export function hasNativeCursorRecordingData( ); } -export function createNativeCursorSmoothingState(): NativeCursorSmoothingState { - return { - cx: 0, - cy: 0, - lastTimeMs: null, - initialized: false, - }; -} - -export function resetNativeCursorSmoothingState(state: NativeCursorSmoothingState) { - state.cx = 0; - state.cy = 0; - state.lastTimeMs = null; - state.initialized = false; -} - export function createNativeCursorMotionBlurState(): NativeCursorMotionBlurState { return { x: 0, @@ -308,49 +287,6 @@ export function resetNativeCursorMotionBlurState(state: NativeCursorMotionBlurSt state.initialized = false; } -export function smoothNativeCursorSample({ - forceSnap = false, - sample, - smoothing, - state, - timeMs, -}: { - forceSnap?: boolean; - sample: CursorRecordingSample; - smoothing: number; - state: NativeCursorSmoothingState; - timeMs: number; -}): CursorRecordingSample { - const clampedSmoothing = clamp(Number.isFinite(smoothing) ? smoothing : 0, 0, 0.98); - const previousTimeMs = state.lastTimeMs; - const shouldSnap = - forceSnap || - clampedSmoothing <= 0 || - !state.initialized || - previousTimeMs === null || - timeMs <= previousTimeMs; - - if (shouldSnap) { - state.cx = sample.cx; - state.cy = sample.cy; - state.lastTimeMs = timeMs; - state.initialized = true; - return sample; - } - - const frameCount = Math.max(1, (timeMs - previousTimeMs) / (1000 / 60)); - const alpha = 1 - Math.pow(clampedSmoothing, frameCount); - state.cx += (sample.cx - state.cx) * alpha; - state.cy += (sample.cy - state.cy) * alpha; - state.lastTimeMs = timeMs; - - return { - ...sample, - cx: state.cx, - cy: state.cy, - }; -} - export function getNativeCursorClickBounceProgress( recordingData: CursorRecordingData | null | undefined, timeMs: number, @@ -592,11 +528,81 @@ export function resolvePrettyNativeCursorAsset( : resolveUntypedPrettyNativeCursorAsset(asset); } +/** + * Infers "arrow" vs "pointer" from a captured bitmap's hotspot, for platforms (macOS) + * that don't tag samples with a `cursorType`. Arrow's hotspot is in the top-left tip; + * the pointing hand's fingertip is in the upper-center band. Anything else stays + * unclassified so it keeps its real captured cursor instead of a themed arrow/pointer. + */ +function classifyCapturedCursorType(asset: NativeCursorAsset): NativeCursorType | null { + if (asset.width <= 0 || asset.height <= 0) { + return null; + } + const hotspotXNorm = asset.hotspotX / asset.width; + const hotspotYNorm = asset.hotspotY / asset.height; + if (hotspotXNorm < 0.33 && hotspotYNorm < 0.33) { + return "arrow"; + } + if (hotspotYNorm < 0.4 && hotspotXNorm >= 0.33 && hotspotXNorm <= 0.6) { + return "pointer"; + } + return null; +} + +/** + * Resolves the theme override for a cursor type, or null when the default theme is active + * or has no art for that type. The asset URL resolves lazily (only when a theme is active) + * so this is safe from tests and non-renderer contexts; a failure degrades to default art. + */ +function resolveThemedCursorAsset( + themeId: string | null | undefined, + cursorType: NativeCursorType, +): PrettyNativeCursorAsset | null { + if (!themeId || themeId === DEFAULT_CURSOR_THEME_ID) { + return null; + } + const themeAsset = getCursorTheme(themeId)?.assets[cursorType]; + if (!themeAsset) { + return null; + } + try { + return { + imageDataUrl: getAssetPath(themeAsset.assetPath), + width: themeAsset.width, + height: themeAsset.height, + hotspotX: themeAsset.hotspotX, + hotspotY: themeAsset.hotspotY, + }; + } catch { + return null; + } +} + export function resolveNativeCursorRenderAsset( asset: NativeCursorAsset, deviceScaleFactor: number, sample?: CursorRecordingSample, + themeId?: string | null, ) { + const cursorType = sample?.cursorType ?? asset.cursorType ?? null; + if (themeId && themeId !== DEFAULT_CURSOR_THEME_ID) { + // A known type uses its override when the theme provides one. Untyped samples + // (common on macOS, where the type isn't tagged) are classified from the captured + // bitmap's hotspot so arrow becomes themed-arrow and hand becomes themed-pointer. + const themedType = cursorType ?? classifyCapturedCursorType(asset); + const themedAsset = themedType ? resolveThemedCursorAsset(themeId, themedType) : null; + if (themedAsset && themedType) { + return { + id: `theme:${themeId}:${themedType}`, + imageDataUrl: themedAsset.imageDataUrl, + width: themedAsset.width, + height: themedAsset.height, + hotspotX: themedAsset.hotspotX, + hotspotY: themedAsset.hotspotY, + }; + } + } + const prettyAsset = resolvePrettyNativeCursorAsset(asset, sample); if (prettyAsset) { return { diff --git a/src/lib/cursorTelemetryBuffer.test.ts b/src/lib/cursorTelemetryBuffer.test.ts index 17174accc..dffb5eec1 100644 --- a/src/lib/cursorTelemetryBuffer.test.ts +++ b/src/lib/cursorTelemetryBuffer.test.ts @@ -2,8 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { type CursorTelemetryPoint, createCursorTelemetryBuffer } from "./cursorTelemetryBuffer"; function sample(tag: number): CursorTelemetryPoint { - // Decouple the timestamp tag from the coordinate fixture so cursor - // points stay inside the normalized [0, 1] range that real samples use. + // Decouple the tag from coordinates so points stay in the normalized [0, 1] range. const normalized = (tag % 100) / 100; return { timeMs: tag, cx: normalized, cy: normalized }; } @@ -121,11 +120,9 @@ describe("createCursorTelemetryBuffer", () => { }); it("discardBatch(id) targets the correct batch even when a later recording sits in front of it", () => { - // Regression test for the rapid Stop → Record → Discard sequence: - // recording A's finalize callback does async work (fixWebmDuration), - // recording B finishes in the meantime, then A's callback resolves - // with discard intent. The discard must drop A — not B, which - // happens to be the *latest* pending batch by the time discard runs. + // Regression for the rapid Stop/Record/Discard sequence: A's finalize callback does + // async work (fixWebmDuration), B finishes meanwhile, then A resolves with discard + // intent. The discard must drop A, not B, which is the latest pending batch by then. const buf = createCursorTelemetryBuffer({ maxActiveSamples: 10 }); buf.startSession(1); @@ -216,8 +213,8 @@ describe("createCursorTelemetryBuffer", () => { expect(buf.pendingCount).toBe(2); expect(warn).not.toHaveBeenCalled(); - // Simulate a misuse where a retry prepends without first draining: - // queue would grow to 3, so the oldest-trailing entry must be evicted. + // Misuse: a retry prepends without draining first, so the queue would grow to 3 + // and the oldest-trailing entry must be evicted. buf.prependBatch({ recordingId: 99, samples: [sample(99)] }); expect(buf.pendingCount).toBe(2); expect(warn).toHaveBeenCalledTimes(1); @@ -232,8 +229,7 @@ describe("createCursorTelemetryBuffer", () => { }); it("sanitizes non-finite or non-positive option values to safe defaults", () => { - // Infinity / NaN / negative would otherwise turn the trim loops - // into infinite loops. The buffer must fall back to defaults. + // Infinity/NaN/negative would turn the trim loops infinite; the buffer must fall back to defaults. const buf = createCursorTelemetryBuffer({ maxActiveSamples: Number.POSITIVE_INFINITY, maxPendingBatches: Number.NaN, diff --git a/src/lib/cursorTelemetryBuffer.ts b/src/lib/cursorTelemetryBuffer.ts index 0c7e0e10e..02d6147bb 100644 --- a/src/lib/cursorTelemetryBuffer.ts +++ b/src/lib/cursorTelemetryBuffer.ts @@ -1,10 +1,7 @@ /** - * A single cursor telemetry sample captured during a recording session. - * - * Coordinates (`cx`, `cy`) are clamped ratios in the `[0, 1]` range, - * normalised against the captured surface's width and height by the - * main-process `sampleCursorPoint()` before being pushed. `timeMs` is the - * offset (in milliseconds) from the recording's start. + * A single cursor telemetry sample. cx/cy are clamped [0,1] ratios of the + * captured surface (normalised in the main process by sampleCursorPoint). + * timeMs is the offset from recording start. */ export interface CursorTelemetryPoint { timeMs: number; @@ -13,9 +10,9 @@ export interface CursorTelemetryPoint { } /** - * A completed batch of cursor samples, tagged with the recording id that - * produced them. The id is supplied at `startSession()` time and travels - * with the batch through the pending queue, retries, and discards. + * A completed batch of cursor samples, tagged with its recording id. The id + * (from startSession) travels with the batch through the queue, retries, and + * discards. */ export interface CursorTelemetryBatch { recordingId: number; @@ -25,87 +22,64 @@ export interface CursorTelemetryBatch { /** * Per-session cursor telemetry buffer with bounded memory. * - * Flow: `startSession(recordingId)` → `push(point)` N times → `endSession()` - * enqueues the collected samples as a completed batch tagged with that - * `recordingId`. The main process later drains batches in FIFO order via - * `takeNextBatch()` to persist them to disk, and can `prependBatch()` on - * write failure to retry without losing order. A discard request keys on - * the recording id so an asynchronous "discard recording A" decision that - * arrives after recording B has already enqueued its batch still drops - * the right one. + * Flow: startSession(recordingId), push(point) N times, endSession() enqueues + * the samples as a batch tagged with that id. The main process drains batches + * FIFO via takeNextBatch() to persist, and prependBatch() on write failure to + * retry without losing order. Discard keys on the recording id so an async + * "discard recording A" that arrives after recording B has enqueued still + * drops the right batch. * - * Memory is bounded by `maxActiveSamples` (ring buffer on the in-progress - * batch) and `maxPendingBatches` (FIFO cap across completed batches). + * Memory bounded by maxActiveSamples (ring buffer on the in-progress batch) + * and maxPendingBatches (FIFO cap across completed batches). */ export interface CursorTelemetryBuffer { /** - * Begin a new recording session under the given `recordingId`. Clears - * any in-progress active samples (without touching already-completed - * pending batches). Safe to call repeatedly — e.g. a rapid Stop → - * Record sequence — and the most recent id wins. + * Begin a new recording session. Clears in-progress active samples but + * leaves completed pending batches. Safe to call repeatedly (e.g. a rapid + * Stop then Record); the most recent id wins. */ startSession(recordingId: number): void; /** - * Append a telemetry sample to the current active session. When the - * active buffer exceeds `maxActiveSamples`, the oldest sample is - * dropped (ring behaviour). + * Append a sample to the active session. Over maxActiveSamples, the oldest + * sample is dropped (ring behaviour). */ push(point: CursorTelemetryPoint): void; /** - * Finalize the active session, moving its samples into the pending - * queue as a single batch tagged with the current recording id. Empty - * sessions are dropped (no empty batch is enqueued). - * - * If the pending queue would exceed `maxPendingBatches`, the oldest - * batches are evicted to bound memory. A `console.warn` is emitted - * whenever at least one batch is dropped so that pathological rapid- - * restart scenarios are observable. + * Finalize the active session into a single pending batch tagged with the + * current recording id. Empty sessions enqueue nothing. Over + * maxPendingBatches, oldest batches are evicted and a warn is logged so + * pathological rapid-restart cases are observable. * - * @returns the number of pending batches dropped by this call (0 under - * normal operation). + * @returns the number of pending batches dropped (0 normally). */ endSession(): number; - /** - * Remove and return the oldest pending batch, or `null` if the queue - * is empty. - */ + /** Remove and return the oldest pending batch, or null if empty. */ takeNextBatch(): CursorTelemetryBatch | null; /** - * Re-insert a batch at the front of the queue, preserving FIFO order - * on retry paths (e.g. when persisting the batch failed and the - * caller wants the next `takeNextBatch()` to yield it again). - * - * Empty batches are ignored. The pending cap is enforced defensively - * — if prepending would push the queue past `maxPendingBatches`, the - * oldest entries are evicted and a `console.warn` is emitted. In - * normal retry usage this trim is a no-op because the caller has just - * removed the batch via `takeNextBatch()`. + * Re-insert a batch at the front, preserving FIFO order on retry (e.g. + * persisting failed and the next takeNextBatch() should yield it again). + * Empty batches are ignored. The pending cap is enforced defensively; in + * normal retry usage the trim is a no-op since the caller just took it. */ prependBatch(batch: CursorTelemetryBatch): void; /** - * Drop the pending batch produced by the given `recordingId`. Used - * when a recording is discarded after its `endSession()` has run but - * before it has been persisted. Returns `true` if a batch was - * removed, `false` otherwise (no matching id, or the batch was - * already drained). + * Drop the pending batch for the given recordingId, when a recording is + * discarded after endSession() but before persistence. Returns true if a + * batch was removed. * - * Keying on the recording id (rather than "the latest pending batch") - * avoids a real bug: when finalizing a recording does asynchronous - * work like `fixWebmDuration`, a quick Stop → Record → Discard - * sequence can interleave such that the latest pending batch belongs - * to a *later* recording than the one being discarded. + * Keys on the recording id rather than "the latest pending batch" to avoid + * a bug: async finalize work (fixWebmDuration) means a quick Stop, Record, + * Discard can leave the latest pending batch belonging to a later recording + * than the one being discarded. */ discardBatch(recordingId: number): boolean; - /** - * Clear both the active and pending state. Intended for tests and - * full teardown paths. - */ + /** Clear active and pending state. For tests and full teardown. */ reset(): void; readonly activeCount: number; @@ -128,11 +102,8 @@ function sanitizeLimit(value: number | undefined, fallback: number): number { } /** - * Create a cursor telemetry buffer. - * - * Numeric options are sanitized: non-finite, negative, or zero values fall - * back to safe defaults so a bad caller cannot disable the memory bounds - * (which would turn the trim loops into infinite loops). + * Create a cursor telemetry buffer. Options are sanitized so a bad caller + * cannot disable the memory bounds (which would make the trim loops infinite). * * @see CursorTelemetryBuffer for the full lifecycle contract. */ diff --git a/src/lib/customFonts.ts b/src/lib/customFonts.ts index af332c139..4cf00c6a3 100644 --- a/src/lib/customFonts.ts +++ b/src/lib/customFonts.ts @@ -1,16 +1,15 @@ -// Google Fonts loading and management utility +// Google Fonts loading and management export interface CustomFont { id: string; - name: string; // Display name - fontFamily: string; // CSS font-family value + name: string; + fontFamily: string; importUrl: string; // Google Fonts @import URL } const STORAGE_KEY = "openscreen_custom_fonts"; const loadedFonts = new Set(); -// Load custom fonts from localStorage export function getCustomFonts(): CustomFont[] { try { const stored = localStorage.getItem(STORAGE_KEY); @@ -21,7 +20,6 @@ export function getCustomFonts(): CustomFont[] { } } -// Save custom fonts to localStorage export function saveCustomFonts(fonts: CustomFont[]): void { try { localStorage.setItem(STORAGE_KEY, JSON.stringify(fonts)); @@ -30,7 +28,7 @@ export function saveCustomFonts(fonts: CustomFont[]): void { } } -// Add a new custom font (throws error if font fails to load) +// Throws if the font fails to load export async function addCustomFont(font: CustomFont): Promise { const fonts = getCustomFonts(); const exists = fonts.some((f) => f.id === font.id || f.fontFamily === font.fontFamily); @@ -39,23 +37,20 @@ export async function addCustomFont(font: CustomFont): Promise { return fonts; } - // Try to load the font first - this will throw if it fails + // Load first so a failure throws before we persist it await loadFont(font); - // Only add to storage if font loaded successfully fonts.push(font); saveCustomFonts(fonts); return fonts; } -// Remove a custom font export function removeCustomFont(fontId: string): CustomFont[] { const fonts = getCustomFonts(); const filtered = fonts.filter((f) => f.id !== fontId); saveCustomFonts(filtered); - // Remove the style element const styleEl = document.getElementById(`custom-font-${fontId}`); if (styleEl) { styleEl.remove(); @@ -68,7 +63,6 @@ export function removeCustomFont(fontId: string): CustomFont[] { // Load a Google Font into the document export function loadFont(font: CustomFont): Promise { return new Promise((resolve, reject) => { - // Skip if already loaded if (loadedFonts.has(font.id)) { resolve(); return; @@ -77,19 +71,16 @@ export function loadFont(font: CustomFont): Promise { try { const styleId = `custom-font-${font.id}`; - // Remove existing style if present const existing = document.getElementById(styleId); if (existing) { existing.remove(); } - // Create style element with @import const style = document.createElement("style"); style.id = styleId; style.textContent = `@import url('${font.importUrl}');`; document.head.appendChild(style); - // Wait for font to load waitForFont(font.fontFamily) .then(() => { loadedFonts.add(font.id); @@ -103,17 +94,15 @@ export function loadFont(font: CustomFont): Promise { }); } -// Wait for a font to be available and verify it loaded +// Wait for a font to load and verify it's actually available function waitForFont(fontFamily: string, timeout = 5000): Promise { return new Promise((resolve, reject) => { - // Use CSS Font Loading API if available if ("fonts" in document) { Promise.race([ document.fonts.load(`16px "${fontFamily}"`), new Promise((_, rej) => setTimeout(() => rej(new Error("Font load timeout")), timeout)), ]) .then(() => { - // Verify the font actually loaded by checking if it's available const isAvailable = document.fonts.check(`16px "${fontFamily}"`); if (isAvailable) { resolve(); @@ -125,14 +114,13 @@ function waitForFont(fontFamily: string, timeout = 5000): Promise { reject(error); }); } else { - // Fallback for browsers without Font Loading API - // Wait a bit and hope for the best + // No Font Loading API: wait a bit and hope for the best setTimeout(() => resolve(), 1000); } }); } -// Load all stored custom fonts on app initialization +// Load all stored custom fonts on app init export function loadAllCustomFonts(): Promise { const fonts = getCustomFonts(); return Promise.all( @@ -144,22 +132,21 @@ export function loadAllCustomFonts(): Promise { ); } -// Generate a unique ID for a font export function generateFontId(name: string): string { return `${name.toLowerCase().replace(/\s+/g, "-")}-${Date.now()}`; } -// Parse Google Fonts @import URL to extract font family name +// Extract the font family from a Google Fonts @import URL export function parseFontFamilyFromImport(importUrl: string): string | null { try { - // Extract from URL like: https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap + // e.g. https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap const url = new URL(importUrl); const familyParam = url.searchParams.get("family"); if (familyParam) { - // Remove weight/style info: "Roboto:wght@400;700" -> "Roboto" + // "Roboto:wght@400;700" -> "Roboto" const fontName = familyParam.split(":")[0]; - // Replace + with spaces: "Open+Sans" -> "Open Sans" + // "Open+Sans" -> "Open Sans" return fontName.replace(/\+/g, " "); } @@ -170,7 +157,7 @@ export function parseFontFamilyFromImport(importUrl: string): string | null { } } -// Validate if a string looks like a Google Fonts import URL +// Does this look like a Google Fonts import URL? export function isValidGoogleFontsUrl(url: string): boolean { try { const urlObj = new URL(url); diff --git a/src/lib/exporter/annotationRenderer.ts b/src/lib/exporter/annotationRenderer.ts index a2ac08a19..1ef55d672 100644 --- a/src/lib/exporter/annotationRenderer.ts +++ b/src/lib/exporter/annotationRenderer.ts @@ -11,11 +11,9 @@ import { let blurScratchCanvas: HTMLCanvasElement | null = null; let blurScratchCtx: CanvasRenderingContext2D | null = null; -// Matches a single code point whose script is Han (including non-BMP -// Extension A-F), Hiragana, Katakana (including halfwidth forms), or -// Hangul. Used to split CJK text at character boundaries during wrap, -// since CJK scripts have no word-separating whitespace. Unicode script -// property escapes require ES2018+; tsconfig target is ES2020. +// Han/Hiragana/Katakana/Hangul code points, to split CJK text at character +// boundaries during wrap (CJK has no word-separating whitespace). Script +// escapes need ES2018+; tsconfig targets ES2020. const CJK_CHAR = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; type GraphemeSegmenter = { @@ -39,10 +37,9 @@ function splitGraphemes(value: string): string[] { } function tokenizeForWrap(line: string): string[] { - // Split Latin text on whitespace (preserving the whitespace as its own token, - // matching the original behavior), and split CJK runs into individual - // characters so each one becomes a breakable unit. This mirrors the editor's - // CSS `word-break: break-word` handling for CJK content. + // Split Latin on whitespace (kept as its own token) and split CJK runs into + // individual chars so each is breakable, mirroring the editor's CSS + // word-break: break-word for CJK. const tokens: string[] = []; let buffer = ""; const chars = Array.from(line); @@ -126,10 +123,8 @@ function renderArrow( const offsetX = padding + (availableWidth - 100 * scale) / 2; const offsetY = padding + (availableHeight - 100 * scale) / 2; - // Apply centering offset ctx.translate(offsetX, offsetY); - // Apply shadow filter ctx.shadowColor = "rgba(0, 0, 0, 0.3)"; ctx.shadowBlur = 8 * scale; ctx.shadowOffsetX = 0; @@ -140,7 +135,7 @@ function renderArrow( ctx.lineCap = "round"; ctx.lineJoin = "round"; - // Draw all paths as a single shape to avoid overlapping shadows/strokes + // One shape so shadows/strokes don't overlap ctx.beginPath(); for (const pathString of paths) { @@ -278,7 +273,7 @@ function renderText( ctx.translate(-transformOriginX, -transformOriginY); ctx.globalAlpha *= animationState.opacity; - // Clip text to annotation box bounds (matches editor's overflow: hidden) + // Clip to box bounds, matching editor's overflow: hidden ctx.beginPath(); ctx.rect(x, y, width, height); ctx.clip(); @@ -414,7 +409,7 @@ async function renderImage( return new Promise((resolve) => { const img = new Image(); img.onload = () => { - // Preserve aspect ratio - contain the image within the bounds + // Contain within bounds, preserving aspect ratio const imgAspect = img.width / img.height; const boxAspect = width / height; @@ -450,12 +445,11 @@ export async function renderAnnotations( currentTimeMs: number, scaleFactor: number = 1.0, ): Promise { - // Filter active annotations at current time const activeAnnotations = annotations.filter( (ann) => currentTimeMs >= ann.startMs && currentTimeMs < ann.endMs, ); - // Sort by z-index (lower first, so higher z-index draws on top) + // Lower z-index first so higher draws on top const sortedAnnotations = [...activeAnnotations].sort((a, b) => a.zIndex - b.zIndex); for (const annotation of sortedAnnotations) { diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 95227844d..7514907e2 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -210,9 +210,8 @@ export class AudioProcessor { } /** - * Audio export has two modes: - * 1) no speed regions -> fast WebCodecs trim-only pipeline - * 2) speed regions present -> pitch-preserving rendered timeline pipeline + * Two modes: no speed regions uses the fast WebCodecs trim-only pipeline; speed + * regions use the pitch-preserving rendered timeline pipeline. */ async process( demuxer: WebDemuxer, @@ -230,7 +229,7 @@ export class AudioProcessor { .sort((a, b) => a.startMs - b.startMs) : []; - // Speed edits must use timeline playback to preserve pitch + // Speed edits need timeline playback to preserve pitch. if (sortedSpeedRegions.length > 0) { const renderedAudioBlob = await this.renderPitchPreservedTimelineAudio( videoUrl, @@ -245,14 +244,14 @@ export class AudioProcessor { return; } - // No speed edits: keep the original demux/decode/encode path with trim timestamp remap. - // The +0.5s buffer mirrors streamingDecoder.decodeAll's read window so the trim-only - // and speed-aware paths agree on how far to read past the validated duration boundary. + // No speed edits: demux/decode/encode with trim timestamp remap. The +0.5s mirrors + // streamingDecoder.decodeAll's read window so both paths read the same distance past + // the validated duration boundary. const readEndSec = validatedDurationSec + 0.5; await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec, exportCodec); } - // Legacy trim-only path. This is still used for projects without speed regions. + // Trim-only path, used for projects without speed regions. private async processTrimOnlyAudio( demuxer: WebDemuxer, muxer: VideoMuxer, @@ -274,7 +273,7 @@ export class AudioProcessor { return; } - // Phase 1: Decode audio from source, skipping trimmed regions + // Phase 1: decode, skipping trimmed regions. const decodedFrames: AudioData[] = []; const decoder = new AudioDecoder({ @@ -325,7 +324,7 @@ export class AudioProcessor { return; } - // Phase 2: Re-encode with timestamps adjusted for trim gaps + // Phase 2: re-encode with timestamps adjusted for trim gaps. const encodedChunks: { chunk: EncodedAudioChunk; meta?: EncodedAudioChunkMetadata }[] = []; const encoder = new AudioEncoder({ @@ -391,7 +390,7 @@ export class AudioProcessor { encoder.close(); } - // Phase 3: Flush encoded chunks to muxer + // Phase 3: flush encoded chunks to muxer. for (const { chunk, meta } of encodedChunks) { if (this.cancelled) break; await muxer.addAudioChunk(chunk, meta); @@ -402,8 +401,8 @@ export class AudioProcessor { ); } - // Speed-aware path that mirrors preview semantics (trim skipping + playbackRate regions) - // preserve pitch through browser media playback behavior to avoid chipmunk effect. + // Speed-aware path mirroring preview semantics (trim skipping + playbackRate). Relies on + // browser media playback to preserve pitch and avoid the chipmunk effect. private async renderPitchPreservedTimelineAudio( videoUrl: string, trimRegions: TrimRegion[], @@ -442,9 +441,8 @@ export class AudioProcessor { await audioContext.resume(); } - // Skip past any initial trim region(s) before recording starts to avoid - // capturing trimmed audio during the first rAF frames of playback. - // Loops to handle back-to-back or overlapping trims at t=0. + // Skip initial trim region(s) before recording so the first rAF frames don't + // capture trimmed audio. Loops to handle back-to-back/overlapping trims at t=0. const effectiveEnd = validatedDurationSec; let startPosition = 0; for (let i = 0; i <= trimRegions.length; i++) { @@ -455,19 +453,19 @@ export class AudioProcessor { } if (startPosition >= effectiveEnd) { - // All content is trimmed — return silent blob + // Everything is trimmed; return a silent blob. return new Blob([], { type: "audio/webm" }); } await this.seekTo(media, startPosition); - // Set initial playback rate for the starting position + // Set initial playback rate for the starting position. const initialSpeedRegion = this.findActiveSpeedRegion(startPosition * 1000, speedRegions); if (initialSpeedRegion) { media.playbackRate = initialSpeedRegion.speed; } - // Start recording only AFTER seeking past trims + // Start recording only after seeking past trims. const recording = this.startAudioRecording(destinationNode.stream); recorder = recording.recorder; recordedBlobPromise = recording.recordedBlobPromise; @@ -500,8 +498,8 @@ export class AudioProcessor { return; } - // Stop playback at validated duration — browser's media.duration - // may be inflated from bad container metadata. + // Stop at validated duration; media.duration can be inflated by bad + // container metadata. if (media.currentTime >= validatedDurationSec) { media.pause(); cleanup(); @@ -520,8 +518,7 @@ export class AudioProcessor { resolve(); return; } - // Pause recording during trim seek to prevent capturing - // silence/noise as the audio element seeks. + // Pause recording during the seek so we don't capture silence/noise. media.pause(); if (recorder?.state === "recording") recorder.pause(); const onSeeked = () => { @@ -591,9 +588,8 @@ export class AudioProcessor { } if (!recordedBlobPromise) { - // Invariant: either an early return above fires, or startAudioRecording ran and - // populated recordedBlobPromise before the playback Promise resolved. Reaching - // here means that contract was broken — fail loud instead of returning silence. + // Either an early return fired or startAudioRecording set this before playback + // resolved. Reaching here means that broke; fail loud rather than return silence. throw new Error("Audio recorder finished without assigning recordedBlobPromise"); } const recordedBlob = await recordedBlobPromise; @@ -603,7 +599,7 @@ export class AudioProcessor { return recordedBlob; } - // Demuxes the rendered speed-adjusted blob and feeds encoded chunks into the MP4 muxer. + // Demux the rendered speed-adjusted blob and feed its chunks into the MP4 muxer. private async muxRenderedAudioBlob( blob: Blob, muxer: VideoMuxer, diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts new file mode 100644 index 000000000..16ba6bde0 --- /dev/null +++ b/src/lib/exporter/frameRenderer.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { drawWebcamFrameImage } from "./webcamFrameDrawing"; + +type DrawCall = + | ["drawImage", unknown, number, number, number, number, number, number, number, number] + | ["restore"] + | ["save"] + | ["scale", number, number] + | ["translate", number, number]; + +function createMockCanvasContext() { + const calls: DrawCall[] = []; + const ctx = { + drawImage: ( + image: CanvasImageSource, + sx: number, + sy: number, + sw: number, + sh: number, + dx: number, + dy: number, + dw: number, + dh: number, + ) => calls.push(["drawImage", image, sx, sy, sw, sh, dx, dy, dw, dh]), + restore: () => calls.push(["restore"]), + save: () => calls.push(["save"]), + scale: (x: number, y: number) => calls.push(["scale", x, y]), + translate: (x: number, y: number) => calls.push(["translate", x, y]), + }; + + return { calls, ctx }; +} + +describe("drawWebcamFrameImage", () => { + it("draws the webcam frame into the layout rect by default", () => { + const { calls, ctx } = createMockCanvasContext(); + const frame = {} as CanvasImageSource; + + drawWebcamFrameImage( + ctx, + frame, + { x: 12, y: 8, width: 640, height: 360 }, + { x: 100, y: 50, width: 320, height: 180 }, + ); + + expect(calls).toEqual([["drawImage", frame, 12, 8, 640, 360, 100, 50, 320, 180]]); + }); + + it("mirrors around the webcam rect without changing the crop", () => { + const { calls, ctx } = createMockCanvasContext(); + const frame = {} as CanvasImageSource; + + drawWebcamFrameImage( + ctx, + frame, + { x: 12, y: 8, width: 640, height: 360 }, + { x: 100, y: 50, width: 320, height: 180 }, + true, + ); + + expect(calls).toEqual([ + ["save"], + ["translate", 420, 50], + ["scale", -1, 1], + ["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180], + ["restore"], + ]); + }); + + it("restores the canvas context if mirrored drawing fails", () => { + const { calls, ctx } = createMockCanvasContext(); + const frame = {} as CanvasImageSource; + const error = new Error("draw failed"); + ctx.drawImage = () => { + calls.push(["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180]); + throw error; + }; + + expect(() => + drawWebcamFrameImage( + ctx, + frame, + { x: 12, y: 8, width: 640, height: 360 }, + { x: 100, y: 50, width: 320, height: 180 }, + true, + ), + ).toThrow(error); + + expect(calls).toEqual([ + ["save"], + ["translate", 420, 50], + ["scale", -1, 1], + ["drawImage", frame, 12, 8, 640, 360, 0, 0, 320, 180], + ["restore"], + ]); + }); +}); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 858215815..d8f3c4312 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -24,19 +24,17 @@ import { lerpRotation3D, } from "@/components/video-editor/types"; import { - AUTO_FOLLOW_RAMP_DISTANCE, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, + AUTO_FOLLOW_PARAMS, DEFAULT_FOCUS, - ZOOM_SCALE_DEADZONE, - ZOOM_TRANSLATION_DEADZONE_PX, } from "@/components/video-editor/videoPlayback/constants"; -import { - adaptiveSmoothFactor, - smoothCursorFocus, -} from "@/components/video-editor/videoPlayback/cursorFollowUtils"; +import { advanceFollowFocus } from "@/components/video-editor/videoPlayback/cursorFollowUtils"; import { clampFocusToScale } from "@/components/video-editor/videoPlayback/focusUtils"; import { findDominantRegion } from "@/components/video-editor/videoPlayback/zoomRegionUtils"; +import { + createZoomSpringState, + resetZoomSpring, + stepZoomSpring, +} from "@/components/video-editor/videoPlayback/zoomSpring"; import { applyZoomTransform, computeFocusFromTransform, @@ -47,21 +45,20 @@ import { import { computeCompositeLayout, getWebcamLayoutPresetDefinition, + reactiveWebcamScale, type Size, type StyledRenderRect, } from "@/lib/compositeLayout"; +import { getSmoothedCursorPath } from "@/lib/cursor/cursorPathSmoothing"; import { createNativeCursorMotionBlurState, - createNativeCursorSmoothingState, getNativeCursorClickBounceProgress, getNativeCursorClickBounceScale, getNativeCursorMotionBlurPx, projectNativeCursorToLocal, resetNativeCursorMotionBlurState, - resetNativeCursorSmoothingState, resolveInterpolatedNativeCursorFrame, resolveNativeCursorRenderAsset, - smoothNativeCursorSample, } from "@/lib/cursor/nativeCursor"; import { BackgroundLoadError, classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { drawCanvasClipPath } from "@/lib/webcamMaskShapes"; @@ -74,6 +71,7 @@ import { resolveLinearGradientAngle, } from "./gradientParser"; import { createThreeDPass, type ThreeDPass } from "./threeDPass"; +import { drawWebcamFrameImage } from "./webcamFrameDrawing"; interface FrameRenderConfig { width: number; @@ -93,11 +91,14 @@ interface FrameRenderConfig { cursorMotionBlur?: number; cursorClickBounce?: number; cursorClipToBounds?: boolean; + cursorTheme?: string; videoWidth: number; videoHeight: number; webcamSize?: Size | null; webcamLayoutPreset?: WebcamLayoutPreset; webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; + webcamMirrored?: boolean; + webcamReactiveZoom?: boolean; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; annotationRegions?: AnnotationRegion[]; @@ -157,10 +158,10 @@ export class FrameRenderer { private layoutCache: LayoutCache | null = null; private currentVideoTime = 0; private motionBlurState: MotionBlurState = createMotionBlurState(); - private nativeCursorSmoothingState = createNativeCursorSmoothingState(); private nativeCursorMotionBlurState = createNativeCursorMotionBlurState(); private smoothedAutoFocus: { cx: number; cy: number } | null = null; private prevAnimationTimeMs: number | null = null; + private zoomSpringState = createZoomSpringState(); private prevTargetProgress = 0; private isLinux = false; @@ -179,22 +180,19 @@ export class FrameRenderer { } async initialize(): Promise { - // Create canvas for rendering const canvas = document.createElement("canvas"); canvas.width = this.config.width; canvas.height = this.config.height; - // Try to set colorSpace if supported (may not be available on all platforms) + // colorSpace isn't available on all platforms try { if (canvas && "colorSpace" in canvas) { canvas.colorSpace = "srgb"; } } catch (error) { - // Silently ignore colorSpace errors on platforms that don't support it console.warn("[FrameRenderer] colorSpace not supported on this platform:", error); } - // Initialize PixiJS with optimized settings for export performance this.app = new Application(); await this.app.init({ canvas, @@ -206,16 +204,14 @@ export class FrameRenderer { autoDensity: true, }); - // Setup containers this.cameraContainer = new Container(); this.videoContainer = new Container(); this.app.stage.addChild(this.cameraContainer); this.cameraContainer.addChild(this.videoContainer); - // Setup background (render separately, not in PixiJS) + // Background renders separately, not in PixiJS await this.setupBackground(); - // Setup blur filter for video container this.blurFilter = new BlurFilter(); this.blurFilter.quality = 5; this.blurFilter.resolution = this.app.renderer.resolution; @@ -223,12 +219,12 @@ export class FrameRenderer { this.motionBlurFilter = new MotionBlurFilter([0, 0], 5, 0); this.videoContainer.filters = [this.blurFilter, this.motionBlurFilter]; - // Setup composite canvas for final output with shadows + // Composite canvas: final output with shadows this.compositeCanvas = document.createElement("canvas"); this.compositeCanvas.width = this.config.width; this.compositeCanvas.height = this.config.height; - // On Linux, getImageData() is called frequently causing frequent CPU readback + // On Linux getImageData() runs frequently, so hint frequent CPU readback this.compositeCtx = this.compositeCanvas.getContext("2d", { willReadFrequently: this.isLinux, }); @@ -245,9 +241,8 @@ export class FrameRenderer { throw new Error("Failed to get 2D context for raster canvas"); } - // Foreground canvas: holds recording + shadow + webcam + cursor + annotations, - // transparent background. The 3D rotation pass operates only on this layer so - // the wallpaper stays flat behind the rotated content (matching preview). + // Foreground (transparent): recording + shadow + webcam + cursor + annotations. + // The 3D pass operates only on this layer so the wallpaper stays flat behind it. this.foregroundCanvas = document.createElement("canvas"); this.foregroundCanvas.width = this.config.width; this.foregroundCanvas.height = this.config.height; @@ -258,7 +253,6 @@ export class FrameRenderer { throw new Error("Failed to get 2D context for foreground canvas"); } - // Setup shadow canvas if needed if (this.config.showShadow) { this.shadowCanvas = document.createElement("canvas"); this.shadowCanvas.width = this.config.width; @@ -272,7 +266,6 @@ export class FrameRenderer { } } - // Setup mask this.maskGraphics = new Graphics(); this.videoContainer.addChild(this.maskGraphics); this.videoContainer.mask = this.maskGraphics; @@ -389,20 +382,18 @@ export class FrameRenderer { this.currentVideoTime = timestamp / 1000000; - // Create or update video sprite from VideoFrame if (!this.videoSprite) { const texture = Texture.from(videoFrame as unknown as TextureSourceLike); this.videoSprite = new Sprite(texture); this.videoContainer.addChild(this.videoSprite); } else { - // Destroy old texture to avoid memory leaks, then create new one + // Destroy old texture before swapping to avoid a leak const oldTexture = this.videoSprite.texture; const newTexture = Texture.from(videoFrame as unknown as TextureSourceLike); this.videoSprite.texture = newTexture; oldTexture.destroy(true); } - // Apply layout this.updateLayout(webcamFrame); const timeMs = this.currentVideoTime * 1000; @@ -419,7 +410,11 @@ export class FrameRenderer { throw new Error("Layout cache not initialized"); } - // Apply transform once with maximum motion intensity from all ticks + // Feed the spring-smoothed transform (appliedScale/x/y) via transformOverride, like the + // preview. Without it applyZoomTransform recomputes the camera from the raw eased target and + // the spring is discarded, so the export snaps to the target every frame while the preview + // glides (very visible for auto-focus, whose target pans with the cursor). It also keeps the + // camera, mask, and cursor (which already read appliedScale/x/y) consistent. applyZoomTransform({ cameraContainer: this.cameraContainer, blurFilter: this.blurFilter, @@ -435,19 +430,24 @@ export class FrameRenderer { motionBlurAmount: this.config.motionBlurAmount ?? 0, motionBlurState: this.motionBlurState, frameTimeMs: timeMs, + transformOverride: { + scale: this.animationState.appliedScale, + x: this.animationState.x, + y: this.animationState.y, + }, }); - // Render the PixiJS stage to its canvas (video only, transparent background) + // Render the PixiJS stage (video only, transparent background) this.app.renderer.render(this.app.stage); - // Skip baking the shadow when the WebGL rotation pass will run — it'd alias to - // a hard edge through bilinear sampling. We re-apply shadow fresh after rotation. + // Skip baking the shadow when the rotation pass will run; bilinear sampling would + // alias it to a hard edge. Re-applied fresh after rotation. const willRotate = !isRotation3DIdentity(this.currentRotation3D); this.compositeWithShadows(webcamFrame, !willRotate); await this.drawNativeCursor(timeMs); - // Render annotations on top of foreground (so they rotate with recording). + // Annotations go on top of foreground so they rotate with the recording if ( this.config.annotationRegions && this.config.annotationRegions.length > 0 && @@ -469,14 +469,14 @@ export class FrameRenderer { ); } - // Apply 3D rotation to foreground only. Wallpaper (on compositeCanvas) is untouched. + // Rotate foreground only; wallpaper (on compositeCanvas) stays untouched if (willRotate && this.threeDPass && this.foregroundCanvas && this.foregroundCtx) { const passCanvas = this.threeDPass.apply(this.foregroundCanvas, this.currentRotation3D); const w = this.foregroundCanvas.width; const h = this.foregroundCanvas.height; this.foregroundCtx.clearRect(0, 0, w, h); if (this.isLinux) { - // drawImage(webglCanvas) is unreliable on Linux/Wayland — use readPixels. + // drawImage(webglCanvas) is unreliable on Linux/Wayland, so use readPixels const pixels = this.threeDPass.readPixels(); const imageData = this.foregroundCtx.createImageData(w, h); imageData.data.set(pixels); @@ -486,9 +486,9 @@ export class FrameRenderer { } } - // Apply shadow fresh on the rotated silhouette (flat path already baked it - // in compositeWithShadows, so guard on willRotate to avoid doubling). - // Same 3-layer filter chain as `main` — keeps the soft Gaussian intact. + // Apply shadow fresh on the rotated silhouette. Flat path already baked it in + // compositeWithShadows, so guard on willRotate to avoid doubling. Same 3-layer + // filter chain as the flat path to keep the soft Gaussian intact. if ( willRotate && this.config.showShadow && @@ -517,24 +517,23 @@ export class FrameRenderer { this.compositeCtx.drawImage(this.shadowCanvas, 0, 0); } } else if (this.compositeCtx && this.foregroundCanvas) { - // Flat path or 3D-without-shadow: stamp foreground directly. + // Flat path or 3D-without-shadow: stamp foreground directly this.compositeCtx.drawImage(this.foregroundCanvas, 0, 0); } } - // The video's actual on-screen boundary, accounting for the zoom camera - // transform. The PIXI mask lives inside cameraContainer, so during zoom the - // visible video extends beyond the static maskRect — a static clip would crop - // it. Mirrors the preview, which clips via the same camera-scaled bounds. + // Video's on-screen boundary including the zoom camera transform. The PIXI mask + // lives inside cameraContainer, so during zoom the visible video extends beyond + // the static maskRect and a static clip would crop it. Mirrors the preview. private cameraAwareMaskRect() { if (!this.layoutCache) return null; const { x: maskX, y: maskY, width: maskW, height: maskH } = this.layoutCache.maskRect; const camS = this.animationState.appliedScale; const camX = this.animationState.x; const camY = this.animationState.y; - // No stage clamping: canvas naturally clips to its bounds, matching CSS inset() behavior. - // Clamping x/y would shift rounded corners to the stage edge rather than the true mask - // boundary, causing preview/export mismatch when zoom/pan pushes the mask off-stage. + // No stage clamping: the canvas clips to its own bounds, matching CSS inset(). + // Clamping x/y would pin rounded corners to the stage edge instead of the true + // mask boundary, mismatching preview/export when zoom/pan pushes the mask off-stage. return { x: camX + camS * maskX, y: camY + camS * maskY, @@ -550,7 +549,6 @@ export class FrameRenderer { } if ((this.config.cursorScale ?? 1) <= 0) { - resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); return; } @@ -560,16 +558,18 @@ export class FrameRenderer { timeMs, ); if (!activeNativeCursor) { - resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); return; } - const displaySample = smoothNativeCursorSample({ - sample: activeNativeCursor.sample, - smoothing: this.config.cursorSmoothing ?? 0, - state: this.nativeCursorSmoothingState, - timeMs, - }); + // Position comes from the precomputed smoothed path (deterministic, matches preview); + // the frame still supplies the cursor image, type, and click timing. + const smoothedPos = getSmoothedCursorPath( + this.config.cursorRecordingData, + this.config.cursorSmoothing ?? 0, + )?.sampleAt(timeMs); + const displaySample = smoothedPos + ? { ...activeNativeCursor.sample, cx: smoothedPos.cx, cy: smoothedPos.cy } + : activeNativeCursor.sample; const projectedPoint = projectNativeCursorToLocal({ cropRegion: this.config.cropRegion, @@ -577,12 +577,16 @@ export class FrameRenderer { sample: displaySample, }); if (!projectedPoint) { - resetNativeCursorSmoothingState(this.nativeCursorSmoothingState); resetNativeCursorMotionBlurState(this.nativeCursorMotionBlurState); return; } - const renderAsset = resolveNativeCursorRenderAsset(activeNativeCursor.asset, 1, displaySample); + const renderAsset = resolveNativeCursorRenderAsset( + activeNativeCursor.asset, + 1, + displaySample, + this.config.cursorTheme, + ); let image: HTMLImageElement; try { image = await this.getCursorImage(renderAsset); @@ -597,8 +601,8 @@ export class FrameRenderer { getNativeCursorClickBounceProgress(this.config.cursorRecordingData, timeMs), ); const appliedScale = this.animationState.appliedScale; - // Normalize cursor size so it appears at the same fraction of the video width - // as in the preview — both paths now use maskRect.width / croppedVideoWidth. + // Normalize cursor size to the same fraction of video width as the preview; + // both paths use maskRect.width / croppedVideoWidth. const sizeNorm = this.layoutCache.videoSize.width > 0 ? this.layoutCache.maskRect.width / this.layoutCache.videoSize.width @@ -611,7 +615,7 @@ export class FrameRenderer { state: this.nativeCursorMotionBlurState, timeMs, }); - // Clip only when explicitly enabled; by default the cursor may overflow the canvas. + // Clip only when explicitly enabled; by default the cursor may overflow the canvas const cursorClip = this.config.cursorClipToBounds === true ? this.cameraAwareMaskRect() : null; this.foregroundCtx.save(); this.foregroundCtx.beginPath(); @@ -673,7 +677,6 @@ export class FrameRenderer { const videoWidth = this.config.videoWidth; const videoHeight = this.config.videoHeight; - // Calculate cropped video dimensions const cropStartX = cropRegion.x; const cropStartY = cropRegion.y; const cropEndX = cropRegion.x + cropRegion.width; @@ -682,9 +685,8 @@ export class FrameRenderer { const croppedVideoWidth = videoWidth * (cropEndX - cropStartX); const croppedVideoHeight = videoHeight * (cropEndY - cropStartY); - // Calculate scale to fit in viewport - // Padding is a percentage (0-100), where 50% ~ 0.8 scale - // Vertical stack ignores padding — it's full-bleed + // Padding is a percentage (0-100), where 50% ~ 0.8 scale. + // Vertical stack is full-bleed, so it ignores padding. const effectivePadding = this.config.webcamLayoutPreset === "vertical-stack" ? 0 : padding; const paddingScale = 1.0 - (effectivePadding / 100) * 0.4; const viewportWidth = width * paddingScale; @@ -703,7 +705,7 @@ export class FrameRenderer { const screenRect = compositeLayout.screenRect; - // Cover mode: scale to fill the rect (may crop), otherwise fit-to-width + // Cover mode scales to fill the rect (may crop), otherwise fit-to-width let scale: number; if (compositeLayout.screenCover) { scale = Math.max( @@ -714,7 +716,6 @@ export class FrameRenderer { scale = screenRect.width / croppedVideoWidth; } - // Position video sprite this.videoSprite.width = videoWidth * scale; this.videoSprite.height = videoHeight * scale; @@ -729,11 +730,10 @@ export class FrameRenderer { this.videoSprite.x = -cropPixelX + coverOffsetX; this.videoSprite.y = -cropPixelY + coverOffsetY; - // Position video container this.videoContainer.x = screenRect.x; this.videoContainer.y = screenRect.y; - // scale border radius by export/preview canvas ratio + // Scale border radius by the export/preview canvas ratio const previewWidth = this.config.previewWidth ?? this.config.width; const previewHeight = this.config.previewHeight ?? this.config.height; const canvasScaleFactor = Math.min(width / previewWidth, height / previewHeight); @@ -748,10 +748,9 @@ export class FrameRenderer { this.maskGraphics.roundRect(0, 0, screenRect.width, screenRect.height, scaledBorderRadius); this.maskGraphics.fill({ color: 0xffffff }); - // Cache layout info. baseOffset is the stage position of the FULL - // (uncropped) video sprite's top-left — matches preview semantics so - // downstream consumers (e.g. cursor highlight) can map normalized - // recording-space coordinates to stage coordinates uniformly: + // baseOffset is the stage position of the full (uncropped) sprite's top-left, matching + // preview semantics, so consumers (e.g. cursor highlight) can map normalized + // recording-space coords to stage coords uniformly: // stagePos = baseOffset + (cx, cy) * (videoWidth, videoHeight) * baseScale this.layoutCache = { stageSize: { width, height }, @@ -794,42 +793,25 @@ export class FrameRenderer { targetFocus = regionFocus; targetProgress = strength; - // Apply adaptive smoothing for auto-follow mode + // Adaptive smoothing for auto-follow mode if (region.focusMode === "auto" && !transition) { const raw = targetFocus; const dtMs = this.prevAnimationTimeMs != null ? timeMs - this.prevAnimationTimeMs : 0; - const framesElapsed = dtMs > 0 ? dtMs / (1000 / 60) : 1; const isZoomingIn = targetProgress < 0.999 && targetProgress >= this.prevTargetProgress; if (targetProgress >= 0.999) { - // Full zoom: adaptive smoothing — moves faster when far, decelerates when close + // Full zoom: move faster when far, decelerate when close const prev = this.smoothedAutoFocus ?? raw; - const baseFactor = adaptiveSmoothFactor( - raw, - prev, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, - AUTO_FOLLOW_RAMP_DISTANCE, - ); - const factor = 1 - Math.pow(1 - baseFactor, Math.max(1, framesElapsed)); - const smoothed = smoothCursorFocus(raw, prev, factor); + const smoothed = advanceFollowFocus(prev, raw, dtMs, AUTO_FOLLOW_PARAMS); this.smoothedAutoFocus = smoothed; targetFocus = smoothed; } else if (isZoomingIn) { - // Zoom-in: track cursor directly so zoom always aims at current cursor - // position; keep ref in sync to avoid snap when full-zoom begins + // Track cursor directly while zooming in; keep ref in sync to avoid a snap + // when full-zoom begins this.smoothedAutoFocus = raw; } else { - // Zoom-out: keep smoothing for continuity — avoids snap at zoom-out start + // Zoom-out: keep smoothing to avoid a snap at the start const prev = this.smoothedAutoFocus ?? raw; - const baseFactor = adaptiveSmoothFactor( - raw, - prev, - AUTO_FOLLOW_SMOOTHING_FACTOR, - AUTO_FOLLOW_SMOOTHING_FACTOR_MAX, - AUTO_FOLLOW_RAMP_DISTANCE, - ); - const factor = 1 - Math.pow(1 - baseFactor, Math.max(1, framesElapsed)); - const smoothed = smoothCursorFocus(raw, prev, factor); + const smoothed = advanceFollowFocus(prev, raw, dtMs, AUTO_FOLLOW_PARAMS); this.smoothedAutoFocus = smoothed; targetFocus = smoothed; } @@ -896,18 +878,24 @@ export class FrameRenderer { focusY: state.focusY, }); - const appliedScale = - Math.abs(projectedTransform.scale - prevScale) < ZOOM_SCALE_DEADZONE - ? projectedTransform.scale - : projectedTransform.scale; - const appliedX = - Math.abs(projectedTransform.x - prevX) < ZOOM_TRANSLATION_DEADZONE_PX - ? projectedTransform.x - : projectedTransform.x; - const appliedY = - Math.abs(projectedTransform.y - prevY) < ZOOM_TRANSLATION_DEADZONE_PX - ? projectedTransform.y - : projectedTransform.y; + // Spring-chase the eased target (same as preview) so exported motion glides past the jerk + // at the steep start of the ease. Stepped by content time; snapped on the first frame or + // any large time jump. + const dtMs = this.prevAnimationTimeMs != null ? timeMs - this.prevAnimationTimeMs : 0; + let appliedScale: number; + let appliedX: number; + let appliedY: number; + if (this.prevAnimationTimeMs == null || dtMs <= 0 || dtMs > 80) { + resetZoomSpring(this.zoomSpringState, projectedTransform); + appliedScale = projectedTransform.scale; + appliedX = projectedTransform.x; + appliedY = projectedTransform.y; + } else { + const sprung = stepZoomSpring(this.zoomSpringState, projectedTransform, dtMs); + appliedScale = sprung.scale; + appliedX = sprung.x; + appliedY = sprung.y; + } state.x = appliedX; state.y = appliedY; @@ -922,10 +910,9 @@ export class FrameRenderer { ); } - // On Linux/Wayland the implicit GPU→2D texture-sharing path - // used by drawImage(webglCanvas) can fail silently (EGL/Ozone), - // producing green/empty frames. Explicit gl.readPixels always - // copies from GPU to CPU memory, bypassing that path. + // On Linux/Wayland the implicit GPU-to-2D texture-sharing path behind + // drawImage(webglCanvas) can fail silently (EGL/Ozone), giving green/empty + // frames. gl.readPixels copies GPU to CPU directly, bypassing that path. private readbackVideoCanvas(): HTMLCanvasElement { const glCanvas = this.app!.canvas as HTMLCanvasElement; const gl = @@ -958,8 +945,8 @@ export class FrameRenderer { return this.rasterCanvas; } - // `applyShadowToRecording` is false when the 3D pass will rotate this canvas - // next — the shadow gets re-applied after rotation to avoid aliasing. + // applyShadowToRecording is false when the 3D pass will rotate this canvas next; + // the shadow is re-applied after rotation to avoid aliasing. private compositeWithShadows( webcamFrame: VideoFrame | null | undefined, applyShadowToRecording: boolean, @@ -982,8 +969,8 @@ export class FrameRenderer { const w = this.compositeCanvas.width; const h = this.compositeCanvas.height; - // Background layer (compositeCanvas): wallpaper only. Stays flat — never - // touched by the 3D rotation pass, matching preview behavior. + // Background (compositeCanvas): wallpaper only. Stays flat, never touched by the + // 3D rotation pass, matching the preview. bgCtx.clearRect(0, 0, w, h); if (this.backgroundSprite) { const bgCanvas = this.backgroundSprite; @@ -999,8 +986,8 @@ export class FrameRenderer { console.warn("[FrameRenderer] No background sprite found during compositing!"); } - // Foreground (transparent): recording + webcam. Shadow only baked here on - // the flat path; the 3D path applies it after rotation (see renderFrame). + // Foreground (transparent): recording + webcam. Shadow baked here only on the + // flat path; the 3D path applies it after rotation (see renderFrame). fgCtx.clearRect(0, 0, w, h); if ( applyShadowToRecording && @@ -1026,54 +1013,34 @@ export class FrameRenderer { shadowCtx.drawImage(videoCanvas, 0, 0, w, h); shadowCtx.restore(); fgCtx.drawImage(this.shadowCanvas, 0, 0, w, h); - // Erase square corners left by PIXI WebGL alpha, then redraw video with explicit - // 2D clip so shadow extends beyond the rounded area but video is precisely clipped. - // The clip is camera-aware so zoom doesn't crop the magnified video. - const shadowClip = - (this.layoutCache?.maskBorderRadius ?? 0) > 0 ? this.cameraAwareMaskRect() : null; - if (shadowClip) { - const { x: smx, y: smy, width: smw, height: smh, br: sbr } = shadowClip; - fgCtx.save(); - fgCtx.globalCompositeOperation = "destination-out"; - fgCtx.beginPath(); - fgCtx.rect(smx, smy, smw, smh); - fgCtx.roundRect(smx, smy, smw, smh, sbr); - fgCtx.fill("evenodd"); - fgCtx.restore(); - fgCtx.save(); - fgCtx.beginPath(); - fgCtx.roundRect(smx, smy, smw, smh, sbr); - fgCtx.clip(); - fgCtx.drawImage(videoCanvas, 0, 0, w, h); - fgCtx.restore(); - } } else { - // Direct path: explicit 2D clip guarantees rounded corners regardless of PIXI - // WebGL alpha. Camera-aware so zoom doesn't crop the magnified video. - const directClip = - (this.layoutCache?.maskBorderRadius ?? 0) > 0 ? this.cameraAwareMaskRect() : null; - if (directClip) { - fgCtx.save(); - fgCtx.beginPath(); - fgCtx.roundRect( - directClip.x, - directClip.y, - directClip.width, - directClip.height, - directClip.br, - ); - fgCtx.clip(); - fgCtx.drawImage(videoCanvas, 0, 0, w, h); - fgCtx.restore(); - } else { - fgCtx.drawImage(videoCanvas, 0, 0, w, h); - } + fgCtx.drawImage(videoCanvas, 0, 0, w, h); } const webcamRect = this.layoutCache?.webcamRect ?? null; if (webcamFrame && webcamRect) { const preset = getWebcamLayoutPresetDefinition(this.config.webcamLayoutPreset); const shape = webcamRect.maskShape ?? this.config.webcamMaskShape ?? "rectangle"; + // Scale the PiP webcam inversely with the eased zoom, anchoring the shrink to the + // docked corner (bottom-right by default) like the preview, so it stays flush to the + // edges instead of drifting toward center. + const reactiveFactor = + this.config.webcamReactiveZoom && this.config.webcamLayoutPreset === "picture-in-picture" + ? reactiveWebcamScale(this.animationState.appliedScale) + : 1; + const camPos = this.config.webcamPosition; + const biasX = (camPos ? camPos.cx >= 0.5 : true) ? 1 : 0; + const biasY = (camPos ? camPos.cy >= 0.5 : true) ? 1 : 0; + const drawRect = + reactiveFactor < 1 + ? { + width: webcamRect.width * reactiveFactor, + height: webcamRect.height * reactiveFactor, + x: webcamRect.x + webcamRect.width * (1 - reactiveFactor) * biasX, + y: webcamRect.y + webcamRect.height * (1 - reactiveFactor) * biasY, + borderRadius: webcamRect.borderRadius * reactiveFactor, + } + : webcamRect; const sourceWidth = ("displayWidth" in webcamFrame && webcamFrame.displayWidth > 0 ? webcamFrame.displayWidth @@ -1093,12 +1060,12 @@ export class FrameRenderer { fgCtx.save(); drawCanvasClipPath( fgCtx, - webcamRect.x, - webcamRect.y, - webcamRect.width, - webcamRect.height, + drawRect.x, + drawRect.y, + drawRect.width, + drawRect.height, shape, - webcamRect.borderRadius, + drawRect.borderRadius, ); if (preset.shadow) { fgCtx.shadowColor = preset.shadow.color; @@ -1109,16 +1076,22 @@ export class FrameRenderer { fgCtx.fillStyle = "#000000"; fgCtx.fill(); fgCtx.clip(); - fgCtx.drawImage( + drawWebcamFrameImage( + fgCtx, webcamFrame as unknown as CanvasImageSource, - sourceCropX, - sourceCropY, - sourceCropWidth, - sourceCropHeight, - webcamRect.x, - webcamRect.y, - webcamRect.width, - webcamRect.height, + { + x: sourceCropX, + y: sourceCropY, + width: sourceCropWidth, + height: sourceCropHeight, + }, + { + x: drawRect.x, + y: drawRect.y, + width: drawRect.width, + height: drawRect.height, + }, + this.config.webcamMirrored, ); fgCtx.restore(); } diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 7c0d2a667..9b06fcd30 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -46,6 +46,8 @@ interface GifExporterConfig { cropRegion: CropRegion; webcamLayoutPreset?: WebcamLayoutPreset; webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; + webcamMirrored?: boolean; + webcamReactiveZoom?: boolean; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; cursorRecordingData?: CursorRecordingData | null; @@ -54,6 +56,7 @@ interface GifExporterConfig { cursorMotionBlur?: number; cursorClickBounce?: number; cursorClipToBounds?: boolean; + cursorTheme?: string; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -135,7 +138,6 @@ export class GifExporter { this.cleanup(); this.cancelled = false; - // Initialize streaming decoder and load video metadata this.streamingDecoder = new StreamingVideoDecoder(); const videoInfo = await this.streamingDecoder.loadMetadata(this.config.videoUrl); let webcamInfo: Awaited> | null = null; @@ -144,7 +146,6 @@ export class GifExporter { webcamInfo = await this.webcamDecoder.loadMetadata(this.config.webcamVideoUrl); } - // Initialize frame renderer this.renderer = new FrameRenderer({ width: this.config.width, height: this.config.height, @@ -163,11 +164,14 @@ export class GifExporter { cursorMotionBlur: this.config.cursorMotionBlur, cursorClickBounce: this.config.cursorClickBounce, cursorClipToBounds: this.config.cursorClipToBounds, + cursorTheme: this.config.cursorTheme, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, webcamLayoutPreset: this.config.webcamLayoutPreset, webcamMaskShape: this.config.webcamMaskShape, + webcamMirrored: this.config.webcamMirrored, + webcamReactiveZoom: this.config.webcamReactiveZoom, webcamSizePreset: this.config.webcamSizePreset, webcamPosition: this.config.webcamPosition, annotationRegions: this.config.annotationRegions, @@ -180,8 +184,7 @@ export class GifExporter { }); await this.renderer.initialize(); - // Initialize GIF encoder - // Loop: 0 = infinite loop, 1 = play once (no loop) + // gif.js repeat: 0 = infinite loop, 1 = play once const repeat = this.config.loop ? 0 : 1; const cores = navigator.hardwareConcurrency || 4; const WORKER_COUNT = Math.max(1, Math.min(8, cores - 1)); @@ -197,14 +200,14 @@ export class GifExporter { dither: "FloydSteinberg", }); - // Calculate effective duration and frame count (excluding trim regions) + // Effective duration and frame count, excluding trim regions const { effectiveDuration, totalFrames } = this.streamingDecoder.getExportMetrics( this.config.frameRate, this.config.trimRegions, this.config.speedRegions, ); - // Calculate frame delay in milliseconds (gif.js uses ms) + // gif.js wants frame delay in ms const frameDelay = Math.round(1000 / this.config.frameRate); console.log("[GifExporter] Original duration:", videoInfo.duration, "s"); @@ -254,7 +257,7 @@ export class GifExporter { })() : null; - // Stream decode and process frames — no seeking! + // Stream decode and process frames, no seeking await this.streamingDecoder.decodeAll( this.config.frameRate, this.config.trimRegions, @@ -274,19 +277,15 @@ export class GifExporter { return; } - // Render the frame with all effects using source timestamp - const sourceTimestampUs = sourceTimestampMs * 1000; // Convert to microseconds + const sourceTimestampUs = sourceTimestampMs * 1000; // us await renderer.renderFrame(videoFrame, sourceTimestampUs, webcamFrame); - // Get the rendered canvas and add to GIF const canvas = renderer.getCanvas(); - // Add frame to GIF encoder with delay this.gif!.addFrame(canvas, { delay: frameDelay, copy: true }); frameIndex++; - // Update progress if (this.config.onProgress) { this.config.onProgress({ currentFrame: frameIndex, @@ -312,7 +311,7 @@ export class GifExporter { this.webcamDecoder?.cancel(); await webcamDecodePromise; - // Update progress to show we're now in the finalizing phase + // Now in the finalizing phase if (this.config.onProgress) { this.config.onProgress({ currentFrame: totalFrames, @@ -323,13 +322,11 @@ export class GifExporter { }); } - // Render the GIF const blob = await new Promise((resolve, _reject) => { this.gif!.on("finished", (blob: Blob) => { resolve(blob); }); - // Track rendering progress this.gif!.on("progress", (progress: number) => { if (this.config.onProgress) { this.config.onProgress({ @@ -343,7 +340,7 @@ export class GifExporter { } }); - // gif.js doesn't have a typed 'error' event, but we can catch errors in the try/catch + // gif.js has no typed 'error' event; the outer try/catch handles failures this.gif!.render(); }); diff --git a/src/lib/exporter/muxer.ts b/src/lib/exporter/muxer.ts index 95d41ad2e..d2a2a93b9 100644 --- a/src/lib/exporter/muxer.ts +++ b/src/lib/exporter/muxer.ts @@ -26,7 +26,6 @@ export class VideoMuxer { } async initialize(): Promise { - // Create the buffer target this.target = new BufferTarget(); this.output = new Output({ @@ -36,19 +35,17 @@ export class VideoMuxer { target: this.target, }); - // Create video source - codec will be deduced from metadata + // Codec is deduced from the chunk metadata. this.videoSource = new EncodedVideoPacketSource("avc"); this.output.addVideoTrack(this.videoSource, { frameRate: this.config.frameRate, }); - // Create audio source if needed if (this.hasAudio) { this.audioSource = new EncodedAudioPacketSource(this.audioCodec); this.output.addAudioTrack(this.audioSource); } - // Start the output to begin accepting media data await this.output.start(); } @@ -57,10 +54,8 @@ export class VideoMuxer { throw new Error("Muxer not initialized"); } - // Convert WebCodecs chunk to Mediabunny packet const packet = EncodedPacket.fromEncodedChunk(chunk); - // Add metadata with the first chunk await this.videoSource.add(packet, meta); } @@ -69,10 +64,8 @@ export class VideoMuxer { throw new Error("Audio not configured for this muxer"); } - // Convert WebCodecs chunk to Mediabunny packet const packet = EncodedPacket.fromEncodedChunk(chunk); - // Add metadata with the first chunk await this.audioSource.add(packet, meta); } diff --git a/src/lib/exporter/streamingDecoder.ts b/src/lib/exporter/streamingDecoder.ts index 752d5cd4c..472da8195 100644 --- a/src/lib/exporter/streamingDecoder.ts +++ b/src/lib/exporter/streamingDecoder.ts @@ -5,9 +5,8 @@ const SOURCE_LOAD_TIMEOUT_MS = 60_000; const EPSILON_SEC = 0.001; /** * Build a full WebCodecs-compatible AV1 codec string from the AV1CodecConfigurationRecord. - * web-demuxer may return a bare "av01" when the WASM-side parser fails to read - * the extradata (e.g. raw OBU sequence header from WebM instead of ISOBMFF av1C box). - * This function parses the record if present, otherwise returns a safe default. + * web-demuxer can return a bare "av01" when the WASM parser fails to read the extradata. + * Parses the record if present, otherwise returns a safe default. * * @see https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-section */ @@ -25,9 +24,8 @@ function buildAV1CodecString(description?: BufferSource): string { // Byte 0: marker (1) | version (7) // Byte 1: seq_profile (3) | seq_level_idx_0 (5) // Byte 2: seq_tier_0 (1) | high_bitdepth (1) | twelve_bit (1) | ... - // The spec says version should be 1, but Chrome/Electron's MediaRecorder - // may write version 127 (0xFF first byte). We accept any version as long - // as the marker bit is set and the record is long enough. + // Spec says version 1, but Chrome/Electron MediaRecorder may write 127 (0xFF), + // so accept any version as long as the marker bit is set and the record is long enough. if (bytes.length < 4) return fallback; if (!(bytes[0] & 0x80)) return fallback; // marker bit must be 1 @@ -71,26 +69,22 @@ const EARLY_DECODE_END_THRESHOLD_SEC = 1; const METADATA_TAIL_TOLERANCE_SEC = 2; const STREAM_DURATION_MATCH_TOLERANCE_SEC = 0.25; const DURATION_DIVERGENCE_THRESHOLD_SEC = 1.5; -// Fallback upper bound for the packet scan when no reliable duration hint is -// available. Explicit end is required (some containers are truncated without -// one), but the hint-derived bound would cap the scan prematurely when -// container/stream duration are missing or corrupt. +// Fallback upper bound for the packet scan when no reliable duration hint exists. +// An explicit end is required (some containers are truncated without one), but a +// hint-derived bound would cap the scan early when duration is missing or corrupt. const SCAN_UNBOUNDED_FALLBACK_SEC = 24 * 60 * 60; /** - * Validate container duration against actual packet timestamps. - * - * Chrome/Electron's MediaRecorder writes WebM containers with unreliable - * Duration fields (often Infinity, 0, or inflated) — especially on Linux. - * This function picks the most trustworthy duration value. + * Pick the most trustworthy duration. Chrome/Electron MediaRecorder writes WebM + * with unreliable Duration fields (often Infinity, 0, or inflated), especially on Linux. * * @param containerDuration Duration from the container-level metadata * @param scannedDuration Duration derived from actual packet timestamps (ground truth) */ export function validateDuration(containerDuration: number, scannedDuration: number): number { if (scannedDuration <= 0) { - // Zero scanned duration means corrupted/empty file — fall back to container - // (downstream shouldFailDecodeEndedEarly will catch truly empty files) + // Corrupted/empty file, fall back to container. + // (downstream shouldFailDecodeEndedEarly catches truly empty files) return Number.isFinite(containerDuration) ? Math.max(containerDuration, 0) : 0; } if (!Number.isFinite(containerDuration) || containerDuration <= 0) { @@ -138,11 +132,8 @@ export function shouldFailDecodeEndedEarly({ } /** - * Loads a video file as an ArrayBuffer, delegating to - * `StreamingVideoDecoder.loadLocalSourceFile` for local paths (Electron IPC) - * and `StreamingVideoDecoder.loadRemoteSourceFile` for remote / blob / data URLs. - * Also returns the `contentType` derived from the blob (empty string for local - * IPC reads where no Content-Type is available). + * Loads a video file as an ArrayBuffer via the local (Electron IPC) or remote loader. + * contentType is empty for local IPC reads, which carry no Content-Type. */ export async function loadFileAsArrayBuffer( videoUrl: string, @@ -229,7 +220,7 @@ export class StreamingVideoDecoder { async loadMetadata(videoUrl: string): Promise { const { file } = await this.loadSourceFile(videoUrl); - // Relative URL so it resolves correctly in both dev (http) and packaged (file://) builds + // Relative URL so it resolves in both dev (http) and packaged (file://) builds const wasmUrl = new URL("./wasm/web-demuxer.wasm", window.location.href).href; this.demuxer = new WebDemuxer({ wasmFilePath: wasmUrl }); await this.withTimeout( @@ -257,12 +248,11 @@ export class StreamingVideoDecoder { const audioStream = mediaInfo.streams.find((s) => s.codec_type_string === "audio"); - // Scan video packets to find the true content boundary. - // MediaRecorder (especially on Linux) writes unreliable container durations. - // Packet timestamps are ground truth — no decode needed, just timestamp reads. - // Pass explicit range because some containers are truncated without one. - // Sanitize because mediaInfo.duration can be NaN/Infinity (Chromium Linux bug), - // which would propagate into demuxer.read() as an invalid endpoint. + // Scan video packets for the true content boundary; MediaRecorder (especially on + // Linux) writes unreliable container durations and packet timestamps are ground truth. + // Pass an explicit range because some containers are truncated without one. + // Sanitize because mediaInfo.duration can be NaN/Infinity (Chromium Linux bug), which + // would reach demuxer.read() as an invalid endpoint. const containerDurationSec = Number.isFinite(mediaInfo.duration) ? mediaInfo.duration : 0; const streamDurationSec = typeof videoStream?.duration === "number" && Number.isFinite(videoStream.duration) @@ -307,12 +297,10 @@ export class StreamingVideoDecoder { return this.metadata; } /** - * Decodes all video frames from the loaded source and invokes a callback for each. - * Handles trimming and speed adjustments, and resamples to the target frame rate. - * On Windows, early decode termination is tolerated to work around driver quirks. + * Decodes all video frames, applying trim/speed and resampling to the target frame rate. * @param targetFrameRate - Desired output frame rate. - * @param trimRegions - Array of time regions to keep (others discarded). - * @param speedRegions - Array of speed adjustments for specific time ranges. + * @param trimRegions - Time regions to keep (others discarded). + * @param speedRegions - Speed adjustments for specific time ranges. * @param onFrame - Async callback receiving each decoded VideoFrame. */ async decodeAll( @@ -331,9 +319,8 @@ export class StreamingVideoDecoder { console.log("[StreamingVideoDecoder] decoderConfig.codec:", decoderConfig.codec); console.log("[StreamingVideoDecoder] decoderConfig.description:", decoderConfig.description); - // web-demuxer may return bare four-character code strings ("av01", "vp08", - // "vp09", "avc1") that WebCodecs rejects. Normalize them to the short or - // full parametrized forms that VideoDecoder accepts. + // web-demuxer can return bare fourcc strings ("av01", "vp08", "vp09", "avc1") + // that WebCodecs rejects; normalize to forms VideoDecoder accepts. if (/^av01$/i.test(decoderConfig.codec)) { decoderConfig.codec = buildAV1CodecString( decoderConfig.description as BufferSource | undefined, @@ -373,7 +360,7 @@ export class StreamingVideoDecoder { ); const frameDurationUs = 1_000_000 / targetFrameRate; - // Async frame queue — decoder pushes, consumer pulls + // Async frame queue: decoder pushes, consumer pulls. const pendingFrames: VideoFrame[] = []; let frameResolve: ((frame: VideoFrame | null) => void) | null = null; let decodeError: Error | null = null; @@ -443,12 +430,12 @@ export class StreamingVideoDecoder { }); }; - // One forward stream through the whole file. - // Pass explicit range because some containers are truncated when no end is provided. + // One forward stream through the whole file. Pass an explicit range because + // some containers are truncated when no end is provided. const readEndSec = this.metadata.duration + 0.5; const reader = this.demuxer.read("video", 0, readEndSec).getReader(); - // Feed chunks to decoder in background with backpressure + // Feed chunks to the decoder in the background with backpressure. const feedPromise = (async () => { try { while (!this.cancelled) { @@ -482,7 +469,7 @@ export class StreamingVideoDecoder { } })(); - // Route decoded frames into segments by timestamp, then deliver with VFR→CFR resampling + // Route decoded frames into segments by timestamp, then deliver with VFR to CFR resampling. let segmentIdx = 0; let segmentFrameIndex = 0; let exportFrameIndex = 0; @@ -681,9 +668,8 @@ export class StreamingVideoDecoder { } /** - * Calculates the effective output duration (in seconds) and total frame count - * for a given combination of trim and speed regions at the target frame rate. - * Requires `loadMetadata()` to have been called first. + * Effective output duration (seconds) and total frame count for the given trim/speed + * regions at the target frame rate. Requires loadMetadata() first. */ getExportMetrics( targetFrameRate: number, diff --git a/src/lib/exporter/threeDPass.ts b/src/lib/exporter/threeDPass.ts index 5733bd05f..657c5ff2a 100644 --- a/src/lib/exporter/threeDPass.ts +++ b/src/lib/exporter/threeDPass.ts @@ -5,10 +5,9 @@ import { rotation3DPerspective, } from "@/components/video-editor/types"; -// CSS uses +y down, WebGL clip space uses +y up. We do all rotation math in CSS -// convention (top-left origin, +y down) to match the preview, then flip -// gl_Position.y at the end so WebGL's clip space lands the input's top edge at -// the top of the output viewport. +// Rotation math is done in CSS convention (+y down) to match the preview, then +// gl_Position.y is flipped so WebGL clip space (+y up) lands the input's top edge +// at the top of the viewport. const VERTEX_SHADER = `#version 300 es in vec2 aPos; in vec2 aUV; @@ -151,10 +150,7 @@ function createProgram(gl: WebGL2RenderingContext): WebGLProgram { export interface ThreeDPass { apply(srcCanvas: HTMLCanvasElement | OffscreenCanvas, rot: Rotation3D): HTMLCanvasElement; - /** - * Reads back the most recent apply() result into a Uint8ClampedArray suitable - * for ImageData. Use this on platforms where drawImage(webglCanvas) is unreliable. - */ + /** Read the last apply() result as ImageData-ready pixels, for platforms where drawImage(webglCanvas) is unreliable. */ readPixels(): Uint8ClampedArray; resize(width: number, height: number): void; destroy(): void; @@ -180,10 +176,9 @@ export function createThreeDPass(width: number, height: number): ThreeDPass { const vao = gl.createVertexArray(); gl.bindVertexArray(vao); - // Quad: two triangles sharing UVs consistently per corner. - // pos.y ranges 0 (top of input) → 1 (bottom of input) following CSS convention. - // UV.y is inverted (1 - pos.y) so that with UNPACK_FLIP_Y_WEBGL the texture - // sample at the top of the input lands at the top of the rendered quad. + // Quad as two triangles. pos.y is 0 (top) to 1 (bottom) per CSS convention; UV.y + // is inverted so that with UNPACK_FLIP_Y_WEBGL the top of the input lands at the + // top of the rendered quad. // TL: pos(0,0) uv(0,1) TR: pos(1,0) uv(1,1) // BL: pos(0,1) uv(0,0) BR: pos(1,1) uv(1,0) const verts = new Float32Array([ @@ -207,7 +202,7 @@ export function createThreeDPass(width: number, height: number): ThreeDPass { 1, 0, 1, - 1, // TR (was 1,0,1,0 — broken) + 1, // TR (was 1,0,1,0, broken) 1, 1, 1, @@ -224,20 +219,17 @@ export function createThreeDPass(width: number, height: number): ThreeDPass { const texture = gl.createTexture(); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); - // Plain bilinear, NO mipmaps. Mipmaps pre-blur the texture for downsampling, but - // at our moderate rotation angles (≤22°) the receding edge would still pick a - // smaller mipmap level, which softens fine details — specifically the few-pixel - // rounded-corner anti-alias ramp and the shadow's Gaussian falloff. The result - // is "rounding looks like a hard corner / shadow looks grimy". Sampling level 0 - // directly preserves the source crispness. + // Plain bilinear, no mipmaps. Even at our moderate angles (<=22deg) the receding + // edge picks a smaller mip level, softening the rounded-corner AA ramp and shadow + // falloff (corners look hard, shadows grimy). Sampling level 0 keeps source crispness. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - // Anisotropic filtering still helps without mipmaps: at oblique viewing angles - // it samples multiple texels along the gradient direction at level 0, recovering - // detail that plain bilinear would lose. Cap to the device max (16× typical). + // Anisotropic filtering still helps without mipmaps: at oblique angles it samples + // multiple texels along the gradient at level 0, recovering detail bilinear loses. + // Cap to the device max (16x typical). const anisoExt = gl.getExtension("EXT_texture_filter_anisotropic") || gl.getExtension("MOZ_EXT_texture_filter_anisotropic") || @@ -263,15 +255,10 @@ export function createThreeDPass(width: number, height: number): ThreeDPass { gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); - // CRITICAL: premultiply on upload. The source 2D canvas stores non-premultiplied - // RGBA (alpha=0 areas have RGB=0). Bilinear filtering between an inside-the-shape - // texel (alpha=1, RGB=color) and an outside texel (alpha=0, RGB=0) in - // non-premultiplied space yields (color/2, alpha=0.5), which the - // premultipliedAlpha:true canvas then interprets as half-strength color — visible - // as a dark halo around rounded corners and softened/grimy shadows. Premultiplying - // at upload time makes the bilinear math operate in linear-light premultiplied - // space, which is exactly the math used for compositing. Edges and shadows then - // reproduce the source crisply. + // Premultiply on upload. The source 2D canvas is non-premultiplied (alpha=0 areas + // have RGB=0), so bilinear filtering across a shape edge in that space gives + // half-strength color, showing as a dark halo on rounded corners and grimy shadows. + // Premultiplying makes the filter math match compositing, so edges stay crisp. gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); gl.texImage2D( gl.TEXTURE_2D, @@ -308,11 +295,9 @@ export function createThreeDPass(width: number, height: number): ThreeDPass { const h = currentSize.height; const buf = new Uint8Array(w * h * 4); gl.readPixels(0, 0, w, h, gl.RGBA, gl.UNSIGNED_BYTE, buf); - // gl.readPixels is bottom-up; flip to top-down for ImageData. We also need - // to un-premultiply the alpha here: the framebuffer holds premultiplied RGBA - // (we set UNPACK_PREMULTIPLY_ALPHA_WEBGL=true on upload), but ImageData / - // putImageData expect non-premultiplied. Without this divide, semi-transparent - // pixels get interpreted as darker than they should be. + // readPixels is bottom-up, so flip to top-down. Also un-premultiply: the + // framebuffer is premultiplied (UNPACK_PREMULTIPLY_ALPHA_WEBGL on upload) but + // ImageData expects non-premultiplied, else semi-transparent pixels read too dark. const rowSize = w * 4; const out = new Uint8ClampedArray(buf.length); for (let row = 0; row < h; row += 1) { diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index 387334177..138d97ecb 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -10,9 +10,9 @@ export interface ExportProgress { currentFrame: number; totalFrames: number; percentage: number; - estimatedTimeRemaining: number; // in seconds - phase?: "extracting" | "finalizing"; // Phase of export - renderProgress?: number; // 0-100, progress of GIF rendering phase + estimatedTimeRemaining: number; // seconds + phase?: "extracting" | "finalizing"; + renderProgress?: number; // 0-100, GIF render phase } export interface ExportResult { diff --git a/src/lib/exporter/videoDecoder.ts b/src/lib/exporter/videoDecoder.ts index 4ed1157ac..505e94f2e 100644 --- a/src/lib/exporter/videoDecoder.ts +++ b/src/lib/exporter/videoDecoder.ts @@ -36,9 +36,7 @@ export class VideoFileDecoder { }); } - /** - * Get video element for seeking - */ + /** The underlying video element, used for seeking. */ getVideoElement(): HTMLVideoElement | null { return this.videoElement; } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index 35c3d559d..a0b063789 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -37,6 +37,8 @@ export interface VideoExporterConfig extends ExportConfig { cropRegion: CropRegion; webcamLayoutPreset?: WebcamLayoutPreset; webcamMaskShape?: import("@/components/video-editor/types").WebcamMaskShape; + webcamMirrored?: boolean; + webcamReactiveZoom?: boolean; webcamSizePreset?: WebcamSizePreset; webcamPosition?: { cx: number; cy: number } | null; cursorRecordingData?: CursorRecordingData | null; @@ -45,6 +47,7 @@ export interface VideoExporterConfig extends ExportConfig { cursorMotionBlur?: number; cursorClickBounce?: number; cursorClipToBounds?: boolean; + cursorTheme?: string; annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; @@ -243,11 +246,14 @@ export class VideoExporter { cursorMotionBlur: this.config.cursorMotionBlur, cursorClickBounce: this.config.cursorClickBounce, cursorClipToBounds: this.config.cursorClipToBounds, + cursorTheme: this.config.cursorTheme, videoWidth: videoInfo.width, videoHeight: videoInfo.height, webcamSize: webcamInfo ? { width: webcamInfo.width, height: webcamInfo.height } : null, webcamLayoutPreset: this.config.webcamLayoutPreset, webcamMaskShape: this.config.webcamMaskShape, + webcamMirrored: this.config.webcamMirrored, + webcamReactiveZoom: this.config.webcamReactiveZoom, webcamSizePreset: this.config.webcamSizePreset, webcamPosition: this.config.webcamPosition, annotationRegions: this.config.annotationRegions, diff --git a/src/lib/exporter/webcamFrameDrawing.ts b/src/lib/exporter/webcamFrameDrawing.ts new file mode 100644 index 000000000..41f98d8f6 --- /dev/null +++ b/src/lib/exporter/webcamFrameDrawing.ts @@ -0,0 +1,43 @@ +interface WebcamFrameCrop { + x: number; + y: number; + width: number; + height: number; +} + +export type WebcamCanvasContext = Pick< + CanvasRenderingContext2D, + "drawImage" | "restore" | "save" | "scale" | "translate" +>; + +export function drawWebcamFrameImage( + ctx: WebcamCanvasContext, + image: CanvasImageSource, + crop: WebcamFrameCrop, + dest: WebcamFrameCrop, + mirrored = false, +) { + if (mirrored) { + ctx.save(); + try { + ctx.translate(dest.x + dest.width, dest.y); + ctx.scale(-1, 1); + ctx.drawImage(image, crop.x, crop.y, crop.width, crop.height, 0, 0, dest.width, dest.height); + } finally { + ctx.restore(); + } + return; + } + + ctx.drawImage( + image, + crop.x, + crop.y, + crop.width, + crop.height, + dest.x, + dest.y, + dest.width, + dest.height, + ); +} diff --git a/src/lib/frameStep.ts b/src/lib/frameStep.ts index dc42d78cb..29d8f49fa 100644 --- a/src/lib/frameStep.ts +++ b/src/lib/frameStep.ts @@ -1,10 +1,7 @@ -/** Duration of a single frame in seconds at 60 FPS (~16.67ms). */ +/** One frame in seconds at 60 FPS (~16.67ms). */ export const FRAME_DURATION_SEC = 1 / 60; -/** - * Compute the new playhead time after stepping one frame forward or backward. - * The result is clamped to the range [0, duration]. - */ +/** New playhead time after stepping one frame, clamped to [0, duration]. */ export function computeFrameStepTime( currentTime: number, duration: number, diff --git a/src/lib/recordingSession.ts b/src/lib/recordingSession.ts index 12a6afd22..f7d69cbf7 100644 --- a/src/lib/recordingSession.ts +++ b/src/lib/recordingSession.ts @@ -21,11 +21,10 @@ export interface StoreRecordedSessionInput { createdAt?: number; cursorCaptureMode?: CursorCaptureMode; /** - * Recording wall-clock duration in milliseconds. Used by the main process - * to patch the WebM Duration header on streamed recordings, since the - * renderer no longer holds the bytes. Browser MediaRecorder writes WebM - * with no/zero duration; without this patch, the editor's seek bar and - * timeline break for any recording that took the streaming path. + * Recording wall-clock duration (ms). The main process patches the WebM Duration + * header on streamed recordings (the renderer no longer holds the bytes). Browser + * MediaRecorder writes no/zero duration, which breaks the editor seek bar and + * timeline for anything that took the streaming path. */ durationMs?: number; } diff --git a/src/lib/userPreferences.test.ts b/src/lib/userPreferences.test.ts index 87ed259f2..8a64295c5 100644 --- a/src/lib/userPreferences.test.ts +++ b/src/lib/userPreferences.test.ts @@ -1,5 +1,11 @@ import { beforeEach, describe, expect, it } from "vitest"; -import { loadUserPreferences, parentDirectoryOf, saveUserPreferences } from "./userPreferences"; +import { + DEFAULT_PREFS, + getProjectFolder, + loadUserPreferences, + parentDirectoryOf, + saveUserPreferences, +} from "./userPreferences"; describe("parentDirectoryOf", () => { it("returns the directory for a POSIX path", () => { @@ -25,6 +31,62 @@ describe("parentDirectoryOf", () => { }); }); +describe("projectFolder preference", () => { + // jsdom's localStorage isn't exposed as a global in this vitest setup, so + // stub it with an in-memory shim before each test. Mirrors what the real + // browser localStorage exposes, scoped to the keys we touch. + beforeEach(() => { + const store = new Map(); + const stub = { + getItem: (key: string) => store.get(key) ?? null, + setItem: (key: string, value: string) => { + store.set(key, String(value)); + }, + removeItem: (key: string) => { + store.delete(key); + }, + clear: () => store.clear(), + key: (i: number) => Array.from(store.keys())[i] ?? null, + get length() { + return store.size; + }, + }; + Object.defineProperty(globalThis, "localStorage", { + value: stub, + configurable: true, + }); + }); + + it("defaults to null when nothing is persisted", () => { + expect(loadUserPreferences().projectFolder).toBeNull(); + expect(getProjectFolder()).toBeUndefined(); + }); + + it("round-trips a saved project folder", () => { + saveUserPreferences({ projectFolder: "/Users/me/Projects/demos" }); + expect(loadUserPreferences().projectFolder).toBe("/Users/me/Projects/demos"); + expect(getProjectFolder()).toBe("/Users/me/Projects/demos"); + }); + + it("ignores non-string persisted values and falls back to the default", () => { + localStorage.setItem("openscreen_user_preferences", JSON.stringify({ projectFolder: 42 })); + expect(loadUserPreferences().projectFolder).toBe(DEFAULT_PREFS.projectFolder); + }); + + it("ignores empty-string persisted values and falls back to the default", () => { + localStorage.setItem("openscreen_user_preferences", JSON.stringify({ projectFolder: "" })); + expect(loadUserPreferences().projectFolder).toBe(DEFAULT_PREFS.projectFolder); + }); + + it("is independent of exportFolder", () => { + saveUserPreferences({ exportFolder: "/Users/me/Downloads" }); + saveUserPreferences({ projectFolder: "/Users/me/Projects/demos" }); + const prefs = loadUserPreferences(); + expect(prefs.exportFolder).toBe("/Users/me/Downloads"); + expect(prefs.projectFolder).toBe("/Users/me/Projects/demos"); + }); +}); + describe("user preferences", () => { beforeEach(() => { localStorage.clear(); diff --git a/src/lib/userPreferences.ts b/src/lib/userPreferences.ts index 128eb73b8..66c5a66a4 100644 --- a/src/lib/userPreferences.ts +++ b/src/lib/userPreferences.ts @@ -29,6 +29,8 @@ export interface UserPreferences { exportFormat: ExportFormat; /** Folder used for the most recent successful export, if any */ exportFolder: string | null; + /** Folder of the most recently opened project, if any */ + projectFolder: string | null; /** Recording HUD control layout */ trayLayout: "horizontal" | "vertical"; } @@ -39,6 +41,7 @@ export const DEFAULT_PREFS: UserPreferences = { exportQuality: DEFAULT_EXPORT_SETTINGS.quality, exportFormat: DEFAULT_EXPORT_SETTINGS.format, exportFolder: null, + projectFolder: null, trayLayout: "horizontal", }; @@ -52,10 +55,7 @@ function safeJsonParse(text: string | null): Record | null { } } -/** - * Load persisted user preferences from localStorage. - * Returns defaults for any missing or invalid fields. - */ +/** Load preferences from localStorage, falling back to defaults for missing or invalid fields. */ export function loadUserPreferences(): UserPreferences { let raw: Record | null = null; try { @@ -91,6 +91,10 @@ export function loadUserPreferences(): UserPreferences { typeof raw.exportFolder === "string" && raw.exportFolder.length > 0 ? raw.exportFolder : DEFAULT_PREFS.exportFolder, + projectFolder: + typeof raw.projectFolder === "string" && raw.projectFolder.length > 0 + ? raw.projectFolder + : DEFAULT_PREFS.projectFolder, trayLayout: raw.trayLayout === "horizontal" || raw.trayLayout === "vertical" ? raw.trayLayout @@ -99,15 +103,10 @@ export function loadUserPreferences(): UserPreferences { } /** - * Extracts the parent directory from a saved file path. Handles both POSIX - * and Windows separators since the path comes from the OS save dialog. - * - * Root directories are preserved with their trailing separator so that the - * value is still a valid directory path: - * "/video.mp4" -> "/" - * "C:\\video.mp4" -> "C:\\" - * - * Returns null if no separator is found. + * Parent directory of a saved file path. Handles both POSIX and Windows + * separators since the path comes from the OS save dialog. Root dirs keep their + * trailing separator so the result stays a valid directory ("/video.mp4" -> "/", + * "C:\\video.mp4" -> "C:\\"). Returns null if no separator is found. */ export function parentDirectoryOf(filePath: string): string | null { const lastSep = Math.max(filePath.lastIndexOf("/"), filePath.lastIndexOf("\\")); @@ -124,24 +123,23 @@ export function parentDirectoryOf(filePath: string): string | null { return filePath.slice(0, lastSep); } -/** - * Returns the remembered export folder as `string | undefined`, suitable for - * passing directly to IPC handlers that treat absence as "use the default". - */ +/** Remembered export folder as `string | undefined`, for IPC handlers that treat absence as "use the default". */ export function getExportFolder(): string | undefined { return loadUserPreferences().exportFolder ?? undefined; } -/** - * Persist user preferences to localStorage. - * Only the explicitly provided fields are updated. - */ +/** Remembered open-project folder as `string | undefined`, for IPC handlers that treat absence as "use the default". */ +export function getProjectFolder(): string | undefined { + return loadUserPreferences().projectFolder ?? undefined; +} + +/** Persist preferences to localStorage; only the provided fields are updated. */ export function saveUserPreferences(partial: Partial): void { const current = loadUserPreferences(); const merged = { ...current, ...partial }; try { localStorage.setItem(PREFS_KEY, JSON.stringify(merged)); } catch { - // localStorage may be unavailable (e.g. private browsing quota exceeded) + // localStorage may be unavailable (e.g. private browsing, quota exceeded) } } diff --git a/src/lib/vite-stubs/empty-node-module.ts b/src/lib/vite-stubs/empty-node-module.ts new file mode 100644 index 000000000..00d8207f7 --- /dev/null +++ b/src/lib/vite-stubs/empty-node-module.ts @@ -0,0 +1,7 @@ +/** + * Empty default export, used as the Vite alias target for Node builtins that + * @xenova/transformers imports. Its env.js reads an empty object as "no filesystem" + * and stays on the browser/remote paths. + */ +const empty = Object.create(null) as Record; +export default empty; diff --git a/src/lib/vite-stubs/onnxruntime-node-stub.ts b/src/lib/vite-stubs/onnxruntime-node-stub.ts new file mode 100644 index 000000000..f13969c22 --- /dev/null +++ b/src/lib/vite-stubs/onnxruntime-node-stub.ts @@ -0,0 +1,10 @@ +/** + * Transformers imports `onnxruntime-node`, then picks web vs node from + * `process.release.name`, which is often `"node"` in Electron's renderer even + * though we need the WASM build. The real `onnxruntime-node` is aliased away (it + * pulls `fs`), so re-export `onnxruntime-web` to give the node branch a working ORT. + */ +import * as ortWeb from "onnxruntime-web"; + +const ort = (ortWeb as { default?: typeof ortWeb }).default ?? ortWeb; +export default ort; diff --git a/src/lib/webcamMaskShapes.ts b/src/lib/webcamMaskShapes.ts index f90e727d2..44d2ca6be 100644 --- a/src/lib/webcamMaskShapes.ts +++ b/src/lib/webcamMaskShapes.ts @@ -16,8 +16,8 @@ export function getCssClipPath(shape: WebcamMaskShape): string | null { } /** - * Draws a Canvas 2D clip path for the given webcam mask shape. - * Call ctx.beginPath() is handled internally; caller should call ctx.clip() after. + * Draws a Canvas 2D clip path for the given webcam mask shape. beginPath is + * handled internally; caller should call ctx.clip() after. */ export function drawCanvasClipPath( ctx: CanvasRenderingContext2D, diff --git a/src/native/client.ts b/src/native/client.ts index 9ff60d357..8d15c324d 100644 --- a/src/native/client.ts +++ b/src/native/client.ts @@ -84,10 +84,11 @@ export const nativeBridgeClient = { existingProjectPath, }, }), - loadProjectFile: () => + loadProjectFile: (projectFolder?: string) => requireNativeBridgeData({ domain: "project", action: "loadProjectFile", + payload: { projectFolder }, }), loadCurrentProjectFile: () => requireNativeBridgeData({ diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 77afa6f48..60075d397 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -165,7 +165,11 @@ export type NativeBridgeRequest = | { domain: "project"; action: "loadProjectFile"; - payload?: EmptyPayload; + payload?: { + /** Folder to pre-fill the open dialog with, usually the user's + * last-opened project folder from userPreferences. */ + projectFolder?: string; + }; requestId?: string; } | { diff --git a/src/utils/aspectRatioUtils.ts b/src/utils/aspectRatioUtils.ts index 5e174ab3c..3fbdcd07c 100644 --- a/src/utils/aspectRatioUtils.ts +++ b/src/utils/aspectRatioUtils.ts @@ -14,9 +14,8 @@ export type AspectRatio = (typeof ASPECT_RATIOS)[number]; const NATIVE_ASPECT_RATIO_FALLBACK = 16 / 9; /** - * Returns the numeric value of an aspect ratio. - * For "native", returns a fallback ratio of 16/9. - * Callers with source/crop context should use getNativeAspectRatioValue(). + * Numeric value of an aspect ratio. "native" returns the 16/9 fallback; + * callers with source/crop context should use getNativeAspectRatioValue(). */ export function getAspectRatioValue(aspectRatio: AspectRatio): number { switch (aspectRatio) { diff --git a/src/utils/platformUtils.ts b/src/utils/platformUtils.ts index 2fb57e1b9..e41145ee4 100644 --- a/src/utils/platformUtils.ts +++ b/src/utils/platformUtils.ts @@ -12,7 +12,7 @@ export const getPlatform = async (): Promise => { return platform; } catch (error) { console.warn("Failed to get platform from Electron, falling back to navigator:", error); - // Fallback for development/testing + // Fallback for dev/testing let fallbackPlatform = "win32"; if (typeof navigator !== "undefined") { if (/Mac|iPhone|iPad|iPod/.test(navigator.platform)) { diff --git a/vite.config.ts b/vite.config.ts index 0779e1358..213e44711 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -28,8 +28,22 @@ export default defineConfig({ resolve: { alias: { "@": path.resolve(__dirname, "src"), + // @xenova/transformers: env.js statically imports fs/path/url; onnx.js imports + // onnxruntime-node (must not be bundled in the renderer — it requires fs). + fs: path.resolve(__dirname, "src/lib/vite-stubs/empty-node-module.ts"), + path: path.resolve(__dirname, "src/lib/vite-stubs/empty-node-module.ts"), + url: path.resolve(__dirname, "src/lib/vite-stubs/empty-node-module.ts"), + "onnxruntime-node": path.resolve(__dirname, "src/lib/vite-stubs/onnxruntime-node-stub.ts"), // re-exports web ORT }, }, + optimizeDeps: { + exclude: ["@xenova/transformers"], + }, + // The captioning worker dynamically imports @xenova/transformers, which makes the + // worker bundle code-split — unsupported by the default "iife" worker format. + worker: { + format: "es", + }, build: { target: "esnext", minify: "terser",