From 3fe89af59370a8a2f2b6ef9d39a16d54d8e16359 Mon Sep 17 00:00:00 2001 From: Wassim Gharbi Date: Sat, 29 Aug 2026 13:10:44 -0700 Subject: [PATCH] Import files over 2 GiB by mounting them instead of copying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #87: "File could not be read! Code=-1" on any file at or above 2 GiB. The message comes from @ffmpeg/util's fetchFile, which slurps the whole File through FileReader.readAsArrayBuffer before handing it to ffmpeg.writeFile. A Uint8Array cannot exceed 2 GiB in Chrome, so that read fails outright at exactly that size — measured in Chromium against slices of one file: 2040 MiB reads, 2048 MiB does not. The reporter's 472 MB file worked and their 3-8 GB files did not, which is the wall exactly. The error reads as "Code=-1" because a modern DOMException has no legacy .code for fetchFile's template to interpolate, so it falls through to the || -1. Mount the File through WORKERFS instead, which is compiled into the core we already ship. It serves reads straight off the Blob — one slice plus a FileReaderSync per avio block — so the media is never materialised as a single ArrayBuffer, and seeks past 2 GiB work. Copying stays the default below 256 MiB: it is ~1.7x faster end to end on a 300 MB file, and the sizes it handles are the ones it already handled. Above that the input is mounted, with a fall back to copying if the mount is rejected. Switching inputs now unmounts or deletes the previous one rather than leaving it resident. Verified in headless Chromium against synthesised WAVs, driving the real lib/ffmpeg.ts through esbuild rather than a replica: - 3.00 GiB / 76 min: pre-fix "File could not be read! Code=-1" in 0.3s; post-fix extracts all 4565.2s of audio in 68s - 2.46 GiB / 8h20m: extracts 30000.0s, 1831 MB of PCM - 300 MB (mount) and 50 MB (copy): correct durations, non-silent audio - mount -> copy -> mount -> copy in one ffmpeg instance: all correct - exportAudio from a mounted 3 GiB input with keep ranges at 4000s/4500s (past the 2 GiB byte offset): valid RIFF/WAVE, exactly 40.0s out for 40s of ranges Co-Authored-By: Claude Opus 5 (1M context) --- lib/ffmpeg.ts | 92 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 5 deletions(-) diff --git a/lib/ffmpeg.ts b/lib/ffmpeg.ts index 6f9be05..4dad836 100644 --- a/lib/ffmpeg.ts +++ b/lib/ffmpeg.ts @@ -6,9 +6,27 @@ import type { TimeRange } from "./types"; const CORE_BASE = "/vendor/ffmpeg"; const INPUT_NAME = "input_video"; +const MOUNT_DIR = "/mnt_input"; + +/** + * Above this, the input is mounted instead of copied into MEMFS. + * + * Copying stays the default because it is measurably faster: MEMFS reads are + * plain memory reads, while a mounted file pays a `Blob.slice` + + * `FileReaderSync` round trip per avio block (~1.7x slower end to end on a + * 300 MB file). But it also holds the whole file in memory several times over + * — the `fetchFile` ArrayBuffer, the structured clone to the worker, and + * MEMFS's own copy, which is a JS-heap `Uint8Array` rather than wasm memory — + * and it cannot work at all at 2 GiB (see `mountInput`). A quarter-gig cap + * keeps the fast path for the sizes it comfortably handles and mounts the rest. + */ +const COPY_INPUT_MAX_BYTES = 256 * 1024 * 1024; let ffmpegPromise: Promise | null = null; let writtenFor: File | null = null; +/** Path `writtenFor`'s media is readable at, and whether it came from a mount. */ +let inputPath = INPUT_NAME; +let inputMounted = false; /** Lazily load a singleton multi-threaded ffmpeg.wasm instance. */ export async function getFFmpeg(): Promise { @@ -63,6 +81,9 @@ export async function releaseFFmpeg(): Promise { // handing out the one we are about to terminate. ffmpegPromise = null; writtenFor = null; + // The worker owns the filesystem, so its mounts and MEMFS files die with it. + inputMounted = false; + inputPath = INPUT_NAME; try { (await pending).terminate(); } catch { @@ -70,13 +91,74 @@ export async function releaseFFmpeg(): Promise { } } +/** + * Expose `file` to ffmpeg via WORKERFS rather than copying it in. + * + * WORKERFS serves reads straight off the `Blob` — one `slice` plus a + * `FileReaderSync` per avio block — so the media is never materialised as a + * single ArrayBuffer. That is what makes multi-gigabyte imports possible at + * all: a `Uint8Array` cannot exceed 2 GiB in Chrome, so `fetchFile`'s + * `FileReader.readAsArrayBuffer` fails outright at exactly that size (measured: + * 2040 MiB reads, 2048 MiB does not). It surfaces as the opaque "File could not + * be read! Code=-1" from `@ffmpeg/util`, because a modern DOMException has no + * legacy `.code` for the template to interpolate. + * + * Mounted as a named blob so the path we hand ffmpeg is always `input_video`, + * whatever the user called their file. Read-only, which is all input needs. + */ +async function mountInput(ffmpeg: FFmpeg, file: File): Promise { + const { FFFSType } = await import("@ffmpeg/ffmpeg"); + try { + await ffmpeg.createDir(MOUNT_DIR); + } catch { + // Already there from a previous file; the unmount in clearInput left it. + } + await ffmpeg.mount( + FFFSType.WORKERFS, + { blobs: [{ name: INPUT_NAME, data: file }] }, + MOUNT_DIR + ); + inputMounted = true; + return `${MOUNT_DIR}/${INPUT_NAME}`; +} + +/** Release the previous input so a second file doesn't stack onto the first. */ +async function clearInput(ffmpeg: FFmpeg): Promise { + if (!writtenFor) return; + const wasMounted = inputMounted; + writtenFor = null; + inputMounted = false; + try { + if (wasMounted) await ffmpeg.unmount(MOUNT_DIR); + else await ffmpeg.deleteFile(INPUT_NAME); + } catch { + // Nothing there to reclaim — mounting/writing the next input still works. + } +} + async function ensureInput(ffmpeg: FFmpeg, file: File): Promise { - if (writtenFor !== file) { - const { fetchFile } = await import("@ffmpeg/util"); - await ffmpeg.writeFile(INPUT_NAME, await fetchFile(file)); - writtenFor = file; + if (writtenFor === file) return inputPath; + await clearInput(ffmpeg); + + if (file.size > COPY_INPUT_MAX_BYTES) { + try { + inputPath = await mountInput(ffmpeg, file); + writtenFor = file; + return inputPath; + } catch (err) { + // No WORKERFS in this core, or the mount was rejected. Copying will + // probably fail too at this size, but it fails with ffmpeg's own error + // rather than ours, so let it try. + console.warn("WORKERFS mount failed, copying input instead:", err); + inputMounted = false; + } } - return INPUT_NAME; + + const { fetchFile } = await import("@ffmpeg/util"); + await ffmpeg.writeFile(INPUT_NAME, await fetchFile(file)); + writtenFor = file; + inputPath = INPUT_NAME; + return inputPath; } /**