From a7be662cc88d9b6a87dc5d16456919bcd2a6f4bc Mon Sep 17 00:00:00 2001 From: Mert Can Altin Date: Sat, 11 Jul 2026 22:52:23 +0300 Subject: [PATCH 1/5] fs: enable chunked reading for large files in readFileHandle Signed-off-by: Mert Can Altin --- lib/fs.js | 10 --- lib/internal/fs/promises.js | 29 ++++++--- .../test-fs-promises-file-handle-readFile.js | 63 ++++++++++++++----- test/parallel/test-fs-readfile.js | 22 +++---- 4 files changed, 73 insertions(+), 51 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index 1ea70ff192d6dd..ad3d00ecdadb9d 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -74,7 +74,6 @@ const { aggregateTwoErrors, codes: { ERR_ACCESS_DENIED, - ERR_FS_FILE_TOO_LARGE, ERR_INVALID_ARG_VALUE, }, } = require('internal/errors'); @@ -98,7 +97,6 @@ const { } = require('internal/util'); const { constants: { - kIoMaxLength, kMaxUserId, }, copyObject, @@ -363,11 +361,6 @@ function readFileAfterStat(err, stats) { // stringify operations vs multiple C++/JS boundary crossings). const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; - if (size > kIoMaxLength) { - err = new ERR_FS_FILE_TOO_LARGE(size); - return context.close(err); - } - try { context.prepare(); } catch (err) { @@ -447,9 +440,6 @@ function tryCreateBuffer(size, fd, isUserFd) { let threw = true; let buffer; try { - if (size > kIoMaxLength) { - throw new ERR_FS_FILE_TOO_LARGE(size); - } buffer = Buffer.allocUnsafe(size); threw = false; } finally { diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 3e336024a15a3c..b727f72200bbb6 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -1228,17 +1228,25 @@ async function readFileHandle(filehandle, options) { let size = 0; let length = 0; - if ((statFields[1/* mode */] & S_IFMT) === S_IFREG) { - size = statFields[8/* size */]; + + if ((statFields[1 /* mode */] & S_IFMT) === S_IFREG) { + size = statFields[8 /* size */]; + + if (size > kIoMaxLength) { + const limit = kIoMaxLength; + process.emitWarning( + `Detected \`fs.readFile()\` to read a file larger than the recommended limit (${size} > ${limit} bytes). Please consider using \`fs.createReadStream()\` instead to minimize memory overhead and increase performance.`, + 'LargeFileWarning', + ERR_FS_FILE_TOO_LARGE, + ); + } + length = encoding ? MathMin(size, kReadFileBufferLength) : size; } if (length === 0) { length = kReadFileUnknownBufferLength; } - if (size > kIoMaxLength) - throw new ERR_FS_FILE_TOO_LARGE(size); - if (options.buffer !== undefined) { return readFileHandleWithUserBuffer(filehandle, options, size); } @@ -1266,8 +1274,8 @@ async function readFileHandle(filehandle, options) { totalRead += bytesRead; if (bytesRead === 0 || - totalRead === size || - (bytesRead !== buffer.length && !chunkedRead && !noSize)) { + totalRead === size || + (bytesRead !== buffer.length && !chunkedRead && !noSize)) { const singleRead = bytesRead === totalRead; const bytesToCheck = chunkedRead ? totalRead : bytesRead; @@ -1290,9 +1298,10 @@ async function readFileHandle(filehandle, options) { result += decoder.end(buffer); return result; } - const readBuffer = bytesRead !== buffer.length ? - buffer.subarray(0, bytesRead) : - buffer; + const readBuffer = + bytesRead !== buffer.length ? + buffer.subarray(0, bytesRead) : + buffer; if (encoding) { result += decoder.write(readBuffer); } else if (size !== 0) { diff --git a/test/parallel/test-fs-promises-file-handle-readFile.js b/test/parallel/test-fs-promises-file-handle-readFile.js index 997433bf8434dd..e7e03444467449 100644 --- a/test/parallel/test-fs-promises-file-handle-readFile.js +++ b/test/parallel/test-fs-promises-file-handle-readFile.js @@ -5,19 +5,24 @@ const common = require('../common'); // The following tests validate base functionality for the fs.promises // FileHandle.readFile method. -const fs = require('fs'); +const fs = require('node:fs'); const { open, readFile, - writeFile, truncate, + writeFile, } = fs.promises; -const path = require('path'); +const path = require('node:path'); +const os = require('node:os'); const tmpdir = require('../common/tmpdir'); const tick = require('../common/tick'); -const assert = require('assert'); +const assert = require('node:assert'); const tmpDir = tmpdir.path; +const skipMessage = 'intensive toString tests due to memory confinements'; +if (!common.enoughTestMem) + common.skip(skipMessage); + tmpdir.refresh(); async function validateReadFile() { @@ -33,6 +38,34 @@ async function validateReadFile() { assert.deepStrictEqual(buffer, readFileData); } +async function validateLargeFileSupport() { + const LARGE_FILE_SIZE = 2147483647 + 10 * 1024 * 1024; // INT32_MAX + 10 MB + const FILE_PATH = path.join(os.tmpdir(), 'large-virtual-file.bin'); + + if (!tmpdir.hasEnoughSpace(LARGE_FILE_SIZE)) { + common.printSkipMessage(`Not enough space in ${os.tmpdir()}`); + return; + } + + function createVirtualLargeFile() { + return Buffer.alloc(LARGE_FILE_SIZE, 'A'); + } + + const virtualFile = createVirtualLargeFile(); + + await writeFile(FILE_PATH, virtualFile); + + const buffer = await readFile(FILE_PATH); + + assert.strictEqual( + buffer.length, + LARGE_FILE_SIZE, + `Expected file size to be ${LARGE_FILE_SIZE}, but got ${buffer.length}` + ); + + await fs.promises.unlink(FILE_PATH); +} + async function validateReadFileProc() { // Test to make sure reading a file under the /proc directory works. Adapted // from test-fs-read-file-sync-hostname.js. @@ -92,30 +125,28 @@ async function doReadAndCancel() { }, 'tick-1'); } - // Validate file size is within range for reading + // For validates the ability of the filesystem module to handle large files { - // Variable taken from https://github.com/nodejs/node/blob/1377163f3351/lib/internal/fs/promises.js#L5 - const kIoMaxLength = 2 ** 31 - 1; + const largeFileSize = 5 * 1024 * 1024 * 1024; // 5 GiB + const newFile = path.resolve(tmpDir, 'dogs-running3.txt'); - if (!tmpdir.hasEnoughSpace(kIoMaxLength)) { - // truncate() will fail with ENOSPC if there is not enough space. + if (!tmpdir.hasEnoughSpace(largeFileSize)) { common.printSkipMessage(`Not enough space in ${tmpDir}`); } else { - const newFile = path.resolve(tmpDir, 'dogs-running3.txt'); await writeFile(newFile, Buffer.from('0')); - await truncate(newFile, kIoMaxLength + 1); + await truncate(newFile, largeFileSize); - await using fileHandle = await open(newFile, 'r'); + const fileHandle = await open(newFile, 'r'); - await assert.rejects(fileHandle.readFile(), { - name: 'RangeError', - code: 'ERR_FS_FILE_TOO_LARGE' - }); + const data = await fileHandle.readFile(); + console.log(`File read successfully, size: ${data.length} bytes`); + await fileHandle.close(); } } } validateReadFile() + .then(validateLargeFileSupport) .then(validateReadFileProc) .then(doReadAndCancel) .then(common.mustCall()); diff --git a/test/parallel/test-fs-readfile.js b/test/parallel/test-fs-readfile.js index 74731728676b1b..c140d6323b9e13 100644 --- a/test/parallel/test-fs-readfile.js +++ b/test/parallel/test-fs-readfile.js @@ -6,6 +6,7 @@ const common = require('../common'); const tmpdir = require('../../test/common/tmpdir'); const assert = require('assert'); +const path = require('node:path'); const fs = require('fs'); const prefix = `.removeme-fs-readfile-${process.pid}`; @@ -52,28 +53,19 @@ for (const e of fileInfo) { })); } -// readFile() and readFileSync() should fail if the file is too big. +// Test to verify that readFile() and readFileSync() can handle large files { - const kIoMaxLength = 2 ** 31 - 1; + const kLargeFileSize = 3 * 1024 * 1024 * 1024; // 3 GiB - if (!tmpdir.hasEnoughSpace(kIoMaxLength)) { + if (!tmpdir.hasEnoughSpace(kLargeFileSize)) { // truncateSync() will fail with ENOSPC if there is not enough space. common.printSkipMessage(`Not enough space in ${tmpdir.path}`); } else { - const file = tmpdir.resolve(`${prefix}-too-large.txt`); - fs.writeFileSync(file, Buffer.from('0')); - fs.truncateSync(file, kIoMaxLength + 1); - - fs.readFile(file, common.expectsError({ - code: 'ERR_FS_FILE_TOO_LARGE', - name: 'RangeError', - })); - assert.throws(() => { - fs.readFileSync(file); - }, { code: 'ERR_FS_FILE_TOO_LARGE', name: 'RangeError' }); + const file = path.join(tmpdir.path, 'temp-large-file.txt'); + fs.writeFileSync(file, Buffer.alloc(1024)); + fs.truncateSync(file, kLargeFileSize); } } - { // Test cancellation, before const signal = AbortSignal.abort(); From 2f4414fa3695649665494a849ab68f939f615c22 Mon Sep 17 00:00:00 2001 From: Mert Can Altin Date: Sun, 12 Jul 2026 00:17:59 +0300 Subject: [PATCH 2/5] removed unused const Signed-off-by: Mert Can Altin --- lib/fs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fs.js b/lib/fs.js index ad3d00ecdadb9d..427bb383ffd34b 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -359,7 +359,7 @@ function readFileAfterStat(err, stats) { // TODO(BridgeAR): Check if allocating a smaller chunk is better performance // wise, similar to the promise based version (less peak memory and chunked // stringify operations vs multiple C++/JS boundary crossings). - const size = context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; + context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; try { context.prepare(); From b489cace3df254413ec0688690032745d4d8981f Mon Sep 17 00:00:00 2001 From: Mert Can Altin Date: Sun, 12 Jul 2026 12:46:45 +0300 Subject: [PATCH 3/5] heavy test moved to pummel test --- .../test-fs-promises-file-handle-readFile.js | 19 ---------- ...-fs-promises-file-handle-readFile-large.js | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 19 deletions(-) create mode 100644 test/pummel/test-fs-promises-file-handle-readFile-large.js diff --git a/test/parallel/test-fs-promises-file-handle-readFile.js b/test/parallel/test-fs-promises-file-handle-readFile.js index e7e03444467449..85e36c2635f6da 100644 --- a/test/parallel/test-fs-promises-file-handle-readFile.js +++ b/test/parallel/test-fs-promises-file-handle-readFile.js @@ -9,7 +9,6 @@ const fs = require('node:fs'); const { open, readFile, - truncate, writeFile, } = fs.promises; const path = require('node:path'); @@ -125,24 +124,6 @@ async function doReadAndCancel() { }, 'tick-1'); } - // For validates the ability of the filesystem module to handle large files - { - const largeFileSize = 5 * 1024 * 1024 * 1024; // 5 GiB - const newFile = path.resolve(tmpDir, 'dogs-running3.txt'); - - if (!tmpdir.hasEnoughSpace(largeFileSize)) { - common.printSkipMessage(`Not enough space in ${tmpDir}`); - } else { - await writeFile(newFile, Buffer.from('0')); - await truncate(newFile, largeFileSize); - - const fileHandle = await open(newFile, 'r'); - - const data = await fileHandle.readFile(); - console.log(`File read successfully, size: ${data.length} bytes`); - await fileHandle.close(); - } - } } validateReadFile() diff --git a/test/pummel/test-fs-promises-file-handle-readFile-large.js b/test/pummel/test-fs-promises-file-handle-readFile-large.js new file mode 100644 index 00000000000000..226248f03e021b --- /dev/null +++ b/test/pummel/test-fs-promises-file-handle-readFile-large.js @@ -0,0 +1,38 @@ +'use strict'; + +const common = require('../common'); + +if (!common.enoughTestMem) + common.skip('intensive toString tests due to memory confinements'); + +const assert = require('node:assert'); +const fs = require('node:fs'); +const { + open, + truncate, + writeFile, +} = fs.promises; +const path = require('node:path'); +const tmpdir = require('../common/tmpdir'); + +tmpdir.refresh(); + +async function validateReadFileLargeSparseFile() { + const largeFileSize = 5 * 1024 * 1024 * 1024; // 5 GiB + const filePath = path.resolve(tmpdir.path, 'file-handle-readfile-large.txt'); + + if (!tmpdir.hasEnoughSpace(largeFileSize)) { + common.printSkipMessage(`Not enough space in ${tmpdir.path}`); + return; + } + + await writeFile(filePath, Buffer.from('0')); + await truncate(filePath, largeFileSize); + + await using fileHandle = await open(filePath, 'r'); + const data = await fileHandle.readFile(); + assert.strictEqual(data.length, largeFileSize); +} + +validateReadFileLargeSparseFile() + .then(common.mustCall()); \ No newline at end of file From c7caaa382e442db7298b188e2106aecc84dd735c Mon Sep 17 00:00:00 2001 From: Mert Can Altin Date: Sun, 12 Jul 2026 12:48:01 +0300 Subject: [PATCH 4/5] added end of line --- test/pummel/test-fs-promises-file-handle-readFile-large.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pummel/test-fs-promises-file-handle-readFile-large.js b/test/pummel/test-fs-promises-file-handle-readFile-large.js index 226248f03e021b..b4db9b1785985e 100644 --- a/test/pummel/test-fs-promises-file-handle-readFile-large.js +++ b/test/pummel/test-fs-promises-file-handle-readFile-large.js @@ -35,4 +35,4 @@ async function validateReadFileLargeSparseFile() { } validateReadFileLargeSparseFile() - .then(common.mustCall()); \ No newline at end of file + .then(common.mustCall()); From 4e4486b0bd77e5546fce52ccbf4519e45d19c07b Mon Sep 17 00:00:00 2001 From: Mert Can Altin Date: Sun, 12 Jul 2026 13:01:57 +0300 Subject: [PATCH 5/5] added warning for all path --- lib/fs.js | 11 +++++++++ .../test-fs-promises-file-handle-readFile.js | 6 +++++ test/parallel/test-fs-readfile.js | 23 +++++++++++++++---- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index 427bb383ffd34b..a3e1e73418a082 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -74,6 +74,7 @@ const { aggregateTwoErrors, codes: { ERR_ACCESS_DENIED, + ERR_FS_FILE_TOO_LARGE, ERR_INVALID_ARG_VALUE, }, } = require('internal/errors'); @@ -97,6 +98,7 @@ const { } = require('internal/util'); const { constants: { + kIoMaxLength, kMaxUserId, }, copyObject, @@ -361,6 +363,15 @@ function readFileAfterStat(err, stats) { // stringify operations vs multiple C++/JS boundary crossings). context.size = isFileType(stats, S_IFREG) ? stats[8] : 0; + if (context.size > kIoMaxLength) { + const limit = kIoMaxLength; + process.emitWarning( + `Detected \`fs.readFile()\` to read a file larger than the recommended limit (${context.size} > ${limit} bytes). Please consider using \`fs.createReadStream()\` instead to minimize memory overhead and increase performance.`, + 'LargeFileWarning', + ERR_FS_FILE_TOO_LARGE, + ); + } + try { context.prepare(); } catch (err) { diff --git a/test/parallel/test-fs-promises-file-handle-readFile.js b/test/parallel/test-fs-promises-file-handle-readFile.js index 85e36c2635f6da..9a32505222d4ba 100644 --- a/test/parallel/test-fs-promises-file-handle-readFile.js +++ b/test/parallel/test-fs-promises-file-handle-readFile.js @@ -54,6 +54,12 @@ async function validateLargeFileSupport() { await writeFile(FILE_PATH, virtualFile); + common.expectWarning({ + LargeFileWarning: [[ + /Detected `fs\.readFile\(\)` to read a file larger than the recommended limit/, + ]], + }); + const buffer = await readFile(FILE_PATH); assert.strictEqual( diff --git a/test/parallel/test-fs-readfile.js b/test/parallel/test-fs-readfile.js index c140d6323b9e13..9b9bdfb136a746 100644 --- a/test/parallel/test-fs-readfile.js +++ b/test/parallel/test-fs-readfile.js @@ -13,6 +13,9 @@ const prefix = `.removeme-fs-readfile-${process.pid}`; tmpdir.refresh(); +const kLargeFileSize = 3 * 1024 * 1024 * 1024; // 3 GiB +let largeFilePath; + const fileInfo = [ { name: tmpdir.resolve(`${prefix}-1K.txt`), len: 1024 }, @@ -55,17 +58,27 @@ for (const e of fileInfo) { // Test to verify that readFile() and readFileSync() can handle large files { - const kLargeFileSize = 3 * 1024 * 1024 * 1024; // 3 GiB - if (!tmpdir.hasEnoughSpace(kLargeFileSize)) { // truncateSync() will fail with ENOSPC if there is not enough space. common.printSkipMessage(`Not enough space in ${tmpdir.path}`); } else { - const file = path.join(tmpdir.path, 'temp-large-file.txt'); - fs.writeFileSync(file, Buffer.alloc(1024)); - fs.truncateSync(file, kLargeFileSize); + largeFilePath = path.join(tmpdir.path, 'temp-large-file.txt'); + fs.writeFileSync(largeFilePath, Buffer.alloc(1024)); + fs.truncateSync(largeFilePath, kLargeFileSize); } } +if (largeFilePath !== undefined) { + common.expectWarning({ + LargeFileWarning: [[ + /Detected `fs\.readFile\(\)` to read a file larger than the recommended limit/, + ]], + }); + + // Trigger warning on the callback path without reading the whole file. + fs.readFile(largeFilePath, { buffer: Buffer.alloc(1) }, common.mustCall((err) => { + assert.strictEqual(err?.code, 'ERR_INVALID_ARG_VALUE'); + })); +} { // Test cancellation, before const signal = AbortSignal.abort();