diff --git a/package-lock.json b/package-lock.json index 03be5af92bb..db798d12e79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@netlify/build": "^36.4.8", "@netlify/build-info": "^11.3.0", "@netlify/config": "^25.2.5", - "@netlify/dev": "^5.0.5", + "@netlify/dev": "^5.1.0", "@netlify/dev-utils": "^6.0.1", "@netlify/edge-bundler": "^16.0.4", "@netlify/edge-functions": "^4.0.0", @@ -26,6 +26,7 @@ "@netlify/images": "^2.0.1", "@netlify/local-functions-proxy": "^2.0.3", "@netlify/redirect-parser": "^16.1.1", + "@netlify/server-dev": "^0.1.1", "@netlify/zip-it-and-ship-it": "^15.5.1", "@octokit/rest": "^22.0.0", "@opentelemetry/api": "^1.8.0", @@ -1788,9 +1789,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1807,9 +1805,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1826,9 +1821,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1845,9 +1837,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1864,9 +1853,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1883,9 +1869,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1902,9 +1885,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1921,9 +1901,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1940,9 +1917,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1965,9 +1939,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1990,9 +1961,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2015,9 +1983,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2040,9 +2005,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2065,9 +2027,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2090,9 +2049,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2115,9 +2071,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3530,9 +3483,9 @@ } }, "node_modules/@netlify/dev": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/@netlify/dev/-/dev-5.0.5.tgz", - "integrity": "sha512-/u9rKpfO86udlbRdWi3Gkox/plWd1uoGSqRBB9qZUARKBg3KvNpPkQA49YvEsrDH0GVHMhCbyrsREwVrxdtJmQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@netlify/dev/-/dev-5.1.0.tgz", + "integrity": "sha512-g4fIVlz62NXshbLMx+UjHkcF/cr28AXAU/xpTJiFntkKddsoO5ahQjtBEoiKSQsBPtUrT7a7vMk6VFlrB3RDrg==", "license": "MIT", "dependencies": { "@netlify/ai": "^1.0.1", @@ -3546,6 +3499,7 @@ "@netlify/images": "2.0.1", "@netlify/redirects": "4.0.2", "@netlify/runtime": "5.0.3", + "@netlify/server-dev": "0.1.1", "@netlify/static": "4.0.0", "ulid": "^3.0.0" }, @@ -4940,6 +4894,53 @@ "node": ">=22.12.0" } }, + "node_modules/@netlify/server-dev": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@netlify/server-dev/-/server-dev-0.1.1.tgz", + "integrity": "sha512-yA+uMfscw2Unz8JvC0dRP9z5P0lnCOy//9YICFF4nzDijYXe80bsnvrblB8wOPYBq0/KmVhl4s0gCP/3roTp4w==", + "license": "MIT", + "dependencies": { + "@netlify/dev-utils": "6.0.1", + "@netlify/serverless-functions-api": "^2.21.2", + "get-port": "^7.1.0" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@netlify/server-dev/node_modules/@netlify/serverless-functions-api": { + "version": "2.21.2", + "resolved": "https://registry.npmjs.org/@netlify/serverless-functions-api/-/serverless-functions-api-2.21.2.tgz", + "integrity": "sha512-hlWyAXqwPXXujQ59fago956xft8EykD9jBncM6h+M/06mBR6E4stO1r1tDgAqRYm+XqgA7ZZNZU5TtBD+ZhLVw==", + "license": "MIT", + "dependencies": { + "@netlify/types": "^2.8.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@netlify/server-dev/node_modules/@netlify/types": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@netlify/types/-/types-2.8.0.tgz", + "integrity": "sha512-8/g0Pt6y6wXj5Ia5eeYLiXhRfWeqZXGXpGFeCiiQdUOem+FPtXdA4+YdGxqzWc7D0AvptKSO01KGeeVWHSu8Kg==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || >=20" + } + }, + "node_modules/@netlify/server-dev/node_modules/get-port": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.2.0.tgz", + "integrity": "sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@netlify/serverless-functions-api": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/@netlify/serverless-functions-api/-/serverless-functions-api-2.18.0.tgz", @@ -6010,9 +6011,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6030,9 +6028,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6050,9 +6045,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6070,9 +6062,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6090,9 +6079,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6110,9 +6096,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6130,9 +6113,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6150,9 +6130,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6422,9 +6399,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6438,9 +6412,6 @@ "cpu": [ "arm" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6454,9 +6425,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6470,9 +6438,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6486,9 +6451,6 @@ "cpu": [ "loong64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6502,9 +6464,6 @@ "cpu": [ "loong64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6518,9 +6477,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6534,9 +6490,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6550,9 +6503,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6566,9 +6516,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6582,9 +6529,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6598,9 +6542,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6614,9 +6555,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ diff --git a/package.json b/package.json index 293b4bf1b31..27daa9258f5 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "@netlify/build": "^36.4.8", "@netlify/build-info": "^11.3.0", "@netlify/config": "^25.2.5", - "@netlify/dev": "^5.0.5", + "@netlify/dev": "^5.1.0", "@netlify/dev-utils": "^6.0.1", "@netlify/edge-bundler": "^16.0.4", "@netlify/edge-functions": "^4.0.0", @@ -68,6 +68,7 @@ "@netlify/images": "^2.0.1", "@netlify/local-functions-proxy": "^2.0.3", "@netlify/redirect-parser": "^16.1.1", + "@netlify/server-dev": "^0.1.1", "@netlify/zip-it-and-ship-it": "^15.5.1", "@octokit/rest": "^22.0.0", "@opentelemetry/api": "^1.8.0", diff --git a/src/utils/proxy-server.ts b/src/utils/proxy-server.ts index 3946666848b..6dee7c11fcb 100644 --- a/src/utils/proxy-server.ts +++ b/src/utils/proxy-server.ts @@ -7,7 +7,7 @@ import type { FunctionsRegistry } from '../lib/functions/registry.js' import { exit, log, NETLIFYDEVERR, type NormalizedCachedConfigConfig } from './command-helpers.js' import { startProxy } from './proxy.js' -import type { LocalState } from './types.js' +import type { LocalState, SiteInfo } from './types.js' import type { ServerSettings } from './types.js' interface InspectSettings { @@ -87,7 +87,7 @@ export const startProxyServer = async ({ settings: ServerSettings offline: boolean site: $TSFixMe - siteInfo: $TSFixMe + siteInfo: SiteInfo projectDir: string repositoryRoot?: string state: LocalState diff --git a/src/utils/proxy.ts b/src/utils/proxy.ts index a027a00b3a7..1c50acff2eb 100644 --- a/src/utils/proxy.ts +++ b/src/utils/proxy.ts @@ -13,8 +13,11 @@ import url from 'url' import util from 'util' import zlib from 'zlib' -import { renderFunctionErrorPage } from '@netlify/dev-utils' +import { FileWatcher, fromWebResponse, mockLocation, renderFunctionErrorPage } from '@netlify/dev-utils' import { ImageHandler } from '@netlify/images' +import { ServerHandler } from '@netlify/server-dev' + +import { runBeforeProcessExit } from './shell.js' import type { AIGatewayContext } from '@netlify/ai/bootstrap' import contentType from 'content-type' import { parseCookie } from 'cookie' @@ -55,7 +58,7 @@ import { NFFunctionName, NFFunctionRoute, NFRequestID, headersForPath, parseHead import { generateRequestID } from './request-id.js' import { createRewriter, onChanges } from './rules-proxy.js' import { signRedirect } from './sign-redirect.js' -import type { Request, Rewriter, ServerSettings } from './types.js' +import type { Request, Rewriter, ServerSettings, SiteInfo } from './types.js' const gunzip = util.promisify(zlib.gunzip) const gzip = util.promisify(zlib.gzip) @@ -805,12 +808,15 @@ const onRequest = async ( imageProxy, proxy, rewriter, + serverHandler, settings, siteInfo, - }: { rewriter: Rewriter; settings: ServerSettings; edgeFunctionsProxy?: EdgeFunctionsProxy } & Record< - string, - $TSFixMe - >, + }: { + rewriter: Rewriter + settings: ServerSettings + edgeFunctionsProxy?: EdgeFunctionsProxy + serverHandler?: ServerHandler + } & Record, req: Request, res: ServerResponse, ) => { @@ -854,6 +860,48 @@ const onRequest = async ( return } + if (serverHandler) { + try { + const requestURL = reqToURL(req, req.url) + const serverMatch = await serverHandler.match(new Request(requestURL)) + + if (serverMatch) { + const staticFile = await getStatic(decodeURIComponent(requestURL.pathname), settings.dist ?? '') + + if (!staticFile) { + const headers = new Headers() + + for (let index = 0; index < req.rawHeaders.length; index += 2) { + headers.append(req.rawHeaders[index], req.rawHeaders[index + 1]) + } + + const response = await serverMatch.handle( + new Request(requestURL, { + body: req.originalBody, + headers, + method: req.method, + }), + ) + + await fromWebResponse(response, res) + + return + } + } + } catch (error) { + // The response may have failed mid-stream, in which case the head is + // out and the only remaining option is dropping the connection. + if (res.headersSent) { + res.destroy() + } else { + res.writeHead(500) + res.end(error instanceof Error ? error.message : 'Failed to serve request from Netlify Server') + } + + return + } + } + const match = await rewriter(req) const options = { match, @@ -956,6 +1004,7 @@ export const startProxy = async function ({ disableEdgeFunctions: boolean getUpdatedConfig: () => Promise aiGatewayContext?: AIGatewayContext | null + siteInfo?: SiteInfo watchIgnore: string[] deployEnvironment: { key: string; value: string; isSecret: boolean; scopes: string[] }[] } & Record) { @@ -999,6 +1048,31 @@ export const startProxy = async function ({ logger: { log, warn, error: logError }, imagesConfig: config.images, }) + + const serverEntryEnabled = + process.env.EXPERIMENTAL_NETLIFY_SERVER === 'true' || Boolean(siteInfo?.feature_flags?.netlify_build_server_entry) + + let serverHandler: ServerHandler | undefined + + if (serverEntryEnabled) { + const serverFileWatcher = new FileWatcher() + + serverHandler = new ServerHandler({ + accountID: siteInfo?.account_id, + fileWatcher: serverFileWatcher, + geolocation: mockLocation, + logger: { log, warn, error: logError }, + projectRoot: projectDir, + siteID: siteInfo?.id, + }) + + const handlerToStop = serverHandler + + runBeforeProcessExit(async () => { + await handlerToStop.stop() + await serverFileWatcher.close() + }) + } const imageProxy = initializeImageProxy({ settings, imageHandler, @@ -1029,6 +1103,7 @@ export const startProxy = async function ({ const onRequestWithOptions = onRequest.bind(undefined, { proxy, rewriter, + serverHandler, settings, addonsUrls, functionsRegistry, @@ -1043,6 +1118,25 @@ export const startProxy = async function ({ ? https.createServer({ cert: settings.https.cert, key: settings.https.key }, onRequestWithOptions) : http.createServer(onRequestWithOptions) const onUpgrade = async function onUpgrade(req: http.IncomingMessage, socket: Duplex, head: Buffer) { + if (serverHandler) { + let handled = false + + try { + handled = await serverHandler.handleUpgrade(req, socket, head) + } catch (error) { + logError( + `Failed to hand over upgrade request to server: ${error instanceof Error ? error.message : String(error)}`, + ) + socket.destroy() + + return + } + + if (handled) { + return + } + } + const match = await rewriter(req) if (match && !match.force404 && isExternal(match)) { const reqUrl = reqToURL(req, req.url) diff --git a/tests/integration/commands/dev/netlify-server.test.ts b/tests/integration/commands/dev/netlify-server.test.ts new file mode 100644 index 00000000000..99d5f7d376a --- /dev/null +++ b/tests/integration/commands/dev/netlify-server.test.ts @@ -0,0 +1,239 @@ +import process from 'process' + +import js from 'dedent' +import fetch from 'node-fetch' +import { describe, test } from 'vitest' +import { WebSocket } from 'ws' + +import { withDevServer } from '../../utils/dev-server.js' +import { withSiteBuilder, type SiteBuilder } from '../../utils/site-builder.js' + +// Serving through Netlify Server requires Node.js 24; on older versions we can +// only test the error surface. +const unsupportedNodeVersion = Number.parseInt(process.versions.node) < 24 + +const env = { EXPERIMENTAL_NETLIFY_SERVER: 'true' } + +const withServerEntry = (builder: SiteBuilder): SiteBuilder => + builder.withContentFile({ + path: 'netlify/server/index.mjs', + // `dedent` converts backslash escapes in the template to the characters + // they name, so the fixture builds CRLF without any escape sequences. + content: js` + import { createHash } from 'node:crypto' + import { createServer } from 'node:http' + + const CRLF = String.fromCharCode(13, 10) + + const server = createServer((req, res) => { + let body = '' + + req.on('data', (chunk) => { + body += chunk + }) + req.on('end', () => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + source: 'server', + method: req.method, + url: req.url, + body, + siteID: req.headers['x-nf-site-id'] ?? null, + }), + ) + }) + }) + + // A hand-rolled WebSocket echo, enough for a single unfragmented text + // frame, so the fixture needs no dependencies. + server.on('upgrade', (req, socket) => { + if (req.url !== '/ws') { + socket.destroy() + + return + } + + const accept = createHash('sha1') + .update(req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11') + .digest('base64') + + socket.write( + [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + 'Sec-WebSocket-Accept: ' + accept, + CRLF, + ].join(CRLF), + ) + + socket.on('data', (frame) => { + const length = frame[1] & 0x7f + const mask = frame.subarray(2, 6) + const payload = Buffer.from(frame.subarray(6, 6 + length).map((byte, index) => byte ^ mask[index % 4])) + const reply = Buffer.from('echo:' + payload.toString()) + + socket.write(Buffer.concat([Buffer.from([0x81, reply.length]), reply])) + }) + }) + + server.listen(process.env.PORT) + `, + }) + +describe.concurrent('command/dev Netlify Server', () => { + test('is inert without the opt-in', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + + await builder.build() + + await withDevServer({ cwd: builder.directory }, async (server) => { + const response = await fetch(`${server.url}/some/path`) + + t.expect(response.status).toBe(404) + }) + }) + }) + + test.skipIf(unsupportedNodeVersion)('serves every path from the server when enabled', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const response = await fetch(`${server.url}/some/deep/path?value=1`) + + t.expect(response.status).toBe(200) + + const result = (await response.json()) as Record + + t.expect(result.source).toBe('server') + t.expect(result.url).toBe('/some/deep/path?value=1') + t.expect(result.siteID).toBe('unlinked') + }) + }) + }) + + test.skipIf(unsupportedNodeVersion)('forwards request methods and bodies', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const response = await fetch(`${server.url}/submit`, { + body: 'name=netlify', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + method: 'POST', + }) + const result = (await response.json()) as Record + + t.expect(result.method).toBe('POST') + t.expect(result.body).toBe('name=netlify') + }) + }) + }) + + test.skipIf(unsupportedNodeVersion)('static files take precedence over the server', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + .withContentFile({ + path: 'public/asset.txt', + content: 'from static file', + }) + .withNetlifyToml({ + config: { + build: { + publish: 'public/', + }, + }, + }) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const staticResponse = await fetch(`${server.url}/asset.txt`) + + t.expect(await staticResponse.text()).toBe('from static file') + + const serverResponse = await fetch(`${server.url}/not-a-static-file`) + const result = (await serverResponse.json()) as Record + + t.expect(result.source).toBe('server') + }) + }) + }) + + test.skipIf(unsupportedNodeVersion)('functions take precedence over the server', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder).withContentFile({ + path: 'netlify/functions/fn.mjs', + content: js` + export default async () => new Response('from-function') + + export const config = { path: '/fn' } + `, + }) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const functionResponse = await fetch(`${server.url}/fn`) + + t.expect(await functionResponse.text()).toBe('from-function') + + const serverResponse = await fetch(`${server.url}/not-the-function`) + const result = (await serverResponse.json()) as Record + + t.expect(result.source).toBe('server') + }) + }) + }) + + test.skipIf(unsupportedNodeVersion)('pipes WebSocket upgrades to the server', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const socket = new WebSocket(`ws://localhost:${String(server.port)}/ws`) + + await new Promise((resolve, reject) => { + socket.once('open', resolve) + socket.once('error', reject) + }) + + const reply = new Promise((resolve) => { + socket.once('message', (message) => { + resolve((message as Buffer).toString('utf8')) + }) + }) + + socket.send('hello') + + t.expect(await reply).toBe('echo:hello') + + socket.close() + }) + }) + }) + + test.runIf(unsupportedNodeVersion)('fails requests with a clear error on Node.js versions below 24', async (t) => { + await withSiteBuilder(t, async (builder) => { + withServerEntry(builder) + + await builder.build() + + await withDevServer({ cwd: builder.directory, env }, async (server) => { + const response = await fetch(`${server.url}/some/path`) + + t.expect(response.status).toBe(500) + t.expect(await response.text()).toContain('Netlify Server requires Node.js 24 or above') + }) + }) + }) +})