diff --git a/sdk/typescript/AGENTS.md b/sdk/typescript/AGENTS.md index fe67f6708..6fdcaa3f2 100644 --- a/sdk/typescript/AGENTS.md +++ b/sdk/typescript/AGENTS.md @@ -41,13 +41,15 @@ accepted and rejected inputs, and each real bug or security boundary. From the SDK directory, run a focused test while iterating, then run the package checks: ```bash -bun test tests-ts/.test.ts -bun test --randomize --seed 12345 +bun test --timeout 30000 tests-ts/.test.ts +pnpm run test --seed 12345 pnpm run types pnpm run format pnpm run test ``` +Tests run in random order by default. To reproduce a failure, use the seed printed in Bun's test summary. + After the implementation is verified, keep the test in the final change only when it provides meaningful, durable regression coverage. If it is merely disposable implementation scaffolding, duplicates existing coverage, or would diff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.toml new file mode 100644 index 000000000..d1639b19a --- /dev/null +++ b/sdk/typescript/bunfig.toml @@ -0,0 +1,2 @@ +[test] +randomize = true diff --git a/sdk/typescript/src/cli.ts b/sdk/typescript/src/cli.ts index 495cecd45..ef9d5bf0d 100644 --- a/sdk/typescript/src/cli.ts +++ b/sdk/typescript/src/cli.ts @@ -7,7 +7,7 @@ import { existsSync, lstatSync, realpathSync, - type Stats, + type BigIntStats, writeSync, } from "node:fs"; import { @@ -598,9 +598,9 @@ async function readPromptFiles( async function readRegularInputFile( path: string, repository: string, - metadata?: Pick, + metadata?: Pick, ): Promise { - const selected = metadata ?? (await lstat(path)); + const selected = metadata ?? (await lstat(path, { bigint: true })); if (!selected.isFile()) { throw new CodexSecurityError("Input files must be regular files."); } @@ -627,7 +627,7 @@ async function readRegularInputFile( (constants.O_NONBLOCK ?? 0), ); try { - const opened = await file.stat(); + const opened = await file.stat({ bigint: true }); if ( !opened.isFile() || opened.dev !== selected.dev || @@ -3263,22 +3263,24 @@ async function runSkill( localDeviceRoot !== normalizedDeviceRoot); if (!windowsNetworkPath) { const path = resolve(directory, input); - const metadata = await lstat(path).catch((error: unknown) => { - if ( - typeof error === "object" && - error !== null && - "code" in error && - (error.code === "ENOENT" || - error.code === "ENOTDIR" || - error.code === "ENAMETOOLONG" || - error.code === "EINVAL") - ) { - return undefined; - } - throw new CodexSecurityError( - "Could not read the finding or issue input.", - ); - }); + const metadata = await lstat(path, { bigint: true }).catch( + (error: unknown) => { + if ( + typeof error === "object" && + error !== null && + "code" in error && + (error.code === "ENOENT" || + error.code === "ENOTDIR" || + error.code === "ENAMETOOLONG" || + error.code === "EINVAL") + ) { + return undefined; + } + throw new CodexSecurityError( + "Could not read the finding or issue input.", + ); + }, + ); if (metadata !== undefined) { if (!metadata.isFile()) { throw new CodexSecurityError( diff --git a/sdk/typescript/tests-ts/api.test.ts b/sdk/typescript/tests-ts/api.test.ts index 695a8e22a..3b621bbee 100644 --- a/sdk/typescript/tests-ts/api.test.ts +++ b/sdk/typescript/tests-ts/api.test.ts @@ -4597,10 +4597,7 @@ describe("CodexSecurity orchestration", () => { expect(before["deep_scan"]).toMatchObject({ workers: index + 2, }); - await Promise.race([ - concurrentScans, - new Promise((resolve) => setTimeout(resolve, 5_000)), - ]); + await concurrentScans; const after = parseToml( await readFile(deepScanConfigPath!, "utf8"), ); @@ -4623,7 +4620,9 @@ describe("CodexSecurity orchestration", () => { try { const results = await Promise.allSettled( clients.map((client, index) => - client.run(repository, { mode: "deep", workers: index + 2 }), + client + .run(repository, { mode: "deep", workers: index + 2 }) + .finally(releaseScans), ), ); for (const result of results) { @@ -4646,6 +4645,7 @@ describe("CodexSecurity orchestration", () => { ), ).toBe(true); } finally { + releaseScans(); await Promise.all(clients.map(async (client) => await client.close())); } }); @@ -6534,20 +6534,17 @@ setInterval(() => {}, 1000); ); const login = client.loginApiKey("secret-key"); void login.catch(() => undefined); - for (let attempt = 0; attempt < 100; attempt += 1) { - const started = await import("node:fs/promises").then(({ stat }) => - stat(ready).catch(() => null), - ); - if (started !== null) break; - await new Promise((resolve) => setTimeout(resolve, 10)); + try { + const deadline = Date.now() + 10_000; + while (!existsSync(ready) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + expect(existsSync(ready), "the fake login process started").toBe(true); + await client.close(); + await expect(login).rejects.toThrow(); + await expect(stat(codexHome)).resolves.toBeDefined(); + } finally { + await client.close(); } - await expect( - import("node:fs/promises").then(({ stat }) => stat(ready)), - ).resolves.toBeDefined(); - await client.close(); - await expect(login).rejects.toThrow(); - await expect( - import("node:fs/promises").then(({ stat }) => stat(codexHome)), - ).resolves.toBeDefined(); - }); + }, 30_000); }); diff --git a/sdk/typescript/tests-ts/cli-skills.test.ts b/sdk/typescript/tests-ts/cli-skills.test.ts index aa6375f8b..60c778245 100644 --- a/sdk/typescript/tests-ts/cli-skills.test.ts +++ b/sdk/typescript/tests-ts/cli-skills.test.ts @@ -13,6 +13,7 @@ import { } from "../src/cli.js"; import type { LinearClientFactory } from "../src/linear.js"; import { capture, dependencies } from "./cli-fixtures.js"; +import { runMockInSubprocess } from "./support/isolated-mock.js"; function linearIssue(identifier: string) { return { @@ -459,11 +460,102 @@ describe("CLI skill commands", () => { } }); + test("rejects input replacements whose numeric file IDs collide", async () => { + if ( + runMockInSubprocess( + import.meta.path, + "rejects input replacements whose numeric file IDs collide", + ) + ) { + return; + } + const root = await mkdtemp(join(tmpdir(), "codex-security-file-identity-")); + const selected = join(root, "finding.txt"); + const replacement = join(root, "replacement.txt"); + const selectedInode = 2n ** 60n; + const replacementInode = selectedInode + 1n; + expect(Number(selectedInode)).toBe(Number(replacementInode)); + await writeFile(selected, "ordinary finding\n"); + await writeFile(replacement, "SYNTHETIC_REPLACEMENT_FINDING\n"); + const canonicalSelected = await filesystem.realpath(selected); + const originalLstat = filesystem.lstat; + const originalOpen = filesystem.open; + let restoreOpenedStat: (() => void) | undefined; + let replaced = false; + const reading = spyOn(filesystem, "lstat").mockImplementation((async ( + ...args: Parameters + ) => { + const metadata = await originalLstat(...args); + if (String(args[0]) === selected) { + metadata.ino = + typeof metadata.ino === "bigint" + ? selectedInode + : Number(selectedInode); + } + return metadata; + }) as typeof filesystem.lstat); + const opening = spyOn(filesystem, "open").mockImplementation( + async (...args: Parameters) => { + if (String(args[0]) !== canonicalSelected) { + return await originalOpen(...args); + } + replaced = true; + const file = await originalOpen(replacement, args[1], args[2]); + const originalStat = file.stat.bind(file); + const openedStat = spyOn(file, "stat").mockImplementation((async ( + ...statArgs: Parameters + ) => { + const metadata = await originalStat(...statArgs); + metadata.ino = + typeof metadata.ino === "bigint" + ? replacementInode + : Number(replacementInode); + return metadata; + }) as typeof file.stat); + restoreOpenedStat = () => openedStat.mockRestore(); + return file; + }, + ); + try { + let started = false; + const stderr = capture(); + const status = await main( + ["validate", "finding.txt"], + capture().stream, + stderr.stream, + dependencies({ + currentDirectory: root, + onCodex: () => { + started = true; + return 0; + }, + }), + ); + expect(replaced).toBe(true); + expect(status).toBe(2); + expect(stderr.text()).not.toContain("SYNTHETIC_REPLACEMENT_FINDING"); + expect(started).toBe(false); + } finally { + restoreOpenedStat?.(); + opening.mockRestore(); + reading.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + }); + test.each( process.platform === "win32" ? ["symbolic link"] : ["symbolic link", "FIFO"], )("rejects finding files replaced with a %s", async (replacement) => { + if ( + runMockInSubprocess( + import.meta.path, + `rejects finding files replaced with a ${replacement}`, + ) + ) { + return; + } const root = await mkdtemp(join(tmpdir(), "codex-security-skill-inputs-")); try { const repository = join(root, "repository"); @@ -475,6 +567,7 @@ describe("CLI skill commands", () => { const canonicalSelected = await filesystem.realpath(selected); const originalOpen = filesystem.open; + let replaced = false; const opening = spyOn(filesystem, "open").mockImplementation( async (...args: Parameters) => { if (String(args[0]) === canonicalSelected) { @@ -482,6 +575,7 @@ describe("CLI skill commands", () => { await rm(selected); if (replacement === "FIFO") execFileSync("mkfifo", [selected]); else await symlink(external, selected); + replaced = true; } return await originalOpen(...args); }, @@ -490,20 +584,20 @@ describe("CLI skill commands", () => { try { let started = false; const stderr = capture(); - expect( - await main( - ["validate", "finding.txt"], - capture().stream, - stderr.stream, - dependencies({ - currentDirectory: repository, - onCodex: () => { - started = true; - return 0; - }, - }), - ), - ).toBe(2); + const status = await main( + ["validate", "finding.txt"], + capture().stream, + stderr.stream, + dependencies({ + currentDirectory: repository, + onCodex: () => { + started = true; + return 0; + }, + }), + ); + expect(replaced, "the file-open replacement hook ran").toBe(true); + expect(status).toBe(2); expect(stderr.text()).not.toContain("SYNTHETIC_EXTERNAL_FINDING"); expect(started).toBe(false); } finally { diff --git a/sdk/typescript/tests-ts/config.test.ts b/sdk/typescript/tests-ts/config.test.ts index 8951c6d62..6cc2c4765 100644 --- a/sdk/typescript/tests-ts/config.test.ts +++ b/sdk/typescript/tests-ts/config.test.ts @@ -62,6 +62,45 @@ function runPinnedCodex(codexHome: string, arguments_: readonly string[]) { ); } +function macOsSandboxUnavailable(): boolean { + if (process.platform !== "darwin") return false; + + // Check the host independently of the generated scan permission profile. + const result = Bun.spawnSync( + [ + "/usr/bin/sandbox-exec", + "-p", + "(version 1) (allow default)", + "/usr/bin/true", + ], + { stdout: "pipe", stderr: "pipe" }, + ); + return ( + result.exitCode !== 0 && + new TextDecoder().decode(result.stderr).trim() === + "sandbox-exec: sandbox_apply: Operation not permitted" + ); +} + +async function scanSandboxFixture() { + const root = await temporaryDirectory(); + const codexHome = join(root, "codex-home"); + const workspace = join(root, "workspace"); + const stateDirectory = join(root, "state"); + await Promise.all( + [codexHome, workspace, stateDirectory].map((path) => mkdir(path)), + ); + await writeCodexConfig( + join(codexHome, "config.toml"), + scanRuntimeCodexConfig( + await mergedCodexConfig({}), + stateDirectory, + codexHome, + ), + ); + return { root, codexHome, workspace }; +} + describe("Codex configuration", () => { test("automatically reviews scan execution approvals by default", async () => { expect(await mergedCodexConfig({})).toMatchObject({ @@ -333,63 +372,64 @@ describe("Codex configuration", () => { }); }); - test("denies writes outside the scan workspace and state directory", async () => { - const root = await temporaryDirectory(); - const codexHome = join(root, "codex-home"); - const workspace = join(root, "workspace"); - const stateDirectory = join(root, "state"); - await Promise.all( - [codexHome, workspace, stateDirectory].map((path) => mkdir(path)), - ); - await writeCodexConfig( - join(codexHome, "config.toml"), - scanRuntimeCodexConfig( - await mergedCodexConfig({}), - stateDirectory, - codexHome, - ), - ); - const node = Bun.which("node"); - expect(node).not.toBeNull(); - const attemptWrite = (path: string) => - runPinnedCodex(codexHome, [ - "sandbox", - "--config", - "permissions.codex_security_scan.network.enabled=true", - "--permission-profile", - "codex_security_scan", - "--cd", - workspace, - node!, - "-e", - "require('node:fs').writeFileSync(process.argv[1], 'probe')", - path, - ]); - - const allowed = join(workspace, "inside.txt"); - const permitted = attemptWrite(allowed); - const outside = join(root, "outside.txt"); - expect(attemptWrite(outside).exitCode).not.toBe(0); - await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" }); - if (permitted.exitCode !== 0) { - const details = new TextDecoder().decode(permitted.stderr); - if ( - process.platform === "linux" && - /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test( - details, - ) - ) { - expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe( - 0, + test("writes scan permissions accepted by the pinned Codex CLI", async () => { + const { codexHome, workspace } = await scanSandboxFixture(); + const result = runPinnedCodex(codexHome, [ + "--cd", + workspace, + "features", + "list", + ]); + expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0); + expect(result.stdout.length).toBeGreaterThan(0); + }); + + test.skipIf(macOsSandboxUnavailable())( + "denies writes outside the scan workspace and state directory", + async () => { + const { root, codexHome, workspace } = await scanSandboxFixture(); + const node = Bun.which("node"); + expect(node).not.toBeNull(); + const attemptWrite = (path: string) => + runPinnedCodex(codexHome, [ + "sandbox", + "--config", + "permissions.codex_security_scan.network.enabled=true", + "--permission-profile", + "codex_security_scan", + "--cd", + workspace, + node!, + "-e", + "require('node:fs').writeFileSync(process.argv[1], 'probe')", + path, + ]); + + const allowed = join(workspace, "inside.txt"); + const permitted = attemptWrite(allowed); + const outside = join(root, "outside.txt"); + expect(attemptWrite(outside).exitCode).not.toBe(0); + await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" }); + if (permitted.exitCode !== 0) { + const details = new TextDecoder().decode(permitted.stderr); + if ( + process.platform === "linux" && + /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test( + details, + ) + ) { + expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe( + 0, + ); + return; + } + throw new Error( + `The pinned Codex CLI rejected an allowed scan write: ${details}`, ); - return; } - throw new Error( - `The pinned Codex CLI rejected an allowed scan write: ${details}`, - ); - } - expect(await readFile(allowed, "utf8")).toBe("probe"); - }); + expect(await readFile(allowed, "utf8")).toBe("probe"); + }, + ); test("writes Windows sandbox settings accepted by the pinned Codex CLI", async () => { const root = await temporaryDirectory(); diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index dc72f52fe..807ecc753 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -1,5 +1,6 @@ import { readFile } from "node:fs/promises"; import { describe, expect, test } from "bun:test"; +import { parse } from "smol-toml"; import { CodexSecurity, CodexSecurityError, VERSION } from "../src/index.js"; import { main } from "../src/cli.js"; @@ -62,7 +63,7 @@ describe("TypeScript package skeleton", () => { } }); - test("uses the default test timeout consistently across CI platforms", async () => { + test("randomizes tests with the default timeout across CI platforms", async () => { const packageJson = JSON.parse( await readFile(new URL("../package.json", import.meta.url), "utf8"), ); @@ -70,10 +71,14 @@ describe("TypeScript package skeleton", () => { new URL("../../../.github/workflows/node-ci.yml", import.meta.url), "utf8", ); + const bunConfig = parse( + await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"), + ); expect(packageJson.scripts.test).toBe( "bun test --timeout 30000 ./tests-ts", ); + expect(bunConfig).toMatchObject({ test: { randomize: true } }); expect(ciWorkflow).toContain( "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}", );