test: randomize SDK tests by default

sdk/typescript/AGENTS.md

@@ -38,9 +38,11 @@ accepted and rejected inputs, and each real bug or security boundary.
 From the SDK directory, run a focused test while iterating, then run the package checks:
 
 ```bash
-bun test tests-ts/<module>.test.ts
-bun test --randomize --seed 12345
+bun test --timeout 30000 tests-ts/<module>.test.ts
+pnpm run test --seed 12345
 pnpm run types
 pnpm run format
 pnpm run test
 ```
+
+Tests run in random order by default. To reproduce a failure, use the seed printed in Bun's test summary.

sdk/typescript/bunfig.toml

@@ -0,0 +1,2 @@
+[test]
+randomize = true

sdk/typescript/tests-ts/api.test.ts

@@ -4578,10 +4578,7 @@ describe("CodexSecurity orchestration", () => {
                       expect(before["deep_scan"]).toMatchObject({
                         workers: index + 2,
                       });
-                      await Promise.race([
-                        concurrentScans,
-                        new Promise((resolve) => setTimeout(resolve, 5_000)),
-                      ]);
+                      await concurrentScans;
                       const after = parseToml(
                         await readFile(deepScanConfigPath!, "utf8"),
                       );
@@ -4604,7 +4601,9 @@ describe("CodexSecurity orchestration", () => {
     try {
       const results = await Promise.allSettled(
         clients.map((client, index) =>
-          client.run(repository, { mode: "deep", workers: index + 2 }),
+          client
+            .run(repository, { mode: "deep", workers: index + 2 })
+            .finally(releaseScans),
         ),
       );
       for (const result of results) {
@@ -4627,6 +4626,7 @@ describe("CodexSecurity orchestration", () => {
         ),
       ).toBe(true);
     } finally {
+      releaseScans();
       await Promise.all(clients.map(async (client) => await client.close()));
     }
   });

sdk/typescript/tests-ts/cli-skills.test.ts

@@ -13,6 +13,7 @@ import {
 } from "../src/cli.js";
 import type { LinearClientFactory } from "../src/linear.js";
 import { capture, dependencies } from "./cli-fixtures.js";
+import { runMockInSubprocess } from "./support/isolated-mock.js";
 
 function linearIssue(identifier: string) {
   return {
@@ -453,6 +454,14 @@ describe("CLI skill commands", () => {
       ? ["symbolic link"]
       : ["symbolic link", "FIFO"],
   )("rejects finding files replaced with a %s", async (replacement) => {
+    if (
+      runMockInSubprocess(
+        import.meta.path,
+        `rejects finding files replaced with a ${replacement}`,
+      )
+    ) {
+      return;
+    }
     const root = await mkdtemp(join(tmpdir(), "codex-security-skill-inputs-"));
     try {
       const repository = join(root, "repository");
@@ -464,13 +473,15 @@ describe("CLI skill commands", () => {
       const canonicalSelected = await filesystem.realpath(selected);
 
       const originalOpen = filesystem.open;
+      let replaced = false;
       const opening = spyOn(filesystem, "open").mockImplementation(
         async (...args: Parameters<typeof filesystem.open>) => {
           if (String(args[0]) === canonicalSelected) {
             opening.mockRestore();
             await rm(selected);
             if (replacement === "FIFO") execFileSync("mkfifo", [selected]);
             else await symlink(external, selected);
+            replaced = true;
           }
           return await originalOpen(...args);
         },
@@ -479,20 +490,20 @@ describe("CLI skill commands", () => {
       try {
         let started = false;
         const stderr = capture();
-        expect(
-          await main(
-            ["validate", "finding.txt"],
-            capture().stream,
-            stderr.stream,
-            dependencies({
-              currentDirectory: repository,
-              onCodex: () => {
-                started = true;
-                return 0;
-              },
-            }),
-          ),
-        ).toBe(2);
+        const status = await main(
+          ["validate", "finding.txt"],
+          capture().stream,
+          stderr.stream,
+          dependencies({
+            currentDirectory: repository,
+            onCodex: () => {
+              started = true;
+              return 0;
+            },
+          }),
+        );
+        expect(replaced, "the file-open replacement hook ran").toBe(true);
+        expect(status).toBe(2);
         expect(stderr.text()).not.toContain("SYNTHETIC_EXTERNAL_FINDING");
         expect(started).toBe(false);
       } finally {

sdk/typescript/tests-ts/config.test.ts

@@ -62,6 +62,45 @@ function runPinnedCodex(codexHome: string, arguments_: readonly string[]) {
   );
 }
 
+function macOsSandboxUnavailable(): boolean {
+  if (process.platform !== "darwin") return false;
+
+  // Check the host independently of the generated scan permission profile.
+  const result = Bun.spawnSync(
+    [
+      "/usr/bin/sandbox-exec",
+      "-p",
+      "(version 1) (allow default)",
+      "/usr/bin/true",
+    ],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  return (
+    result.exitCode !== 0 &&
+    new TextDecoder().decode(result.stderr).trim() ===
+      "sandbox-exec: sandbox_apply: Operation not permitted"
+  );
+}
+
+async function scanSandboxFixture() {
+  const root = await temporaryDirectory();
+  const codexHome = join(root, "codex-home");
+  const workspace = join(root, "workspace");
+  const stateDirectory = join(root, "state");
+  await Promise.all(
+    [codexHome, workspace, stateDirectory].map((path) => mkdir(path)),
+  );
+  await writeCodexConfig(
+    join(codexHome, "config.toml"),
+    scanRuntimeCodexConfig(
+      await mergedCodexConfig({}),
+      stateDirectory,
+      codexHome,
+    ),
+  );
+  return { root, codexHome, workspace };
+}
+
 describe("Codex configuration", () => {
   test("automatically reviews scan execution approvals by default", async () => {
     expect(await mergedCodexConfig({})).toMatchObject({
@@ -333,63 +372,64 @@ describe("Codex configuration", () => {
     });
   });
 
-  test("denies writes outside the scan workspace and state directory", async () => {
-    const root = await temporaryDirectory();
-    const codexHome = join(root, "codex-home");
-    const workspace = join(root, "workspace");
-    const stateDirectory = join(root, "state");
-    await Promise.all(
-      [codexHome, workspace, stateDirectory].map((path) => mkdir(path)),
-    );
-    await writeCodexConfig(
-      join(codexHome, "config.toml"),
-      scanRuntimeCodexConfig(
-        await mergedCodexConfig({}),
-        stateDirectory,
-        codexHome,
-      ),
-    );
-    const node = Bun.which("node");
-    expect(node).not.toBeNull();
-    const attemptWrite = (path: string) =>
-      runPinnedCodex(codexHome, [
-        "sandbox",
-        "--config",
-        "permissions.codex_security_scan.network.enabled=true",
-        "--permission-profile",
-        "codex_security_scan",
-        "--cd",
-        workspace,
-        node!,
-        "-e",
-        "require('node:fs').writeFileSync(process.argv[1], 'probe')",
-        path,
-      ]);
-
-    const allowed = join(workspace, "inside.txt");
-    const permitted = attemptWrite(allowed);
-    const outside = join(root, "outside.txt");
-    expect(attemptWrite(outside).exitCode).not.toBe(0);
-    await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" });
-    if (permitted.exitCode !== 0) {
-      const details = new TextDecoder().decode(permitted.stderr);
-      if (
-        process.platform === "linux" &&
-        /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test(
-          details,
-        )
-      ) {
-        expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe(
-          0,
+  test("writes scan permissions accepted by the pinned Codex CLI", async () => {
+    const { codexHome, workspace } = await scanSandboxFixture();
+    const result = runPinnedCodex(codexHome, [
+      "--cd",
+      workspace,
+      "features",
+      "list",
+    ]);
+    expect(result.exitCode, new TextDecoder().decode(result.stderr)).toBe(0);
+    expect(result.stdout.length).toBeGreaterThan(0);
+  });
+
+  test.skipIf(macOsSandboxUnavailable())(
+    "denies writes outside the scan workspace and state directory",
+    async () => {
+      const { root, codexHome, workspace } = await scanSandboxFixture();
+      const node = Bun.which("node");
+      expect(node).not.toBeNull();
+      const attemptWrite = (path: string) =>
+        runPinnedCodex(codexHome, [
+          "sandbox",
+          "--config",
+          "permissions.codex_security_scan.network.enabled=true",
+          "--permission-profile",
+          "codex_security_scan",
+          "--cd",
+          workspace,
+          node!,
+          "-e",
+          "require('node:fs').writeFileSync(process.argv[1], 'probe')",
+          path,
+        ]);
+
+      const allowed = join(workspace, "inside.txt");
+      const permitted = attemptWrite(allowed);
+      const outside = join(root, "outside.txt");
+      expect(attemptWrite(outside).exitCode).not.toBe(0);
+      await expect(stat(outside)).rejects.toMatchObject({ code: "ENOENT" });
+      if (permitted.exitCode !== 0) {
+        const details = new TextDecoder().decode(permitted.stderr);
+        if (
+          process.platform === "linux" &&
+          /bwrap: (?:setting up uid map: Permission denied|loopback: Failed RTM_NEWADDR: Operation not permitted)/u.test(
+            details,
+          )
+        ) {
+          expect(runPinnedCodex(codexHome, ["features", "list"]).exitCode).toBe(
+            0,
+          );
+          return;
+        }
+        throw new Error(
+          `The pinned Codex CLI rejected an allowed scan write: ${details}`,
         );
-        return;
       }
-      throw new Error(
-        `The pinned Codex CLI rejected an allowed scan write: ${details}`,
-      );
-    }
-    expect(await readFile(allowed, "utf8")).toBe("probe");
-  });
+      expect(await readFile(allowed, "utf8")).toBe("probe");
+    },
+  );
 
   test("writes Windows sandbox settings accepted by the pinned Codex CLI", async () => {
     const root = await temporaryDirectory();

sdk/typescript/tests-ts/skeleton.test.ts

@@ -1,5 +1,6 @@
 import { readFile } from "node:fs/promises";
 import { describe, expect, test } from "bun:test";
+import { parse } from "smol-toml";
 import { CodexSecurity, CodexSecurityError, VERSION } from "../src/index.js";
 import { main } from "../src/cli.js";
 
@@ -62,18 +63,22 @@ describe("TypeScript package skeleton", () => {
     }
   });
 
-  test("uses the default test timeout consistently across CI platforms", async () => {
+  test("randomizes tests with the default timeout across CI platforms", async () => {
     const packageJson = JSON.parse(
       await readFile(new URL("../package.json", import.meta.url), "utf8"),
     );
     const ciWorkflow = await readFile(
       new URL("../../../.github/workflows/node-ci.yml", import.meta.url),
       "utf8",
     );
+    const bunConfig = parse(
+      await readFile(new URL("../bunfig.toml", import.meta.url), "utf8"),
+    );
 
     expect(packageJson.scripts.test).toBe(
       "bun test --timeout 30000 ./tests-ts",
     );
+    expect(bunConfig).toMatchObject({ test: { randomize: true } });
     expect(ciWorkflow).toContain(
       "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}",
     );