diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index c6398851..b8704c94 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -66,6 +66,7 @@ jobs: run: pnpm --dir sdk/typescript run audit:prod - name: Typecheck + if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' run: pnpm --dir sdk/typescript run types - name: Test @@ -75,9 +76,23 @@ jobs: TMP: ${{ runner.temp }} TMPDIR: ${{ runner.temp }} CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" - run: pnpm --dir sdk/typescript run test + run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }} + + - name: Upload test reports + if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-22-test-reports + overwrite: true + path: | + sdk/typescript/reports/junit.xml + sdk/typescript/coverage/lcov.info + if-no-files-found: warn + retention-days: 14 - name: Check formatting + if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' run: pnpm --dir sdk/typescript run format - name: Pack @@ -89,21 +104,6 @@ jobs: shell: bash run: pnpm run check:package ../../dist/*.tgz - - name: Smoke-test Node.js runtime - working-directory: sdk/typescript - shell: bash - run: | - set -euo pipefail - node --input-type=module --eval ' - import { CodexSecurity } from "@openai/codex-security"; - - if (typeof CodexSecurity !== "function") { - throw new Error("The SDK does not export CodexSecurity."); - } - ' - node bin/codex-security.mjs --version - node bin/codex-security.mjs --help - windows-test: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }} runs-on: windows-latest @@ -162,16 +162,6 @@ jobs: CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts - - name: Typecheck - if: matrix.shard == 7 - working-directory: sdk/typescript - run: pnpm run types - - - name: Check formatting - if: matrix.shard == 7 - working-directory: sdk/typescript - run: pnpm run format - windows-verify: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify runs-on: windows-latest @@ -210,21 +200,6 @@ jobs: shell: bash run: pnpm run check:package ../../dist/*.tgz - - name: Smoke-test Node.js runtime - working-directory: sdk/typescript - shell: bash - run: | - set -euo pipefail - node --input-type=module --eval ' - import { CodexSecurity } from "@openai/codex-security"; - - if (typeof CodexSecurity !== "function") { - throw new Error("The SDK does not export CodexSecurity."); - } - ' - node bin/codex-security.mjs --version - node bin/codex-security.mjs --help - windows: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} runs-on: ubuntu-latest diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml new file mode 100644 index 00000000..86747639 --- /dev/null +++ b/.github/workflows/test-quality.yml @@ -0,0 +1,158 @@ +name: test-quality + +on: + pull_request: + paths: + - .github/workflows/test-quality.yml + workflow_dispatch: + schedule: + - cron: "23 9 * * 1" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CODEX_SECURITY_INTEGRATION: "0" + CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" + CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }} + +jobs: + runner: + name: ${{ matrix.os }} / ${{ matrix.mode }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + mode: [baseline, isolated, parallel, randomized] + include: + - mode: baseline + args: "" + - mode: isolated + args: --isolate + - mode: parallel + args: --parallel=2 + - mode: randomized + args: --isolate --randomize --seed=${{ github.event_name == 'pull_request' && 1 || github.run_number }} + - os: windows-latest + mode: shard-1 + args: --shard=1/7 + - os: windows-latest + mode: shard-2 + args: --shard=2/7 + - os: windows-latest + mode: shard-3 + args: --shard=3/7 + - os: windows-latest + mode: shard-4 + args: --shard=4/7 + - os: windows-latest + mode: shard-5 + args: --shard=5/7 + - os: windows-latest + mode: shard-6 + args: --shard=6/7 + - os: windows-latest + mode: shard-7 + args: --shard=7/7 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - name: Set up pnpm + run: npm install --global pnpm@11.9.0 --no-audit --no-fund + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - name: Install dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Prepare private Windows test root + if: runner.os == 'Windows' + id: windows-temp + shell: pwsh + run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1 + - name: Test runner mode + env: + TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }} + run: | + node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})" + pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml + - name: Upload runner report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runner-${{ matrix.os }}-${{ matrix.mode }} + overwrite: true + path: sdk/typescript/reports/runner-*.xml + if-no-files-found: error + retention-days: 14 + + compare: + name: Runner inventory and timing + if: always() && !cancelled() + needs: [runner] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: runner-* + merge-multiple: true + path: reports + - name: Compare inventories and outcomes + shell: bash + run: | + comparison_status=0 + for os in ubuntu-latest windows-latest; do + for mode in isolated parallel randomized; do + python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + done + done + python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + exit "$comparison_status" + + mutation: + name: Pure-module mutation trial + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + package_json_file: sdk/typescript/package.json + cache: true + cache_dependency_path: sdk/typescript/pnpm-lock.yaml + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - name: Install dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Run mutation trial + run: pnpm --dir sdk/typescript run test:mutation + - name: Upload mutation report + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mutation-report + overwrite: true + path: sdk/typescript/reports/mutation/ + if-no-files-found: warn + retention-days: 14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 303eb270..e4e6015f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,3 +47,6 @@ Maintainers update package dependencies and the committed lockfile in the canonical repository. The public release workflow installs that locked graph, tests the package, and publishes a verified artifact with npm provenance. GitHub Actions dependencies are maintained separately in this repository. + +See the [SDK testing guide](sdk/typescript/TESTING.md) for local checks, +test conventions, and the required and experimental CI jobs. diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignore index d39addef..350fb263 100644 --- a/sdk/typescript/.gitignore +++ b/sdk/typescript/.gitignore @@ -1,5 +1,6 @@ /dist/ /node_modules +/coverage/ /reports/ /.stryker-tmp/ /private_release/dist/ diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md new file mode 100644 index 00000000..781797a2 --- /dev/null +++ b/sdk/typescript/TESTING.md @@ -0,0 +1,93 @@ +# Testing the SDK and CLI + +Use the pnpm version in `package.json` and Bun 1.3.14, matching CI. Run these +commands from `sdk/typescript`: + +```sh +pnpm install --frozen-lockfile +bun test --timeout 30000 ./tests-ts/worker-progress.test.ts +pnpm run types +pnpm run format +pnpm run test +pnpm run test:ci +pnpm pack --pack-destination ../../dist +pnpm run test:package +``` + +For CI's full archive inspection, pass the exact `.tgz` path printed by +`pnpm pack` to `pnpm run check:package`. + +The local test commands pass a 30-second per-test timeout explicitly. Windows +CI and the Windows runner experiment allow 120 seconds for slower native +credential and document checks. `test:ci` writes `reports/junit.xml` and +`coverage/lcov.info`. Coverage measures loaded +JavaScript and TypeScript, not the Python helpers or child processes. It is +diagnostic for now. Use several successful CI runs to establish a baseline +before proposing a coverage floor. + +## Writing tests + +- Test observable results, failures, cancellation, and cleanup. Prefer a + regression case that fails before a fix over assertions about private calls + or exact prose. +- Keep fixtures synthetic and independent. Use real temporary directories, + Git repositories, SQLite databases, and installed packages when those + boundaries are the behavior under test. Do not use live model credentials. +- Use the typed `TestClient` and `createApiTestFixtures` helpers for API tests. + Do not add a production abstraction solely to support a mock. +- Restore spies, timers, and environment changes. Tests that change the process + cwd or install persistent ESM module mocks use `runTestInSubprocess`. + Per-file Bun isolation does not isolate process-wide state inside one file. +- Keep shared behavior enabled on Linux, macOS, and Windows. The constrained + PowerShell test changes machine-wide policy and runs alone, only on an + explicitly enabled GitHub-hosted Windows runner. +- Add property tests for meaningful invariants, with accepted and rejected + inputs. Keep example-based regression tests for readable failure cases. +- Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values + that are not strings. The JUnit comparison rejects duplicate identities. + +Property tests use a fixed default seed. Fast-check prints the seed, shrink +path, and counterexample on failure. To replay one property, select its file +and test name, then set `CODEX_SECURITY_PROPERTY_SEED` and +`CODEX_SECURITY_PROPERTY_PATH` to the reported values. Set +`CODEX_SECURITY_PROPERTY_RUNS` to increase the case count. Pure properties +default to 100 cases; filesystem contract properties default to 20. + +## GitHub Actions + +`node-ci` retains the required `ubuntu-latest / node-22`, +`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu +Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime +lanes still test and inspect an installed package. Package inspection includes +a strict NodeNext TypeScript consumer and the actual installed CLI. Failed +tests block CI; a failed diagnostic upload does not. + +The separate `test-quality` workflow runs weekly, can be dispatched manually, +and runs on pull requests that change its workflow file. It exercises Bun's +native `--isolate`, `--parallel=2`, randomized +ordering, and seven-way Windows sharding. It compares test identities and +outcomes against an unsharded run and records timings in the job summary. +Pull requests replay seed 1; scheduled and manual runs use the workflow run +number for both property cases and test ordering. +It is not a required check or part of the release trigger. + +Keep the current file-balanced Windows runner until the native runner has +matching inventories and acceptable Windows timings. Before promotion, compare +the slowest native shard with the current required shards on the same commit. +Keep the machine-policy test serial. Do not replace the full required suite +with `--changed`: Python files, schemas, fixtures, and workflows loaded at +runtime are not necessarily part of Bun's import graph. + +## Mutation testing + +```sh +pnpm run test:mutation +pnpm exec stryker run --mutate src/worker-progress.ts +``` + +The initial Stryker trial covers progress parsing, safe error messages, and +pure cost arithmetic. It runs a small Bun suite without live services and +writes HTML and JSON under `reports/mutation`. Review surviving mutants for +missing behavior assertions or equivalent changes. There is no score gate yet; +set one only after the trial has a stable, useful baseline. Do not make a +surviving mutant disappear by adding assertions about implementation details. diff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.toml new file mode 100644 index 00000000..22686602 --- /dev/null +++ b/sdk/typescript/bunfig.toml @@ -0,0 +1,3 @@ +[test] +coverageSkipTestFiles = true +coveragePathIgnorePatterns = ["tests-ts/**", "**/node_modules/**", "dist/**"] diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 82c6aa10..05669a71 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -49,6 +49,7 @@ "lint": "tsc --noEmit", "prepack": "node --run build", "test": "bun test --timeout 30000 ./tests-ts", + "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && bun test --timeout 30000 ./tests-ts --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml", "test:mutation": "stryker run", "test:package": "node scripts/smoke-package.mjs", "types": "pnpm run generate:models:check && tsc --noEmit" diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.py new file mode 100644 index 00000000..6849ad31 --- /dev/null +++ b/sdk/typescript/scripts/compare-test-reports.py @@ -0,0 +1,80 @@ +"""Compare Bun JUnit inventories before changing the required CI runner.""" + +import argparse +from collections import Counter +from glob import glob +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + + +def read_report(path: Path) -> tuple[Counter, float]: + root = ET.parse(path).getroot() + cases = Counter() + identities = set() + for case in root.iter("testcase"): + status = "passed" + if case.find("skipped") is not None: + status = "skipped" + if case.find("failure") is not None or case.find("error") is not None: + status = "failed" + identity = ( + case.get("file", "").replace("\\", "/").removeprefix("./"), + case.get("classname", ""), + case.get("name", ""), + ) + if identity in identities: + raise ValueError(f"{path}: duplicate test identity: {' > '.join(identity)}") + identities.add(identity) + cases[(*identity, status)] += 1 + if not cases: + raise ValueError(f"{path}: no test cases") + if int(root.get("tests", str(sum(cases.values())))) != sum(cases.values()): + raise ValueError(f"{path}: reported test count does not match test cases") + if any(key[-1] == "failed" for key in cases) or any( + int(node.get(field, "0")) + for node in root.iter() + if node.tag in ("testsuite", "testsuites") + for field in ("failures", "errors") + ): + raise ValueError(f"{path}: test run failed") + seconds = float(root.get("time", "0")) + skipped = sum(count for key, count in cases.items() if key[-1] == "skipped") + print(f"| {path.name} | {sum(cases.values())} | {skipped} | {seconds:.2f} |") + return cases, seconds + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidates", nargs="+", help="JUnit files or glob patterns") + args = parser.parse_args() + print("| Report | Cases | Skipped | Seconds |") + print("| --- | ---: | ---: | ---: |") + baseline, _ = read_report(args.baseline) + candidates = Counter() + durations = [] + for pattern in args.candidates: + paths = sorted(glob(pattern)) + if not paths: + raise ValueError(f"No reports match {pattern}") + for path in paths: + cases, seconds = read_report(Path(path)) + candidates.update(cases) + durations.append(seconds) + missing, extra = baseline - candidates, candidates - baseline + if missing or extra: + for label, difference in (("Missing", missing), ("Extra", extra)): + for identity, count in sorted(difference.items()): + print(f"{label} ({count}): {' > '.join(identity)}", file=sys.stderr) + return 1 + print(f"\nIdentical test inventory and outcomes. Slowest candidate: {max(durations):.2f}s; combined test time: {sum(durations):.2f}s.\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, ET.ParseError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index 8d891cef..e6ffb4d9 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -1763,7 +1763,7 @@ describe("CLI", () => { }); test.each([false, true])( - "subscribes to session details only with TTY stdin: %s", + "subscribes to session details only with TTY stdin: %p", async (stdinTTY) => { const descriptor = Object.getOwnPropertyDescriptor( process.stdin, diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index ce3fa8f4..5fafb3d9 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -125,7 +125,7 @@ describe("scan cost", () => { [{ cache_write_input_tokens: 0, cache_write_tokens: 15 }, 15], [{ cache_write_input_tokens: 0, cache_write_tokens: 80 }, 0], ] as const)( - "keeps workbench cache-write normalization aligned with SDK usage", + "keeps workbench cache-write normalization aligned with SDK usage for %j as %p tokens", async (cacheWrites, expectedCacheWrites) => { const { PLUGIN_ROOT } = await import("./plugin-root.js"); const python = Bun.which("python3") ?? Bun.which("python"); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index b4a0469c..cd09ed73 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -154,7 +154,7 @@ describe("deep scan workbench ownership", () => { [0.5, 0.5], [96, 96], ] as const)( - "resolves the configured discovery deadline %s as %s hours", + "resolves the configured discovery deadline %p as %p hours", async (configuredHours, expectedHours) => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-deadline-config-")), @@ -271,7 +271,7 @@ describe("deep scan workbench ownership", () => { }); test.each([false, true] as const)( - "backfills and repairs discovery deadline migration when already recorded: %s", + "backfills and repairs discovery deadline migration when already recorded: %p", (migrationRecorded) => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); @@ -327,7 +327,7 @@ describe("deep scan workbench ownership", () => { ["scoped_path", false], ["repository", true], ] as const)( - "returns an honest partial %s report with existing deferred work %s when saturated discovery exceeds its cost limit", + "returns an honest partial %s report with existing deferred work %p when saturated discovery exceeds its cost limit", async (inventoryStrategy, existingDeferred) => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-budget-recovery-")), diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index 2c62c042..4e21f8d5 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -385,7 +385,7 @@ describe("multiscan", () => { }); test.each([false, true])( - "continues scanning when a progress observer fails %s", + "continues scanning when a progress observer fails %p", async (asynchronous) => { const paths = await fixture(); const source = await repository(paths.root, "observer-failure"); @@ -1388,7 +1388,7 @@ describe("multiscan", () => { }); test.each([false, true])( - "never removes a replacement lock when owner creation fails (owner published: %s)", + "never removes a replacement lock when owner creation fails (owner published: %p)", async (ownerPublished) => { const paths = await fixture(); const source = await repository(paths.root, "owner-creation-race"); diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index 25ee45cd..778e6d93 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -3,6 +3,40 @@ import { describe, expect, test } from "bun:test"; import { CodexSecurity, CodexSecurityError, VERSION } from "../src/index.js"; import { main } from "../src/cli.js"; +interface WorkflowStep { + name?: string; + uses?: string; + with?: Record; + run?: string; + if?: string; + env?: Record; + "continue-on-error"?: boolean; +} + +interface WorkflowJob { + name?: string; + needs?: string[]; + strategy?: { matrix: Record }; + steps: WorkflowStep[]; +} + +async function workflow(name: string): Promise<{ + on: Record; + env?: Record; + jobs: Record; +}> { + return Bun.YAML.parse( + await readFile( + new URL(`../../../.github/workflows/${name}`, import.meta.url), + "utf8", + ), + ) as { + on: Record; + env?: Record; + jobs: Record; + }; +} + function capture(): { stream: Pick; text: () => string; @@ -48,43 +82,159 @@ describe("TypeScript package skeleton", () => { }); test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => { - const ciWorkflow = await readFile( - new URL("../../../.github/workflows/node-ci.yml", import.meta.url), - "utf8", + const { jobs } = await workflow("node-ci.yml"); + expect(jobs["test"]?.name).toBe( + "${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", ); - - expect(ciWorkflow).toContain( - "${{ matrix.node == '22.13.0' && '22' || matrix.node }}", + expect(jobs["test"]?.strategy?.matrix).toEqual({ + os: ["ubuntu-latest", "macos-latest"], + node: ["22.13.0"], + include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({ + os: "ubuntu-latest", + node, + })), + }); + expect(jobs["windows"]?.name).toBe( + "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", ); - expect(ciWorkflow).toContain('node: ["22.13.0"]'); - for (const version of ["24.0.0", "24", "26.0.0", "26"]) { - expect(ciWorkflow).toContain(`node: "${version}"`); - } + expect(jobs["windows"]?.needs).toEqual(["windows-test", "windows-verify"]); + expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([ + "22.13.0", + "24", + ]); }); test("keeps the default and Windows CI test timeouts", async () => { const packageJson = JSON.parse( await readFile(new URL("../package.json", import.meta.url), "utf8"), ); - const ciWorkflow = await readFile( - new URL("../../../.github/workflows/node-ci.yml", import.meta.url), - "utf8", - ); + const { jobs } = await workflow("node-ci.yml"); expect(packageJson.scripts.test).toBe( "bun test --timeout 30000 ./tests-ts", ); - expect(ciWorkflow).toContain( - "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}", + expect(packageJson.scripts["test:ci"]).toContain( + `${packageJson.scripts.test} `, ); - expect(ciWorkflow).toContain( - "run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts", + expect(jobs["windows-test"]?.steps).toContainEqual( + expect.objectContaining({ + run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}", + }), ); - expect(ciWorkflow).toContain( - "name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", + }); + + test("runs shared static checks once and keeps report upload non-blocking", async () => { + const { jobs } = await workflow("node-ci.yml"); + const steps = Object.values(jobs).flatMap((job) => job.steps); + for (const name of ["Typecheck", "Check formatting"]) { + expect(steps.filter((step) => step.name === name)).toEqual([ + expect.objectContaining({ + if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'", + }), + ]); + } + expect(steps.find((step) => step.name === "Test")).not.toHaveProperty( + "continue-on-error", ); - expect(ciWorkflow).toContain("run: pnpm --dir sdk/typescript run test"); - expect(ciWorkflow).not.toContain("--timeout 60000"); + expect( + steps.find((step) => step.name === "Upload test reports"), + ).toMatchObject({ + "continue-on-error": true, + }); + for (const name of ["test", "windows-verify"]) { + expect(jobs[name]?.steps).toContainEqual( + expect.objectContaining({ + run: "pnpm run check:package ../../dist/*.tgz", + }), + ); + } + }); + + test("keeps machine-wide policy changes out of parallel and experimental runs", async () => { + const ci = await workflow("node-ci.yml"); + const windows = ci.jobs["windows-test"]!.steps; + expect( + windows.find((step) => step.name === "Test shard ${{ matrix.shard }}") + ?.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"], + ).toBe("false"); + expect( + windows.find( + (step) => step.name === "Test machine-wide PowerShell policy", + ), + ).toMatchObject({ + if: "matrix.shard == 3 && runner.environment == 'github-hosted'", + env: { CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" }, + run: "bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts", + }); + const quality = await workflow("test-quality.yml"); + expect(Object.keys(quality.on).sort()).toEqual([ + "pull_request", + "schedule", + "workflow_dispatch", + ]); + expect(quality.on["pull_request"]).toEqual({ + paths: [".github/workflows/test-quality.yml"], + }); + expect(quality.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"]).toBe( + "false", + ); + expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0"); + for (let shard = 1; shard <= 7; shard += 1) { + expect( + quality.jobs["runner"]?.strategy?.matrix["include"], + ).toContainEqual({ + os: "windows-latest", + mode: `shard-${shard}`, + args: `--shard=${shard}/7`, + }); + } + }); + + test("keeps runner modes distinct and report uploads rerunnable", async () => { + const ci = await workflow("node-ci.yml"); + const quality = await workflow("test-quality.yml"); + const runner = quality.jobs["runner"]!; + const seed = + "${{ github.event_name == 'pull_request' && 1 || github.run_number }}"; + expect(quality.env?.["CODEX_SECURITY_PROPERTY_SEED"]).toBe(seed); + for (const [mode, args] of [ + ["baseline", ""], + ["isolated", "--isolate"], + ["parallel", "--parallel=2"], + ["randomized", `--isolate --randomize --seed=${seed}`], + ] as const) { + expect(runner.strategy?.matrix["include"]).toContainEqual({ + mode, + args, + }); + } + const command = runner.steps.find( + (step) => step.name === "Test runner mode", + )?.run; + expect(command).toContain( + "${{ runner.os == 'Windows' && '--timeout=120000' || '' }}", + ); + expect(command).toContain("${{ matrix.args }}"); + expect(command).not.toContain("--seed="); + + const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)] + .flatMap((job) => job.steps) + .filter((step) => step.uses?.startsWith("actions/upload-artifact@")); + expect(uploads).toHaveLength(3); + for (const upload of uploads) { + expect(upload.with?.["overwrite"]).toBe(true); + } + expect( + uploads.find((step) => step.name === "Upload mutation report"), + ).toMatchObject({ "continue-on-error": true }); + expect( + uploads.find((step) => step.name === "Upload runner report"), + ).not.toHaveProperty("continue-on-error"); + expect( + quality.jobs["mutation"]?.steps.find( + (step) => step.name === "Run mutation trial", + ), + ).not.toHaveProperty("continue-on-error"); }); test("builds packages without a preinstalled package manager and provides a production audit", async () => { @@ -103,14 +253,17 @@ describe("TypeScript package skeleton", () => { test("keeps production dependency audits non-blocking in CI and releases", async () => { for (const workflowName of ["node-ci.yml", "node-release.yml"]) { - const workflow = await readFile( - new URL(`../../../.github/workflows/${workflowName}`, import.meta.url), - "utf8", - ); - - expect(workflow).toMatch( - /- name: Audit production dependencies\n(?:\s+if: [^\n]+\n)?\s+continue-on-error: true\n\s+run: (?:sfw )?pnpm --dir sdk\/typescript run audit:prod/u, - ); + const { jobs } = await workflow(workflowName); + const audits = Object.values(jobs) + .flatMap((job) => job.steps) + .filter((step) => step.name === "Audit production dependencies"); + expect(audits.length).toBeGreaterThan(0); + for (const audit of audits) { + expect(audit["continue-on-error"]).toBe(true); + expect(audit.run).toMatch( + /^(?:sfw )?pnpm --dir sdk\/typescript run audit:prod$/u, + ); + } } }); diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts new file mode 100644 index 00000000..80dfff8c --- /dev/null +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -0,0 +1,178 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, test } from "bun:test"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +async function fixtures() { + const root = await mkdtemp(join(tmpdir(), "codex-security-test-reports-")); + directories.push(root); + return { + root, + async report( + name: string, + cases: string[], + failures = 0, + count = cases.length, + ) { + const path = join(root, name); + await writeFile( + path, + `${cases.join("")}`, + ); + return path; + }, + }; +} + +function testcase(name: string, status = "") { + return `${status}`; +} + +async function compare(baseline: string, ...candidates: string[]) { + const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + if (python === null) throw new Error("A Python interpreter is required."); + const child = Bun.spawn({ + cmd: [ + python, + "-I", + "-B", + fileURLToPath( + new URL("../scripts/compare-test-reports.py", import.meta.url), + ), + baseline, + ...candidates, + ], + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [status, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + return { status, stdout, stderr }; +} + +describe("JUnit inventory comparison", () => { + test("runs every workflow comparison before reporting a mismatch", async () => { + const fixture = await fixtures(); + const workflow = Bun.YAML.parse( + await readFile( + new URL("../../../.github/workflows/test-quality.yml", import.meta.url), + "utf8", + ), + ) as { + jobs: { compare: { steps: Array<{ name?: string; run?: string }> } }; + }; + const script = workflow.jobs.compare.steps.find( + (step) => step.name === "Compare inventories and outcomes", + )!.run!; + const expected = [ + ...["ubuntu-latest", "windows-latest"].flatMap((os) => + ["isolated", "parallel", "randomized"].map( + (mode) => `reports/runner-${os}-${mode}.xml`, + ), + ), + "reports/runner-windows-latest-shard-*.xml", + ]; + const mock = `python3() { + printf '%s\\n' "$3" + [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]] +}`; + const summary = join(fixture.root, "summary.md"); + for (const failedReport of ["", expected[0]!]) { + await writeFile(summary, ""); + const result = spawnSync( + "bash", + ["-e", "-o", "pipefail", "-c", `${mock}\n${script}`], + { + cwd: fixture.root, + encoding: "utf8", + env: { + ...process.env, + GITHUB_STEP_SUMMARY: "summary.md", + CODEX_SECURITY_TEST_FAIL_REPORT: failedReport, + }, + timeout: 10_000, + }, + ); + expect(result.status, result.stderr).toBe(failedReport === "" ? 0 : 1); + expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual( + expected, + ); + } + }); + + test("merges native shards without depending on test order", async () => { + const fixture = await fixtures(); + const passed = testcase("accepts & preserves"); + const skipped = testcase("platform-only", ""); + const baseline = await fixture.report("baseline.xml", [passed, skipped]); + await fixture.report("shard-1.xml", [skipped]); + await fixture.report("shard-2.xml", [passed]); + const result = await compare(baseline, join(fixture.root, "shard-*.xml")); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Identical test inventory and outcomes"); + expect(result.stdout).toContain("combined test time: 2.50s"); + }); + + test("rejects ambiguous test identities even when totals match", async () => { + const fixture = await fixtures(); + const first = testcase("same parameterized name"); + for (const [name, repeated] of [ + ["same-outcome", first], + ["different-outcome", testcase("same parameterized name", "")], + ] as const) { + const baseline = await fixture.report(`${name}-baseline.xml`, [ + first, + repeated, + ]); + const candidate = await fixture.report(`${name}-candidate.xml`, [ + first, + repeated, + ]); + const result = await compare(baseline, candidate); + expect(result.status).toBe(1); + expect(result.stderr).toContain("duplicate test identity"); + } + }); + + test("rejects dropped, duplicated, skipped, failed, or incomplete results", async () => { + const fixture = await fixtures(); + const first = testcase("first"); + const second = testcase("second"); + const baseline = await fixture.report("baseline.xml", [first, second]); + for (const [name, cases, failures, count] of [ + ["missing", [first], 0, 1], + ["duplicate", [first, second, second], 0, 3], + ["skipped", [first, testcase("second", "")], 0, 2], + ["failed", [first, testcase("second", "")], 1, 2], + ["summary-failed", [first, second], 1, 2], + ["incomplete", [first], 0, 2], + ["empty", [], 0, 0], + ] as const) { + const candidate = await fixture.report( + `${name}.xml`, + [...cases], + failures, + count, + ); + expect((await compare(baseline, candidate)).status, name).toBe(1); + } + expect( + (await compare(baseline, join(fixture.root, "absent-*.xml"))).status, + ).toBe(1); + }); +});