From 04d5c56159a8886a4d1b4668e0cfa617bab10ab6 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 12:17:05 -0700 Subject: [PATCH 1/7] ci: report test coverage and exercise native Bun runners --- .github/workflows/node-ci.yml | 56 ++---- .github/workflows/test-quality.yml | 150 +++++++++++++++++ CONTRIBUTING.md | 3 + sdk/typescript/.gitignore | 1 + sdk/typescript/TESTING.md | 83 +++++++++ sdk/typescript/bunfig.toml | 3 + sdk/typescript/package.json | 1 + .../scripts/compare-test-reports.py | 77 +++++++++ sdk/typescript/tests-ts/skeleton.test.ts | 159 +++++++++++++++--- sdk/typescript/tests-ts/test-reports.test.ts | 100 +++++++++++ 10 files changed, 564 insertions(+), 69 deletions(-) create mode 100644 .github/workflows/test-quality.yml create mode 100644 sdk/typescript/TESTING.md create mode 100644 sdk/typescript/bunfig.toml create mode 100644 sdk/typescript/scripts/compare-test-reports.py create mode 100644 sdk/typescript/tests-ts/test-reports.test.ts diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index ae0998420..d61df1abe 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -66,6 +66,7 @@ jobs: run: pnpm --dir sdk/typescript run audit:prod - name: Typecheck + if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' run: pnpm --dir sdk/typescript run types - name: Test @@ -75,9 +76,22 @@ jobs: TMP: ${{ runner.temp }} TMPDIR: ${{ runner.temp }} CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" - run: pnpm --dir sdk/typescript run test + run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }} + + - name: Upload test reports + if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-22-test-reports + path: | + sdk/typescript/reports/junit.xml + sdk/typescript/coverage/lcov.info + if-no-files-found: warn + retention-days: 14 - name: Check formatting + if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' run: pnpm --dir sdk/typescript run format - name: Pack @@ -89,21 +103,6 @@ jobs: shell: bash run: pnpm run check:package ../../dist/*.tgz - - name: Smoke-test Node.js runtime - working-directory: sdk/typescript - shell: bash - run: | - set -euo pipefail - node --input-type=module --eval ' - import { CodexSecurity } from "@openai/codex-security"; - - if (typeof CodexSecurity !== "function") { - throw new Error("The SDK does not export CodexSecurity."); - } - ' - node bin/codex-security.mjs --version - node bin/codex-security.mjs --help - windows-test: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }} runs-on: windows-latest @@ -162,16 +161,6 @@ jobs: CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" run: bun test --timeout 30000 ./tests-ts/windows-machine-policy.test.ts - - name: Typecheck - if: matrix.shard == 7 - working-directory: sdk/typescript - run: pnpm run types - - - name: Check formatting - if: matrix.shard == 7 - working-directory: sdk/typescript - run: pnpm run format - windows-verify: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify runs-on: windows-latest @@ -210,21 +199,6 @@ jobs: shell: bash run: pnpm run check:package ../../dist/*.tgz - - name: Smoke-test Node.js runtime - working-directory: sdk/typescript - shell: bash - run: | - set -euo pipefail - node --input-type=module --eval ' - import { CodexSecurity } from "@openai/codex-security"; - - if (typeof CodexSecurity !== "function") { - throw new Error("The SDK does not export CodexSecurity."); - } - ' - node bin/codex-security.mjs --version - node bin/codex-security.mjs --help - windows: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} runs-on: ubuntu-latest diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml new file mode 100644 index 000000000..6695a3926 --- /dev/null +++ b/.github/workflows/test-quality.yml @@ -0,0 +1,150 @@ +name: test-quality + +on: + workflow_dispatch: + schedule: + - cron: "23 9 * * 1" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CODEX_SECURITY_INTEGRATION: "0" + CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" + CODEX_SECURITY_PROPERTY_SEED: ${{ github.run_number }} + +jobs: + runner: + name: ${{ matrix.os }} / ${{ matrix.mode }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + mode: [baseline, isolated, parallel, randomized] + include: + - mode: baseline + args: "" + - mode: isolated + args: --isolate + - mode: parallel + args: --parallel=2 + - mode: randomized + args: --isolate --randomize + - os: windows-latest + mode: shard-1 + args: --shard=1/7 + - os: windows-latest + mode: shard-2 + args: --shard=2/7 + - os: windows-latest + mode: shard-3 + args: --shard=3/7 + - os: windows-latest + mode: shard-4 + args: --shard=4/7 + - os: windows-latest + mode: shard-5 + args: --shard=5/7 + - os: windows-latest + mode: shard-6 + args: --shard=6/7 + - os: windows-latest + mode: shard-7 + args: --shard=7/7 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - name: Set up pnpm + run: npm install --global pnpm@11.9.0 --no-audit --no-fund + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - name: Install dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Prepare private Windows test root + if: runner.os == 'Windows' + id: windows-temp + shell: pwsh + run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1 + - name: Test runner mode + env: + TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }} + run: | + node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})" + pnpm --dir sdk/typescript run test ${{ matrix.args }} --seed=${{ github.run_number }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml + - name: Upload runner report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runner-${{ matrix.os }}-${{ matrix.mode }} + path: sdk/typescript/reports/runner-*.xml + if-no-files-found: error + retention-days: 14 + + compare: + name: Runner inventory and timing + if: always() && !cancelled() + needs: [runner] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: runner-* + merge-multiple: true + path: reports + - name: Compare inventories and outcomes + shell: bash + run: | + for os in ubuntu-latest windows-latest; do + for mode in isolated parallel randomized; do + python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" + done + done + python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" + + mutation: + name: Pure-module mutation trial + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + package_json_file: sdk/typescript/package.json + cache: true + cache_dependency_path: sdk/typescript/pnpm-lock.yaml + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.3.14" + - name: Install dependencies + run: pnpm --dir sdk/typescript install --frozen-lockfile + - name: Run mutation trial + run: pnpm --dir sdk/typescript run test:mutation + - name: Upload mutation report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mutation-report + path: sdk/typescript/reports/mutation/ + if-no-files-found: warn + retention-days: 14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 303eb270a..e4e6015f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,3 +47,6 @@ Maintainers update package dependencies and the committed lockfile in the canonical repository. The public release workflow installs that locked graph, tests the package, and publishes a verified artifact with npm provenance. GitHub Actions dependencies are maintained separately in this repository. + +See the [SDK testing guide](sdk/typescript/TESTING.md) for local checks, +test conventions, and the required and experimental CI jobs. diff --git a/sdk/typescript/.gitignore b/sdk/typescript/.gitignore index d39addef6..350fb263f 100644 --- a/sdk/typescript/.gitignore +++ b/sdk/typescript/.gitignore @@ -1,5 +1,6 @@ /dist/ /node_modules +/coverage/ /reports/ /.stryker-tmp/ /private_release/dist/ diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md new file mode 100644 index 000000000..d638b9214 --- /dev/null +++ b/sdk/typescript/TESTING.md @@ -0,0 +1,83 @@ +# Testing the SDK and CLI + +Use the pnpm version in `package.json` and Bun 1.3.14, matching CI. Run these +commands from `sdk/typescript`: + +```sh +pnpm install --frozen-lockfile +bun test --timeout 30000 ./tests-ts/worker-progress.test.ts +pnpm run types +pnpm run format +pnpm run test +pnpm run test:ci +pnpm pack --pack-destination ../../dist +pnpm run check:package ../../dist/*.tgz +``` + +The test commands pass a 30-second per-test timeout explicitly. `test:ci` writes +`reports/junit.xml` and `coverage/lcov.info`. Coverage measures loaded +JavaScript and TypeScript, not the Python helpers or child processes. It is +diagnostic for now. Use several successful CI runs to establish a baseline +before proposing a coverage floor. + +## Writing tests + +- Test observable results, failures, cancellation, and cleanup. Prefer a + regression case that fails before a fix over assertions about private calls + or exact prose. +- Keep fixtures synthetic and independent. Use real temporary directories, + Git repositories, SQLite databases, and installed packages when those + boundaries are the behavior under test. Do not use live model credentials. +- Use the typed `TestClient` and `createApiTestFixtures` helpers for API tests. + Do not add a production abstraction solely to support a mock. +- Restore spies, timers, and environment changes. Tests that change the process + cwd or install persistent ESM module mocks use `runTestInSubprocess`. + Per-file Bun isolation does not isolate process-wide state inside one file. +- Keep shared behavior enabled on Linux, macOS, and Windows. The constrained + PowerShell test changes machine-wide policy and runs alone, only on an + explicitly enabled GitHub-hosted Windows runner. +- Add property tests for meaningful invariants, with accepted and rejected + inputs. Keep example-based regression tests for readable failure cases. + +Property tests use a fixed default seed. Fast-check prints the seed, shrink +path, and counterexample on failure. To replay one property, select its file +and test name, then set `CODEX_SECURITY_PROPERTY_SEED` and +`CODEX_SECURITY_PROPERTY_PATH` to the reported values. Set +`CODEX_SECURITY_PROPERTY_RUNS` to increase the case count. Pure properties +default to 100 cases; filesystem contract properties default to 20. + +## GitHub Actions + +`node-ci` retains the required `ubuntu-latest / node-22`, +`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu +Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime +lanes still test and inspect an installed package. Package inspection includes +a strict NodeNext TypeScript consumer and the actual installed CLI. Failed +tests block CI; a failed diagnostic upload does not. + +The separate `test-quality` workflow runs weekly and can be dispatched +manually. It exercises Bun's native `--isolate`, `--parallel=2`, randomized +ordering, and seven-way Windows sharding. It compares test identities and +outcomes against an unsharded run and records timings in the job summary. +It is not a required check or part of the release trigger. + +Keep the current file-balanced Windows runner until the native runner has +matching inventories and acceptable Windows timings. Before promotion, compare +the slowest native shard with the current required shards on the same commit. +Keep the machine-policy test serial. Do not replace the full required suite +with `--changed`: Python files, schemas, fixtures, and workflows loaded at +runtime are not necessarily part of Bun's import graph. + +## Mutation testing + +```sh +pnpm run test:mutation +pnpm exec stryker run --mutate src/worker-progress.ts +``` + +The initial Stryker trial covers progress parsing, safe error messages, and +pure cost arithmetic. It runs a small Bun suite without live services and +writes HTML and JSON under `reports/mutation`. Review surviving mutants for +missing behavior assertions or equivalent changes. There is no score gate yet; +set one only after the trial has a stable, useful baseline. Do not make a +surviving mutant disappear by adding assertions about implementation details. diff --git a/sdk/typescript/bunfig.toml b/sdk/typescript/bunfig.toml new file mode 100644 index 000000000..226866027 --- /dev/null +++ b/sdk/typescript/bunfig.toml @@ -0,0 +1,3 @@ +[test] +coverageSkipTestFiles = true +coveragePathIgnorePatterns = ["tests-ts/**", "**/node_modules/**", "dist/**"] diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 82c6aa106..05669a710 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -49,6 +49,7 @@ "lint": "tsc --noEmit", "prepack": "node --run build", "test": "bun test --timeout 30000 ./tests-ts", + "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && bun test --timeout 30000 ./tests-ts --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml", "test:mutation": "stryker run", "test:package": "node scripts/smoke-package.mjs", "types": "pnpm run generate:models:check && tsc --noEmit" diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.py new file mode 100644 index 000000000..ce9fe8b2e --- /dev/null +++ b/sdk/typescript/scripts/compare-test-reports.py @@ -0,0 +1,77 @@ +"""Compare Bun JUnit inventories before changing the required CI runner.""" + +import argparse +from collections import Counter +from glob import glob +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + + +def read_report(path: Path) -> tuple[Counter, float]: + root = ET.parse(path).getroot() + cases = Counter() + for case in root.iter("testcase"): + status = "passed" + if case.find("skipped") is not None: + status = "skipped" + if case.find("failure") is not None or case.find("error") is not None: + status = "failed" + identity = ( + case.get("file", "").replace("\\", "/").removeprefix("./"), + case.get("classname", ""), + case.get("name", ""), + status, + ) + cases[identity] += 1 + if not cases: + raise ValueError(f"{path}: no test cases") + if int(root.get("tests", str(sum(cases.values())))) != sum(cases.values()): + raise ValueError(f"{path}: reported test count does not match test cases") + if any(key[-1] == "failed" for key in cases) or any( + int(node.get(field, "0")) + for node in root.iter() + if node.tag in ("testsuite", "testsuites") + for field in ("failures", "errors") + ): + raise ValueError(f"{path}: test run failed") + seconds = float(root.get("time", "0")) + skipped = sum(count for key, count in cases.items() if key[-1] == "skipped") + print(f"| {path.name} | {sum(cases.values())} | {skipped} | {seconds:.2f} |") + return cases, seconds + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("baseline", type=Path) + parser.add_argument("candidates", nargs="+", help="JUnit files or glob patterns") + args = parser.parse_args() + print("| Report | Cases | Skipped | Seconds |") + print("| --- | ---: | ---: | ---: |") + baseline, _ = read_report(args.baseline) + candidates = Counter() + durations = [] + for pattern in args.candidates: + paths = sorted(glob(pattern)) + if not paths: + raise ValueError(f"No reports match {pattern}") + for path in paths: + cases, seconds = read_report(Path(path)) + candidates.update(cases) + durations.append(seconds) + missing, extra = baseline - candidates, candidates - baseline + if missing or extra: + for label, difference in (("Missing", missing), ("Extra", extra)): + for identity, count in sorted(difference.items()): + print(f"{label} ({count}): {' > '.join(identity)}", file=sys.stderr) + return 1 + print(f"\nIdentical test inventory and outcomes. Slowest candidate: {max(durations):.2f}s; combined test time: {sum(durations):.2f}s.\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, ET.ParseError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index dc72f52fe..db320d59d 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -3,6 +3,38 @@ import { describe, expect, test } from "bun:test"; import { CodexSecurity, CodexSecurityError, VERSION } from "../src/index.js"; import { main } from "../src/cli.js"; +interface WorkflowStep { + name?: string; + run?: string; + if?: string; + env?: Record; + "continue-on-error"?: boolean; +} + +interface WorkflowJob { + name?: string; + needs?: string[]; + strategy?: { matrix: Record }; + steps: WorkflowStep[]; +} + +async function workflow(name: string): Promise<{ + on: Record; + env?: Record; + jobs: Record; +}> { + return Bun.YAML.parse( + await readFile( + new URL(`../../../.github/workflows/${name}`, import.meta.url), + "utf8", + ), + ) as { + on: Record; + env?: Record; + jobs: Record; + }; +} + function capture(): { stream: Pick; text: () => string; @@ -48,40 +80,108 @@ describe("TypeScript package skeleton", () => { }); test("pins each Node.js minimum and preserves protected and latest LTS checks", async () => { - const ciWorkflow = await readFile( - new URL("../../../.github/workflows/node-ci.yml", import.meta.url), - "utf8", + const { jobs } = await workflow("node-ci.yml"); + expect(jobs["test"]?.name).toBe( + "${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", ); - - expect(ciWorkflow).toContain( - "${{ matrix.node == '22.13.0' && '22' || matrix.node }}", + expect(jobs["test"]?.strategy?.matrix).toEqual({ + os: ["ubuntu-latest", "macos-latest"], + node: ["22.13.0"], + include: ["24.0.0", "24", "26.0.0", "26"].map((node) => ({ + os: "ubuntu-latest", + node, + })), + }); + expect(jobs["windows"]?.name).toBe( + "windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", ); - expect(ciWorkflow).toContain('node: ["22.13.0"]'); - for (const version of ["24.0.0", "24", "26.0.0", "26"]) { - expect(ciWorkflow).toContain(`node: "${version}"`); - } + expect(jobs["windows"]?.needs).toEqual(["windows-test", "windows-verify"]); + expect(jobs["windows-test"]?.strategy?.matrix["node"]).toEqual([ + "22.13.0", + "24", + ]); }); test("uses the default test timeout consistently across CI platforms", async () => { const packageJson = JSON.parse( await readFile(new URL("../package.json", import.meta.url), "utf8"), ); - const ciWorkflow = await readFile( - new URL("../../../.github/workflows/node-ci.yml", import.meta.url), - "utf8", - ); + const { jobs } = await workflow("node-ci.yml"); expect(packageJson.scripts.test).toBe( "bun test --timeout 30000 ./tests-ts", ); - expect(ciWorkflow).toContain( - "run: node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}", + expect(packageJson.scripts["test:ci"]).toContain( + `${packageJson.scripts.test} `, ); - expect(ciWorkflow).toContain( - "name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}", + expect(jobs["windows-test"]?.steps).toContainEqual( + expect.objectContaining({ + run: "node sdk/typescript/scripts/run-windows-ci-tests.mjs ${{ matrix.shard }}", + }), ); - expect(ciWorkflow).toContain("run: pnpm --dir sdk/typescript run test"); - expect(ciWorkflow).not.toContain("--timeout 60000"); + }); + + test("runs shared static checks once and keeps report upload non-blocking", async () => { + const { jobs } = await workflow("node-ci.yml"); + const steps = Object.values(jobs).flatMap((job) => job.steps); + for (const name of ["Typecheck", "Check formatting"]) { + expect(steps.filter((step) => step.name === name)).toEqual([ + expect.objectContaining({ + if: "matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'", + }), + ]); + } + expect(steps.find((step) => step.name === "Test")).not.toHaveProperty( + "continue-on-error", + ); + expect( + steps.find((step) => step.name === "Upload test reports"), + ).toMatchObject({ + "continue-on-error": true, + }); + for (const name of ["test", "windows-verify"]) { + expect(jobs[name]?.steps).toContainEqual( + expect.objectContaining({ + run: "pnpm run check:package ../../dist/*.tgz", + }), + ); + } + }); + + test("keeps machine-wide policy changes out of parallel and experimental runs", async () => { + const ci = await workflow("node-ci.yml"); + const windows = ci.jobs["windows-test"]!.steps; + expect( + windows.find((step) => step.name === "Test shard ${{ matrix.shard }}") + ?.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"], + ).toBe("false"); + expect( + windows.find( + (step) => step.name === "Test machine-wide PowerShell policy", + ), + ).toMatchObject({ + if: "matrix.shard == 3 && runner.environment == 'github-hosted'", + env: { CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true" }, + run: "bun test --timeout 30000 ./tests-ts/windows-machine-policy.test.ts", + }); + const quality = await workflow("test-quality.yml"); + expect(Object.keys(quality.on).sort()).toEqual([ + "schedule", + "workflow_dispatch", + ]); + expect(quality.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"]).toBe( + "false", + ); + expect(quality.env?.["CODEX_SECURITY_INTEGRATION"]).toBe("0"); + for (let shard = 1; shard <= 7; shard += 1) { + expect( + quality.jobs["runner"]?.strategy?.matrix["include"], + ).toContainEqual({ + os: "windows-latest", + mode: `shard-${shard}`, + args: `--shard=${shard}/7`, + }); + } }); test("builds packages without a preinstalled package manager and provides a production audit", async () => { @@ -100,14 +200,17 @@ describe("TypeScript package skeleton", () => { test("keeps production dependency audits non-blocking in CI and releases", async () => { for (const workflowName of ["node-ci.yml", "node-release.yml"]) { - const workflow = await readFile( - new URL(`../../../.github/workflows/${workflowName}`, import.meta.url), - "utf8", - ); - - expect(workflow).toMatch( - /- name: Audit production dependencies\n(?:\s+if: [^\n]+\n)?\s+continue-on-error: true\n\s+run: (?:sfw )?pnpm --dir sdk\/typescript run audit:prod/u, - ); + const { jobs } = await workflow(workflowName); + const audits = Object.values(jobs) + .flatMap((job) => job.steps) + .filter((step) => step.name === "Audit production dependencies"); + expect(audits.length).toBeGreaterThan(0); + for (const audit of audits) { + expect(audit["continue-on-error"]).toBe(true); + expect(audit.run).toMatch( + /^(?:sfw )?pnpm --dir sdk\/typescript run audit:prod$/u, + ); + } } }); diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts new file mode 100644 index 000000000..41b626d18 --- /dev/null +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -0,0 +1,100 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, test } from "bun:test"; + +const directories: string[] = []; +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +async function fixtures() { + const root = await mkdtemp(join(tmpdir(), "codex-security-test-reports-")); + directories.push(root); + return { + root, + async report( + name: string, + cases: string[], + failures = 0, + count = cases.length, + ) { + const path = join(root, name); + await writeFile( + path, + `${cases.join("")}`, + ); + return path; + }, + }; +} + +function testcase(name: string, status = "") { + return `${status}`; +} + +function compare(baseline: string, ...candidates: string[]) { + const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); + if (python === null) throw new Error("A Python interpreter is required."); + return spawnSync( + python, + [ + "-I", + "-B", + fileURLToPath( + new URL("../scripts/compare-test-reports.py", import.meta.url), + ), + baseline, + ...candidates, + ], + { encoding: "utf8" }, + ); +} + +describe("JUnit inventory comparison", () => { + test("merges native shards without depending on test order", async () => { + const fixture = await fixtures(); + const passed = testcase("accepts & preserves"); + const skipped = testcase("platform-only", ""); + const baseline = await fixture.report("baseline.xml", [passed, skipped]); + await fixture.report("shard-1.xml", [skipped]); + await fixture.report("shard-2.xml", [passed]); + const result = compare(baseline, join(fixture.root, "shard-*.xml")); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain("Identical test inventory and outcomes"); + expect(result.stdout).toContain("combined test time: 2.50s"); + }); + + test("rejects dropped, duplicated, skipped, failed, or incomplete results", async () => { + const fixture = await fixtures(); + const first = testcase("first"); + const second = testcase("second"); + const baseline = await fixture.report("baseline.xml", [first, second]); + for (const [name, cases, failures, count] of [ + ["missing", [first], 0, 1], + ["duplicate", [first, second, second], 0, 3], + ["skipped", [first, testcase("second", "")], 0, 2], + ["failed", [first, testcase("second", "")], 1, 2], + ["summary-failed", [first, second], 1, 2], + ["incomplete", [first], 0, 2], + ["empty", [], 0, 0], + ] as const) { + const candidate = await fixture.report( + `${name}.xml`, + [...cases], + failures, + count, + ); + expect(compare(baseline, candidate).status, name).toBe(1); + } + expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe( + 1, + ); + }); +}); From 567d64472f856a97d2f0d457e8cd5be5a87d3216 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 12:36:30 -0700 Subject: [PATCH 2/7] ci: make runner comparisons distinct and rerunnable --- .github/workflows/node-ci.yml | 1 + .github/workflows/test-quality.yml | 6 ++-- sdk/typescript/TESTING.md | 7 +++- .../scripts/compare-test-reports.py | 7 ++-- sdk/typescript/tests-ts/cost.test.ts | 2 +- .../tests-ts/deep-scan-workbench.test.ts | 6 ++-- sdk/typescript/tests-ts/multiscan.test.ts | 4 +-- sdk/typescript/tests-ts/skeleton.test.ts | 32 +++++++++++++++++++ sdk/typescript/tests-ts/test-reports.test.ts | 21 ++++++++++++ 9 files changed, 75 insertions(+), 11 deletions(-) diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index d61df1abe..46e9c3fda 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -84,6 +84,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: node-22-test-reports + overwrite: true path: | sdk/typescript/reports/junit.xml sdk/typescript/coverage/lcov.info diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 6695a3926..94087156c 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -35,7 +35,7 @@ jobs: - mode: parallel args: --parallel=2 - mode: randomized - args: --isolate --randomize + args: --isolate --randomize --seed=${{ github.run_number }} - os: windows-latest mode: shard-1 args: --shard=1/7 @@ -83,12 +83,13 @@ jobs: TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }} run: | node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})" - pnpm --dir sdk/typescript run test ${{ matrix.args }} --seed=${{ github.run_number }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml + pnpm --dir sdk/typescript run test ${{ matrix.args }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml - name: Upload runner report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: runner-${{ matrix.os }}-${{ matrix.mode }} + overwrite: true path: sdk/typescript/reports/runner-*.xml if-no-files-found: error retention-days: 14 @@ -145,6 +146,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: mutation-report + overwrite: true path: sdk/typescript/reports/mutation/ if-no-files-found: warn retention-days: 14 diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md index d638b9214..f97062a05 100644 --- a/sdk/typescript/TESTING.md +++ b/sdk/typescript/TESTING.md @@ -11,9 +11,12 @@ pnpm run format pnpm run test pnpm run test:ci pnpm pack --pack-destination ../../dist -pnpm run check:package ../../dist/*.tgz +pnpm run test:package ``` +For CI's full archive inspection, pass the exact `.tgz` path printed by +`pnpm pack` to `pnpm run check:package`. + The test commands pass a 30-second per-test timeout explicitly. `test:ci` writes `reports/junit.xml` and `coverage/lcov.info`. Coverage measures loaded JavaScript and TypeScript, not the Python helpers or child processes. It is @@ -38,6 +41,8 @@ before proposing a coverage floor. explicitly enabled GitHub-hosted Windows runner. - Add property tests for meaningful invariants, with accepted and rejected inputs. Keep example-based regression tests for readable failure cases. +- Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values + that are not strings. The JUnit comparison rejects duplicate identities. Property tests use a fixed default seed. Fast-check prints the seed, shrink path, and counterexample on failure. To replay one property, select its file diff --git a/sdk/typescript/scripts/compare-test-reports.py b/sdk/typescript/scripts/compare-test-reports.py index ce9fe8b2e..6849ad312 100644 --- a/sdk/typescript/scripts/compare-test-reports.py +++ b/sdk/typescript/scripts/compare-test-reports.py @@ -11,6 +11,7 @@ def read_report(path: Path) -> tuple[Counter, float]: root = ET.parse(path).getroot() cases = Counter() + identities = set() for case in root.iter("testcase"): status = "passed" if case.find("skipped") is not None: @@ -21,9 +22,11 @@ def read_report(path: Path) -> tuple[Counter, float]: case.get("file", "").replace("\\", "/").removeprefix("./"), case.get("classname", ""), case.get("name", ""), - status, ) - cases[identity] += 1 + if identity in identities: + raise ValueError(f"{path}: duplicate test identity: {' > '.join(identity)}") + identities.add(identity) + cases[(*identity, status)] += 1 if not cases: raise ValueError(f"{path}: no test cases") if int(root.get("tests", str(sum(cases.values())))) != sum(cases.values()): diff --git a/sdk/typescript/tests-ts/cost.test.ts b/sdk/typescript/tests-ts/cost.test.ts index 520efe90c..8a9ff8dd7 100644 --- a/sdk/typescript/tests-ts/cost.test.ts +++ b/sdk/typescript/tests-ts/cost.test.ts @@ -121,7 +121,7 @@ describe("scan cost", () => { [{ cache_write_input_tokens: 0, cache_write_tokens: 15 }, 15], [{ cache_write_input_tokens: 0, cache_write_tokens: 80 }, 0], ] as const)( - "keeps workbench cache-write normalization aligned with SDK usage", + "keeps workbench cache-write normalization aligned with SDK usage for %j as %p tokens", async (cacheWrites, expectedCacheWrites) => { const { PLUGIN_ROOT } = await import("./plugin-root.js"); const python = Bun.which("python3") ?? Bun.which("python"); diff --git a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts index b4a0469c8..cd09ed730 100644 --- a/sdk/typescript/tests-ts/deep-scan-workbench.test.ts +++ b/sdk/typescript/tests-ts/deep-scan-workbench.test.ts @@ -154,7 +154,7 @@ describe("deep scan workbench ownership", () => { [0.5, 0.5], [96, 96], ] as const)( - "resolves the configured discovery deadline %s as %s hours", + "resolves the configured discovery deadline %p as %p hours", async (configuredHours, expectedHours) => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-deadline-config-")), @@ -271,7 +271,7 @@ describe("deep scan workbench ownership", () => { }); test.each([false, true] as const)( - "backfills and repairs discovery deadline migration when already recorded: %s", + "backfills and repairs discovery deadline migration when already recorded: %p", (migrationRecorded) => { const python = Bun.which("python3") ?? Bun.which("python"); expect(python).not.toBeNull(); @@ -327,7 +327,7 @@ describe("deep scan workbench ownership", () => { ["scoped_path", false], ["repository", true], ] as const)( - "returns an honest partial %s report with existing deferred work %s when saturated discovery exceeds its cost limit", + "returns an honest partial %s report with existing deferred work %p when saturated discovery exceeds its cost limit", async (inventoryStrategy, existingDeferred) => { const root = await realpath( await mkdtemp(join(tmpdir(), "codex-security-deep-budget-recovery-")), diff --git a/sdk/typescript/tests-ts/multiscan.test.ts b/sdk/typescript/tests-ts/multiscan.test.ts index 2c62c0422..4e21f8d5a 100644 --- a/sdk/typescript/tests-ts/multiscan.test.ts +++ b/sdk/typescript/tests-ts/multiscan.test.ts @@ -385,7 +385,7 @@ describe("multiscan", () => { }); test.each([false, true])( - "continues scanning when a progress observer fails %s", + "continues scanning when a progress observer fails %p", async (asynchronous) => { const paths = await fixture(); const source = await repository(paths.root, "observer-failure"); @@ -1388,7 +1388,7 @@ describe("multiscan", () => { }); test.each([false, true])( - "never removes a replacement lock when owner creation fails (owner published: %s)", + "never removes a replacement lock when owner creation fails (owner published: %p)", async (ownerPublished) => { const paths = await fixture(); const source = await repository(paths.root, "owner-creation-race"); diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index db320d59d..de90a8140 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -5,6 +5,8 @@ import { main } from "../src/cli.js"; interface WorkflowStep { name?: string; + uses?: string; + with?: Record; run?: string; if?: string; env?: Record; @@ -184,6 +186,36 @@ describe("TypeScript package skeleton", () => { } }); + test("keeps runner modes distinct and report uploads rerunnable", async () => { + const ci = await workflow("node-ci.yml"); + const quality = await workflow("test-quality.yml"); + const runner = quality.jobs["runner"]!; + for (const [mode, args] of [ + ["baseline", ""], + ["isolated", "--isolate"], + ["parallel", "--parallel=2"], + ["randomized", "--isolate --randomize --seed=${{ github.run_number }}"], + ] as const) { + expect(runner.strategy?.matrix["include"]).toContainEqual({ + mode, + args, + }); + } + const command = runner.steps.find( + (step) => step.name === "Test runner mode", + )?.run; + expect(command).toContain("${{ matrix.args }}"); + expect(command).not.toContain("--seed="); + + const uploads = [...Object.values(ci.jobs), ...Object.values(quality.jobs)] + .flatMap((job) => job.steps) + .filter((step) => step.uses?.startsWith("actions/upload-artifact@")); + expect(uploads).toHaveLength(3); + for (const upload of uploads) { + expect(upload.with?.["overwrite"]).toBe(true); + } + }); + test("builds packages without a preinstalled package manager and provides a production audit", async () => { const packageJson = JSON.parse( await readFile(new URL("../package.json", import.meta.url), "utf8"), diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts index 41b626d18..ee56eddb8 100644 --- a/sdk/typescript/tests-ts/test-reports.test.ts +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -71,6 +71,27 @@ describe("JUnit inventory comparison", () => { expect(result.stdout).toContain("combined test time: 2.50s"); }); + test("rejects ambiguous test identities even when totals match", async () => { + const fixture = await fixtures(); + const first = testcase("same parameterized name"); + for (const [name, repeated] of [ + ["same-outcome", first], + ["different-outcome", testcase("same parameterized name", "")], + ] as const) { + const baseline = await fixture.report(`${name}-baseline.xml`, [ + first, + repeated, + ]); + const candidate = await fixture.report(`${name}-candidate.xml`, [ + first, + repeated, + ]); + const result = compare(baseline, candidate); + expect(result.status).toBe(1); + expect(result.stderr).toContain("duplicate test identity"); + } + }); + test("rejects dropped, duplicated, skipped, failed, or incomplete results", async () => { const fixture = await fixtures(); const first = testcase("first"); From 958ed07b4c65846f9495173105304b1ecdf9fd52 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 12:46:49 -0700 Subject: [PATCH 3/7] ci: keep optional mutation report uploads non-blocking --- .github/workflows/test-quality.yml | 1 + sdk/typescript/tests-ts/skeleton.test.ts | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 94087156c..553fdffea 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -143,6 +143,7 @@ jobs: run: pnpm --dir sdk/typescript run test:mutation - name: Upload mutation report if: always() + continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: mutation-report diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index de90a8140..3a2e22895 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -214,6 +214,17 @@ describe("TypeScript package skeleton", () => { for (const upload of uploads) { expect(upload.with?.["overwrite"]).toBe(true); } + expect( + uploads.find((step) => step.name === "Upload mutation report"), + ).toMatchObject({ "continue-on-error": true }); + expect( + uploads.find((step) => step.name === "Upload runner report"), + ).not.toHaveProperty("continue-on-error"); + expect( + quality.jobs["mutation"]?.steps.find( + (step) => step.name === "Run mutation trial", + ), + ).not.toHaveProperty("continue-on-error"); }); test("builds packages without a preinstalled package manager and provides a production audit", async () => { From 3ecd6cc86e83431201e7fe6ef92e4285b958210f Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 12:51:07 -0700 Subject: [PATCH 4/7] ci: validate quality workflow changes before merge --- .github/workflows/test-quality.yml | 3 +++ sdk/typescript/TESTING.md | 5 +++-- sdk/typescript/tests-ts/skeleton.test.ts | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 553fdffea..71a2555e6 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -1,6 +1,9 @@ name: test-quality on: + pull_request: + paths: + - .github/workflows/test-quality.yml workflow_dispatch: schedule: - cron: "23 9 * * 1" diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md index f97062a05..9f7273e7d 100644 --- a/sdk/typescript/TESTING.md +++ b/sdk/typescript/TESTING.md @@ -60,8 +60,9 @@ lanes still test and inspect an installed package. Package inspection includes a strict NodeNext TypeScript consumer and the actual installed CLI. Failed tests block CI; a failed diagnostic upload does not. -The separate `test-quality` workflow runs weekly and can be dispatched -manually. It exercises Bun's native `--isolate`, `--parallel=2`, randomized +The separate `test-quality` workflow runs weekly, can be dispatched manually, +and runs on pull requests that change its workflow file. It exercises Bun's +native `--isolate`, `--parallel=2`, randomized ordering, and seven-way Windows sharding. It compares test identities and outcomes against an unsharded run and records timings in the job summary. It is not a required check or part of the release trigger. diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index 3a2e22895..5f1bf6adb 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -168,9 +168,13 @@ describe("TypeScript package skeleton", () => { }); const quality = await workflow("test-quality.yml"); expect(Object.keys(quality.on).sort()).toEqual([ + "pull_request", "schedule", "workflow_dispatch", ]); + expect(quality.on["pull_request"]).toEqual({ + paths: [".github/workflows/test-quality.yml"], + }); expect(quality.env?.["CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST"]).toBe( "false", ); From 28a5562cb0667564e55cabf5461ee5866c506504 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 13:38:05 -0700 Subject: [PATCH 5/7] ci: collect every runner comparison before failing --- .github/workflows/test-quality.yml | 6 ++- sdk/typescript/tests-ts/test-reports.test.ts | 51 +++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index 71a2555e6..beb535cd4 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -114,12 +114,14 @@ jobs: - name: Compare inventories and outcomes shell: bash run: | + comparison_status=0 for os in ubuntu-latest windows-latest; do for mode in isolated parallel randomized; do - python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" + python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 done done - python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" + python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1 + exit "$comparison_status" mutation: name: Pure-module mutation trial diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts index ee56eddb8..daef4f4c1 100644 --- a/sdk/typescript/tests-ts/test-reports.test.ts +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -58,6 +58,55 @@ function compare(baseline: string, ...candidates: string[]) { } describe("JUnit inventory comparison", () => { + test("runs every workflow comparison before reporting a mismatch", async () => { + const fixture = await fixtures(); + const workflow = Bun.YAML.parse( + await readFile( + new URL("../../../.github/workflows/test-quality.yml", import.meta.url), + "utf8", + ), + ) as { + jobs: { compare: { steps: Array<{ name?: string; run?: string }> } }; + }; + const script = workflow.jobs.compare.steps.find( + (step) => step.name === "Compare inventories and outcomes", + )!.run!; + const expected = [ + ...["ubuntu-latest", "windows-latest"].flatMap((os) => + ["isolated", "parallel", "randomized"].map( + (mode) => `reports/runner-${os}-${mode}.xml`, + ), + ), + "reports/runner-windows-latest-shard-*.xml", + ]; + const mock = `python3() { + printf '%s\\n' "$3" + [[ "$3" != "$CODEX_SECURITY_TEST_FAIL_REPORT" ]] +}`; + const summary = join(fixture.root, "summary.md"); + for (const failedReport of ["", expected[0]!]) { + await writeFile(summary, ""); + const result = spawnSync( + "bash", + ["-e", "-o", "pipefail", "-c", `${mock}\n${script}`], + { + cwd: fixture.root, + encoding: "utf8", + env: { + ...process.env, + GITHUB_STEP_SUMMARY: "summary.md", + CODEX_SECURITY_TEST_FAIL_REPORT: failedReport, + }, + timeout: 10_000, + }, + ); + expect(result.status, result.stderr).toBe(failedReport === "" ? 0 : 1); + expect((await readFile(summary, "utf8")).trim().split(/\r?\n/u)).toEqual( + expected, + ); + } + }); + test("merges native shards without depending on test order", async () => { const fixture = await fixtures(); const passed = testcase("accepts & preserves"); From 51630f10514a045747105e69938052a8f0977394 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Mon, 17 Aug 2026 14:38:20 -0700 Subject: [PATCH 6/7] test: await report subprocesses and replay the PR seed --- .github/workflows/test-quality.yml | 4 +-- sdk/typescript/TESTING.md | 2 ++ sdk/typescript/tests-ts/skeleton.test.ts | 5 ++- sdk/typescript/tests-ts/test-reports.test.ts | 32 ++++++++++++-------- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-quality.yml b/.github/workflows/test-quality.yml index beb535cd4..d3561aa0b 100644 --- a/.github/workflows/test-quality.yml +++ b/.github/workflows/test-quality.yml @@ -18,7 +18,7 @@ concurrency: env: CODEX_SECURITY_INTEGRATION: "0" CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false" - CODEX_SECURITY_PROPERTY_SEED: ${{ github.run_number }} + CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }} jobs: runner: @@ -38,7 +38,7 @@ jobs: - mode: parallel args: --parallel=2 - mode: randomized - args: --isolate --randomize --seed=${{ github.run_number }} + args: --isolate --randomize --seed=${{ github.event_name == 'pull_request' && 1 || github.run_number }} - os: windows-latest mode: shard-1 args: --shard=1/7 diff --git a/sdk/typescript/TESTING.md b/sdk/typescript/TESTING.md index 9f7273e7d..0a154094c 100644 --- a/sdk/typescript/TESTING.md +++ b/sdk/typescript/TESTING.md @@ -65,6 +65,8 @@ and runs on pull requests that change its workflow file. It exercises Bun's native `--isolate`, `--parallel=2`, randomized ordering, and seven-way Windows sharding. It compares test identities and outcomes against an unsharded run and records timings in the job summary. +Pull requests replay seed 1; scheduled and manual runs use the workflow run +number for both property cases and test ordering. It is not a required check or part of the release trigger. Keep the current file-balanced Windows runner until the native runner has diff --git a/sdk/typescript/tests-ts/skeleton.test.ts b/sdk/typescript/tests-ts/skeleton.test.ts index 5f1bf6adb..4b75d0d55 100644 --- a/sdk/typescript/tests-ts/skeleton.test.ts +++ b/sdk/typescript/tests-ts/skeleton.test.ts @@ -194,11 +194,14 @@ describe("TypeScript package skeleton", () => { const ci = await workflow("node-ci.yml"); const quality = await workflow("test-quality.yml"); const runner = quality.jobs["runner"]!; + const seed = + "${{ github.event_name == 'pull_request' && 1 || github.run_number }}"; + expect(quality.env?.["CODEX_SECURITY_PROPERTY_SEED"]).toBe(seed); for (const [mode, args] of [ ["baseline", ""], ["isolated", "--isolate"], ["parallel", "--parallel=2"], - ["randomized", "--isolate --randomize --seed=${{ github.run_number }}"], + ["randomized", `--isolate --randomize --seed=${seed}`], ] as const) { expect(runner.strategy?.matrix["include"]).toContainEqual({ mode, diff --git a/sdk/typescript/tests-ts/test-reports.test.ts b/sdk/typescript/tests-ts/test-reports.test.ts index daef4f4c1..80dfff8c7 100644 --- a/sdk/typescript/tests-ts/test-reports.test.ts +++ b/sdk/typescript/tests-ts/test-reports.test.ts @@ -39,12 +39,12 @@ function testcase(name: string, status = "") { return `${status}`; } -function compare(baseline: string, ...candidates: string[]) { +async function compare(baseline: string, ...candidates: string[]) { const python = Bun.which("python3") ?? Bun.which("python") ?? Bun.which("py"); if (python === null) throw new Error("A Python interpreter is required."); - return spawnSync( - python, - [ + const child = Bun.spawn({ + cmd: [ + python, "-I", "-B", fileURLToPath( @@ -53,8 +53,16 @@ function compare(baseline: string, ...candidates: string[]) { baseline, ...candidates, ], - { encoding: "utf8" }, - ); + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const [status, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + return { status, stdout, stderr }; } describe("JUnit inventory comparison", () => { @@ -114,7 +122,7 @@ describe("JUnit inventory comparison", () => { const baseline = await fixture.report("baseline.xml", [passed, skipped]); await fixture.report("shard-1.xml", [skipped]); await fixture.report("shard-2.xml", [passed]); - const result = compare(baseline, join(fixture.root, "shard-*.xml")); + const result = await compare(baseline, join(fixture.root, "shard-*.xml")); expect(result.status, result.stderr).toBe(0); expect(result.stdout).toContain("Identical test inventory and outcomes"); expect(result.stdout).toContain("combined test time: 2.50s"); @@ -135,7 +143,7 @@ describe("JUnit inventory comparison", () => { first, repeated, ]); - const result = compare(baseline, candidate); + const result = await compare(baseline, candidate); expect(result.status).toBe(1); expect(result.stderr).toContain("duplicate test identity"); } @@ -161,10 +169,10 @@ describe("JUnit inventory comparison", () => { failures, count, ); - expect(compare(baseline, candidate).status, name).toBe(1); + expect((await compare(baseline, candidate)).status, name).toBe(1); } - expect(compare(baseline, join(fixture.root, "absent-*.xml")).status).toBe( - 1, - ); + expect( + (await compare(baseline, join(fixture.root, "absent-*.xml"))).status, + ).toBe(1); }); }); From e43f5554f507dd97b39206b2b2224014e05bf40c Mon Sep 17 00:00:00 2001 From: mldangelo-oai <269034524+mldangelo-oai@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:27:43 -0700 Subject: [PATCH 7/7] test: give session detail cases distinct report names --- sdk/typescript/tests-ts/cli.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/typescript/tests-ts/cli.test.ts b/sdk/typescript/tests-ts/cli.test.ts index 8d891cefb..e6ffb4d90 100644 --- a/sdk/typescript/tests-ts/cli.test.ts +++ b/sdk/typescript/tests-ts/cli.test.ts @@ -1763,7 +1763,7 @@ describe("CLI", () => { }); test.each([false, true])( - "subscribes to session details only with TTY stdin: %s", + "subscribes to session details only with TTY stdin: %p", async (stdinTTY) => { const descriptor = Object.getOwnPropertyDescriptor( process.stdin,