ci: report test quality and exercise native Bun runners

.github/workflows/node-ci.yml

@@ -66,6 +66,7 @@ jobs:
         run: pnpm --dir sdk/typescript run audit:prod
 
       - name: Typecheck
+        if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
         run: pnpm --dir sdk/typescript run types
 
       - name: Test
@@ -75,9 +76,23 @@ jobs:
           TMP: ${{ runner.temp }}
           TMPDIR: ${{ runner.temp }}
           CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
-        run: pnpm --dir sdk/typescript run test
+        run: pnpm --dir sdk/typescript run ${{ matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0' && 'test:ci' || 'test' }}
+
+      - name: Upload test reports
+        if: always() && matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: node-22-test-reports
+          overwrite: true
+          path: |
+            sdk/typescript/reports/junit.xml
+            sdk/typescript/coverage/lcov.info
+          if-no-files-found: warn
+          retention-days: 14
 
       - name: Check formatting
+        if: matrix.os == 'ubuntu-latest' && matrix.node == '22.13.0'
         run: pnpm --dir sdk/typescript run format
 
       - name: Pack
@@ -89,21 +104,6 @@ jobs:
         shell: bash
         run: pnpm run check:package ../../dist/*.tgz
 
-      - name: Smoke-test Node.js runtime
-        working-directory: sdk/typescript
-        shell: bash
-        run: |
-          set -euo pipefail
-          node --input-type=module --eval '
-            import { CodexSecurity } from "@openai/codex-security";
-
-            if (typeof CodexSecurity !== "function") {
-              throw new Error("The SDK does not export CodexSecurity.");
-            }
-          '
-          node bin/codex-security.mjs --version
-          node bin/codex-security.mjs --help
-
   windows-test:
     name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / tests-${{ matrix.shard }}
     runs-on: windows-latest
@@ -162,16 +162,6 @@ jobs:
           CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "true"
         run: bun test --timeout 120000 ./tests-ts/windows-machine-policy.test.ts
 
-      - name: Typecheck
-        if: matrix.shard == 7
-        working-directory: sdk/typescript
-        run: pnpm run types
-
-      - name: Check formatting
-        if: matrix.shard == 7
-        working-directory: sdk/typescript
-        run: pnpm run format
-
   windows-verify:
     name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} / verify
     runs-on: windows-latest
@@ -210,21 +200,6 @@ jobs:
         shell: bash
         run: pnpm run check:package ../../dist/*.tgz
 
-      - name: Smoke-test Node.js runtime
-        working-directory: sdk/typescript
-        shell: bash
-        run: |
-          set -euo pipefail
-          node --input-type=module --eval '
-            import { CodexSecurity } from "@openai/codex-security";
-
-            if (typeof CodexSecurity !== "function") {
-              throw new Error("The SDK does not export CodexSecurity.");
-            }
-          '
-          node bin/codex-security.mjs --version
-          node bin/codex-security.mjs --help
-
   windows:
     name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }}
     runs-on: ubuntu-latest

.github/workflows/test-quality.yml

@@ -0,0 +1,158 @@
+name: test-quality
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/test-quality.yml
+  workflow_dispatch:
+  schedule:
+    - cron: "23 9 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CODEX_SECURITY_INTEGRATION: "0"
+  CODEX_SECURITY_ALLOW_MACHINE_POLICY_TEST: "false"
+  CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }}
+
+jobs:
+  runner:
+    name: ${{ matrix.os }} / ${{ matrix.mode }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        mode: [baseline, isolated, parallel, randomized]
+        include:
+          - mode: baseline
+            args: ""
+          - mode: isolated
+            args: --isolate
+          - mode: parallel
+            args: --parallel=2
+          - mode: randomized
+            args: --isolate --randomize --seed=${{ github.event_name == 'pull_request' && 1 || github.run_number }}
+          - os: windows-latest
+            mode: shard-1
+            args: --shard=1/7
+          - os: windows-latest
+            mode: shard-2
+            args: --shard=2/7
+          - os: windows-latest
+            mode: shard-3
+            args: --shard=3/7
+          - os: windows-latest
+            mode: shard-4
+            args: --shard=4/7
+          - os: windows-latest
+            mode: shard-5
+            args: --shard=5/7
+          - os: windows-latest
+            mode: shard-6
+            args: --shard=6/7
+          - os: windows-latest
+            mode: shard-7
+            args: --shard=7/7
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+        with:
+          node-version: "22.13.0"
+      - name: Set up pnpm
+        run: npm install --global pnpm@11.9.0 --no-audit --no-fund
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.14"
+      - name: Install dependencies
+        run: pnpm --dir sdk/typescript install --frozen-lockfile
+      - name: Prepare private Windows test root
+        if: runner.os == 'Windows'
+        id: windows-temp
+        shell: pwsh
+        run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
+      - name: Test runner mode
+        env:
+          TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
+          TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
+          TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
+        run: |
+          node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})"
+          pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml
+      - name: Upload runner report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: runner-${{ matrix.os }}-${{ matrix.mode }}
+          overwrite: true
+          path: sdk/typescript/reports/runner-*.xml
+          if-no-files-found: error
+          retention-days: 14
+
+  compare:
+    name: Runner inventory and timing
+    if: always() && !cancelled()
+    needs: [runner]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: runner-*
+          merge-multiple: true
+          path: reports
+      - name: Compare inventories and outcomes
+        shell: bash
+        run: |
+          comparison_status=0
+          for os in ubuntu-latest windows-latest; do
+            for mode in isolated parallel randomized; do
+              python3 sdk/typescript/scripts/compare-test-reports.py "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+            done
+          done
+          python3 sdk/typescript/scripts/compare-test-reports.py reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+          exit "$comparison_status"
+
+  mutation:
+    name: Pure-module mutation trial
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
+        with:
+          node-version: "22.13.0"
+      - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+        with:
+          package_json_file: sdk/typescript/package.json
+          cache: true
+          cache_dependency_path: sdk/typescript/pnpm-lock.yaml
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.14"
+      - name: Install dependencies
+        run: pnpm --dir sdk/typescript install --frozen-lockfile
+      - name: Run mutation trial
+        run: pnpm --dir sdk/typescript run test:mutation
+      - name: Upload mutation report
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: mutation-report
+          overwrite: true
+          path: sdk/typescript/reports/mutation/
+          if-no-files-found: warn
+          retention-days: 14

CONTRIBUTING.md

@@ -47,3 +47,6 @@ Maintainers update package dependencies and the committed lockfile in the
 canonical repository. The public release workflow installs that locked graph,
 tests the package, and publishes a verified artifact with npm provenance.
 GitHub Actions dependencies are maintained separately in this repository.
+
+See the [SDK testing guide](sdk/typescript/TESTING.md) for local checks,
+test conventions, and the required and experimental CI jobs.

sdk/typescript/.gitignore

@@ -1,5 +1,6 @@
 /dist/
 /node_modules
+/coverage/
 /reports/
 /.stryker-tmp/
 /private_release/dist/

sdk/typescript/TESTING.md

@@ -0,0 +1,93 @@
+# Testing the SDK and CLI
+
+Use the pnpm version in `package.json` and Bun 1.3.14, matching CI. Run these
+commands from `sdk/typescript`:
+
+```sh
+pnpm install --frozen-lockfile
+bun test --timeout 30000 ./tests-ts/worker-progress.test.ts
+pnpm run types
+pnpm run format
+pnpm run test
+pnpm run test:ci
+pnpm pack --pack-destination ../../dist
+pnpm run test:package
+```
+
+For CI's full archive inspection, pass the exact `.tgz` path printed by
+`pnpm pack` to `pnpm run check:package`.
+
+The local test commands pass a 30-second per-test timeout explicitly. Windows
+CI and the Windows runner experiment allow 120 seconds for slower native
+credential and document checks. `test:ci` writes `reports/junit.xml` and
+`coverage/lcov.info`. Coverage measures loaded
+JavaScript and TypeScript, not the Python helpers or child processes. It is
+diagnostic for now. Use several successful CI runs to establish a baseline
+before proposing a coverage floor.
+
+## Writing tests
+
+- Test observable results, failures, cancellation, and cleanup. Prefer a
+  regression case that fails before a fix over assertions about private calls
+  or exact prose.
+- Keep fixtures synthetic and independent. Use real temporary directories,
+  Git repositories, SQLite databases, and installed packages when those
+  boundaries are the behavior under test. Do not use live model credentials.
+- Use the typed `TestClient` and `createApiTestFixtures` helpers for API tests.
+  Do not add a production abstraction solely to support a mock.
+- Restore spies, timers, and environment changes. Tests that change the process
+  cwd or install persistent ESM module mocks use `runTestInSubprocess`.
+  Per-file Bun isolation does not isolate process-wide state inside one file.
+- Keep shared behavior enabled on Linux, macOS, and Windows. The constrained
+  PowerShell test changes machine-wide policy and runs alone, only on an
+  explicitly enabled GitHub-hosted Windows runner.
+- Add property tests for meaningful invariants, with accepted and rejected
+  inputs. Keep example-based regression tests for readable failure cases.
+- Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values
+  that are not strings. The JUnit comparison rejects duplicate identities.
+
+Property tests use a fixed default seed. Fast-check prints the seed, shrink
+path, and counterexample on failure. To replay one property, select its file
+and test name, then set `CODEX_SECURITY_PROPERTY_SEED` and
+`CODEX_SECURITY_PROPERTY_PATH` to the reported values. Set
+`CODEX_SECURITY_PROPERTY_RUNS` to increase the case count. Pure properties
+default to 100 cases; filesystem contract properties default to 20.
+
+## GitHub Actions
+
+`node-ci` retains the required `ubuntu-latest / node-22`,
+`macos-latest / node-22`, and `windows-latest / node-22` checks. Its Ubuntu
+Node 22 job runs static checks and uploads JUnit and LCOV. All supported runtime
+lanes still test and inspect an installed package. Package inspection includes
+a strict NodeNext TypeScript consumer and the actual installed CLI. Failed
+tests block CI; a failed diagnostic upload does not.
+
+The separate `test-quality` workflow runs weekly, can be dispatched manually,
+and runs on pull requests that change its workflow file. It exercises Bun's
+native `--isolate`, `--parallel=2`, randomized
+ordering, and seven-way Windows sharding. It compares test identities and
+outcomes against an unsharded run and records timings in the job summary.
+Pull requests replay seed 1; scheduled and manual runs use the workflow run
+number for both property cases and test ordering.
+It is not a required check or part of the release trigger.
+
+Keep the current file-balanced Windows runner until the native runner has
+matching inventories and acceptable Windows timings. Before promotion, compare
+the slowest native shard with the current required shards on the same commit.
+Keep the machine-policy test serial. Do not replace the full required suite
+with `--changed`: Python files, schemas, fixtures, and workflows loaded at
+runtime are not necessarily part of Bun's import graph.
+
+## Mutation testing
+
+```sh
+pnpm run test:mutation
+pnpm exec stryker run --mutate src/worker-progress.ts
+```
+
+The initial Stryker trial covers progress parsing, safe error messages, and
+pure cost arithmetic. It runs a small Bun suite without live services and
+writes HTML and JSON under `reports/mutation`. Review surviving mutants for
+missing behavior assertions or equivalent changes. There is no score gate yet;
+set one only after the trial has a stable, useful baseline. Do not make a
+surviving mutant disappear by adding assertions about implementation details.

sdk/typescript/bunfig.toml

@@ -0,0 +1,3 @@
+[test]
+coverageSkipTestFiles = true
+coveragePathIgnorePatterns = ["tests-ts/**", "**/node_modules/**", "dist/**"]

sdk/typescript/package.json

@@ -49,6 +49,7 @@
     "lint": "tsc --noEmit",
     "prepack": "node --run build",
     "test": "bun test --timeout 30000 ./tests-ts",
+    "test:ci": "node -e \"require('node:fs').mkdirSync('reports',{recursive:true})\" && bun test --timeout 30000 ./tests-ts --coverage --coverage-reporter=text --coverage-reporter=lcov --reporter=junit --reporter-outfile=reports/junit.xml",
     "test:mutation": "stryker run",
     "test:package": "node scripts/smoke-package.mjs",
     "types": "pnpm run generate:models:check && tsc --noEmit"