From 130152c0eb2a69b5cad50d87ef92ed65e2c9cc9d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 13 Aug 2026 10:45:18 -0400 Subject: [PATCH 01/11] spec: scaffold for DSPX-4372 --- spec/DSPX-4372.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 spec/DSPX-4372.md diff --git a/spec/DSPX-4372.md b/spec/DSPX-4372.md new file mode 100644 index 000000000..b76bd3865 --- /dev/null +++ b/spec/DSPX-4372.md @@ -0,0 +1,34 @@ +--- +ticket: DSPX-4372 +title: +status: draft +authors: [dmihalcik@virtru.com] +branches: [opentdf/tests:DSPX-4372] +prs: [] +created: 2026-08-13 +updated: 2026-08-13 +--- + +# DSPX-4372 + +## Summary + + +## Problem / Motivation +_Why does this work need to happen? What is the user/business pain?_ + +## Proposed Solution +_What will you build, at a functional level? Sketch the approach._ + +## Inputs / Outputs / Contracts +_Function signatures, data shapes, API contracts, CLI flags._ + +## Edge Cases & Constraints +_Boundary conditions, error states, performance limits, security considerations._ + +## Out of Scope +_What this work item explicitly does not cover._ + +## Acceptance Criteria +- [ ] _Clear, testable condition_ +- [ ] _…_ From 27d7ea19fd2f0176cace357f96de7756fdb2e24d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 13 Aug 2026 11:14:37 -0400 Subject: [PATCH 02/11] feat(xtest): paired A/B SDK performance regression benchmarks Nothing in this repo measured the cost of an SDK operation, so a performance regression in any SDK shipped silently. The obvious design -- record timings, store them, compare to last week -- does not work on GitHub-hosted runners. CPU models vary, tenancy is shared, and steal time is unbounded, so run-to-run variation on identical code exceeds any regression worth catching. A historical gate would produce false alarms until people muted it. So no history is stored and no absolute number is ever compared. Each cell runs the newest installed release and the branch build on the *same runner*, paired within randomized interleaved rounds. Runner speed is a shared term that cancels in the per-round ratio. Verdicts come from the median log-ratio with a BCa bootstrap CI, a one-sided Wilcoxon signed-rank test, and Benjamini-Hochberg control across the run. A cell fails only when the CI lower bound clears 1.15x *and* the adjusted p clears 0.05: the interval clause cannot fire on noise, and the p clause cannot fire on a trivial effect that got lucky across ~14 comparisons. Two guards make the verdict honest rather than merely computed: - An A/A control per SDK compares the baseline against itself through the identical pipeline, so its true ratio is 1.0 and anything it reports is the harness's own error. If it trips, the run reports but does not fail. Its interval width is the empirical noise floor; if that is wider than the threshold the run had no power, and no cell may report PASS. "We could not tell" must never be reported as "no regression". - Rounds stop on attained CI precision, never on significance. Stopping when p drops below alpha is optional stopping and inflates the false positive rate well past nominal. This is easy to "optimize" away -- significance-stopping finishes sooner -- and doing so silently invalidates every number the job produces. Confounders are pinned rather than hoped away: same plaintext, same RSA attribute, one container and target mode for both arms, and for decrypt both arms read one baseline-produced ciphertext, since letting each arm read its own output would measure two different files. A cell skips with a stated reason when the arms disagree on a feature in the measured path. Wall clock and peak RSS gate the build; CPU time is reported but never fails. Measurement uses Popen + os.wait4 rather than getrusage(CHILDREN), whose ru_maxrss is a process-lifetime high-water mark and so has meaningless deltas. CI runs nightly and on manual dispatch, one runner per SDK, serial -- the xdist guard is a hard error because parallel workers contending for the CPU under measurement would invalidate everything. Never on PRs. The harness tests demonstrate the gate catching a planted 25% slowdown and ignoring a planted 3% one; a gate never shown to do both is not yet known to work. --- .github/workflows/check.yml | 13 + .github/workflows/pr-lint.yaml | 1 + .github/workflows/xtest.yml | 227 ++++++++++++++ .gitignore | 1 + spec/DSPX-4372.md | 240 ++++++++++++++- xtest/conftest.py | 199 +++++++++++- xtest/fixtures/bench.py | 407 ++++++++++++++++++++++++ xtest/perf/__init__.py | 12 + xtest/perf/_launcher.py | 146 +++++++++ xtest/perf/cells.py | 84 +++++ xtest/perf/measure.py | 254 +++++++++++++++ xtest/perf/report.py | 263 ++++++++++++++++ xtest/perf/runner.py | 431 ++++++++++++++++++++++++++ xtest/perf/stats.py | 544 +++++++++++++++++++++++++++++++++ xtest/pyproject.toml | 8 +- xtest/tdfs.py | 91 +++++- xtest/test_bench_arms.py | 135 ++++++++ xtest/test_bench_measure.py | 233 ++++++++++++++ xtest/test_bench_runner.py | 447 +++++++++++++++++++++++++++ xtest/test_bench_stats.py | 344 +++++++++++++++++++++ xtest/test_benchmarks.py | 91 ++++++ xtest/test_sdk_commands.py | 149 +++++++++ xtest/uv.lock | 86 ++++++ 23 files changed, 4390 insertions(+), 16 deletions(-) create mode 100644 xtest/fixtures/bench.py create mode 100644 xtest/perf/__init__.py create mode 100644 xtest/perf/_launcher.py create mode 100644 xtest/perf/cells.py create mode 100644 xtest/perf/measure.py create mode 100644 xtest/perf/report.py create mode 100644 xtest/perf/runner.py create mode 100644 xtest/perf/stats.py create mode 100644 xtest/test_bench_arms.py create mode 100644 xtest/test_bench_measure.py create mode 100644 xtest/test_bench_runner.py create mode 100644 xtest/test_bench_stats.py create mode 100644 xtest/test_benchmarks.py create mode 100644 xtest/test_sdk_commands.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5d52208aa..7e42f1435 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,6 +34,19 @@ jobs: uv run ruff format --check . uv run pyright working-directory: xtest + # The benchmark harness's own tests: statistics, measurement, and the + # CLI command builders. No platform and no SDK builds required, so the + # part of the gate that has to be *correct* is checked on every PR + # rather than only when the nightly benchmark runs. + # --frozen --no-build: resolve nothing and build nothing, so a + # dependency cannot slip in an unlocked version or a setup script on a + # runner that already has everything installed from the step above. + - name: Test xtest benchmark harness + run: >- + uv run --frozen --no-build pytest --no-header -q + test_bench_stats.py test_bench_measure.py test_bench_runner.py + test_bench_arms.py test_sdk_commands.py + working-directory: xtest - name: Lint and test otdf-local run: | uv sync diff --git a/.github/workflows/pr-lint.yaml b/.github/workflows/pr-lint.yaml index 6da092109..7ade36efc 100644 --- a/.github/workflows/pr-lint.yaml +++ b/.github/workflows/pr-lint.yaml @@ -29,6 +29,7 @@ jobs: java web xtest + perf ci dependabot env: diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 0f8e2f439..740a3648d 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -33,6 +33,11 @@ on: type: boolean default: false description: "Enable DPoP nonce challenge on KAS instances" + run-benchmarks: + required: false + type: boolean + default: false + description: "Run the SDK performance regression benchmarks (adds ~45m per SDK)" workflow_call: inputs: platform-ref: @@ -59,6 +64,10 @@ on: required: false type: boolean default: false + run-benchmarks: + required: false + type: boolean + default: false schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -748,6 +757,224 @@ jobs: ${{ steps.kas-km2.outputs.log-file }} if-no-files-found: ignore + # Paired A/B performance regression benchmark. + # + # Absolute timings from a GitHub-hosted runner are not comparable to + # timings from any other runner -- CPU model, tenancy, and steal time all + # vary more than any regression worth catching. So nothing is compared to + # history. Instead both builds under comparison run on *this* runner, in + # the same interleaved round, and only their ratio is reported. Runner + # speed divides out. + # + # Never runs on pull requests: 30 minutes of serial measurement is too slow + # for a PR gate, and a PR runner is the noisiest place to measure. + bench: + timeout-minutes: 45 + runs-on: ubuntu-latest + needs: resolve-versions + # Nightly cron only, not the Mon/Wed or weekly ones: three runs a week of + # the same comparison would tell us nothing the first one did not. + if: >- + github.event.schedule == '30 6 * * *' || + ((github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') + && inputs.run-benchmarks) + permissions: + contents: read + packages: read + strategy: + # One runner per SDK. Two SDKs on one runner would contend for the very + # CPU being measured. + fail-fast: false + matrix: + sdk: [go, java, js] + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: opentdf/tests + path: otdftests + persist-credentials: false + + - name: load extra keys from file + id: load-extra-keys + run: |- + echo "EXTRA_KEYS=$(jq -c > "${GITHUB_OUTPUT}" + + ######## SPIN UP PLATFORM BACKEND ############# + # Pinned to main, and to the default KAS only. We are measuring SDK + # regressions, so the server is held constant; and the six extra KAS + # instances the ABAC tests need would draw background CPU on the runner + # doing the measuring, which is noise rather than merely waste. + - name: Check out and start up platform with deps/containers + id: run-platform + uses: opentdf/platform/test/start-up-with-containers@18b8070f7ae1e3547234342f42d0d686dc77788f # keycloak-26.4 (opentdf/platform#3792) + with: + platform-ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} + bootstrap-ref: main + ec-tdf-enabled: true + extra-keys: ${{ steps.load-extra-keys.outputs.EXTRA_KEYS }} + log-type: json + pqc-enabled: true + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: bufbuild/buf-action@fd21066df7214747548607aaa45548ba2b9bc1ff # v1.4.0 + if: matrix.sdk == 'java' + with: + setup_only: true + token: ${{ secrets.BUF_TOKEN }} + version: "1.56.0" + + - name: Set up JDK + if: matrix.sdk == 'java' + uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 + with: + java-version: "11" + distribution: "adopt" + server-id: github + + - name: Set up Node 22 + if: matrix.sdk == 'js' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: "22.x" + + - name: Capture platform otdfctl location + if: matrix.sdk == 'go' + id: platform-otdfctl + run: |- + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD) || { + echo "::error::Failed to get SHA from platform checkout at $PLATFORM_DIR" + exit 1 + } + echo "sha=$sha" >> "$GITHUB_OUTPUT" + else + echo "dir=" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" + fi + env: + PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} + + ######## INSTALL BOTH ARMS OF THE COMPARISON ############# + # The whole design rests on this step laying down two builds side by + # side under sdk//dist/: the branch head (candidate) and the + # newest release (baseline). Arm selection picks them up from there. + - name: Configure ${{ matrix.sdk }} sdk + id: configure-sdk + uses: ./otdftests/xtest/setup-cli-tool + with: + path: otdftests/xtest/sdk + sdk: ${{ matrix.sdk }} + version-info: "${{ needs.resolve-versions.outputs[matrix.sdk] }}" + platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} + platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + + - name: Cache Go modules + if: matrix.sdk == 'go' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }} + restore-keys: | + go-${{ runner.os }}- + + - name: Cache npm + if: matrix.sdk == 'js' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }} + restore-keys: | + npm-${{ runner.os }}- + + - name: Cache Maven repository + if: matrix.sdk == 'java' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}- + + - name: point java heads at the platform under test + if: matrix.sdk == 'java' && fromJson(steps.configure-sdk.outputs.heads)[0] != null + run: |- + for row in $(echo "$java_version_info" | jq -c '.[]'); do + TAG=$(echo "$row" | jq -r '.tag') + HEAD=$(echo "$row" | jq -r '.head') + if [[ "$HEAD" == "true" ]]; then + echo "PLATFORM_BRANCH=$platform_ref" > "otdftests/xtest/sdk/java/${TAG}.env" + fi + done + env: + java_version_info: ${{ needs.resolve-versions.outputs.java }} + platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} + + - name: Build the ${{ matrix.sdk }} cli + if: fromJson(steps.configure-sdk.outputs.heads)[0] != null + run: make + working-directory: otdftests/xtest/sdk/${{ matrix.sdk }} + env: + BUF_INPUT_HTTPS_USERNAME: opentdf-bot + BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + + ######## MEASURE ############# + # --locked --no-build: install exactly what uv.lock pins, and run no + # setup scripts doing it. A benchmark that measured a differently + # resolved dependency set would be measuring the wrong thing anyway. + - name: Install test dependencies + run: uv sync --locked --no-build + working-directory: otdftests/xtest + + # Deliberately serial: no -n / --dist. Parallel pytest workers contend + # for the CPU under measurement and would invalidate every number here. + # conftest.py refuses to run --bench under xdist for the same reason. + - name: Run performance benchmarks + id: bench + run: |- + uv run --frozen --no-build pytest -ra -v \ + --bench \ + --sdks "$BENCH_SDK" \ + --bench-budget-seconds 1500 \ + --bench-out test-results/benchmarks \ + --html "test-results/bench-${BENCH_SDK}.html" \ + --self-contained-html \ + test_benchmarks.py + working-directory: otdftests/xtest + env: + BENCH_SDK: ${{ matrix.sdk }} + PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" + SCHEMA_FILE: "manifest.schema.json" + PLATFORM_TAG: main + OTDFCTL_HEADS: ${{ steps.configure-sdk.outputs.heads }} + # The benchmark never touches the audit-log fixture; asserting on + # logs would also add file IO to the measured path. + DISABLE_AUDIT_ASSERTIONS: "1" + + # Raw per-round samples, not just the verdict. Re-analysing a + # surprising result offline beats re-running a 30-minute job to look at + # the same numbers again. + - name: Upload benchmark results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: success() || failure() + with: + name: ${{ job.status == 'success' && '✅' || '❌' }} bench-${{ matrix.sdk }} + path: | + otdftests/xtest/test-results/benchmarks/*.json + otdftests/xtest/test-results/*.html + if-no-files-found: warn + + - name: Upload server logs on failure + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: bench-server-logs-${{ matrix.sdk }} + path: ${{ steps.run-platform.outputs.platform-log-file }} + if-no-files-found: ignore + publish-results: runs-on: ubuntu-latest needs: xct diff --git a/.gitignore b/.gitignore index a1bb32382..8eab78bda 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ vulnerability/tilt_modules/ /xtest/node_modules/ /xtest/tilt_modules/ /xtest/tmp/ +/xtest/test-results/ /xtest/sdk/js/web/dist/ /xtest/.helm diff --git a/spec/DSPX-4372.md b/spec/DSPX-4372.md index b76bd3865..dcd1740b2 100644 --- a/spec/DSPX-4372.md +++ b/spec/DSPX-4372.md @@ -1,6 +1,6 @@ --- ticket: DSPX-4372 -title: +title: Statistically valid SDK performance regression benchmarks status: draft authors: [dmihalcik@virtru.com] branches: [opentdf/tests:DSPX-4372] @@ -9,26 +9,248 @@ created: 2026-08-13 updated: 2026-08-13 --- -# DSPX-4372 +# DSPX-4372 — Statistically valid SDK performance regression benchmarks ## Summary +A nightly, per-SDK CI job measures the branch build against the newest release +**on the same runner, in the same interleaved round**, and fails the job on a +confirmed wall-clock or peak-RSS regression. CPU time is measured and reported +but never gates. + +Nothing is compared to history. Absolute timings from a GitHub-hosted runner +are not comparable to timings from any other runner, so the only quantity the +job reports is a *ratio between two builds measured under identical +conditions*. ## Problem / Motivation -_Why does this work need to happen? What is the user/business pain?_ + +Nothing in this repo measures the cost of an SDK operation. `tdfs.SDK.encrypt` +and `decrypt` return `None`, there is no timing code, and CI produces no +machine-readable durations. A performance regression in any SDK ships +silently, and is found by a customer rather than by us. + +The ticket asks for "performance metric tests ... (memory usage, wall clock +time, cpu usage)" and already points at the design that makes them work: +*"comparing main to release of the SDKs ... separate jobs from the existing job +matrix so we can have encrypt from different versions running on the same +instance."* + +The naive alternative — record timings, store them, compare to last week — does +not work here. GitHub-hosted runners vary in CPU model, are shared tenancy, and +suffer unbounded steal time. Run-to-run variation on identical code exceeds any +regression worth catching, so a historical gate produces false alarms until +people mute it, at which point it is worse than nothing. ## Proposed Solution -_What will you build, at a functional level? Sketch the approach._ + +### Paired A/B on one runner + +A **cell** is one operation at one payload size for one SDK, e.g. +`go-encrypt-1MiB`. Each cell runs *rounds*; each round runs both arms once: + +- **baseline** — the newest installed release (`go@v0.36.0`) +- **candidate** — the branch build (`go@main`) + +Runner speed, thermal state, and noisy neighbours are shared within a round and +cancel in the per-round ratio. Order within the round is randomized from a +seeded RNG so neither arm systematically inherits the other's cache state. + +### Statistics + +Per round, on the log scale: `d_i = ln(candidate_i) - ln(baseline_i)`. + +| Quantity | Method | +|---|---| +| Point estimate | median of `d_i`, exponentiated | +| Interval | BCa percentile bootstrap, 95%, 10 000 resamples | +| Test | one-sided Wilcoxon signed-rank (`alternative="greater"`) | +| Multiplicity | Benjamini–Hochberg across the run's gated cells | + +**Decision rule: regression iff `ci_low > threshold` AND BH-adjusted +`p < 0.05`.** Default threshold 1.15 (+15%). The conjunction is deliberate and +neither clause is redundant: the interval clause cannot fire on pure noise — +that would require excluding an effect that is not there — and the p clause +cannot fire on a real-but-trivial effect surviving by luck across ~14 cells. +Symmetrically, `ci_high < 1/threshold` reports an improvement, which is +informational and never fails. + +### The A/A control + +Each SDK runs one extra cell comparing the **baseline against itself** through +the identical pipeline. Its true ratio is 1.0 by construction, so whatever it +reports is the harness's own error. Two things come out of it: + +- If it *trips* — reports an effect past the threshold — the runner is too noisy + or the harness is biased. The whole run is downgraded: verdicts are reported, + the build is not failed. +- Its interval width is the run's empirical **noise floor**. If that is not + tighter than the threshold, the run had no power to detect the effect being + gated on, and no cell may report `PASS` — only `INCONCLUSIVE`. "We could not + tell" must never be reported as "no regression". + +Controls run **first** in each SDK's cell list. A run that overruns its budget +loses whatever is at the end; losing one comparison leaves the rest +trustworthy, losing the control leaves nothing trustworthy at all. + +### Stopping rule + +Rounds continue until the bootstrap CI half-width falls below +`ln(threshold)/3`, bounded below by `--bench-min-rounds` (20), above by +`--bench-max-rounds` (60), and by a shared wall-clock budget. + +**Stopping is on precision, never on significance.** Peeking at the p-value and +stopping when it drops below alpha is optional stopping: it inflates the +false-positive rate well past nominal, because each round is a fresh chance to +cross the line and the loop only ever stops on the lucky side. Attained CI width +is driven by dispersion rather than location, so it is approximately ancillary +to the effect being tested. This is easy to "optimize away" — stopping on +significance finishes sooner — and doing so silently invalidates every number +the job produces. ## Inputs / Outputs / Contracts -_Function signatures, data shapes, API contracts, CLI flags._ + +### New modules + +| Path | Responsibility | +|---|---| +| `xtest/perf/measure.py` | one invocation → wall ns, CPU s, peak RSS bytes | +| `xtest/perf/_launcher.py` | forks the measured command from an empty process, so its RSS is its own | +| `xtest/perf/stats.py` | log-ratios, BCa CI, Wilcoxon, BH, decision rule | +| `xtest/perf/runner.py` | paired round loop, warm-up, stopping rule, budget | +| `xtest/perf/cells.py` | the experiment matrix | +| `xtest/perf/report.py` | JSON artifact + `$GITHUB_STEP_SUMMARY` markdown | +| `xtest/fixtures/bench.py` | arm selection, payloads, ciphertexts, comparability guards | +| `xtest/test_benchmarks.py` | the cells (needs a platform) | +| `xtest/test_bench_stats.py` | statistics, offline | +| `xtest/test_bench_measure.py` | measurement primitive, offline | +| `xtest/test_bench_runner.py` | round loop and gate, offline | +| `xtest/test_sdk_commands.py` | the `XT_WITH_*` CLI contract, offline | + +`xtest/tdfs.py` gains `SDK.encrypt_command` / `SDK.decrypt_command` — argv+env +builders extracted from the existing `encrypt`/`decrypt`, whose behaviour is +unchanged — and `SDK.semver()`. The benchmark drives the CLI through the same +builders the functional tests use, so the `XT_WITH_*` contract cannot drift +between them. + +### Measurement primitive + +```python +@dataclass(frozen=True, slots=True) +class Sample: + wall_ns: int # perf_counter_ns around the call + cpu_s: float # ru_utime + ru_stime + max_rss_bytes: int # ru_maxrss, unit-normalized + exit_code: int + rss_floor_bytes: int # RSS of the process that forked it +``` + +`os.wait4`, not `resource.getrusage(RUSAGE_CHILDREN)`: the latter's +`ru_maxrss` is a process-lifetime high-water mark, so deltas are meaningless. +rusage folds in reaped descendants, so the `java`/`node` process behind each +`cli.sh` shim is counted. `ru_maxrss` is KiB on Linux and bytes on macOS; +normalized on `sys.platform`. + +The command is **not** forked from the pytest process. On Linux a child +inherits the parent's resident-set accounting and `execve` does not clear it, +so `ru_maxrss` comes back as `max(the child's true peak, the parent's RSS at +fork time)`. Measured from a pytest process holding numpy, scipy and a +session's worth of samples, every SDK invocation reported *pytest's* footprint +— about 165 MiB on a CI runner — instead of its own. That does not look +broken; it looks like a stable ratio of 1.000, which reads as "no regression" +forever. `posix_spawn` and `sh -c 'exec …'` were measured and are equally +contaminated: an exec is too late, the accounting is already latched. So +`perf/_launcher.py` runs as a small `python -I -S` process holding nothing and +forks the real command itself, reporting its own RSS as the floor under the +reading. It also puts the command in its own process group, so a timeout kills +the whole tree rather than just the shim. + +### CLI options + +`--bench` (opt-in; without it `test_benchmarks.py` collects nothing), +`--bench-baseline`, `--bench-candidate`, `--bench-threshold` (1.15), +`--bench-min-rounds` (20), `--bench-max-rounds` (60), `--bench-warmup` (5), +`--bench-budget-seconds` (1500), `--bench-seed` (0), `--bench-out` +(`test-results/benchmarks`), `--bench-no-gate`. + +### Outputs + +- `test-results/benchmarks/.json` — **every raw per-round sample** + alongside the derived statistics, runner metadata, seed, and thresholds. + Re-analysing a surprising result offline is the difference between + understanding a red build and re-running a 30-minute job to look at the same + numbers again. +- `$GITHUB_STEP_SUMMARY` — one row per (cell, metric): baseline median, + candidate median, ratio with CI, adjusted p, verdict, plus the noise floor. +- Exit status: `pytest_sessionfinish` fails the session on a confirmed + regression. + +### CI + +New `bench` job in `.github/workflows/xtest.yml`: matrix over +`sdk: [go, java, js]`, one runner each, `timeout-minutes: 45`, platform pinned +to the `main` SHA, default KAS only, **serial** (no `-n`). Triggers on the +nightly cron and on `workflow_dispatch`/`workflow_call` with +`run-benchmarks: true`. Never on pull requests. ## Edge Cases & Constraints -_Boundary conditions, error states, performance limits, security considerations._ + +| Threat to validity | Handling | +|---|---| +| Runner CPU heterogeneity | Both arms on one runner; ratios, not absolutes | +| Noisy neighbours, steal time | Paired interleaved rounds; median + Wilcoxon; A/A gate | +| Thermal and slow drift | Randomized within-round order; pairing differences it out | +| Page cache, first `go build`, npx resolve | Warm-up rounds discarded | +| JVM/npx startup dominating small payloads | Reported separately per payload size; 1 KiB *is* the startup cell | +| Platform/KAS latency in the decrypt path | Shared by both arms in a round; cancels | +| Arms differing in function, not speed | Container, target mode, and attribute pinned; cell skipped if the arms disagree on `SDK.supports()` for anything in the measured path | +| Decrypt arms reading different ciphertexts | Both arms decrypt one baseline-produced file | +| ~14 simultaneous comparisons | BH correction plus an effect threshold | +| Optional-stopping bias | Stop on precision, never significance; min-round floor | +| xdist contention | `--bench` under xdist is a hard `UsageError`; CI runs serial | +| Peak RSS inheriting the measuring process's memory | The command is forked from an empty launcher, not from pytest | +| Peak RSS pinned at the measurement floor | Both arms clip to the same number, so the ratio is 1.000 with a tight interval — the most convincing PASS the harness can emit, carrying no information. A floored cell is forced `INCONCLUSIVE` and excluded from the BH correction, like the control | +| A failing operation | Any non-zero exit aborts the cell with captured stderr; a benchmark over the error path is worse than no benchmark | + +Payload sizes are 1 KiB / 1 MiB / 32 MiB because they separate two regimes that +fail independently: at 1 KiB nearly all cost is process startup, so a throughput +regression is invisible; at 32 MiB crypto and IO dominate, so a startup +regression is invisible. + +Budget: java is the worst case at roughly 1.0 s/op small and 2.5 s at 32 MiB, +giving ~19 s per round across both operations and both arms; 45 rounds plus the +control lands around 16 minutes, inside the 30-minute target and the 45-minute +job timeout. Go and JS are substantially cheaper. ## Out of Scope -_What this work item explicitly does not cover._ + +Historical trend storage; gh-pages dashboards or `github-action-benchmark`; +flamegraphs and profiling artifacts; cross-SDK comparison (go vs java is not a +regression signal); cross-platform-version performance comparison; perf gating +on pull requests; nano and other container types; in-process microbenchmarks or +`pytest-benchmark` semantics. ## Acceptance Criteria -- [ ] _Clear, testable condition_ -- [ ] _…_ + +- [x] Wall clock, CPU time, and peak RSS are measured per invocation, with + descendant processes folded in and RSS units normalized across platforms. +- [x] Both arms are measured on one runner, paired within randomized + interleaved rounds, with warm-up rounds discarded. +- [x] Decrypt cells compare two arms reading the *same* baseline-produced + ciphertext; encrypt cells pin container, target mode, and attribute. +- [x] A cell is skipped with a stated reason when a build is missing or the two + arms disagree on a feature in the measured path. +- [x] The verdict uses a robust CI, a one-sided signed-rank test, a minimum + effect threshold, and BH multiplicity control across the run. +- [x] An A/A control runs per SDK; if it trips, the run reports but does not + fail; if its interval is wider than the threshold, no cell reports PASS. +- [x] The round loop stops on attained precision, never on significance, and + refuses a verdict below the minimum usable round count. +- [x] Raw per-round samples are written to JSON and a summary table to + `$GITHUB_STEP_SUMMARY`. +- [x] A confirmed wall-clock or peak-RSS regression fails the job; CPU time + never does. +- [x] The offline harness tests demonstrate the gate catching a planted 25% + slowdown and *ignoring* a planted 3% one. +- [ ] A `workflow_dispatch` run with `run-benchmarks: true` produces step + summaries and artifacts for all three SDKs inside the 45-minute timeout. diff --git a/xtest/conftest.py b/xtest/conftest.py index 4f39176be..903c54724 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -24,6 +24,8 @@ import tdfs from otdfctl import OpentdfCommandLineTool +from perf import report, stats +from perf.cells import cells_for logging.basicConfig(level=os.environ.get("LOGLEVEL", "DEBUG")) @@ -35,8 +37,16 @@ def pytest_report_header() -> list[str]: and pytest does not show captured output for skipped tests. Echoing the detected version and feature set into the report header makes it visible in CI even when every gated test skips. + + Detection probes the platform over HTTP, which fails when there is no + platform -- running only the offline unit tests, for instance. That is a + header, not a test result, so report the failure and carry on rather than + breaking collection for tests that never needed a platform. """ - pfs = tdfs.get_platform_features() + try: + pfs = tdfs.get_platform_features() + except Exception as e: # a header must never break collection + return [f"platform features unavailable: {e}"] return [ f"platform version: {pfs.version} (semver={pfs.semver})", f"detected features: {', '.join(sorted(pfs.features))}", @@ -52,6 +62,7 @@ def pytest_report_header() -> list[str]: "fixtures.keys", "fixtures.audit", "fixtures.encryption", + "fixtures.bench", ] @@ -144,6 +155,83 @@ def pytest_addoption(parser: pytest.Parser): help="select which sdks to run for encrypt only; accepts same format as --sdks", type=sdk_spec_type, ) + _add_benchmark_options(parser) + + +def _add_benchmark_options(parser: pytest.Parser): + """Options for the SDK performance regression benchmarks. + + Grouped separately because they configure an experiment rather than + selecting tests, and because none of them do anything without --bench. + """ + group = parser.getgroup("benchmarks", "SDK performance regression benchmarks") + group.addoption( + "--bench", + action="store_true", + help="run the performance regression benchmarks (they are long, so they " + "are opt-in and collect nothing otherwise)", + ) + group.addoption( + "--bench-baseline", + help="build to compare against, e.g. go@v0.29.0; defaults to the newest " + "installed release of each sdk", + ) + group.addoption( + "--bench-candidate", + help="build under test, e.g. go@main; defaults to the installed " + "unreleased build of each sdk", + ) + group.addoption( + "--bench-threshold", + type=float, + default=stats.DEFAULT_THRESHOLD, + help="smallest slowdown ratio worth failing on (default: %(default)s, " + "i.e. 15%% slower)", + ) + group.addoption( + "--bench-min-rounds", + type=int, + default=20, + help="paired rounds to run before the stopping rule may fire " + "(default: %(default)s)", + ) + group.addoption( + "--bench-max-rounds", + type=int, + default=60, + help="hard cap on paired rounds per cell (default: %(default)s)", + ) + group.addoption( + "--bench-warmup", + type=int, + default=5, + help="paired rounds discarded before measuring, to pay one-time costs " + "like page cache and package resolution (default: %(default)s)", + ) + group.addoption( + "--bench-budget-seconds", + type=float, + default=1500.0, + help="wall-clock allowance shared by every cell (default: %(default)s)", + ) + group.addoption( + "--bench-seed", + type=int, + default=0, + help="seed for payload generation, round ordering, and the bootstrap; " + "fixing it makes a run reproducible (default: %(default)s)", + ) + group.addoption( + "--bench-out", + type=Path, + default=Path("test-results/benchmarks"), + help="directory for the JSON result artifact (default: %(default)s)", + ) + group.addoption( + "--bench-no-gate", + action="store_true", + help="measure and report, but never fail the run on a regression", + ) def pytest_generate_tests(metafunc: pytest.Metafunc): @@ -231,6 +319,115 @@ def sdk_specs_opt(names: list[str]) -> list[str]: containers = list(typing.get_args(tdfs.container_type)) metafunc.parametrize("container", containers) + if "bench_cell" in metafunc.fixturenames: + _parametrize_bench_cells(metafunc) + + +def _parametrize_bench_cells(metafunc: pytest.Metafunc): + """Fan the benchmark module out over its cells. + + Without --bench there is nothing to fan out over, and the items are + dropped wholesale in :func:`pytest_collection_modifyitems` rather than + parametrized here. Parametrizing over an empty list would *not* collect + zero items: pytest's default ``empty_parameter_set_mark`` turns an empty + set into one skipped item per test, so every ordinary run would carry + benchmark skips it never asked for. + """ + if not metafunc.config.getoption("--bench"): + return + + # --sdks may be version-qualified (go@main); benchmark arms come from + # --bench-baseline/--bench-candidate instead, so only the name matters. + specs = metafunc.config.getoption("--sdks") or " ".join( + typing.get_args(tdfs.sdk_type) + ) + names = list(dict.fromkeys(s.split("@", 1)[0] for s in str(specs).split())) + cells = cells_for(names) + metafunc.config.stash[report.CELLS_KEY] = cells + metafunc.parametrize("bench_cell", cells, ids=[c.id for c in cells]) + + +def pytest_configure(config: pytest.Config): + if not config.getoption("--bench", default=False): + return + # Parallel workers contend for the CPU the benchmark is measuring, which + # turns every number into noise. The CI step also omits -n; this guard is + # what stops a later edit from silently invalidating the whole job. + distributed = getattr(config, "workerinput", None) is not None or bool( + config.getoption("numprocesses", default=None) + ) + if distributed: + raise pytest.UsageError( + "--bench cannot run under pytest-xdist: parallel workers compete " + "for the CPU being measured. Drop -n / --dist." + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Drop the benchmark cells entirely unless --bench asked for them. + + Deselected rather than skipped: a 20-minute cell has no business in the + regular integration matrix, and a skip would report it as a test that + exists and was declined rather than one that was never in scope. + """ + if config.getoption("--bench", default=False): + return + keep, drop = [], [] + for item in items: + (drop if item.get_closest_marker("benchmark") else keep).append(item) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int): + """Analyse every recorded cell, write the artifacts, and gate the run. + + The gate lives here rather than in the cells because it is a run-level + decision: the multiplicity correction spans all cells, and the A/A control + can invalidate the lot. Artifacts are written first and unconditionally -- + a run that is about to fail is exactly the run whose raw numbers someone + will want to read. + """ + del exitstatus # the benchmark's own verdict is independent of test outcomes + config = session.config + if not config.getoption("--bench", default=False): + return + recorder = config.stash.get(report.RECORDER_KEY, None) + if recorder is None or not (recorder.results or recorder.skipped): + return + + # Imported here, not at module scope: importing a pytest plugin from a + # conftest before pytest registers it costs the plugin its assertion + # rewriting, which the fixture module's own asserts rely on. + from fixtures import bench + + bench_config = bench.config_from_options(config) + recorder.metadata = bench.runner_metadata(config) + gate = recorder.gate(bench_config) + + cells = config.stash.get(report.CELLS_KEY, []) + name = "-".join(dict.fromkeys(c.sdk for c in cells)) or "benchmarks" + out_dir = cast(Path, config.getoption("--bench-out")) + json_path = report.write_json( + out_dir / f"{name}.json", recorder, bench_config, gate + ) + + summary = report.markdown(recorder, bench_config, gate) + report.append_step_summary(summary) + reporter = config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_sep("=", "benchmark results") + reporter.write_line(gate.summary) + reporter.write_line(f"raw samples and statistics: {json_path}") + + if config.getoption("--bench-no-gate", default=False): + return + if gate.should_fail: + session.exitstatus = pytest.ExitCode.TESTS_FAILED + def pytest_runtest_setup(item: pytest.Item): if not item.config.getoption("--skip-released-pairs", default=False): diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py new file mode 100644 index 000000000..d5194b380 --- /dev/null +++ b/xtest/fixtures/bench.py @@ -0,0 +1,407 @@ +"""Fixtures for the SDK performance regression benchmarks. + +The experiment matrix, the payload files, the arm selection, and the shared +time budget all live here. The measurement loop itself is in ``perf/runner.py`` +and the statistics in ``perf/stats.py``; this module is the glue that turns +pytest's world (options, fixtures, SDK discovery) into the runner's world +(two arms and a config). +""" + +from __future__ import annotations + +import os +import platform +import random +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +import pytest + +import abac +import tdfs +from perf import report +from perf.cells import PAYLOADS, BenchCell +from perf.runner import Arm, BenchConfig, Budget, Invocation + + +class ArmSelectionError(Exception): + """The two builds a comparison needs are not both installed.""" + + +def select_arms( + sdk: str, + *, + baseline_spec: str | None = None, + candidate_spec: str | None = None, +) -> tuple[tdfs.SDK, tdfs.SDK]: + """Pick (baseline, candidate) builds for one SDK. + + By default the candidate is the branch build (``main``) and the baseline + is the newest installed release, which is exactly what the CI setup action + lays down side by side. Explicit specs override either side, for + reproducing a comparison or for pinning a specific release. + + Raises: + ArmSelectionError: if either side is missing or the two resolve to the + same build (a comparison of a build against itself is only + meaningful as the explicit A/A control). + """ + installed = tdfs.all_versions_of(sdk) # pyright: ignore[reportArgumentType] + if not installed: + raise ArmSelectionError(f"no {sdk} builds installed under sdk/{sdk}/dist/") + + def resolve(spec: str, role: str) -> tdfs.SDK: + try: + matches = tdfs.parse_sdk_spec(spec) + except (FileNotFoundError, ValueError) as e: + raise ArmSelectionError(f"{role} {spec!r}: {e}") from e + if len(matches) != 1: + raise ArmSelectionError( + f"{role} {spec!r} resolved to {len(matches)} builds; " + "name one exactly, e.g. go@v0.29.0" + ) + return matches[0] + + if candidate_spec: + candidate = resolve(candidate_spec, "candidate") + else: + heads = [s for s in installed if not s.is_released()] + if not heads: + raise ArmSelectionError( + f"no unreleased {sdk} build to test; installed: " + f"{', '.join(sorted(s.version for s in installed))}" + ) + # Prefer 'main' when several branch builds are present. + candidate = next((s for s in heads if s.version == "main"), heads[0]) + + if baseline_spec: + baseline = resolve(baseline_spec, "baseline") + else: + # Final releases only. A release candidate parses to the same semver + # as its final release, so including them leaves `max` breaking a tie + # on whatever order the directory listing happened to produce -- and a + # baseline that is silently an rc is a baseline nobody chose. + releases = [s for s in installed if s.is_final_release()] + if not releases: + raise ArmSelectionError( + f"no final {sdk} release to compare against (prereleases do " + f"not count); installed: " + f"{', '.join(sorted(s.version for s in installed))}" + ) + baseline = max(releases, key=lambda s: s.semver() or (0, 0, 0)) + + if baseline == candidate: + raise ArmSelectionError( + f"baseline and candidate are both {baseline}; nothing to compare" + ) + return baseline, candidate + + +# --- Session-scoped configuration ------------------------------------------- + + +def config_from_options(config: pytest.Config) -> BenchConfig: + """Build a :class:`BenchConfig` from the ``--bench-*`` options. + + Every option has a default, so ``getoption`` never returns None here; the + casts are for the type checker, which cannot see the parser setup. + """ + + def as_int(name: str) -> int: + return int(cast(int, config.getoption(name))) + + def as_float(name: str) -> float: + return float(cast(float, config.getoption(name))) + + try: + return BenchConfig( + min_rounds=as_int("--bench-min-rounds"), + max_rounds=as_int("--bench-max-rounds"), + warmup=as_int("--bench-warmup"), + budget_seconds=as_float("--bench-budget-seconds"), + seed=as_int("--bench-seed"), + threshold=as_float("--bench-threshold"), + ) + except ValueError as e: + raise pytest.UsageError(f"invalid benchmark options: {e}") from e + + +@pytest.fixture(scope="session") +def bench_config(request: pytest.FixtureRequest) -> BenchConfig: + """Round-loop and analysis settings, from the --bench-* options.""" + return config_from_options(request.config) + + +@pytest.fixture(scope="session") +def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: + """Generate one plaintext file per payload size, shared by both arms. + + Content is pseudo-random but seeded, so a rerun measures byte-identical + input. Random rather than repetitive because compressible input would let + an SDK that happens to compress look faster for reasons unrelated to the + crypto path. + + One RNG per payload rather than one stream shared across them: ``tmp_dir`` + persists between runs, so a partially cached set skips some ``randbytes`` + calls and shifts the stream for every payload after it. Deriving each + payload's bytes from the seed *and* its label keeps the promise above true + whether the cache is empty, full, or half there. + """ + out: dict[str, Path] = {} + for payload in PAYLOADS: + path = tmp_dir / f"bench-plain-{payload.label}.bin" + if not path.is_file() or path.stat().st_size != payload.n_bytes: + rng = random.Random(f"{bench_config.seed}:{payload.label}") + path.write_bytes(rng.randbytes(payload.n_bytes)) + out[payload.label] = path + return out + + +@pytest.fixture(scope="session") +def bench_budget(request: pytest.FixtureRequest, bench_config: BenchConfig) -> Budget: + """One wall-clock allowance shared by every cell in the session. + + Divided evenly as cells start, so a cell that stops early on precision + donates its unused time to the ones after it instead of leaving the last + cell starved by whatever the first ones happened to spend. + """ + n_cells = max(1, len(_selected_cells(request.config))) + return Budget(bench_config.budget_seconds, n_cells) + + +@pytest.fixture(scope="session") +def bench_recorder(request: pytest.FixtureRequest) -> report.BenchmarkRecorder: + """The session-wide collector that the end-of-run gate reads.""" + return report.recorder_for(request.config) + + +def _selected_cells(config: pytest.Config) -> list[BenchCell]: + """Cells this session will run, cached on the config by the parametrizer.""" + return config.stash.get(report.CELLS_KEY, []) + + +# --- Module-scoped experiment inputs ---------------------------------------- + + +@dataclass(frozen=True, slots=True) +class BenchArms: + baseline: tdfs.SDK + candidate: tdfs.SDK + + +class ArmResolver: + """Resolves and memoizes the two builds to compare, per SDK. + + Resolution is lazy so that a missing build skips one SDK's cells with a + readable reason instead of erroring out every cell in the module. + """ + + def __init__(self, baseline_spec: str | None, candidate_spec: str | None) -> None: + self._baseline_spec = baseline_spec + self._candidate_spec = candidate_spec + self._cache: dict[str, BenchArms] = {} + + def __call__(self, sdk: str) -> BenchArms: + cached = self._cache.get(sdk) + if cached is None: + baseline, candidate = select_arms( + sdk, + baseline_spec=_spec_for(self._baseline_spec, sdk), + candidate_spec=_spec_for(self._candidate_spec, sdk), + ) + cached = self._cache[sdk] = BenchArms(baseline, candidate) + return cached + + +@pytest.fixture(scope="module") +def bench_arms(request: pytest.FixtureRequest) -> ArmResolver: + """Resolver for the (baseline, candidate) pair of any SDK in the run.""" + return ArmResolver( + cast(str | None, request.config.getoption("--bench-baseline")), + cast(str | None, request.config.getoption("--bench-candidate")), + ) + + +def _spec_for(spec: str | None, sdk: str) -> str | None: + """Return ``spec`` only if it names this SDK, so one flag can cover a run.""" + if not spec: + return None + return spec if spec.split("@", 1)[0] == sdk else None + + +#: Features whose presence changes what an encrypt or decrypt actually *does*. +#: If the two arms disagree on one of these they are not performing the same +#: operation, and a timing difference between them is a difference in work, +#: not in speed. +_COMPARABILITY_FEATURES: tuple[tdfs.feature_type, ...] = ( + "hexless", + "hexaflexible", + "autoconfigure", +) + + +def comparability_problem(arms: BenchArms) -> str | None: + """Return why these two builds cannot be fairly compared, or None.""" + for feature in _COMPARABILITY_FEATURES: + if arms.baseline.supports(feature) != arms.candidate.supports(feature): + supporter, other = ( + (arms.baseline, arms.candidate) + if arms.baseline.supports(feature) + else (arms.candidate, arms.baseline) + ) + return ( + f"{supporter} supports [{feature}] and {other} does not, so the " + "two arms would not be doing the same work" + ) + return None + + +def pinned_target_mode(arms: BenchArms) -> tdfs.container_version | None: + """Pick one container version both arms emit, or None for their default. + + Letting each arm choose its own target would compare two output formats. + ``None`` is only returned when neither arm can be told which to use, in + which case :func:`comparability_problem` has already established that they + agree on the relevant features and will pick the same one. + """ + if not ( + arms.baseline.supports("hexaflexible") + and arms.candidate.supports("hexaflexible") + ): + return None + if arms.baseline.supports("hexless") and arms.candidate.supports("hexless"): + return "4.3.0" + return "4.2.2" + + +class CiphertextFactory: + """Baseline-produced ciphertexts for the decrypt cells, made on demand. + + Both arms of a decrypt comparison must read the *same* file. If each arm + decrypted its own output, a difference in how the two builds *write* a TDF + would show up as a difference in how fast they read one. + """ + + def __init__( + self, + payloads: dict[str, Path], + tmp_dir: Path, + attr_values: list[str], + ) -> None: + self._payloads = payloads + self._tmp_dir = tmp_dir + self._attr_values = attr_values + self._cache: dict[tuple[str, str], Path] = {} + + def __call__(self, arms: BenchArms, payload_label: str) -> Path: + key = (str(arms.baseline), payload_label) + cached = self._cache.get(key) + if cached is not None: + return cached + ct_file = self._tmp_dir / f"bench-ct-{arms.baseline}-{payload_label}.tdf" + arms.baseline.encrypt( + self._payloads[payload_label], + ct_file, + container="ztdf", + attr_values=self._attr_values, + target_mode=pinned_target_mode(arms), + ) + assert ct_file.is_file() + self._cache[key] = ct_file + return ct_file + + +@pytest.fixture(scope="module") +def bench_ciphertexts( + bench_payloads: dict[str, Path], + tmp_dir: Path, + attribute_default_rsa: abac.Attribute, +) -> CiphertextFactory: + """Ciphertext source for decrypt cells. + + Pinned to the explicit RSA attribute so both arms wrap with RSA regardless + of what base key the platform happens to have configured -- an arm that + silently switched to EC would look slower for reasons that have nothing to + do with a regression. + """ + return CiphertextFactory(bench_payloads, tmp_dir, attribute_default_rsa.value_fqns) + + +def build_arms( + cell: BenchCell, + arms: BenchArms, + *, + pt_file: Path, + ct_file: Path | None, + tmp_dir: Path, + attr_values: list[str], +) -> tuple[Arm, Arm]: + """Turn a cell plus its two builds into two ready-to-run invocations. + + Everything that is not the build under test is pinned identically across + the arms: same plaintext, same attribute (so both wrap with RSA), same + container, same target mode. A functional difference between the builds + that changed any of these would otherwise show up as a speed difference. + + For decrypt, both arms read the *same* ``ct_file``, produced once by the + baseline. Letting each arm decrypt its own output would compare the cost + of reading two different files. + + In a control cell both arms are the baseline build, so the pair differs + only in the output path -- exactly the harness overhead the A/A cell + exists to measure. + """ + baseline_sdk = arms.baseline + candidate_sdk = arms.baseline if cell.control else arms.candidate + target_mode = pinned_target_mode(arms) + + def invocation(sdk: tdfs.SDK, role: str) -> Invocation: + out = tmp_dir / f"bench-{cell.id}-{role}" + if cell.operation == "encrypt": + out = out.with_suffix(".tdf") + argv, env = sdk.encrypt_command( + pt_file, + out, + container="ztdf", + attr_values=attr_values, + target_mode=target_mode, + ) + else: + if ct_file is None: + raise ValueError(f"{cell.id} is a decrypt cell but has no ciphertext") + out = out.with_suffix(".untdf") + argv, env = sdk.decrypt_command(ct_file, out, container="ztdf") + return Invocation(argv, env, out) + + return ( + Arm("baseline", str(baseline_sdk), invocation(baseline_sdk, "baseline")), + Arm("candidate", str(candidate_sdk), invocation(candidate_sdk, "candidate")), + ) + + +def runner_metadata(config: pytest.Config) -> dict[str, object]: + """Machine facts worth keeping alongside the numbers. + + Absolute timings are not comparable across runners, which is why nothing + here feeds the decision rule. It is recorded so that a human reading an + old artifact can tell what they are looking at. + """ + return { + "python": platform.python_version(), + "platform": platform.platform(), + "processor": platform.processor() or "unknown", + "cpu_count": os.cpu_count(), + "runner_os": os.environ.get("RUNNER_OS", ""), + "runner_arch": os.environ.get("RUNNER_ARCH", ""), + "github_run_id": os.environ.get("GITHUB_RUN_ID", ""), + "platform_version": _platform_version(), + "seed": config.getoption("--bench-seed"), + } + + +def _platform_version() -> str: + try: + return tdfs.get_platform_features().version or "unknown" + except Exception: # pragma: no cover - reporting must not break the run + return "unknown" diff --git a/xtest/perf/__init__.py b/xtest/perf/__init__.py new file mode 100644 index 000000000..5ff6e1479 --- /dev/null +++ b/xtest/perf/__init__.py @@ -0,0 +1,12 @@ +"""Performance regression benchmarking for the OpenTDF SDK CLIs. + +The suite compares two SDK builds -- typically the latest release against +``main`` -- by running them against each other on the same machine at the same +time. See ``perf/stats.py`` for why the comparison is structured that way. + +Modules: +- ``measure``: wall-clock / CPU / peak-RSS for a single CLI invocation. +- ``stats``: the paired statistical comparison and its decision rule. +- ``runner``: the round loop that produces paired samples. +- ``report``: JSON artifacts and GitHub step-summary markdown. +""" diff --git a/xtest/perf/_launcher.py b/xtest/perf/_launcher.py new file mode 100644 index 000000000..aab0326ce --- /dev/null +++ b/xtest/perf/_launcher.py @@ -0,0 +1,146 @@ +"""Run one command and report its resource usage, isolated from the caller. + +Why this extra process exists +----------------------------- +On Linux a forked child inherits the parent's resident-set accounting, and +``execve`` does not clear it. ``ru_maxrss`` from ``wait4`` therefore comes back +as ``max(the child's true peak, the parent's RSS at fork time)``. + +Measured straight from a pytest process holding numpy, scipy and a session's +worth of samples, every SDK invocation reports *pytest's* footprint -- about +165 MiB on a CI runner -- instead of its own. Every cell cheaper than that +reports the same number, so a peak-RSS comparison between two builds becomes a +comparison between two readings of the harness. It does not look broken: it +looks like a stable ratio of 1.000, which reads as "no regression". + +``posix_spawn`` and ``sh -c 'exec ...'`` do not help. Both were measured on +Linux and both inherit the same floor; an exec is too late, the accounting is +already latched. The only fix is to fork the measured command from a process +that is holding nothing, which is what this one is for. + +Its own RSS is the floor under every reading it produces, so it reports that +alongside them: a measurement sitting at the floor is censored, not small. + +Private protocol -- :mod:`perf.measure` is the only caller:: + + -I -S _launcher.py [args...] + +One line of space-separated integers is written to ````:: + + + + +``maxrss_raw`` is passed through in whatever unit the platform uses; the caller +normalizes it. ``floor_bytes`` is already bytes. +""" + +import os +import signal +import sys +import time + + +class _Timeout(Exception): + """Raised in the alarm handler to interrupt a blocking wait.""" + + +def _on_alarm(_signum: int, _frame: object) -> None: + raise _Timeout + + +def _current_rss_bytes() -> int: + """This process's resident size right now -- the floor for its children.""" + try: + with open("/proc/self/statm", "rb") as f: + pages = int(f.read().split()[1]) + except OSError, IndexError, ValueError: + # macOS has no /proc. Its ru_maxrss is already bytes, and it does not + # show the inheritance above, so a high-water reading is close enough. + import resource + + return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + return pages * os.sysconf("SC_PAGE_SIZE") + + +def _spawn(command: list[str], err_w: int) -> int: + """Fork and exec ``command``, reporting a failed exec down ``err_w``.""" + pid = os.fork() + if pid != 0: + return pid + try: + # Its own process group, so a timeout kills the whole tree instead of + # just the shim -- leaving a wedged JVM behind would hold the runner + # until the job timeout. + os.setpgid(0, 0) + os.execvp(command[0], command) + # BaseException, not Exception: this is a forked child, and letting a + # SystemExit or a KeyboardInterrupt unwind past here would run the + # *parent's* cleanup -- atexit handlers, buffered output -- a second time, + # from a process that only exists to exec. NOSONAR + except BaseException as e: # noqa: BLE001 - nothing may escape into a fork + try: + os.write(err_w, str(getattr(e, "errno", 0) or 0).encode()) + except OSError: + pass + os._exit(127) + + +def _kill_tree(pid: int) -> None: + try: + os.killpg(pid, signal.SIGKILL) + except OSError: + # setpgid may not have run yet; the bare process is all there is. + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + + +def main(argv: list[str]) -> int: + result_path, timeout_arg = argv[1], argv[2] + command = argv[3:] + timeout = None if timeout_arg == "-" else float(timeout_arg) + + # Closed by a successful exec; carries an errno if the exec never happened. + err_r, err_w = os.pipe() + + floor = _current_rss_bytes() + started = time.perf_counter_ns() + pid = _spawn(command, err_w) + os.close(err_w) + + timed_out = False + if timeout is not None: + signal.signal(signal.SIGALRM, _on_alarm) + signal.setitimer(signal.ITIMER_REAL, timeout) + try: + _, status, ru = os.wait4(pid, 0) + except _Timeout: + timed_out = True + _kill_tree(pid) + _, status, ru = os.wait4(pid, 0) + finally: + if timeout is not None: + signal.setitimer(signal.ITIMER_REAL, 0) + elapsed = time.perf_counter_ns() - started + + exec_errno = os.read(err_r, 32) or b"0" + os.close(err_r) + + fields = ( + status, + elapsed, + int(ru.ru_utime * 1e6), + int(ru.ru_stime * 1e6), + int(ru.ru_maxrss), + floor, + int(timed_out), + int(exec_errno), + ) + with open(result_path, "w") as f: + f.write(" ".join(str(v) for v in fields)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/xtest/perf/cells.py b/xtest/perf/cells.py new file mode 100644 index 000000000..4a1d4ba47 --- /dev/null +++ b/xtest/perf/cells.py @@ -0,0 +1,84 @@ +"""The benchmark's experiment matrix. + +Kept free of pytest and of ``tdfs`` so that both the conftest parametrizer and +the reporting layer can name a cell without importing each other. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +operation_type = Literal["encrypt", "decrypt"] + + +@dataclass(frozen=True, slots=True) +class Payload: + """One payload size regime. + + The sizes separate two failure modes that hide each other. At 1 KiB + essentially all the cost is process startup -- JVM boot, npx resolution, + TLS handshake, token fetch -- so a throughput regression is invisible. At + 32 MiB the crypto and IO dominate and a startup regression is invisible. + A benchmark at one size only will miss half the regressions it claims to + cover. + """ + + label: str + n_bytes: int + + +PAYLOADS: tuple[Payload, ...] = ( + Payload("1KiB", 1024), + Payload("1MiB", 2**20), + Payload("32MiB", 32 * 2**20), +) + +#: Payload used for the A/A control. Mid-size: large enough that startup noise +#: does not dominate it, small enough that the control is not a big slice of +#: the budget. +CONTROL_PAYLOAD = PAYLOADS[1] + + +@dataclass(frozen=True, slots=True) +class BenchCell: + """One comparison: an operation at a payload size, for one SDK.""" + + sdk: str + operation: operation_type + payload: Payload + #: A/A control -- the same build in both arms, through the same pipeline. + #: Its true ratio is 1.0 by construction, so whatever it reports is the + #: harness's own error. + control: bool = False + + @property + def id(self) -> str: + suffix = "-control" if self.control else "" + return f"{self.sdk}-{self.operation}-{self.payload.label}{suffix}" + + def __str__(self) -> str: + return self.id + + +def cells_for(sdks: list[str]) -> list[BenchCell]: + """Build the full cell list for a run, each SDK's control cell first. + + One control per SDK rather than one per run: a control measures a + particular SDK's harness path, and go's noise floor says nothing about + java's. + + Controls run first because a run that overruns its time budget loses + whatever is at the end. Losing one comparison leaves the rest + trustworthy; losing the control leaves nothing trustworthy at all, since + without a noise floor no cell may report PASS. + """ + cells: list[BenchCell] = [] + for sdk in sdks: + cells.append(BenchCell(sdk, "encrypt", CONTROL_PAYLOAD, control=True)) + cells += [ + BenchCell(sdk, op, payload) + for op in ("encrypt", "decrypt") + for payload in PAYLOADS + ] + return cells diff --git a/xtest/perf/measure.py b/xtest/perf/measure.py new file mode 100644 index 000000000..0385a9895 --- /dev/null +++ b/xtest/perf/measure.py @@ -0,0 +1,254 @@ +"""Resource measurement for a single SDK CLI invocation. + +Measures wall-clock time, CPU time, and peak resident memory for one child +process and everything it spawns. The SDK CLIs are bash shims that exec a Go +binary, a JVM, or node, so "everything it spawns" is the interesting part. + +Why ``os.wait4`` rather than ``resource.getrusage`` +--------------------------------------------------- +``resource.getrusage(RUSAGE_CHILDREN)`` reports a *process-lifetime* high-water +mark for ``ru_maxrss``. Subtracting successive readings does not give the peak +of the most recent child -- once one big child has run, every later delta reads +zero. ``os.wait4`` returns rusage for the specific child being reaped, which is +what we actually want. + +The kernel folds a child's reaped descendants into its rusage, so the shim's +``java``/``node``/``otdfctl`` grandchild is included: CPU times sum and +``ru_maxrss`` takes the maximum. Both are the right aggregation here. + +Why the command is not spawned from this process +------------------------------------------------ +A forked child inherits the parent's resident-set accounting on Linux, and +exec does not clear it, so ``ru_maxrss`` would report the *measuring* process's +memory whenever the measured command uses less. Everything therefore goes +through :mod:`perf._launcher`, which holds nothing and forks the real command +itself. That module's docstring has the measurements behind this. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import threading +from dataclasses import dataclass, fields +from pathlib import Path +from typing import IO + +#: ``ru_maxrss`` is kilobytes on Linux and bytes on macOS. There is no portable +#: way to ask, so branch on the platform. +_RSS_SCALE = 1 if sys.platform == "darwin" else 1024 + +#: How much of a failing child's stderr to keep for the error message. +_STDERR_TAIL_BYTES = 4000 + +_LAUNCHER = Path(__file__).with_name("_launcher.py") + +#: Grace period on top of the launcher's own timeout, before this process +#: gives up on it. Only reachable if the launcher itself wedges. +_LAUNCHER_GRACE_S = 30.0 + + +class MeasurementError(RuntimeError): + """A measured invocation failed, timed out, or could not be measured. + + Benchmarking an operation that does not succeed is worse than not + benchmarking it: a build that errors out early looks fast. + """ + + +@dataclass(frozen=True, slots=True) +class Sample: + """Resource usage of one CLI invocation.""" + + #: Wall-clock duration including fork/exec, which is part of the real cost. + wall_ns: int + #: User + system CPU seconds, summed over the process tree. + cpu_s: float + #: Peak resident set size in bytes, maximum over the process tree. + max_rss_bytes: int + exit_code: int + #: RSS of the launcher at fork time. A child cannot be measured below the + #: memory of whatever forked it, so a reading at this value is censored + #: rather than small. Reported so that fact is visible. + rss_floor_bytes: int = 0 + + @property + def rss_is_floored(self) -> bool: + """Whether peak RSS is indistinguishable from the measurement floor.""" + return self.rss_floor_bytes > 0 and self.max_rss_bytes <= self.rss_floor_bytes + + @property + def wall_s(self) -> float: + return self.wall_ns / 1e9 + + def metric(self, name: str) -> float: + """Return a metric by name, for generic iteration over metric sets.""" + match name: + case "wall": + return float(self.wall_ns) + case "cpu": + return self.cpu_s + case "rss": + return float(self.max_rss_bytes) + case _: + raise KeyError(f"unknown metric {name!r}") + + +#: Metrics a :class:`Sample` can report, in display order. +METRICS: tuple[str, ...] = ("wall", "cpu", "rss") + +#: Human-facing labels and units, keyed by metric name. +METRIC_LABELS: dict[str, tuple[str, str]] = { + "wall": ("wall clock", "ms"), + "cpu": ("cpu time", "s"), + "rss": ("peak rss", "MiB"), +} + + +def format_metric(name: str, value: float) -> str: + """Render a raw metric value in its display unit.""" + match name: + case "wall": + return f"{value / 1e6:.1f} ms" + case "cpu": + return f"{value:.3f} s" + case "rss": + return f"{value / 2**20:.1f} MiB" + case _: + raise KeyError(f"unknown metric {name!r}") + + +def measure( + argv: list[str], + env: dict[str, str] | None = None, + cwd: Path | None = None, + *, + timeout_s: float | None = 600.0, + check: bool = True, +) -> Sample: + """Run ``argv`` once and return its resource usage. + + Args: + argv: command to run, already fully constructed. + env: complete environment for the child (not merged with the parent's). + cwd: working directory for the child. + timeout_s: kill the child after this long. ``None`` waits forever, + which will hang a CI job on a wedged CLI. + check: raise :class:`MeasurementError` on a non-zero exit. + + Raises: + MeasurementError: on non-zero exit (when ``check``), on timeout, or if + the process could not be started. + """ + if not hasattr(os, "wait4"): # pragma: no cover - Unix-only test suite + raise MeasurementError( + "per-process rusage requires os.wait4, which this platform lacks" + ) + + # Redirect to real files rather than pipes: nothing here drains a pipe, so + # a child that fills its stdout buffer would deadlock against us. + with ( + tempfile.TemporaryFile() as out, + tempfile.TemporaryFile() as err, + tempfile.TemporaryDirectory() as scratch, + ): + result_path = Path(scratch) / "rusage" + launcher_argv = [ + sys.executable, + "-I", + "-S", + str(_LAUNCHER), + str(result_path), + "-" if timeout_s is None else repr(float(timeout_s)), + *argv, + ] + try: + proc = subprocess.Popen( + launcher_argv, stdout=out, stderr=err, env=env, cwd=cwd + ) + except OSError as e: # pragma: no cover - our own interpreter is missing + raise MeasurementError( + f"could not start the measurement launcher: {e}" + ) from e + + # The launcher enforces the real timeout and kills the measured process + # group. This only covers the launcher itself wedging. + backstop: threading.Timer | None = None + if timeout_s is not None: + backstop = threading.Timer(timeout_s + _LAUNCHER_GRACE_S, proc.kill) + backstop.daemon = True + backstop.start() + try: + proc.wait() + finally: + if backstop is not None: + backstop.cancel() + + reading = _read_result(result_path, argv) + if reading.exec_errno: + raise MeasurementError( + f"could not start {argv[0]!r}: {os.strerror(reading.exec_errno)}" + ) + if reading.timed_out: + raise MeasurementError( + f"{argv[0]!r} exceeded the {timeout_s:g}s measurement timeout" + ) + + sample = Sample( + wall_ns=reading.wall_ns, + cpu_s=(reading.utime_us + reading.stime_us) / 1e6, + max_rss_bytes=reading.maxrss_raw * _RSS_SCALE, + exit_code=os.waitstatus_to_exitcode(reading.status), + rss_floor_bytes=reading.floor_bytes, + ) + if check and sample.exit_code != 0: + raise MeasurementError( + f"{' '.join(argv)} exited {sample.exit_code}\nstderr: {_tail(err)}" + ) + return sample + + +@dataclass(frozen=True, slots=True) +class _Reading: + """The launcher's raw report, before it is turned into a :class:`Sample`.""" + + status: int + wall_ns: int + utime_us: int + stime_us: int + maxrss_raw: int + floor_bytes: int + timed_out: int + exec_errno: int + + +#: Integers the launcher writes, one per :class:`_Reading` field. +_RESULT_FIELD_COUNT = len(fields(_Reading)) + + +def _read_result(path: Path, argv: list[str]) -> _Reading: + """Parse the launcher's report, or say clearly that it never made one.""" + try: + fields = [int(v) for v in path.read_text().split()] + except (OSError, ValueError) as e: + raise MeasurementError( + f"the measurement launcher did not report on {argv[0]!r}: {e}" + ) from e + if len(fields) != _RESULT_FIELD_COUNT: + raise MeasurementError( + f"the measurement launcher reported {len(fields)} fields for " + f"{argv[0]!r}, expected {_RESULT_FIELD_COUNT}" + ) + return _Reading(*fields) + + +def _tail(fh: IO[bytes]) -> str: + """Read the last few KiB of a temp file, for error messages.""" + try: + fh.seek(0, os.SEEK_END) + fh.seek(max(0, fh.tell() - _STDERR_TAIL_BYTES)) + return fh.read().decode(errors="replace").strip() + except OSError: # pragma: no cover - defensive + return "" diff --git a/xtest/perf/report.py b/xtest/perf/report.py new file mode 100644 index 000000000..a100044ae --- /dev/null +++ b/xtest/perf/report.py @@ -0,0 +1,263 @@ +"""Collecting benchmark results and turning them into artifacts. + +Cells do not assert. Each one records its raw samples here and moves on, +because the decision rule needs every cell before it can decide anything: +the multiplicity correction is computed across the run, and the A/A control +can invalidate the whole thing. The gate therefore runs once, at session +finish, from :meth:`BenchmarkRecorder.gate`. + +Two artifacts come out: + +- A JSON file per run, holding **every raw per-round sample** alongside the + derived statistics. Re-analysing a surprising result offline is the + difference between understanding a red build and re-running a 30-minute job + to look at the same numbers again. +- A markdown table for ``$GITHUB_STEP_SUMMARY``, so the answer is on the job + page rather than buried in log output. +""" + +from __future__ import annotations + +import json +import math +import os +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from perf import stats +from perf.cells import BenchCell +from perf.measure import METRIC_LABELS, METRICS, format_metric +from perf.runner import BenchConfig, CellResult, analyze + +#: Cells the session intends to run. Set by the conftest parametrizer, read by +#: the budget and arm-resolution fixtures. +CELLS_KEY: pytest.StashKey[list[BenchCell]] = pytest.StashKey() + +#: The session's recorder, reachable from both fixtures and session hooks. +RECORDER_KEY: pytest.StashKey[BenchmarkRecorder] = pytest.StashKey() + + +def recorder_for(config: pytest.Config) -> BenchmarkRecorder: + """Return the session's recorder, creating it on first use.""" + existing = config.stash.get(RECORDER_KEY, None) + if existing is not None: + return existing + recorder = BenchmarkRecorder() + config.stash[RECORDER_KEY] = recorder + return recorder + + +@dataclass(slots=True) +class BenchmarkRecorder: + """Session-wide collector for cell results, skips, and failures.""" + + results: list[CellResult] = field(default_factory=list) + #: cell id -> why it did not run. Reported so a quiet run is visibly + #: quiet rather than indistinguishable from a clean one. + skipped: dict[str, str] = field(default_factory=dict) + metadata: dict[str, object] = field(default_factory=dict) + + def record(self, result: CellResult) -> None: + self.results.append(result) + + def skip(self, cell_id: str, reason: str) -> None: + self.skipped[cell_id] = reason + + def gate(self, config: BenchConfig) -> stats.GateResult: + return analyze(self.results, config) + + +def to_dict( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> dict[str, object]: + """Serialize a whole run, raw samples included.""" + return { + "schema": 1, + "metadata": recorder.metadata, + "config": { + "min_rounds": config.min_rounds, + "max_rounds": config.max_rounds, + "warmup": config.warmup, + "budget_seconds": config.budget_seconds, + "seed": config.seed, + "threshold": config.threshold, + "confidence": config.confidence, + "n_resamples": config.n_resamples, + "gated_metrics": list(config.gated_metrics), + }, + "noise_floor": _noise_dict(gate.noise), + # Per-control floors as well as the run-level worst case: with several + # SDKs in one run, "which one was noisy" is the first question a + # surprising verdict raises. + "noise_floor_by_control": { + k: _noise_dict(n) for k, n in gate.noise_by_control.items() + }, + "trustworthy": gate.trustworthy, + "regressions": gate.regressions, + "improvements": gate.improvements, + "summary": gate.summary, + "skipped": recorder.skipped, + "cells": [ + { + "id": r.cell_id, + "baseline": r.baseline_label, + "candidate": r.candidate_label, + "control": r.control, + "n_rounds": r.n_rounds, + "n_warmup": r.n_warmup, + "elapsed_s": round(r.elapsed_s, 3), + "stopped_because": r.stopped_because, + "rss_floor_bytes": r.rss_floor_bytes, + "samples": r.samples, + "metrics": { + m: _comparison_dict(gate.comparisons[f"{r.cell_id}/{m}"]) + for m in METRICS + if f"{r.cell_id}/{m}" in gate.comparisons + }, + } + for r in recorder.results + ], + } + + +def _noise_dict(noise: stats.NoiseFloor | None) -> dict[str, object]: + if noise is None: + return {"assessed": False} + return { + "assessed": True, + "tripped": noise.tripped, + "underpowered": noise.underpowered, + "width_ratio": _jsonable(noise.width_ratio), + "detail": noise.detail, + } + + +def _comparison_dict(c: stats.PairedComparison) -> dict[str, object]: + return { + "n_rounds": c.n_rounds, + "baseline_median": _jsonable(c.baseline_median), + "candidate_median": _jsonable(c.candidate_median), + "ratio": _jsonable(c.ratio), + "ci_low": _jsonable(c.ci_low), + "ci_high": _jsonable(c.ci_high), + "p_value": _jsonable(c.p_value), + "p_adjusted": _jsonable(c.p_adjusted), + "verdict": str(c.verdict), + "note": c.note, + } + + +def _jsonable(v: float | None) -> float | None: + """JSON has no NaN or infinity; emit null rather than invalid JSON.""" + if v is None or not math.isfinite(v): + return None + return v + + +def write_json( + path: Path, + recorder: BenchmarkRecorder, + config: BenchConfig, + gate: stats.GateResult, +) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(to_dict(recorder, config, gate), indent=2)) + return path + + +def markdown( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> str: + """Render the run as a GitHub step summary.""" + threshold_pct = (config.threshold - 1) * 100 + lines = [ + "## SDK performance regression benchmark", + "", + f"Paired A/B on one runner. A cell fails only if the 95% CI lower " + f"bound exceeds **{config.threshold:.2f}x** (+{threshold_pct:.0f}%) " + f"*and* the BH-adjusted p < {stats.DEFAULT_ALPHA}.", + "", + f"**{gate.summary}**", + "", + ] + + noise = gate.noise + if noise is not None and noise.detail: + lines += [f"> {noise.detail}", ""] + elif noise is not None and math.isfinite(noise.width_ratio): + lines += [ + f"A/A noise floor: +/-{(noise.width_ratio - 1) * 100:.1f}% " + f"(the smallest effect this run could resolve).", + "", + ] + + lines += [ + "| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for result in recorder.results: + for metric in METRICS: + key = f"{result.cell_id}/{metric}" + c = gate.comparisons.get(key) + if c is None: + continue + gated = metric in config.gated_metrics and not result.control + label = METRIC_LABELS[metric][0] + ("" if gated else " (ungated)") + lines.append( + f"| {result.cell_id} | {label} " + f"| {format_metric(metric, c.baseline_median)} " + f"| {format_metric(metric, c.candidate_median)} " + f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " + f"| {_verdict_cell(c)} |" + ) + + if recorder.skipped: + lines += ["", "### Not measured", ""] + lines += [f"- `{cid}`: {why}" for cid, why in sorted(recorder.skipped.items())] + + lines += [ + "", + f"seed {config.seed}; warm-up {config.warmup} rounds; " + f"{config.min_rounds}-{config.max_rounds} measured rounds per cell; " + "stopping on attained CI width, never on significance.", + ] + return "\n".join(lines) + "\n" + + +def _ratio_cell(c: stats.PairedComparison) -> str: + if not math.isfinite(c.ratio): + return "-" + if not (math.isfinite(c.ci_low) and math.isfinite(c.ci_high)): + return f"{c.ratio:.3f}x" + return f"{c.ratio:.3f}x [{c.ci_low:.3f}, {c.ci_high:.3f}]" + + +def _p_cell(c: stats.PairedComparison) -> str: + p = c.p_adjusted if c.p_adjusted is not None else c.p_value + if p is None or not math.isfinite(p): + return "-" + return f"{p:.3f}" if p >= 0.001 else "<0.001" + + +_VERDICT_ICONS = { + stats.Verdict.PASS: "PASS", + stats.Verdict.REGRESSION: "**REGRESSION**", + stats.Verdict.IMPROVED: "IMPROVED", + stats.Verdict.INCONCLUSIVE: "inconclusive", +} + + +def _verdict_cell(c: stats.PairedComparison) -> str: + text = _VERDICT_ICONS[c.verdict] + return f"{text} ({c.note})" if c.note else text + + +def append_step_summary(text: str) -> None: + """Append to ``$GITHUB_STEP_SUMMARY`` when running in Actions.""" + target = os.environ.get("GITHUB_STEP_SUMMARY") + if not target: + return + with open(target, "a", encoding="utf-8") as f: + f.write(text) diff --git a/xtest/perf/runner.py b/xtest/perf/runner.py new file mode 100644 index 000000000..af69ff68b --- /dev/null +++ b/xtest/perf/runner.py @@ -0,0 +1,431 @@ +"""The paired round loop that produces comparable samples for one cell. + +A *cell* is one operation at one payload size, measured for two SDK builds. +The loop runs both arms once per round, in a randomized order, until it has +either enough precision or no more time. + +Why rounds rather than "run A 30 times, then B 30 times" +-------------------------------------------------------- +A shared runner drifts: a noisy neighbour arrives, the CPU thermally throttles, +the page cache warms. Run all of A and then all of B and every one of those +effects lands entirely on one arm and shows up as a difference between builds. +Interleaving means both arms see the same conditions within a round, and the +per-round ratio differences it out. + +The order *within* a round is randomized because a fixed order is itself a +confounder -- whichever arm runs second inherits the first one's cache state. + +Why stopping on precision and not on significance +------------------------------------------------- +The loop stops when the confidence interval is narrow enough, never when the +p-value gets small. Peeking at the p-value and stopping the moment it drops +below alpha is optional stopping: it inflates the false-positive rate well +past the nominal level, because you get a fresh chance to cross the line every +round and only ever stop on the lucky side. Attained CI width, in contrast, is +driven by the dispersion of the differences rather than their location, so it +is approximately ancillary to the effect being tested and stopping on it does +not bias the verdict. + +This distinction is easy to "optimize away" -- stopping on significance would +finish sooner -- and doing so silently invalidates every result the job +produces. Do not. +""" + +from __future__ import annotations + +import os +import random +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np + +from perf import stats +from perf.measure import METRICS, MeasurementError, Sample, measure + +#: Target CI half-width on the log scale, as a fraction of the log threshold. +#: At 1/3, an interval centred on "no change" is comfortably clear of the +#: threshold, so a PASS is a real statement about precision rather than a +#: shrug. Tighter costs rounds superlinearly; looser makes PASS meaningless. +PRECISION_FRACTION = 1 / 3 + +#: Bootstrap resamples for the between-round precision check. Far fewer than +#: the final analysis uses: this only needs to answer "is the interval roughly +#: narrow enough yet", and it runs after every round. +_INTERIM_RESAMPLES = 1000 + +#: The A/A control metric that assesses an SDK's noise floor. Wall clock is +#: the most sensitive of the gated metrics to runner noise, which makes it the +#: honest choice of canary. +_CONTROL_METRIC = "wall" + + +class BudgetExhausted(RuntimeError): + """The time budget ran out before the cell could collect usable rounds.""" + + +@dataclass(frozen=True, slots=True) +class BenchConfig: + """Knobs for the round loop and the analysis that follows it.""" + + min_rounds: int = 20 + max_rounds: int = 60 + warmup: int = 5 + budget_seconds: float = 1500.0 + seed: int = 0 + threshold: float = stats.DEFAULT_THRESHOLD + confidence: float = 0.95 + n_resamples: int = stats.DEFAULT_BOOTSTRAP_RESAMPLES + #: Per-invocation timeout. A wedged CLI must not eat the whole job. + timeout_s: float = 600.0 + #: Metrics whose verdict can fail the build. CPU time is measured and + #: reported but excluded: it is the noisiest of the three on a shared + #: runner, and a real CPU regression shows up in wall clock anyway. + gated_metrics: tuple[str, ...] = ("wall", "rss") + + def __post_init__(self) -> None: + if self.min_rounds < stats.MIN_USABLE_ROUNDS: + raise ValueError( + f"min_rounds must be at least {stats.MIN_USABLE_ROUNDS}, " + f"below which no verdict is possible" + ) + if self.max_rounds < self.min_rounds: + raise ValueError("max_rounds must not be below min_rounds") + if self.warmup < 0: + raise ValueError("warmup must not be negative") + if self.threshold <= 1.0: + raise ValueError("threshold is a ratio above 1.0, e.g. 1.15 for 15%") + unknown = set(self.gated_metrics) - set(METRICS) + if unknown: + raise ValueError(f"unknown gated metrics: {sorted(unknown)}") + + @property + def target_half_width_log(self) -> float: + """CI half-width, on the log scale, that ends the round loop.""" + return float(np.log(self.threshold)) * PRECISION_FRACTION + + +@dataclass(frozen=True, slots=True) +class Invocation: + """One fully-built CLI call, ready to run repeatedly.""" + + argv: list[str] + #: CLI-specific overrides, merged over ``os.environ`` at run time. + env: dict[str, str] = field(default_factory=dict) + #: Removed before each measured run, so every round starts from the same + #: state rather than measuring an overwrite in round 2 onwards. + output: Path | None = None + + def child_env(self) -> dict[str, str]: + return dict(os.environ) | self.env + + +@dataclass(frozen=True, slots=True) +class Arm: + """One side of a comparison.""" + + #: ``"baseline"`` or ``"candidate"``; identifies the role, not the build. + name: str + #: The build under this role, e.g. ``"go@v0.29.0"``. + label: str + invocation: Invocation + + +@dataclass(slots=True) +class CellResult: + """Everything one cell measured, before any verdict is assigned. + + Raw per-round vectors are kept in full. Re-analysing a surprising result + offline is the difference between understanding a red build and re-running + a 30-minute job to look at it again. + """ + + cell_id: str + baseline_label: str + candidate_label: str + #: ``samples[arm_name][metric]`` is the per-round vector, warm-up excluded. + samples: dict[str, dict[str, list[float]]] + n_warmup: int + elapsed_s: float + stopped_because: str + #: True for the A/A control, where both arms are the same build. + control: bool = False + #: Which SDK's control cell assesses this cell's noise floor. A run may + #: measure several SDKs, and each has its own harness path and its own + #: floor; go's says nothing about java's. + sdk: str = "" + #: Highest measurement floor seen in this cell -- the RSS of the process + #: that forked each invocation. See :mod:`perf._launcher`. + rss_floor_bytes: int = 0 + + @property + def rss_censored_reason(self) -> str | None: + """Why this cell's peak RSS cannot be compared, or None if it can. + + A command whose peak sits at the floor was not measured, it was + clipped, and both arms clip to the same value. The resulting ratio is + 1.000 with a tight interval, which is the most convincing-looking + PASS the harness can emit and means nothing at all. + """ + if self.rss_floor_bytes <= 0: + return None + rss = [v for arm in self.samples.values() for v in arm.get("rss", [])] + if not rss or min(rss) > self.rss_floor_bytes: + return None + return ( + f"peak rss reaches the {self.rss_floor_bytes / 2**20:.0f} MiB " + "measurement floor, so the two arms are not distinguishable" + ) + + @property + def n_rounds(self) -> int: + first = next(iter(self.samples.values()), {}) + return len(next(iter(first.values()), [])) + + def metric_pair(self, metric: str) -> tuple[list[float], list[float]]: + """Return ``(baseline, candidate)`` vectors for one metric.""" + return self.samples["baseline"][metric], self.samples["candidate"][metric] + + def compare(self, metric: str, config: BenchConfig) -> stats.PairedComparison: + baseline, candidate = self.metric_pair(metric) + return stats.compare( + baseline, + candidate, + confidence=config.confidence, + seed=config.seed, + n_resamples=config.n_resamples, + ) + + +def _empty_samples() -> dict[str, dict[str, list[float]]]: + return {arm: {m: [] for m in METRICS} for arm in ("baseline", "candidate")} + + +def run_cell( + cell_id: str, + baseline: Arm, + candidate: Arm, + config: BenchConfig, + *, + deadline: float | None = None, + control: bool = False, + sdk: str = "", + clock: Callable[[], float] = time.monotonic, + run: Callable[..., Sample] = measure, +) -> CellResult: + """Run one cell's paired rounds and return its raw samples. + + Args: + cell_id: stable identifier, also the per-cell RNG seed material so + that two cells do not share an interleaving order. + baseline: the arm the candidate is compared against. + candidate: the arm under test. For an A/A control this is the same + build as ``baseline``, running through the identical path. + deadline: absolute ``clock()`` value past which no new round starts. + control: records that this is the A/A cell; does not change the loop. + sdk: which SDK this cell belongs to, so that the analysis can pair it + with the right control. Only matters in a multi-SDK run. + clock: injectable monotonic clock. + run: injectable measurement function, for testing the loop itself. + + Raises: + MeasurementError: if any invocation fails. A benchmark over an + operation that errors out is measuring the error path. + BudgetExhausted: if the deadline passed before ``min_rounds``, or + during warm-up. + """ + arms = (baseline, candidate) + # Seeded per cell so a rerun reproduces the interleaving exactly, but the + # cells do not all share one order (which would correlate their noise). + rng = random.Random(f"{config.seed}:{cell_id}") + samples = _empty_samples() + round_durations: list[float] = [] + rss_floor = 0 + started = clock() + + def one_round(into: dict[str, dict[str, list[float]]]) -> None: + order = list(arms) + rng.shuffle(order) + for arm in order: + inv = arm.invocation + if inv.output is not None: + inv.output.unlink(missing_ok=True) + try: + sample = run(inv.argv, inv.child_env(), timeout_s=config.timeout_s) + except MeasurementError as e: + raise MeasurementError(f"{cell_id}: {arm.label} failed: {e}") from e + nonlocal rss_floor + rss_floor = max(rss_floor, sample.rss_floor_bytes) + for metric in METRICS: + into[arm.name][metric].append(sample.metric(metric)) + + for i in range(config.warmup): + # Warm-up rounds pay the one-time costs -- page cache, `go build` + # cache, npx package resolution, JIT warm-up -- that would otherwise + # land unevenly and show up as a difference between builds. Their + # samples are collected into a throwaway dict and dropped. + # + # The deadline is checked here too, and not only in the measured loop + # below. The budget's end is absolute, so warm-ups that overrun it + # spend the *following* cells' time and then reach the measured loop + # with nothing left -- paying the full cost of the cell and producing + # no data. Better to give up here and say why. + if deadline is not None and clock() >= deadline: + raise BudgetExhausted( + f"{cell_id}: budget ran out after {i} of {config.warmup} " + f"warm-up rounds ({clock() - started:.0f}s), " + "before any measurement began" + ) + one_round(_empty_samples()) + + stopped_because = "max_rounds" + for _ in range(config.max_rounds): + round_start = clock() + if deadline is not None and round_start >= deadline: + stopped_because = "budget" + break + if deadline is not None and round_durations: + # Do not start a round we cannot finish: a half-measured round is + # unpaired data, and unpaired data is exactly what this design + # exists to avoid. + expected = float(np.median(round_durations)) + if round_start + expected > deadline: + stopped_because = "budget" + break + one_round(samples) + round_durations.append(clock() - round_start) + + n = len(samples["baseline"]["wall"]) + if n >= config.min_rounds and _precise_enough(samples, config): + stopped_because = "precision" + break + + elapsed = clock() - started + n = len(samples["baseline"]["wall"]) + if n < stats.MIN_USABLE_ROUNDS: + raise BudgetExhausted( + f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " + f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + ) + return CellResult( + cell_id=cell_id, + baseline_label=baseline.label, + candidate_label=candidate.label, + samples=samples, + n_warmup=config.warmup, + elapsed_s=elapsed, + stopped_because=stopped_because, + control=control, + sdk=sdk, + rss_floor_bytes=rss_floor, + ) + + +def _precise_enough( + samples: dict[str, dict[str, list[float]]], config: BenchConfig +) -> bool: + """True once every gated metric's CI is narrow enough to decide on. + + Deliberately looks only at interval *width*, never at where the interval + sits or at any p-value -- see the module docstring. + """ + for metric in config.gated_metrics: + c = stats.compare( + samples["baseline"][metric], + samples["candidate"][metric], + confidence=config.confidence, + seed=config.seed, + n_resamples=_INTERIM_RESAMPLES, + ) + # `not (a <= b)` rather than `a > b`, which is not the same thing when + # a is NaN: an unusable interval must read as "keep going", and + # `NaN > b` is False, which would end the loop and call it precise. + if not c.ci_half_width_log <= config.target_half_width_log: # NOSONAR + return False + return True + + +class Budget: + """Shares one wall-clock allowance across a run's cells. + + Cells are measured one at a time, so a cell that stops early on precision + should hand its unused time to the cells after it rather than letting the + last cell get squeezed by whatever the first ones happened to use. + """ + + def __init__( + self, + total_seconds: float, + n_cells: int, + *, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if n_cells <= 0: + raise ValueError("a budget needs at least one cell to divide across") + self._clock = clock + self._end = clock() + total_seconds + self._cells_left = n_cells + + def next_deadline(self) -> float: + """Absolute deadline for the next cell: an even share of what is left.""" + now = self._clock() + share = max(0.0, self._end - now) / max(1, self._cells_left) + self._cells_left = max(0, self._cells_left - 1) + return now + share + + @property + def remaining_s(self) -> float: + return max(0.0, self._end - self._clock()) + + +def analyze(results: Sequence[CellResult], config: BenchConfig) -> stats.GateResult: + """Turn every cell's raw samples into one gate decision. + + ``GateResult.comparisons`` is keyed by ``"/"`` and holds + the *finalized* comparisons -- the ones carrying adjusted p-values and + verdicts. Ungated metrics are included so they appear in the report, but + they cannot fail the build, and they are corrected separately from the + gated ones: adjusting across tests nobody gates on only makes a real + regression harder to confirm. + + Each SDK's cells are paired with *that SDK's* control. A run measuring go + and java has two harness paths and two noise floors, and judging java's + cells against go's control would be judging them against a floor that was + never measured for them. + """ + comparisons: dict[str, stats.PairedComparison] = {} + gated: set[str] = set() + censored: dict[str, str] = {} + control_keys: set[str] = set() + controls: dict[str, str] = {} + control_for_sdk = { + r.sdk: f"{r.cell_id}/{_CONTROL_METRIC}" for r in results if r.control + } + + for result in results: + floored = result.rss_censored_reason + for metric in METRICS: + key = f"{result.cell_id}/{metric}" + comparisons[key] = result.compare(metric, config) + control_key = control_for_sdk.get(result.sdk) + if control_key is not None: + controls[key] = control_key + if metric == "rss" and floored is not None: + censored[key] = floored + continue + if result.control: + control_keys.add(key) + elif metric in config.gated_metrics: + gated.add(key) + + return stats.apply_multiplicity_control( + comparisons, + gated=gated, + controls=controls, + control_keys=control_keys, + censored=censored, + threshold=config.threshold, + alpha=stats.DEFAULT_ALPHA, + ) diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py new file mode 100644 index 000000000..2f3e49255 --- /dev/null +++ b/xtest/perf/stats.py @@ -0,0 +1,544 @@ +"""Paired statistical comparison of two SDK builds. + +Why this shape +-------------- +Absolute timings from a GitHub-hosted runner are not comparable across runs: +CPU models vary, tenancy is shared, and steal time is unbounded. Storing a +baseline and diffing against it produces false alarms until people mute the +job. So we never compare across runs. Both builds are measured on the *same* +runner, interleaved in time, and the statistic is the within-round *ratio*. +Runner speed is then a shared factor that divides out. + +Everything here is a pure function over sample vectors, so the decision logic +is testable without a platform, an SDK, or a subprocess. + +The scale +--------- +Comparisons use the log-ratio ``d_i = ln(candidate_i) - ln(baseline_i)`` of the +i-th paired round. Logs make ratios symmetric (a 2x slowdown and a 2x speedup +are equal and opposite) and additive, which is what the median and the +bootstrap want. Results are exponentiated back to ratios for reporting. + +The decision rule +----------------- +A cell is a regression iff **both**: + +1. the lower bound of the 95% CI on the ratio exceeds ``threshold``, and +2. the Benjamini-Hochberg adjusted one-sided p-value is below ``alpha``. + +Requiring both is deliberate, and neither clause is redundant: + +- Clause 1 alone would fire on a real-but-trivial effect measured precisely + enough -- a reproducible 0.5% slowdown is not worth a red build. + It cannot fire on pure noise, since that would require the interval to + exclude an effect that is not there. +- Clause 2 alone would fire on noise roughly ``alpha`` of the time per cell, + and a run has enough cells that "roughly alpha" becomes "most nights". + BH adjustment across cells controls the false discovery rate. + +Together they answer the only question worth gating on: is the slowdown both +real and large enough to care about? +""" + +from __future__ import annotations + +import math +import warnings +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import StrEnum +from typing import cast + +import numpy as np +from scipy import stats as _scipy_stats + +# Rounds below this cannot support a meaningful interval. The one-sided +# signed-rank test cannot reach p < 0.05 at all below n=5, so anything less is +# reported as INCONCLUSIVE rather than given a verdict. +MIN_USABLE_ROUNDS = 5 + +#: A vector of per-round measurements for one arm of one cell. Accepts a plain +#: list from the runner or an array from a test's synthetic data generator. +type Samples = Sequence[float] | np.ndarray + +DEFAULT_ALPHA = 0.05 +DEFAULT_THRESHOLD = 1.15 +DEFAULT_BOOTSTRAP_RESAMPLES = 10000 + + +class Verdict(StrEnum): + """Outcome for a single comparison cell.""" + + PASS = "PASS" + REGRESSION = "REGRESSION" + IMPROVED = "IMPROVED" + INCONCLUSIVE = "INCONCLUSIVE" + + +@dataclass(frozen=True, slots=True) +class PairedComparison: + """The statistical summary of one (cell, metric) comparison. + + Ratios are candidate-over-baseline: 1.20 means the candidate took 20% + longer (or used 20% more memory) than the baseline. + """ + + n_rounds: int + baseline_median: float + candidate_median: float + ratio: float + ci_low: float + ci_high: float + p_value: float + #: Set by :func:`apply_multiplicity_control` once every cell is known. + p_adjusted: float | None = None + verdict: Verdict = Verdict.INCONCLUSIVE + note: str = "" + + @property + def ci_half_width_log(self) -> float: + """Half-width of the CI on the log scale, the run's attained precision.""" + if not (math.isfinite(self.ci_low) and math.isfinite(self.ci_high)): + return math.inf + if self.ci_low <= 0 or self.ci_high <= 0: + return math.inf + return (math.log(self.ci_high) - math.log(self.ci_low)) / 2 + + +def log_ratios(baseline: Samples, candidate: Samples) -> np.ndarray: + """Return per-round log-ratios ``ln(candidate) - ln(baseline)``. + + The two vectors must be the same length: entry i of each comes from the + same round, which is what makes the comparison paired. Non-positive + measurements cannot be log-transformed and indicate a broken measurement + rather than a fast one, so they are rejected outright. + """ + b = np.asarray(baseline, dtype=float) + c = np.asarray(candidate, dtype=float) + if b.shape != c.shape: + raise ValueError( + f"paired vectors must be the same length, got {b.shape} and {c.shape}" + ) + if b.size == 0: + return np.empty(0, dtype=float) + if not (np.all(np.isfinite(b)) and np.all(np.isfinite(c))): + raise ValueError("measurements must all be finite") + if np.any(b <= 0) or np.any(c <= 0): + raise ValueError("measurements must all be positive to take a log-ratio") + return np.log(c) - np.log(b) + + +def _bootstrap_ci( + d: np.ndarray, *, confidence: float, seed: int, n_resamples: int +) -> tuple[float, float]: + """Percentile-bootstrap CI on the median log-ratio. + + Returns log-scale bounds. BCa is preferred but degenerates when the + jackknife acceleration is undefined (every value identical), so fall back + to the basic percentile method there. + """ + if np.allclose(d, d[0]): + # A perfectly constant difference has no sampling variability to + # estimate; the interval is the point itself. + return float(d[0]), float(d[0]) + try: + with warnings.catch_warnings(): + # SciPy announces a degenerate BCa interval (DegenerateDataWarning, + # a RuntimeWarning) through the warnings machinery and returns NaN + # bounds rather than raising. Promote it, or the fallback below is + # only reachable via the isfinite check and every other degenerate + # case prints a warning nobody reads. + warnings.simplefilter("error", RuntimeWarning) + res = _scipy_stats.bootstrap( + (d,), + np.median, + confidence_level=confidence, + method="BCa", + n_resamples=n_resamples, + rng=np.random.default_rng(seed), + ) + low = float(res.confidence_interval.low) + high = float(res.confidence_interval.high) + if math.isfinite(low) and math.isfinite(high): + return low, high + except ValueError, RuntimeWarning: + pass + res = _scipy_stats.bootstrap( + (d,), + np.median, + confidence_level=confidence, + method="percentile", + n_resamples=n_resamples, + rng=np.random.default_rng(seed), + ) + return float(res.confidence_interval.low), float(res.confidence_interval.high) + + +def _one_sided_p(d: np.ndarray) -> float: + """One-sided Wilcoxon signed-rank p-value for "candidate is slower". + + Signed-rank rather than a t-test because latency distributions are + skewed and occasionally have a stray outlier round; we do not want a + single stalled invocation to drive the verdict. + """ + if np.all(d == 0): + # No difference whatsoever. Wilcoxon rejects an all-zero input. + return 1.0 + # scipy's stubs type the result as an opaque tuple-like; index and cast. + return cast(float, _scipy_stats.wilcoxon(d, alternative="greater")[1]) + + +def compare( + baseline: Samples, + candidate: Samples, + *, + confidence: float = 0.95, + seed: int = 0, + n_resamples: int = DEFAULT_BOOTSTRAP_RESAMPLES, +) -> PairedComparison: + """Compute the paired comparison for one metric of one cell. + + The returned comparison has no final verdict yet: ``p_adjusted`` is unset + and ``verdict`` is INCONCLUSIVE until :func:`apply_multiplicity_control` + has seen every cell in the run. + """ + d = log_ratios(baseline, candidate) + n = int(d.size) + b_med = float(np.median(baseline)) if n else math.nan + c_med = float(np.median(candidate)) if n else math.nan + + if n < MIN_USABLE_ROUNDS: + return PairedComparison( + n_rounds=n, + baseline_median=b_med, + candidate_median=c_med, + ratio=math.exp(float(np.median(d))) if n else math.nan, + ci_low=math.nan, + ci_high=math.nan, + p_value=math.nan, + note=f"only {n} usable rounds; need at least {MIN_USABLE_ROUNDS}", + ) + + lo_log, hi_log = _bootstrap_ci( + d, confidence=confidence, seed=seed, n_resamples=n_resamples + ) + return PairedComparison( + n_rounds=n, + baseline_median=b_med, + candidate_median=c_med, + ratio=math.exp(float(np.median(d))), + ci_low=math.exp(lo_log), + ci_high=math.exp(hi_log), + p_value=_one_sided_p(d), + ) + + +@dataclass(frozen=True, slots=True) +class NoiseFloor: + """What the A/A control says about this runner's measurement noise. + + The A/A control compares the baseline build against *itself* through the + identical pipeline, so its true ratio is exactly 1.0 by construction. Any + apparent effect it reports is pure measurement noise, which makes it a + direct, run-specific check on whether the verdicts can be believed. + """ + + #: True if the A/A comparison itself looked like a regression. The gate + #: is then unreliable and the run must not fail the build on its findings. + tripped: bool + #: CI half-width on the log scale, as an equivalent ratio (e.g. 1.04). + width_ratio: float + #: True if the noise floor is wider than the effect we claim to detect. + underpowered: bool + detail: str = "" + + +def assess_noise_floor( + control: PairedComparison | None, *, threshold: float = DEFAULT_THRESHOLD +) -> NoiseFloor: + """Judge whether this runner was quiet enough to trust the verdicts. + + Two independent failure modes: + + - The control *tripped*: A/A produced an apparent effect past the + threshold. Something is systematically biased (ordering, caching, + thermal drift) and every verdict in the run is suspect. + - The run is *underpowered*: the control's interval is wider than the + effect size we are gating on, so a real regression of that size could + not have been distinguished from noise. A PASS here means "we could not + tell", which must not be reported as "no regression". + """ + if control is None: + return NoiseFloor( + tripped=False, + width_ratio=math.nan, + underpowered=True, + detail="no A/A control cell was run", + ) + if control.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite( + control.ci_half_width_log + ): + return NoiseFloor( + tripped=False, + width_ratio=math.nan, + underpowered=True, + detail=f"A/A control did not produce a usable interval ({control.note})", + ) + + width_ratio = math.exp(control.ci_half_width_log) + # The control's true ratio is 1.0. If its interval excludes the threshold + # in either direction, the pipeline is measuring a difference that cannot + # exist. + tripped = control.ci_low > threshold or control.ci_high < 1 / threshold + underpowered = control.ci_half_width_log >= math.log(threshold) + + detail = "" + if tripped: + detail = ( + f"A/A control reported ratio {control.ratio:.3f} " + f"[{control.ci_low:.3f}, {control.ci_high:.3f}] against a true 1.000; " + "runner is too noisy or the harness is biased" + ) + elif underpowered: + detail = ( + f"A/A noise floor +/-{(width_ratio - 1) * 100:.1f}% is not tighter than " + f"the {(threshold - 1) * 100:.0f}% detection threshold" + ) + return NoiseFloor( + tripped=tripped, + width_ratio=width_ratio, + underpowered=underpowered, + detail=detail, + ) + + +def _worst_noise(noises: Iterable[NoiseFloor]) -> NoiseFloor | None: + """The least reassuring control in the run, or None if there were none. + + Worst case rather than average: a single tripped control means the + harness may be biased on this runner, and averaging that away with two + quiet ones is exactly the reassurance the control exists to withhold. + """ + + def rank(n: NoiseFloor) -> tuple[bool, bool, float]: + # A NaN width is an unusable interval, which is worse than any real one. + width = n.width_ratio if math.isfinite(n.width_ratio) else math.inf + return (n.tripped, n.underpowered, width) + + return max(noises, key=rank, default=None) + + +def benjamini_hochberg(p_values: Sequence[float]) -> list[float]: + """BH-adjusted p-values, controlling the false discovery rate. + + NaNs (cells with too few rounds to test) pass through untouched and are + excluded from the adjustment, so an unmeasurable cell neither gains nor + confers significance. + """ + p = np.asarray(p_values, dtype=float) + out = p.copy() + testable = np.isfinite(p) + if not testable.any(): + return out.tolist() + out[testable] = _scipy_stats.false_discovery_control(p[testable], method="bh") + return out.tolist() + + +@dataclass(slots=True) +class GateResult: + """The run-level outcome after every cell has been compared.""" + + comparisons: dict[str, PairedComparison] = field(default_factory=dict) + #: The run-level noise floor: the *worst* of the per-control assessments, + #: since one biased control means the harness may be biased everywhere. + noise: NoiseFloor | None = None + #: Every control's own assessment, keyed by its comparison key. A run with + #: several SDKs has one control each, and go's noise floor says nothing + #: about java's. + noise_by_control: dict[str, NoiseFloor] = field(default_factory=dict) + #: Keys of cells that are confirmed regressions on a gated metric. + regressions: list[str] = field(default_factory=list) + improvements: list[str] = field(default_factory=list) + #: True if the run may fail the build. False when the A/A control tripped: + #: we still report, but a gate we cannot trust must not turn the build red. + trustworthy: bool = True + summary: str = "" + + @property + def should_fail(self) -> bool: + return self.trustworthy and bool(self.regressions) + + +def apply_multiplicity_control( + comparisons: dict[str, PairedComparison], + *, + gated: set[str] | None = None, + controls: Mapping[str, str] | None = None, + control_keys: set[str] | None = None, + censored: dict[str, str] | None = None, + threshold: float = DEFAULT_THRESHOLD, + alpha: float = DEFAULT_ALPHA, +) -> GateResult: + """Assign final verdicts to every cell and decide the run's outcome. + + Args: + comparisons: cell key -> comparison, for every measured (cell, metric). + gated: keys allowed to fail the build. Keys outside this set are still + given a verdict and reported, but never counted as a regression. + ``None`` means every key is gated. + controls: comparison key -> the A/A control key that assesses *its* + noise floor. A run measuring several SDKs has one control each, + and a cell judged against another SDK's control is judged against + a noise floor that was never measured for it. A key absent from + this mapping has no control, which is treated as underpowered. + control_keys: every key belonging to a control cell. Kept out of the + multiplicity correction and out of the regression and improvement + tallies: an A/A cell is not a hypothesis about the candidate. + censored: keys whose measurement is known to be invalid, mapped to why. + Reported as INCONCLUSIVE and never counted as a regression or an + improvement. A censored reading that happens to land inside the + threshold is otherwise indistinguishable from a real PASS, which + is the more dangerous of the two ways to be wrong. + threshold: minimum ratio worth calling a regression, e.g. 1.15. + alpha: false discovery rate for the BH adjustment. + + Returns: + A :class:`GateResult` whose ``comparisons`` hold the finalized + verdicts. The input mapping is not mutated. + """ + keys = list(comparisons) + controls = controls or {} + # The keys actually doing the assessing: one metric of one control cell + # per SDK. A control cell's other metrics are still control keys -- kept + # out of the gate -- but they are not anybody's noise floor. + assessors = set(controls.values()) + control_keys = control_keys or assessors + censored = censored or {} + + noise_by_control = { + ck: assess_noise_floor(comparisons.get(ck), threshold=threshold) + for ck in assessors + } + #: What a cell with no control of its own is judged against: nothing, which + #: `assess_noise_floor` calls underpowered, so it may report at worst + #: INCONCLUSIVE rather than a PASS nobody measured the power for. + uncontrolled = assess_noise_floor(None, threshold=threshold) + noise = _worst_noise(noise_by_control.values()) or uncontrolled + + # Neither a control nor a censored reading is a hypothesis about the + # candidate, so neither may dilute the correction applied to the cells + # that are. The gated keys are corrected as their own family for the same + # reason: adjusting them against metrics nobody gates on only makes a real + # regression harder to confirm. Ungated metrics still get a family of + # their own so that they carry a reportable verdict. + adjustable = [k for k in keys if k not in control_keys and k not in censored] + gated_family = [k for k in adjustable if gated is None or k in gated] + rest = [k for k in adjustable if k not in set(gated_family)] + p_adj: dict[str, float] = {} + for family in (gated_family, rest): + p_adj.update( + zip( + family, + benjamini_hochberg([comparisons[k].p_value for k in family]), + strict=True, + ) + ) + + result = GateResult(noise=noise, noise_by_control=noise_by_control) + for key in keys: + c = comparisons[key] + pa = p_adj.get(key) + if key in censored: + verdict, note = Verdict.INCONCLUSIVE, censored[key] + else: + verdict, note = _verdict_for( + c, + pa, + threshold=threshold, + alpha=alpha, + noise=noise_by_control.get(controls.get(key, ""), uncontrolled), + is_control=key in control_keys, + ) + result.comparisons[key] = PairedComparison( + n_rounds=c.n_rounds, + baseline_median=c.baseline_median, + candidate_median=c.candidate_median, + ratio=c.ratio, + ci_low=c.ci_low, + ci_high=c.ci_high, + p_value=c.p_value, + p_adjusted=pa, + verdict=verdict, + note=note or c.note, + ) + if key in control_keys: + continue + if verdict is Verdict.REGRESSION and (gated is None or key in gated): + result.regressions.append(key) + elif verdict is Verdict.IMPROVED: + result.improvements.append(key) + + result.trustworthy = not noise.tripped + result.summary = _summarize(result, noise, threshold) + return result + + +def _verdict_for( + c: PairedComparison, + p_adjusted: float | None, + *, + threshold: float, + alpha: float, + noise: NoiseFloor, + is_control: bool, +) -> tuple[Verdict, str]: + if c.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite(c.ci_low): + return Verdict.INCONCLUSIVE, c.note or "no usable interval" + + p = c.p_value if is_control else p_adjusted + if p is None or not math.isfinite(p): + return Verdict.INCONCLUSIVE, "no p-value" + + if c.ci_low > threshold and p < alpha: + return Verdict.REGRESSION, "" + if c.ci_high < 1 / threshold and p > 1 - alpha: + return Verdict.IMPROVED, "" + + # Not a regression. But "we looked and found nothing" only counts as PASS + # if we could have found something. Without the power to resolve an effect + # of `threshold`, the honest answer is that we do not know. + if not is_control and noise.underpowered: + return Verdict.INCONCLUSIVE, noise.detail + if c.ci_half_width_log >= math.log(threshold): + return ( + Verdict.INCONCLUSIVE, + f"interval +/-{(math.exp(c.ci_half_width_log) - 1) * 100:.1f}% is wider " + f"than the {(threshold - 1) * 100:.0f}% threshold", + ) + return Verdict.PASS, "" + + +def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: + if noise.tripped: + return ( + f"INCONCLUSIVE: the A/A control failed its own comparison. {noise.detail}. " + "Verdicts below are reported but not gated." + ) + if result.regressions: + return ( + f"{len(result.regressions)} confirmed regression(s) past the " + f"{(threshold - 1) * 100:.0f}% threshold: {', '.join(result.regressions)}" + ) + inconclusive = [ + k for k, c in result.comparisons.items() if c.verdict is Verdict.INCONCLUSIVE + ] + if inconclusive: + return ( + f"No confirmed regressions. {len(inconclusive)} cell(s) INCONCLUSIVE " + f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + ) + return ( + f"No regressions. All cells resolved within the " + f"{(threshold - 1) * 100:.0f}% threshold " + f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + ) diff --git a/xtest/pyproject.toml b/xtest/pyproject.toml index a21816fd8..182e5e55c 100644 --- a/xtest/pyproject.toml +++ b/xtest/pyproject.toml @@ -46,6 +46,8 @@ dependencies = [ "smmap>=5.0.3", "typing_extensions>=4.15.0", "urllib3>=2.7.0", + "numpy>=2.5.2", + "scipy>=1.18.0", ] [project.optional-dependencies] @@ -79,7 +81,7 @@ ignore = [ ] [tool.ruff.lint.isort] -known-first-party = ["abac", "tdfs", "otdfctl", "assertions", "fixtures"] +known-first-party = ["abac", "tdfs", "otdfctl", "assertions", "fixtures", "perf"] [tool.ruff.format] quote-style = "double" @@ -100,3 +102,7 @@ testpaths = ["."] python_files = ["test_*.py"] python_functions = ["test_*"] addopts = "-ra -v" +markers = [ + "benchmark: paired A/B performance cell; only collected under --bench", + "no_audit_logs: opt this test out of the default audit-log assertions", +] diff --git a/xtest/tdfs.py b/xtest/tdfs.py index d09ca470d..b73bc508c 100644 --- a/xtest/tdfs.py +++ b/xtest/tdfs.py @@ -505,7 +505,26 @@ def is_released(self) -> bool: ) ) - def encrypt( + def is_final_release(self) -> bool: + """True only for a plain ``vX.Y.Z`` tag -- no prerelease, no build metadata. + + :meth:`is_released` accepts ``v0.29.0-rc.1``, and :meth:`semver` + parses it to the same ``(0, 29, 0)`` as the final release, so ordering + by semver alone leaves the two tied and directory-listing order breaks + the tie. Callers that must not pick a release candidate by accident -- + choosing a benchmark baseline, for one -- want this instead. + """ + return bool(re.fullmatch(r"(?:sdk/)?v?\d+\.\d+\.\d+", self.version)) + + def semver(self) -> tuple[int, int, int] | None: + """Parsed (major, minor, patch), or None for branch builds like 'main'. + + Lets callers order the installed versions -- picking the newest + release as a benchmark baseline, for instance. + """ + return _parse_semver(self.version.removeprefix("sdk/")) + + def encrypt_command( self, pt_file: Path, ct_file: Path, @@ -515,7 +534,17 @@ def encrypt( assert_value: str = "", policy_mode: str = "encrypted", target_mode: container_version | None = None, - ): + ) -> tuple[list[str], dict[str, str]]: + """Build the argv and CLI-specific env vars for an encrypt invocation. + + Split out from :meth:`encrypt` so that callers which need to run the + command themselves -- the benchmark harness measures resource usage + around it -- share this one definition of the `XT_WITH_*` contract + instead of keeping a second copy that drifts. + + The returned env holds only the CLI-specific overrides; merge it over + ``os.environ`` before handing it to a subprocess. + """ use_ecwrap = container == "ztdf-ecwrap" fmt = simple_container(container) c = [ @@ -541,6 +570,29 @@ def encrypt( if use_ecwrap: local_env |= {"XT_WITH_ECWRAP": "true"} + return c, local_env + + def encrypt( + self, + pt_file: Path, + ct_file: Path, + mime_type: str = "application/octet-stream", + container: container_type = "ztdf", + attr_values: list[str] | None = None, + assert_value: str = "", + policy_mode: str = "encrypted", + target_mode: container_version | None = None, + ): + c, local_env = self.encrypt_command( + pt_file, + ct_file, + mime_type=mime_type, + container=container, + attr_values=attr_values, + assert_value=assert_value, + policy_mode=policy_mode, + target_mode=target_mode, + ) logger.debug(f"enc [{' '.join([fmt_env(local_env)] + c)}]") env = dict(os.environ) env |= local_env @@ -554,7 +606,7 @@ def encrypt( result.returncode, c, output=result.stdout, stderr=result.stderr ) - def decrypt( + def decrypt_command( self, ct_file: Path, rt_file: Path, @@ -562,10 +614,15 @@ def decrypt( assert_keys: str = "", verify_assertions: bool = True, ecwrap: bool = False, - expect_error: bool = False, kasallowlist: str = "", ignore_kas_allowlist: bool = False, - ): + ) -> tuple[list[str], dict[str, str]]: + """Build the argv and CLI-specific env vars for a decrypt invocation. + + See :meth:`encrypt_command` for why this is separate. ``expect_error`` + has no counterpart here: it selects how the caller runs the command, + not what the command is. + """ fmt = simple_container(container) c = [ @@ -587,6 +644,30 @@ def decrypt( local_env |= {"XT_WITH_KAS_ALLOWLIST": kasallowlist} if ignore_kas_allowlist: local_env |= {"XT_WITH_IGNORE_KAS_ALLOWLIST": "true"} + return c, local_env + + def decrypt( + self, + ct_file: Path, + rt_file: Path, + container: container_type = "ztdf", + assert_keys: str = "", + verify_assertions: bool = True, + ecwrap: bool = False, + expect_error: bool = False, + kasallowlist: str = "", + ignore_kas_allowlist: bool = False, + ): + c, local_env = self.decrypt_command( + ct_file, + rt_file, + container=container, + assert_keys=assert_keys, + verify_assertions=verify_assertions, + ecwrap=ecwrap, + kasallowlist=kasallowlist, + ignore_kas_allowlist=ignore_kas_allowlist, + ) logger.info(f"dec [{' '.join([fmt_env(local_env)] + c)}]") env = dict(os.environ) env |= local_env diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py new file mode 100644 index 000000000..bf7a3194e --- /dev/null +++ b/xtest/test_bench_arms.py @@ -0,0 +1,135 @@ +"""Unit tests for benchmark arm selection and payload generation. + +Both decide *what* gets measured, before any measuring happens, and both fail +quietly when they get it wrong: a baseline that is silently a release +candidate, or a payload whose bytes changed between two runs that claim to be +comparable. Neither shows up as an error -- only as numbers that mean +something other than what the report says they mean. + +No platform and no real SDK; the builds are stub ``cli.sh`` trees in +``tmp_path``. +""" + +from pathlib import Path + +import pytest + +import tdfs +from fixtures import bench +from perf.cells import PAYLOADS +from perf.runner import BenchConfig + + +def install(root: Path, sdk: str, *versions: str) -> None: + """Lay down a stub build tree, as ``otdf-sdk-mgr install`` would.""" + for version in versions: + cli = root / "sdk" / sdk / "dist" / version / "cli.sh" + cli.parent.mkdir(parents=True) + cli.write_text("#!/bin/sh\nexit 0\n") + + +@pytest.fixture +def cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """`SDK.__init__` resolves `cli.sh` relative to the cwd, so move there.""" + monkeypatch.chdir(tmp_path) + return tmp_path + + +class TestFinalRelease: + @pytest.mark.parametrize("version", ["v0.29.0", "0.29.0"]) + def test_accepts_a_plain_tag(self, cwd: Path, version: str): + install(cwd, "go", version) + assert tdfs.SDK("go", version).is_final_release() + + @pytest.mark.parametrize( + "version", ["v0.29.0-rc.1", "v0.29.0+build.5", "main", "DSPX-1234"] + ) + def test_rejects_anything_else(self, cwd: Path, version: str): + install(cwd, "go", version) + assert not tdfs.SDK("go", version).is_final_release() + + +class TestBaselineSelection: + def test_picks_the_newest_final_release(self, cwd: Path): + install(cwd, "go", "main", "v0.28.0", "v0.29.0") + baseline, candidate = bench.select_arms("go") + assert baseline.version == "v0.29.0" + assert candidate.version == "main" + + def test_a_release_candidate_never_becomes_the_baseline(self, cwd: Path): + # An rc parses to the same semver as its final release, so ordering by + # semver alone leaves the two tied and the directory listing breaks + # the tie -- a baseline nobody chose, differing run to run. + install(cwd, "go", "main", "v0.29.0", "v0.29.0-rc.1", "v0.30.0-rc.1") + baseline, _ = bench.select_arms("go") + assert baseline.version == "v0.29.0" + + def test_no_final_release_is_a_clear_refusal(self, cwd: Path): + install(cwd, "go", "main", "v0.30.0-rc.1") + with pytest.raises(bench.ArmSelectionError, match="no final go release"): + bench.select_arms("go") + + def test_no_branch_build_is_a_clear_refusal(self, cwd: Path): + install(cwd, "go", "v0.29.0") + with pytest.raises(bench.ArmSelectionError, match="no unreleased go build"): + bench.select_arms("go") + + def test_explicit_specs_win(self, cwd: Path): + install(cwd, "go", "main", "v0.28.0", "v0.29.0") + baseline, candidate = bench.select_arms( + "go", baseline_spec="go@v0.28.0", candidate_spec="go@v0.29.0" + ) + assert (baseline.version, candidate.version) == ("v0.28.0", "v0.29.0") + + def test_refuses_to_compare_a_build_against_itself(self, cwd: Path): + install(cwd, "go", "main", "v0.29.0") + with pytest.raises(bench.ArmSelectionError, match="nothing to compare"): + bench.select_arms("go", baseline_spec="go@main", candidate_spec="go@main") + + +#: The fixture body, called directly: these tests are about the bytes it +#: writes, not about pytest's fixture wiring. +make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] + + +class TestPayloads: + def test_every_size_is_generated(self, tmp_path: Path): + out = make_payloads(tmp_path, BenchConfig(seed=1)) + for payload in PAYLOADS: + assert out[payload.label].stat().st_size == payload.n_bytes + + def test_a_seed_reproduces_the_bytes(self, tmp_path: Path): + a = read_all(make_payloads(subdir(tmp_path, "a"), BenchConfig(seed=1))) + b = read_all(make_payloads(subdir(tmp_path, "b"), BenchConfig(seed=1))) + assert a == b + + def test_a_different_seed_changes_them(self, tmp_path: Path): + a = read_all(make_payloads(subdir(tmp_path, "a"), BenchConfig(seed=1))) + b = read_all(make_payloads(subdir(tmp_path, "b"), BenchConfig(seed=2))) + assert a != b + + def test_a_partial_cache_still_reproduces_the_bytes(self, tmp_path: Path): + # tmp_dir persists between runs. With one RNG stream shared across the + # payloads, skipping a cached file shifts every payload after it, so a + # rerun measures different input than the run it is compared against. + first = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + (tmp_path / f"bench-plain-{PAYLOADS[0].label}.bin").unlink() + second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + assert first == second + + def test_a_truncated_cache_entry_is_regenerated(self, tmp_path: Path): + first = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + path = tmp_path / f"bench-plain-{PAYLOADS[1].label}.bin" + path.write_bytes(b"truncated") + second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + assert first == second + + +def read_all(paths: dict[str, Path]) -> dict[str, bytes]: + return {label: p.read_bytes() for label, p in paths.items()} + + +def subdir(root: Path, name: str) -> Path: + out = root / name + out.mkdir() + return out diff --git a/xtest/test_bench_measure.py b/xtest/test_bench_measure.py new file mode 100644 index 000000000..d7990df80 --- /dev/null +++ b/xtest/test_bench_measure.py @@ -0,0 +1,233 @@ +"""Unit tests for ``perf/measure.py``. + +No platform and no SDK: these drive small Python and shell children with known +resource profiles, so they run in ``check.yml`` next to the stats tests. + +Tolerances are deliberately loose. The point is to catch a primitive that is +plain wrong -- reporting kilobytes as bytes, missing a grandchild's CPU, +returning the parent's memory instead of the child's -- not to assert precise +timings on a shared runner. +""" + +import os +import subprocess +import sys +import textwrap + +import pytest + +from perf import measure +from perf.measure import MeasurementError, Sample + +pytestmark = pytest.mark.skipif( + not hasattr(os, "wait4"), + reason="per-process rusage requires os.wait4", +) + + +def python_child(body: str) -> list[str]: + """Build an argv running a short Python snippet as a child process.""" + return [sys.executable, "-c", textwrap.dedent(body)] + + +class TestWallClock: + def test_tracks_sleep_duration(self): + s = measure.measure(python_child("import time; time.sleep(0.25)")) + assert 0.2 < s.wall_s < 1.5 + + def test_wall_s_matches_wall_ns(self): + s = measure.measure(python_child("pass")) + assert s.wall_s == pytest.approx(s.wall_ns / 1e9) + + +class TestCpuTime: + def test_sleeping_child_burns_almost_no_cpu(self): + s = measure.measure(python_child("import time; time.sleep(0.5)")) + # Interpreter startup costs some CPU, but far less than the wall time. + assert s.cpu_s < s.wall_s + + def test_busy_child_burns_cpu_close_to_wall_time(self): + s = measure.measure( + python_child( + """ + import time + end = time.perf_counter() + 0.5 + while time.perf_counter() < end: + pass + """ + ) + ) + assert s.cpu_s > 0.3 + + def test_includes_grandchild_cpu(self): + # The SDK shims are bash wrappers around a real binary, so a + # measurement that misses grandchildren would report near-zero CPU for + # every SDK operation. + busy = ( + "import time\n" + "end = time.perf_counter() + 0.5\n" + "while time.perf_counter() < end: pass\n" + ) + s = measure.measure( + [ + "/bin/sh", + "-c", + f"{sys.executable} -c {subprocess.list2cmdline([busy])}", + ] + ) + assert s.cpu_s > 0.3, "grandchild CPU was not folded into the parent's rusage" + + +def ballast_child(mb: int) -> list[str]: + """A child that allocates ``mb`` MiB and touches every page of it.""" + return python_child( + f""" + ballast = bytearray({mb} * 1024 * 1024) + ballast[::4096] = b'x' * len(ballast[::4096]) + """ + ) + + +class TestPeakRss: + def test_reports_ballast_in_bytes(self): + # Allocate ~200 MB and confirm the figure is in bytes, not kilobytes. + # Getting the unit wrong is a 1024x error that a loose bound catches. + s = measure.measure( + python_child( + """ + ballast = bytearray(200 * 1024 * 1024) + ballast[::4096] = b'x' * len(ballast[::4096]) + """ + ) + ) + assert 150 * 2**20 < s.max_rss_bytes < 1200 * 2**20 + + def test_larger_allocation_reports_larger_peak(self): + def peak(mb: int) -> int: + return measure.measure(ballast_child(mb)).max_rss_bytes + + assert peak(200) > peak(20) + 100 * 2**20 + + def test_a_fat_measuring_process_does_not_inflate_a_small_child(self): + # The failure this guards against does not look like a failure. On + # Linux a forked child inherits the parent's resident-set accounting + # and exec does not clear it, so every command cheaper than the pytest + # process reported the pytest process's memory instead of its own -- + # a stable, plausible number that reads as "no regression" forever. + # + # Holding real ballast here is the only way to reproduce it: with a + # slim parent the bug is invisible, which is exactly why it reached CI. + lean = measure.measure(ballast_child(20)).max_rss_bytes + ballast = bytearray(400 * 2**20) + try: + ballast[::4096] = b"x" * len(ballast[::4096]) + fat = measure.measure(ballast_child(20)).max_rss_bytes + finally: + del ballast + assert fat < lean + 100 * 2**20, ( + f"measuring from a 400 MiB process reported {fat / 2**20:.0f} MiB " + f"for a child that reads {lean / 2**20:.0f} MiB from a lean one" + ) + + def test_reports_the_floor_under_the_reading(self): + # A peak RSS cannot be measured below the memory of whatever forked + # the process, so the floor travels with the sample and callers can + # tell a censored reading from a genuinely small one. + s = measure.measure(ballast_child(200)) + assert 0 < s.rss_floor_bytes < 100 * 2**20 + assert not s.rss_is_floored + + def test_a_reading_at_the_floor_is_marked_censored(self): + floored = Sample( + wall_ns=1, + cpu_s=0.0, + max_rss_bytes=12 * 2**20, + exit_code=0, + rss_floor_bytes=12 * 2**20, + ) + assert floored.rss_is_floored + assert not Sample( + wall_ns=1, + cpu_s=0.0, + max_rss_bytes=80 * 2**20, + exit_code=0, + rss_floor_bytes=12 * 2**20, + ).rss_is_floored + + def test_includes_grandchild_memory(self): + alloc = ballast_child(200)[2] + s = measure.measure( + [ + "/bin/sh", + "-c", + f"{sys.executable} -c {subprocess.list2cmdline([alloc])}", + ] + ) + assert s.max_rss_bytes > 150 * 2**20 + + +class TestFailureHandling: + def test_non_zero_exit_raises_by_default(self): + with pytest.raises(MeasurementError, match="exited 3"): + measure.measure(python_child("raise SystemExit(3)")) + + def test_error_includes_child_stderr(self): + with pytest.raises(MeasurementError, match="disaster strikes"): + measure.measure( + python_child( + "import sys; sys.stderr.write('disaster strikes'); sys.exit(1)" + ) + ) + + def test_check_false_returns_the_failing_sample(self): + s = measure.measure(python_child("raise SystemExit(7)"), check=False) + assert s.exit_code == 7 + + def test_missing_executable_raises(self): + with pytest.raises(MeasurementError, match="could not start"): + measure.measure(["/nonexistent/definitely-not-a-real-binary"]) + + def test_timeout_kills_and_raises(self): + with pytest.raises(MeasurementError, match="measurement timeout"): + measure.measure(python_child("import time; time.sleep(30)"), timeout_s=0.5) + + def test_large_output_does_not_deadlock(self): + # os.wait4 does not drain pipes. If stdout were a pipe, a child writing + # more than the pipe buffer would block forever and hang the job. + s = measure.measure( + python_child("import sys; sys.stdout.write('x' * 5_000_000)") + ) + assert s.exit_code == 0 + + +class TestMetricAccess: + def test_metric_lookup_matches_fields(self): + s = Sample(wall_ns=1_500_000, cpu_s=0.25, max_rss_bytes=2**20, exit_code=0) + assert s.metric("wall") == 1_500_000 + assert s.metric("cpu") == 0.25 + assert s.metric("rss") == 2**20 + + def test_unknown_metric_raises(self): + s = Sample(wall_ns=1, cpu_s=1.0, max_rss_bytes=1, exit_code=0) + with pytest.raises(KeyError): + s.metric("bogus") + + def test_every_declared_metric_is_retrievable(self): + s = Sample(wall_ns=1, cpu_s=1.0, max_rss_bytes=1, exit_code=0) + for name in measure.METRICS: + assert isinstance(s.metric(name), float) + assert name in measure.METRIC_LABELS + + def test_formatting(self): + cases = [ + ("wall", 1_500_000.0, "1.5 ms"), + ("cpu", 0.25, "0.250 s"), + ("rss", float(2**21), "2.0 MiB"), + ] + assert [measure.format_metric(n, v) for n, v, _ in cases] == [ + e for _, _, e in cases + ] + + def test_formatting_rejects_unknown_metric(self): + with pytest.raises(KeyError): + measure.format_metric("bogus", 1.0) diff --git a/xtest/test_bench_runner.py b/xtest/test_bench_runner.py new file mode 100644 index 000000000..3b9e9f4f1 --- /dev/null +++ b/xtest/test_bench_runner.py @@ -0,0 +1,447 @@ +"""Tests for the paired round loop and the gate it feeds. + +No subprocesses and no platform: the measurement function and the clock are +both injected, so a whole 40-round cell runs in microseconds and a planted +regression is exactly the size we planted. + +The last class here is the one that matters most. A benchmark gate that has +never been shown to catch a planted regression -- and to *ignore* a trivial +one -- is not yet known to work. +""" + +from __future__ import annotations + +import math +import random +from collections.abc import Callable +from pathlib import Path + +import pytest + +from perf import stats +from perf.measure import Sample +from perf.runner import ( + Arm, + BenchConfig, + Budget, + BudgetExhausted, + Invocation, + analyze, + run_cell, +) + +BASELINE_WALL_S = 1.0 +BASELINE_RSS = 100_000_000 +BASELINE_CPU = 0.8 + + +def arm(role: str, key: str, output: Path | None = None) -> Arm: + """An arm whose argv is a single token, so ``FakeRuns`` can recognize it. + + ``role`` is what the runner keys samples by ("baseline"/"candidate"); + ``key`` is the stand-in for the build. + """ + return Arm(role, f"sdk@{key}", Invocation([key], {}, output)) + + +def config(**overrides: object) -> BenchConfig: + """A config small enough to run fast, still valid for a verdict.""" + base: dict[str, object] = { + "min_rounds": stats.MIN_USABLE_ROUNDS, + "max_rounds": 40, + "warmup": 2, + "n_resamples": 400, + } + return BenchConfig(**(base | overrides)) # pyright: ignore[reportArgumentType] + + +class FakeRuns: + """A stand-in for ``measure`` that returns scripted, noisy samples. + + ``ratio_for`` maps the invocation's first argv element to a multiplier on + the baseline cost, so a caller plants an effect by naming the arms. + """ + + def __init__( + self, + ratio_for: dict[str, float], + *, + noise: float = 0.05, + seed: int = 7, + rss_floor: int = 0, + ) -> None: + self.ratio_for = ratio_for + self.noise = noise + #: Readings below this clip up to it, the way a real measurement floor + #: behaves: the number is the floor's, not the command's. + self.rss_floor = rss_floor + self.rng = random.Random(seed) + #: Every argv[0] seen, in call order. The interleaving is visible here. + self.calls: list[str] = [] + #: Simulated seconds consumed, for tests that drive the clock from it. + self.elapsed = 0.0 + + def __call__( + self, argv: list[str], env: dict[str, str], **kwargs: object + ) -> Sample: + del env, kwargs + key = argv[0] + self.calls.append(key) + ratio = self.ratio_for[key] + # Lognormal jitter: latency is positive and multiplicative, so noise + # on the log scale is the honest model of a noisy runner. + jitter = math.exp(self.rng.gauss(0.0, self.noise)) + wall = BASELINE_WALL_S * ratio * jitter + self.elapsed += wall + return Sample( + wall_ns=int(wall * 1e9), + cpu_s=BASELINE_CPU * ratio * jitter, + max_rss_bytes=max(int(BASELINE_RSS * ratio * jitter), self.rss_floor), + exit_code=0, + rss_floor_bytes=self.rss_floor, + ) + + +def clock_from(runs: FakeRuns) -> Callable[[], float]: + """A clock that advances only as simulated work happens.""" + return lambda: runs.elapsed + + +def run( + ratio: float, + *, + cfg: BenchConfig | None = None, + noise: float = 0.05, + seed: int = 7, + cell_id: str = "cell", + control: bool = False, + sdk: str = "", + rss_floor: int = 0, +): + """Run one cell where the candidate costs ``ratio`` times the baseline.""" + runs = FakeRuns( + {"base": 1.0, "cand": ratio}, noise=noise, seed=seed, rss_floor=rss_floor + ) + result = run_cell( + cell_id, + arm("baseline", "base"), + arm("candidate", "cand"), + cfg or config(), + control=control, + sdk=sdk, + clock=clock_from(runs), + run=runs, + ) + return result, runs + + +class TestRoundLoop: + def test_arms_are_paired_every_round(self): + _, runs = run(1.0) + assert runs.calls.count("base") == runs.calls.count("cand") + # Every consecutive pair holds one of each: that is what pairing means. + pairs = [set(runs.calls[i : i + 2]) for i in range(0, len(runs.calls), 2)] + assert all(p == {"base", "cand"} for p in pairs) + + def test_order_within_rounds_is_shuffled(self): + _, runs = run(1.0) + firsts = runs.calls[::2] + assert "base" in firsts and "cand" in firsts, ( + "a fixed within-round order lets the second arm inherit the first " + "one's cache state" + ) + + def test_warmup_rounds_are_discarded(self): + cfg = config(warmup=3, max_rounds=stats.MIN_USABLE_ROUNDS) + result, runs = run(1.0, cfg=cfg) + assert result.n_rounds == stats.MIN_USABLE_ROUNDS + assert result.n_warmup == 3 + # Warm-up rounds ran, they just are not in the samples. + assert len(runs.calls) == 2 * (3 + stats.MIN_USABLE_ROUNDS) + + def test_interleaving_is_reproducible_for_a_seed(self): + _, a = run(1.0) + _, b = run(1.0) + assert a.calls == b.calls + + def test_cells_do_not_share_an_interleaving(self): + _, a = run(1.0, cell_id="encrypt-1KiB") + _, b = run(1.0, cell_id="decrypt-1KiB") + assert a.calls != b.calls, "cells sharing one order would correlate their noise" + + def test_output_is_removed_before_each_run(self, tmp_path: Path): + out = tmp_path / "out.tdf" + out.write_bytes(b"stale") + seen: list[bool] = [] + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + + def observe(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: + if argv[0] == "base": + # What the arm that owns this output sees when it starts. + seen.append(out.exists()) + out.write_bytes(b"produced") + return runs(argv, env, **kwargs) + + run_cell( + "cell", + arm("baseline", "base", out), + arm("candidate", "cand"), + config(max_rounds=stats.MIN_USABLE_ROUNDS), + clock=clock_from(runs), + run=observe, + ) + assert not any(seen), "a stale output makes round 2 measure an overwrite" + + def test_samples_are_collected_for_every_metric(self): + result, _ = run(1.0) + for name in ("baseline", "candidate"): + for metric in ("wall", "cpu", "rss"): + assert len(result.samples[name][metric]) == result.n_rounds + + +class TestStopping: + def test_stops_early_on_precision_when_quiet(self): + result, _ = run(1.0, noise=0.005) + assert result.stopped_because == "precision" + assert result.n_rounds < 40 + + def test_runs_to_max_rounds_when_noisy(self): + result, _ = run(1.0, noise=0.4) + assert result.stopped_because == "max_rounds" + assert result.n_rounds == 40 + + def test_never_stops_before_min_rounds(self): + cfg = config(min_rounds=25, max_rounds=40) + result, _ = run(1.0, cfg=cfg, noise=0.0001) + assert result.n_rounds >= 25 + + def test_deadline_stops_the_loop(self): + runs = FakeRuns({"base": 1.0, "cand": 1.0}, noise=0.3) + clock = clock_from(runs) + result = run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=0, max_rounds=200), + deadline=clock() + 60.0, # each round costs ~2 simulated seconds + clock=clock, + run=runs, + ) + assert result.stopped_because == "budget" + assert result.elapsed_s <= 60.0, "a round we could not finish was started" + + def test_warmup_gives_up_when_the_budget_runs_out(self): + # The budget's end is absolute, so warm-ups that run past it are + # spending the *following* cells' time -- and then reaching the + # measured loop with nothing left, paying the whole cost of the cell + # for no data at all. Stop at the deadline and say where it went. + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + clock = clock_from(runs) + with pytest.raises(BudgetExhausted, match="warm-up"): + run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=10), + deadline=clock() + 4.0, # each round costs ~2 simulated seconds + clock=clock, + run=runs, + ) + assert len(runs.calls) < 2 * 10, "warm-up ran past its own deadline" + + def test_budget_below_min_usable_rounds_refuses_a_verdict(self): + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + clock = clock_from(runs) + with pytest.raises(BudgetExhausted, match="below the"): + run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=0), + deadline=clock() + 4.0, + clock=clock, + run=runs, + ) + + +class TestBenchConfigValidation: + def test_rejects_min_rounds_below_the_usable_floor(self): + with pytest.raises(ValueError, match="min_rounds"): + BenchConfig(min_rounds=stats.MIN_USABLE_ROUNDS - 1) + + def test_rejects_max_below_min(self): + with pytest.raises(ValueError, match="max_rounds"): + BenchConfig(min_rounds=20, max_rounds=10) + + def test_rejects_a_threshold_that_is_not_a_ratio(self): + with pytest.raises(ValueError, match="ratio above 1.0"): + BenchConfig(threshold=0.9) + + def test_rejects_unknown_gated_metrics(self): + with pytest.raises(ValueError, match="unknown gated metrics"): + BenchConfig(gated_metrics=("wall", "iops")) + + def test_target_half_width_is_a_third_of_the_log_threshold(self): + cfg = BenchConfig(threshold=1.15) + assert cfg.target_half_width_log == pytest.approx(math.log(1.15) / 3) + + +class TestBudget: + def test_divides_remaining_time_evenly(self): + now = 1000.0 + budget = Budget(300.0, 3, clock=lambda: now) + assert budget.next_deadline() == pytest.approx(now + 100.0) + + def test_unused_time_flows_to_later_cells(self): + now = [0.0] + budget = Budget(300.0, 3, clock=lambda: now[0]) + budget.next_deadline() + now[0] = 10.0 # the first cell stopped early on precision + # 290s left over two cells, not the 100s it would have got by + # dividing up front. + assert budget.next_deadline() == pytest.approx(155.0) + + def test_never_hands_out_a_deadline_in_the_past(self): + now = [0.0] + budget = Budget(10.0, 2, clock=lambda: now[0]) + now[0] = 60.0 + assert budget.next_deadline() == pytest.approx(60.0) + assert budget.remaining_s == 0.0 + + def test_rejects_a_budget_with_no_cells(self): + with pytest.raises(ValueError, match="at least one cell"): + Budget(10.0, 0) + + +class TestGateOnPlantedEffects: + """The critical check: does the gate fire when it should, and only then? + + Each case runs a real cell through the real statistics; only the + measurement is simulated. The A/A control cell is included exactly as a + live run would include it, so the noise floor is assessed the same way. + """ + + def gate(self, candidate_ratio: float, *, noise: float = 0.05, seed: int = 11): + cfg = config(max_rounds=40) + control, _ = run( + 1.0, cfg=cfg, noise=noise, seed=seed, cell_id="aa", control=True + ) + measured, _ = run( + candidate_ratio, cfg=cfg, noise=noise, seed=seed + 1, cell_id="encrypt" + ) + return analyze([control, measured], cfg) + + def test_planted_25_percent_slowdown_is_caught(self): + gate = self.gate(1.25) + assert gate.should_fail + assert "encrypt/wall" in gate.regressions + c = gate.comparisons["encrypt/wall"] + assert c.verdict is stats.Verdict.REGRESSION + assert c.ci_low > 1.15, "the interval must exclude the threshold, not just 1.0" + assert c.ratio == pytest.approx(1.25, rel=0.1) + + def test_planted_3_percent_slowdown_is_ignored(self): + gate = self.gate(1.03) + assert not gate.should_fail + assert gate.comparisons["encrypt/wall"].verdict is not stats.Verdict.REGRESSION + + def test_no_effect_does_not_fire(self): + gate = self.gate(1.0) + assert not gate.should_fail + assert not gate.regressions + + def test_planted_speedup_is_reported_not_failed(self): + gate = self.gate(0.7) + assert not gate.should_fail + assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.IMPROVED + assert "encrypt/wall" in gate.improvements + + def test_the_control_cell_never_fails_the_build(self): + # Both arms of the control are the same build, so any verdict it + # reaches is the harness's own error, not a regression in anything. + gate = self.gate(1.25) + assert not any(k.startswith("aa/") for k in gate.regressions) + + def test_a_regression_in_an_ungated_metric_does_not_fail(self): + cfg = config(max_rounds=40, gated_metrics=("wall",)) + control, _ = run(1.0, cfg=cfg, cell_id="aa", control=True) + measured, _ = run(1.4, cfg=cfg, seed=12, cell_id="encrypt") + gate = analyze([control, measured], cfg) + # CPU time moved with everything else and is reported as such; it + # simply is not allowed to turn the build red. + assert gate.comparisons["encrypt/cpu"].verdict is stats.Verdict.REGRESSION + assert "encrypt/cpu" not in gate.regressions + assert "encrypt/wall" in gate.regressions + + def test_rss_pinned_to_the_measurement_floor_cannot_report_pass(self): + # A command whose peak sits at the floor is not measured, it is + # clipped -- and both arms clip to the same number. That produces a + # ratio of exactly 1.000 with a vanishing interval, which is the most + # convincing PASS the harness can emit and carries no information. + cfg = config(max_rounds=40) + floor = 4 * BASELINE_RSS + control, _ = run(1.0, cfg=cfg, cell_id="aa", control=True, rss_floor=floor) + measured, _ = run(1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=floor) + gate = analyze([control, measured], cfg) + + rss = gate.comparisons["encrypt/rss"] + assert rss.ratio == pytest.approx(1.0), "the floor clipped both arms" + assert rss.verdict is stats.Verdict.INCONCLUSIVE + assert "floor" in rss.note + assert "encrypt/rss" not in gate.regressions + assert "encrypt/rss" not in gate.improvements + # Wall clock is untouched by a memory floor and still does its job. + assert "encrypt/wall" in gate.regressions + + def test_rss_above_the_floor_is_still_gated(self): + cfg = config(max_rounds=40) + control, _ = run( + 1.0, cfg=cfg, cell_id="aa", control=True, rss_floor=BASELINE_RSS // 10 + ) + measured, _ = run( + 1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=BASELINE_RSS // 10 + ) + gate = analyze([control, measured], cfg) + assert "encrypt/rss" in gate.regressions + + def test_each_sdk_is_judged_against_its_own_control(self): + # One control per SDK: they are different harness paths with different + # floors. Judging go's cells against java's control judges them + # against a noise floor that was never measured for them -- and with + # a single run-level control, whichever SDK happened to be last wins. + cfg = config(max_rounds=40) + go_aa, _ = run( + 1.0, cfg=cfg, noise=0.02, cell_id="go-aa", control=True, sdk="go" + ) + go_cell, _ = run( + 1.0, cfg=cfg, noise=0.02, seed=12, cell_id="go-encrypt", sdk="go" + ) + # java's runner was noisy enough that it could not resolve the + # threshold; its cells must not claim a clean bill of health. + java_aa, _ = run( + 1.0, + cfg=cfg, + noise=0.5, + seed=13, + cell_id="java-aa", + control=True, + sdk="java", + ) + java_cell, _ = run( + 1.0, cfg=cfg, noise=0.02, seed=14, cell_id="java-encrypt", sdk="java" + ) + gate = analyze([go_aa, go_cell, java_aa, java_cell], cfg) + + assert len(gate.noise_by_control) == 2, "one noise floor per SDK" + assert gate.comparisons["go-encrypt/wall"].verdict is stats.Verdict.PASS + assert ( + gate.comparisons["java-encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + ), "java's own control had no power, whatever go's control managed" + + def test_a_run_with_no_control_cannot_report_pass(self): + cfg = config(max_rounds=40) + measured, _ = run(1.0, cfg=cfg, cell_id="encrypt") + gate = analyze([measured], cfg) + assert gate.noise is not None and gate.noise.underpowered + assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + assert not gate.should_fail, "an unassessed run warns; it does not fail" diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py new file mode 100644 index 000000000..20f077da0 --- /dev/null +++ b/xtest/test_bench_stats.py @@ -0,0 +1,344 @@ +"""Unit tests for the benchmark decision logic in ``perf/stats.py``. + +These are pure-function tests: no platform, no SDK, no subprocess. They run in +``check.yml`` alongside lint, because a regression gate whose statistics are +wrong is worse than no gate at all -- it either cries wolf until it is muted, +or stays quiet while performance rots. + +The two properties that matter most are covered by +``test_pure_noise_false_positive_rate_is_controlled`` (the gate does not fire +on a runner that is merely noisy) and +``test_planted_regression_is_detected`` (it does fire on a real slowdown). +""" + +import math + +import numpy as np +import pytest + +from perf import stats +from perf.stats import Verdict + +# Typical CI-runner dispersion for a CLI invocation: roughly +/-8% round to +# round. Large enough to be realistic, small enough that 30 rounds can resolve +# a 15% effect. +NOISE_SIGMA = 0.08 +ROUNDS = 30 +# Bootstrap resamples for tests. Lower than the production default to keep the +# repeated-trial tests quick; the estimates are still stable to ~1%. +RESAMPLES = 999 + + +def synth( + rng: np.random.Generator, + true_ratio: float, + *, + n: int = ROUNDS, + sigma: float = NOISE_SIGMA, +) -> tuple[np.ndarray, np.ndarray]: + """Return (baseline, candidate) samples with a known multiplicative effect. + + Both arms get independent lognormal noise around a shared base cost, which + is the structure the real harness produces: a per-round shared component + (runner speed) that cancels, plus independent per-invocation jitter. + """ + base = 1.0 * rng.lognormal(0.0, sigma, n) + baseline = base * rng.lognormal(0.0, sigma, n) + candidate = base * true_ratio * rng.lognormal(0.0, sigma, n) + return baseline, candidate + + +def gate_one( + comparison: stats.PairedComparison, + *, + control: stats.PairedComparison | None = None, + threshold: float = 1.15, +) -> stats.GateResult: + """Run a single comparison through the full run-level gate.""" + cells = {"cell": comparison} + if control is not None: + cells["control"] = control + return stats.apply_multiplicity_control( + cells, + controls=all_under_one_control(cells) if control is not None else None, + threshold=threshold, + ) + + +def all_under_one_control( + cells: dict[str, stats.PairedComparison], key: str = "control" +) -> dict[str, str]: + """Map every cell to the single control cell, as a one-SDK run does.""" + return dict.fromkeys(cells, key) + + +def quiet_control(seed: int = 7) -> stats.PairedComparison: + """An A/A control from a well-behaved runner, tight enough to have power.""" + rng = np.random.default_rng(seed) + # More rounds and lower jitter than a real cell, so the control does not + # itself become the limiting factor in tests about other things. + b, c = synth(rng, 1.0, n=80, sigma=0.03) + return stats.compare(b, c, seed=seed, n_resamples=RESAMPLES) + + +class TestLogRatios: + def test_recovers_exact_ratio(self): + d = stats.log_ratios([2.0, 4.0], [3.0, 6.0]) + assert np.allclose(np.exp(d), 1.5) + + def test_rejects_mismatched_lengths(self): + with pytest.raises(ValueError, match="same length"): + stats.log_ratios([1.0, 2.0], [1.0]) + + @pytest.mark.parametrize("bad", [0.0, -1.0]) + def test_rejects_non_positive(self, bad: float): + # A zero or negative duration is a broken measurement, not a fast one. + with pytest.raises(ValueError, match="positive"): + stats.log_ratios([1.0, bad], [1.0, 1.0]) + + def test_rejects_non_finite(self): + with pytest.raises(ValueError, match="finite"): + stats.log_ratios([1.0, math.inf], [1.0, 1.0]) + + def test_empty_is_empty(self): + assert stats.log_ratios([], []).size == 0 + + +class TestCompare: + def test_point_estimate_tracks_true_ratio(self): + rng = np.random.default_rng(0) + b, c = synth(rng, 1.25, n=200) + r = stats.compare(b, c, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.25, rel=0.05) + + def test_interval_covers_truth(self): + rng = np.random.default_rng(1) + b, c = synth(rng, 1.25, n=200) + r = stats.compare(b, c, seed=1, n_resamples=RESAMPLES) + assert r.ci_low < 1.25 < r.ci_high + + def test_too_few_rounds_yields_no_interval(self): + rng = np.random.default_rng(2) + b, c = synth(rng, 2.0, n=3) + r = stats.compare(b, c, seed=2, n_resamples=RESAMPLES) + assert r.n_rounds == 3 + assert math.isnan(r.ci_low) + assert "at least" in r.note + + def test_identical_inputs_give_unit_ratio_and_no_significance(self): + v = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + r = stats.compare(v, v, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.0) + assert r.p_value == 1.0 + + def test_constant_offset_has_degenerate_interval(self): + # Every round shows exactly a 2x slowdown: there is no sampling + # variability, so the interval collapses onto the point estimate + # rather than blowing up in the BCa jackknife. + b = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + c = [2 * x for x in b] + r = stats.compare(b, c, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(2.0) + assert r.ci_low == pytest.approx(2.0) + assert r.ci_high == pytest.approx(2.0) + + def test_single_outlier_round_does_not_dominate(self): + # One stalled invocation out of 30 must not manufacture a regression; + # this is why the estimator is a median and the test is signed-rank. + rng = np.random.default_rng(3) + b, c = synth(rng, 1.0) + c = c.copy() + c[0] *= 50 + r = stats.compare(b, c, seed=3, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.0, abs=0.1) + assert ( + gate_one(r, control=quiet_control()).comparisons["cell"].verdict + is not Verdict.REGRESSION + ) + + +class TestDecisionRule: + def test_planted_regression_is_detected(self): + rng = np.random.default_rng(10) + b, c = synth(rng, 1.30, n=60) + g = gate_one( + stats.compare(b, c, seed=10, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.REGRESSION + assert g.regressions == ["cell"] + assert g.should_fail + + def test_trivial_but_real_slowdown_does_not_fire(self): + # A reproducible 3% slowdown, measured precisely enough to be + # statistically significant, is deliberately not a build failure. + rng = np.random.default_rng(11) + b, c = synth(rng, 1.03, n=400, sigma=0.02) + r = stats.compare(b, c, seed=11, n_resamples=RESAMPLES) + assert r.p_value < 0.05, "precondition: the effect is statistically real" + g = gate_one(r, control=quiet_control()) + assert g.comparisons["cell"].verdict is not Verdict.REGRESSION + assert not g.should_fail + + def test_planted_speedup_is_reported_but_never_fails(self): + rng = np.random.default_rng(12) + b, c = synth(rng, 0.70, n=60) + g = gate_one( + stats.compare(b, c, seed=12, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.IMPROVED + assert not g.should_fail + + def test_borderline_effect_without_power_is_inconclusive_not_pass(self): + # A 15% effect with only a handful of very noisy rounds: the honest + # answer is "cannot tell", never "no regression". + rng = np.random.default_rng(13) + b, c = synth(rng, 1.15, n=6, sigma=0.35) + g = gate_one( + stats.compare(b, c, seed=13, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.INCONCLUSIVE + + def test_pure_noise_false_positive_rate_is_controlled(self): + # The property the whole design exists to guarantee: on a runner with + # no real effect, the gate must almost never fire. Nominal alpha is + # 0.05, but the threshold clause should push the realized rate far + # below that. + trials, fired = 200, 0 + for seed in range(trials): + rng = np.random.default_rng(1000 + seed) + b, c = synth(rng, 1.0) + g = gate_one( + stats.compare(b, c, seed=seed, n_resamples=RESAMPLES), + control=quiet_control(), + ) + fired += g.should_fail + assert fired / trials <= 0.02, ( + f"gate fired on {fired}/{trials} pure-noise runs; " + "it will be muted in production at this rate" + ) + + def test_detects_regression_across_realistic_noise(self): + # The complement of the false-positive test: a 30% regression must be + # caught reliably, not just on a lucky seed. + trials, caught = 40, 0 + for seed in range(trials): + rng = np.random.default_rng(2000 + seed) + b, c = synth(rng, 1.30, n=40) + g = gate_one( + stats.compare(b, c, seed=seed, n_resamples=RESAMPLES), + control=quiet_control(), + ) + caught += g.should_fail + assert caught / trials >= 0.90, ( + f"only caught {caught}/{trials} real 30% regressions" + ) + + +class TestNoiseFloor: + def test_clean_control_is_trusted(self): + n = stats.assess_noise_floor(quiet_control(), threshold=1.15) + assert not n.tripped + assert not n.underpowered + + def test_missing_control_is_underpowered(self): + n = stats.assess_noise_floor(None, threshold=1.15) + assert n.underpowered + assert "no A/A control" in n.detail + + def test_wide_control_marks_run_underpowered(self): + rng = np.random.default_rng(20) + b, c = synth(rng, 1.0, n=8, sigma=0.5) + n = stats.assess_noise_floor( + stats.compare(b, c, seed=20, n_resamples=RESAMPLES), threshold=1.15 + ) + assert n.underpowered + + def test_biased_control_disables_the_gate(self): + # A control that reports a large effect against a true ratio of 1.0 + # means the harness or the runner is systematically biased. Real cells + # must still be reported, but must not turn the build red. + rng = np.random.default_rng(21) + cb, cc = synth(rng, 1.40, n=60) # A/A that "found" 40%: impossible + control = stats.compare(cb, cc, seed=21, n_resamples=RESAMPLES) + rng2 = np.random.default_rng(22) + b, c = synth(rng2, 1.40, n=60) + g = gate_one( + stats.compare(b, c, seed=22, n_resamples=RESAMPLES), control=control + ) + + assert g.noise is not None and g.noise.tripped + assert not g.trustworthy + assert g.comparisons["cell"].verdict is Verdict.REGRESSION + assert not g.should_fail, "an untrustworthy gate must not fail the build" + assert "A/A control failed" in g.summary + + def test_underpowered_run_cannot_report_pass(self): + rng = np.random.default_rng(23) + noisy_control = stats.compare( + *synth(np.random.default_rng(24), 1.0, n=8, sigma=0.4), + seed=24, + n_resamples=RESAMPLES, + ) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=23, n_resamples=RESAMPLES), control=noisy_control + ) + assert g.comparisons["cell"].verdict is Verdict.INCONCLUSIVE + assert not g.should_fail + + +class TestMultiplicityControl: + def test_bh_is_monotone_and_bounded(self): + raw = [0.001, 0.01, 0.03, 0.2, 0.7] + adj = stats.benjamini_hochberg(raw) + pairs = zip(adj, raw, strict=True) + assert all(a >= r - 1e-12 for a, r in pairs), "adjustment never shrinks p" + assert adj == sorted(adj), "monotone in the sorted input" + assert all(a <= 1.0 for a in adj) + + def test_bh_passes_nan_through(self): + adj = stats.benjamini_hochberg([0.01, float("nan"), 0.02]) + assert math.isnan(adj[1]) + assert all(math.isfinite(a) for a in (adj[0], adj[2])) + + def test_correction_suppresses_lone_lucky_cell(self): + # 20 pure-noise cells: without BH one of them firing is expected. + cells = {} + for i in range(20): + rng = np.random.default_rng(3000 + i) + b, c = synth(rng, 1.0) + cells[f"cell{i}"] = stats.compare(b, c, seed=i, n_resamples=RESAMPLES) + cells["control"] = quiet_control() + g = stats.apply_multiplicity_control( + cells, controls=all_under_one_control(cells) + ) + assert not g.should_fail + assert g.regressions == [] + + def test_control_is_excluded_from_the_gate(self): + rng = np.random.default_rng(30) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=30, n_resamples=RESAMPLES), control=quiet_control() + ) + assert "control" not in g.regressions + assert g.comparisons["control"].p_adjusted is None + + def test_ungated_metric_is_reported_but_cannot_fail(self): + rng = np.random.default_rng(31) + b, c = synth(rng, 1.5, n=60) + cells = { + "wall": stats.compare(b, c, seed=31, n_resamples=RESAMPLES), + "cpu": stats.compare(b, c, seed=32, n_resamples=RESAMPLES), + "control": quiet_control(), + } + g = stats.apply_multiplicity_control( + cells, gated={"wall"}, controls=all_under_one_control(cells) + ) + assert g.comparisons["cpu"].verdict is Verdict.REGRESSION + assert g.regressions == ["wall"], "cpu is reported but never gates" + + def test_empty_run_is_not_a_failure(self): + g = stats.apply_multiplicity_control({}) + assert not g.should_fail + assert g.regressions == [] diff --git a/xtest/test_benchmarks.py b/xtest/test_benchmarks.py new file mode 100644 index 000000000..0e4197455 --- /dev/null +++ b/xtest/test_benchmarks.py @@ -0,0 +1,91 @@ +"""SDK performance regression cells. + +One test per cell: an operation at a payload size, measuring the newest +installed release against the branch build on the same runner, in the same +round, in a randomized order. + +**These tests do not assert.** Each one records its raw samples and passes. +The verdict cannot be reached cell by cell: the multiplicity correction is +computed across every gated cell in the run, and the A/A control can +invalidate all of them at once. The gate therefore runs once in +``pytest_sessionfinish`` (see ``conftest.py``), which fails the session on a +confirmed regression. + +Nothing is collected here without ``--bench``; see ``conftest.py``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import NoReturn + +import pytest + +import abac +from fixtures import bench +from perf import report, runner +from perf.cells import BenchCell + +pytestmark = pytest.mark.benchmark + + +def test_sdk_performance( + bench_cell: BenchCell, + bench_config: runner.BenchConfig, + bench_arms: bench.ArmResolver, + bench_payloads: dict[str, Path], + bench_ciphertexts: bench.CiphertextFactory, + bench_budget: runner.Budget, + bench_recorder: report.BenchmarkRecorder, + attribute_default_rsa: abac.Attribute, + tmp_dir: Path, +) -> None: + """Measure one cell and record it; the session-wide gate decides. + + A cell that cannot be measured -- a missing build, two builds that would + not be doing the same work, a budget that ran out -- is skipped *and* + recorded as skipped, so that a quiet report is visibly quiet rather than + indistinguishable from a clean one. + """ + + def bail(reason: str) -> NoReturn: + bench_recorder.skip(bench_cell.id, reason) + pytest.skip(reason) + + try: + arms = bench_arms(bench_cell.sdk) + except bench.ArmSelectionError as e: + bail(str(e)) + + problem = bench.comparability_problem(arms) + if problem: + bail(problem) + + ct_file = ( + bench_ciphertexts(arms, bench_cell.payload.label) + if bench_cell.operation == "decrypt" + else None + ) + baseline, candidate = bench.build_arms( + bench_cell, + arms, + pt_file=bench_payloads[bench_cell.payload.label], + ct_file=ct_file, + tmp_dir=tmp_dir, + attr_values=attribute_default_rsa.value_fqns, + ) + + try: + result = runner.run_cell( + bench_cell.id, + baseline, + candidate, + bench_config, + deadline=bench_budget.next_deadline(), + control=bench_cell.control, + sdk=bench_cell.sdk, + ) + except runner.BudgetExhausted as e: + bail(str(e)) + + bench_recorder.record(result) diff --git a/xtest/test_sdk_commands.py b/xtest/test_sdk_commands.py new file mode 100644 index 000000000..9b01c0fc4 --- /dev/null +++ b/xtest/test_sdk_commands.py @@ -0,0 +1,149 @@ +"""Unit tests for the SDK CLI command builders in ``tdfs.py``. + +``SDK.encrypt_command`` / ``SDK.decrypt_command`` are the single definition of +the ``XT_WITH_*`` contract: both ``SDK.encrypt``/``SDK.decrypt`` and the +benchmark harness build their invocations through them. Pinning the argv and +env here means a change to that contract shows up as a failing assertion +rather than as a benchmark silently measuring a different operation than the +functional suite. + +No platform and no real SDK -- the builders only need ``cli.sh`` to exist, so +these run against a stub tree in ``tmp_path``. +""" + +from pathlib import Path + +import pytest + +import tdfs + + +@pytest.fixture +def sdk(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tdfs.SDK: + """An SDK pointing at a stub ``cli.sh`` that is never executed.""" + cli = tmp_path / "sdk" / "go" / "dist" / "main" / "cli.sh" + cli.parent.mkdir(parents=True) + cli.write_text("#!/bin/sh\nexit 0\n") + monkeypatch.chdir(tmp_path) + return tdfs.SDK("go", "main") + + +class TestEncryptCommand: + def test_positional_arguments(self, sdk: tdfs.SDK): + argv, _ = sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + assert argv == [sdk.path, "encrypt", "in.txt", "out.tdf", "ztdf"] + + def test_mime_type_defaults_on(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + assert env == {"XT_WITH_MIME_TYPE": "application/octet-stream"} + + def test_empty_mime_type_omits_the_variable(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf"), mime_type="") + assert "XT_WITH_MIME_TYPE" not in env + + def test_attributes_are_comma_joined(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), + Path("out.tdf"), + attr_values=[ + "https://e.com/attr/a/value/1", + "https://e.com/attr/b/value/2", + ], + ) + assert env["XT_WITH_ATTRIBUTES"] == ( + "https://e.com/attr/a/value/1,https://e.com/attr/b/value/2" + ) + + def test_empty_attribute_list_omits_the_variable(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf"), attr_values=[]) + assert "XT_WITH_ATTRIBUTES" not in env + + def test_assertions(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), assert_value="[{}]" + ) + assert env["XT_WITH_ASSERTIONS"] == "[{}]" + + def test_target_mode(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), target_mode="4.3.0" + ) + assert env["XT_WITH_TARGET_MODE"] == "4.3.0" + + def test_ecwrap_container_maps_to_ztdf_plus_a_flag(self, sdk: tdfs.SDK): + argv, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), container="ztdf-ecwrap" + ) + assert argv[-1] == "ztdf", "the CLI format argument is the simple container" + assert env["XT_WITH_ECWRAP"] == "true" + + def test_target_mode_survives_ecwrap(self, sdk: tdfs.SDK): + # The XT_WITH_TARGET_MODE guard tests the *simplified* format, and + # ztdf-ecwrap simplifies to ztdf, so target mode applies to both. + _, env = sdk.encrypt_command( + Path("in.txt"), + Path("out.tdf"), + container="ztdf-ecwrap", + target_mode="4.3.0", + ) + assert env["XT_WITH_TARGET_MODE"] == "4.3.0" + assert env["XT_WITH_ECWRAP"] == "true" + + +class TestDecryptCommand: + def test_positional_arguments(self, sdk: tdfs.SDK): + argv, env = sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + assert argv == [sdk.path, "decrypt", "in.tdf", "out.txt", "ztdf"] + assert env == {}, "a plain decrypt sets no XT_WITH_* overrides" + + def test_assertion_verification_keys(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), assert_keys="{keys}" + ) + assert env["XT_WITH_ASSERTION_VERIFICATION_KEYS"] == "{keys}" + + def test_verify_assertions_only_set_when_disabled(self, sdk: tdfs.SDK): + _, on = sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + _, off = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), verify_assertions=False + ) + assert "XT_WITH_VERIFY_ASSERTIONS" not in on + assert off["XT_WITH_VERIFY_ASSERTIONS"] == "false" + + def test_ecwrap_flag(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command(Path("in.tdf"), Path("out.txt"), ecwrap=True) + assert env["XT_WITH_ECWRAP"] == "true" + + def test_kas_allowlist(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command( + Path("in.tdf"), + Path("out.txt"), + kasallowlist="http://localhost:8080", + ignore_kas_allowlist=True, + ) + assert env["XT_WITH_KAS_ALLOWLIST"] == "http://localhost:8080" + assert env["XT_WITH_IGNORE_KAS_ALLOWLIST"] == "true" + + def test_ecwrap_container_maps_to_ztdf(self, sdk: tdfs.SDK): + argv, _ = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), container="ztdf-ecwrap" + ) + assert argv[-1] == "ztdf" + + +class TestDeterminism: + def test_builders_are_pure(self, sdk: tdfs.SDK): + # The benchmark builds a command once and runs it many times; a + # builder that mutated shared state would make round N differ from + # round 1 and show up as a phantom regression. + args = (Path("in.txt"), Path("out.tdf")) + kwargs = {"container": "ztdf-ecwrap", "attr_values": ["a"]} + first = sdk.encrypt_command(*args, **kwargs) + second = sdk.encrypt_command(*args, **kwargs) + assert first == second + + def test_no_side_effects_on_the_filesystem(self, sdk: tdfs.SDK, tmp_path: Path): + sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + assert not (tmp_path / "out.tdf").exists() + assert not (tmp_path / "out.txt").exists() diff --git a/xtest/uv.lock b/xtest/uv.lock index 5ee4556c6..c2452db75 100644 --- a/xtest/uv.lock +++ b/xtest/uv.lock @@ -313,6 +313,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "numpy" +version = "2.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/f8/c3b222bf075b50afd8e949a07a15c4b312a4a84bd8102a332bcd953cbbb4/numpy-2.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d787cf769c3baeb5f6235e778edb52c08dfa923789b5958f28e6450f96107cb1", size = 16885180, upload-time = "2026-08-09T13:46:03.939Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/2c1d4b1987795a92b5bbf7c24fe249ab96aa2573ab0d7604802c189d7b86/numpy-2.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:24b9dc2e3d84aa58523798805194e23e736f3f6ce2d1a5b92583ae734e6dbda8", size = 11907878, upload-time = "2026-08-09T13:46:07.045Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ee/d08226fc858044355983a6e5b94f08ff6f3969e0a2b160a4a89f0ddb3445/numpy-2.5.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9e9413326d726c2545bfa65d2c0876871e8d8386e77f992c1d426e180bbd4323", size = 5354922, upload-time = "2026-08-09T13:46:10.04Z" }, + { url = "https://files.pythonhosted.org/packages/94/f0/6d3d933056440ebbc5e6bad92065fc6c26a48a84a36b1208580e94eea76c/numpy-2.5.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:60e902ac295855348a5ca2ea4c89108989a9f5fddfad3dfc0a8f36b10358567e", size = 6679168, upload-time = "2026-08-09T13:46:12.275Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3b/ecd49dd90033cceb2704d88ca905d4d7d89b0e8c739608754ffd325fa820/numpy-2.5.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e500dc868e9313530ce12ba470fe50ff3afe3d62993ed6eff652dacd555b65", size = 15624501, upload-time = "2026-08-09T13:46:15.322Z" }, + { url = "https://files.pythonhosted.org/packages/c7/99/461bd36dbdfac6c1c53efa370bd55a83227542d0d118f1677dbf1a3dacd5/numpy-2.5.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318b9a4c845dbea06708a29c84ee429cc3065048db34cdb799047643492050ee", size = 16713701, upload-time = "2026-08-09T13:46:18.949Z" }, + { url = "https://files.pythonhosted.org/packages/f9/9c/2b251df9e8a5d647b62b0cbc1b90a91850c1cf4859ecb532fd0b4eacff6c/numpy-2.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34c319e2963be042673fb46570501b2f06c41924e17e3563d58646b4380dfb68", size = 16986065, upload-time = "2026-08-09T13:46:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/8f/25/20de43f53ff1390534a124475055a19f01fe10c920a0fd11b8e18d6d6052/numpy-2.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f06571a052127dc1b4e8b83029b4d1b20daa2b64a31cdd181fc6bc774e9000eb", size = 18470031, upload-time = "2026-08-09T13:46:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/56/5e/0c577ca308d6da5eb79b546ba10bbe5b60148192194e2da060913b1de4f1/numpy-2.5.2-cp314-cp314-win32.whl", hash = "sha256:2cc779226e476d1e1f08c74068c419e60f41a9e0e069c92f6671d31d5c985e98", size = 6121028, upload-time = "2026-08-09T13:46:30.046Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/7bcbd5b11f94199073320410cddcbb80cee62415bfeb540874b265c2d922/numpy-2.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:7587f53dfbd5edc0f7b87c6217b4c6d2d1f2ef9c3da70bc1315e7db5f8d7ec9d", size = 12597627, upload-time = "2026-08-09T13:46:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/87/bc/4d0b06fba0da90ccc75af62823cb9dcedb6c9ea0cffa058cb2c9ee773a77/numpy-2.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:3e4c367352d3747784248a227fbec218e193b56f7e6692e3b64fc805478ecfdf", size = 10680414, upload-time = "2026-08-09T13:46:36.036Z" }, + { url = "https://files.pythonhosted.org/packages/cd/17/f429aac9dc08833a0d0f188eba38c532a751b1a1f2ca6018a37b455cb321/numpy-2.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b879fb674276e331513fb136b78dbc6bd3c848309e0d841cfd63be3896c4cfc1", size = 12026967, upload-time = "2026-08-09T13:46:39.084Z" }, + { url = "https://files.pythonhosted.org/packages/ca/9f/d0849de96a2a4ceaa16662f18ee13eaa9c0aa418269fdc8c4857c56b11da/numpy-2.5.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:fd0d703772bba096843785bd38371e31bb4a0c1151497ad5739d182114a73f7f", size = 5473874, upload-time = "2026-08-09T13:46:42.075Z" }, + { url = "https://files.pythonhosted.org/packages/89/3c/8df216d4a4a5422a3de045301cf7df8ea47286d76f5cb7160b0128ac26b7/numpy-2.5.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:3a2f061cebd9e3d23bdcfaaded5e2293a4c6a5b60fa42df85d410a725ce621bf", size = 6789276, upload-time = "2026-08-09T13:46:44.387Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3a/20d7e9891c4ddfadd6ff8d95bf4b29f353d8e1770553de2099880551dfb9/numpy-2.5.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6df895598c0edcb41030126c89e0f353b07d93238116143b7405e937359736c4", size = 15659154, upload-time = "2026-08-09T13:46:47.538Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d6/f3aa3d2688bf501b858835c6bd087ae9b51a56ae6fca8e2b0990abd177af/numpy-2.5.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ab3d4a901f844ea836c3e80bf463c6a27d7f3c14e8e292fcf28d348b25b9bce", size = 16748909, upload-time = "2026-08-09T13:46:51.442Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8f/1c5cae8d2baf86ab802ae97a00be55bc7e21ebc11b12bbc33376c5f05342/numpy-2.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cebc2d6dbb605a7703d59751dea4bd6b0ab127a5a4338a6f432df1936fef8b26", size = 17027685, upload-time = "2026-08-09T13:46:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/5c/27/71d3467404aedc1c24ce79610f91b52b0b0f466c43a701aa56fc75c145ab/numpy-2.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eaca7ff36f0f52e2111ec71f169d8fd3e889e7ddc0d2592e0d703fd8d3ce8fac", size = 18501181, upload-time = "2026-08-09T13:46:59.09Z" }, + { url = "https://files.pythonhosted.org/packages/14/2f/42921d27c40aea7e077f4a423ae509fd9220b028cd787bafefd8ab2b3a5f/numpy-2.5.2-cp314-cp314t-win32.whl", hash = "sha256:ddf47472af2e4280d79bac82304f5e80150211f1b9e614b760061d5fdfbb6eba", size = 6271085, upload-time = "2026-08-09T13:47:01.903Z" }, + { url = "https://files.pythonhosted.org/packages/75/e6/bad5f5d56de9b1971bac959963dda276d35c40f1854475005434bbe08692/numpy-2.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:44ef9675d908e65f9953063837c3277730f3f4437615a4cdab67b366cabaf884", size = 12787971, upload-time = "2026-08-09T13:47:04.963Z" }, + { url = "https://files.pythonhosted.org/packages/df/05/f608795cb34391acd67e38d94a3c36abd8d8576293a3a80727d7595c372c/numpy-2.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:eaa088384c46f519dacb93b7ec483a6d6b19a4a2085ae4f25ab9b1c43d387d1e", size = 10750306, upload-time = "2026-08-09T13:47:07.976Z" }, + { url = "https://files.pythonhosted.org/packages/33/c6/28de0191c5f82b7d42a0a51390ba98587048aa93a39fafb05bdbe6e8d00c/numpy-2.5.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:078f9b027b478c9379b9677babbf0f8b8f1ecfada27636d7b9a93990c638739f", size = 16885274, upload-time = "2026-08-09T13:47:11.439Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/973ca116000d244897e468ea1aff30b589e5022e3c8744b71706fe33bd57/numpy-2.5.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:50a68f4bacd8a2b33d8da3d2269d0d78500f86ea582e4786dc10f5ef2c2c6842", size = 11907846, upload-time = "2026-08-09T13:47:15.128Z" }, + { url = "https://files.pythonhosted.org/packages/78/d9/8c4b3937ef204cb2fd88d389ccd0f265a2ffb11f35a01d2064cf46714bd6/numpy-2.5.2-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:e79aba74ffaf5f78a050d777c184cddf8fdffabab38acf5f3ef1fecbc17895d6", size = 5354892, upload-time = "2026-08-09T13:47:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/b6ee65ea2999fdb7023935e108e6fb776ee4082aa15f159acfa857e578c8/numpy-2.5.2-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:9a0731745a72a184490a582fb4af2533512bd071ace67785b5fdffc0ae58dce8", size = 6679309, upload-time = "2026-08-09T13:47:20.456Z" }, + { url = "https://files.pythonhosted.org/packages/43/f3/acb18d8b137a393c8e7803a8c994c9e64bde3930692a69d826993113a159/numpy-2.5.2-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ec954036759bcee3aa484f8603bd9c14f3e776293b85578b8734c2d72777c69", size = 15625850, upload-time = "2026-08-09T13:47:24.365Z" }, + { url = "https://files.pythonhosted.org/packages/a9/bf/a8e9bb0db815a0e265b5744ebedd3af0bd5faad8604e5b50a1cd012f3c91/numpy-2.5.2-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc649493697006bc90614a5f0bbc8cb3cb1866715c474e473694968d7e6b99ab", size = 16713664, upload-time = "2026-08-09T13:47:27.965Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c3/6e913736b3dd6582344af32418b5fb9dab34282e8a8174ae1d54ceb0fc13/numpy-2.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cf7de32f486e4ac9e2d93b810f9e9ac72a728dd46a32a0bb403222f27f653514", size = 16986749, upload-time = "2026-08-09T13:47:31.541Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/7d3b23eff5c7428ef6c01e6f7052bb60d504c4d33e317b36b8959c24ad97/numpy-2.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2ffa7bacab3e2ee1b19ed31766bb60bb380b68c23f051e199c5cc598afd68710", size = 18470495, upload-time = "2026-08-09T13:47:35.364Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a4/68a321d825374f6eb677ffe8ef8c6b9a328304e6fd2e39d9530822776607/numpy-2.5.2-cp315-cp315-win32.whl", hash = "sha256:6b588cc8f902d6bff201c19fd00c43ab8545671e3554d014e12e14139e5e8617", size = 6120696, upload-time = "2026-08-09T13:47:38.561Z" }, + { url = "https://files.pythonhosted.org/packages/c8/23/deafbb1700f79fae9cd1e91220f133d124cc267de1b584da3fbf6db2f6cd/numpy-2.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:07d4e89f3a9ab0a9ba24264ccdb642b3dd951b2281e8883a5481a4aa79cc31a7", size = 12597324, upload-time = "2026-08-09T13:47:41.401Z" }, + { url = "https://files.pythonhosted.org/packages/33/cd/3272ba105e3bbbdaeb11357eda31e7a6825ffe159e8171665660299a948f/numpy-2.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:a610dc7e3c52edd39c2bc2375ff9c3fd59cb3ad00e4472d36f83bc1457145788", size = 10680466, upload-time = "2026-08-09T13:47:44.873Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/58370637b1bb70a5c9ce2b43f4b521ccb224e36ccb76a6596b17ae4b447c/numpy-2.5.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:40f4d451aed46a8046a1aae41c4e55fb3612273df9c502480135e1501576a34b", size = 16993947, upload-time = "2026-08-09T13:47:48.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/93/2abcb807712b289d6d60fe4cf30532f98974a8396d885650f3ba5a13026e/numpy-2.5.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c081cbe16ba1ab53078e5ff29013621e33c509eedab055775d956427712c236e", size = 12025331, upload-time = "2026-08-09T13:47:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3a/2898e003a5fbaf87e76c039b4ee1f5eb390471b4ffe74887c1f34c4e791e/numpy-2.5.2-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:0090ccdd57ec2703e9b49d0bf554767370581c1dd0a6b2bb2b2d9def317d042a", size = 5472336, upload-time = "2026-08-09T13:47:55.403Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/23f69d07c544597b29758b31b55c27dc9d541012a2c1496189fef702aec2/numpy-2.5.2-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:6a9bb119fb8dd21ba30b3f0e555b7e2b081bd9883af21ec9c1c633d161cda3a8", size = 6788387, upload-time = "2026-08-09T13:47:58.192Z" }, + { url = "https://files.pythonhosted.org/packages/15/ea/c0dbdbcf22f43782510a3e492dd3da73c6112b69cac8929d16d127536fc4/numpy-2.5.2-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a839318485284a6fb31be4f8f2c91c8f2cb22f4543c4a8903f12b0671ffe07cc", size = 15667096, upload-time = "2026-08-09T13:48:01.562Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5e/29c73c31748cdb0f7566642125ba17fd5b56780cddf891b085dab27e4466/numpy-2.5.2-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba0a474801b8dc67b66bf465548abc90e82b44d2611b5770f33008dcabffe8ec", size = 16751730, upload-time = "2026-08-09T13:48:05.706Z" }, + { url = "https://files.pythonhosted.org/packages/47/95/02501e8454796bb58dadf7a99d3181e0b464bf264e1003039572f9779fac/numpy-2.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0a4035ae1129ff8777f08bfbd44f1e5d8e9c049ce0c2dd78fc0d92c13e7251c0", size = 17038686, upload-time = "2026-08-09T13:48:09.627Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b5/53a681d91b5c82687067d8ea5035e02d917b5509d6f334cb06484a954714/numpy-2.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:77843ca236b777e67f8d6b3660ea116e499612703a0ecd7093f316201eb9d8e2", size = 18507727, upload-time = "2026-08-09T13:48:13.744Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/6e11443f7b64ee376c860506091103bf68f92d2cab9e8d96d4501babf07c/numpy-2.5.2-cp315-cp315t-win32.whl", hash = "sha256:7354826bc6f8f69402e9b7fe28d15fcd34feebd74f856f111585c5b0c9fb0251", size = 6269775, upload-time = "2026-08-09T13:48:17.543Z" }, + { url = "https://files.pythonhosted.org/packages/f1/18/195d6b86cd72dbbc501edfa778005fa6b87afd34c153e46028cd3a0938f4/numpy-2.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:e5651f3f87add730ee6608d915009e19c911fba0cb000c7e3ea994b7d768eb12", size = 12782559, upload-time = "2026-08-09T13:48:21.023Z" }, + { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -592,6 +643,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, ] +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + [[package]] name = "smmap" version = "5.0.3" @@ -652,6 +734,7 @@ dependencies = [ { name = "jsonschema" }, { name = "jsonschema-specifications" }, { name = "markupsafe" }, + { name = "numpy" }, { name = "packaging" }, { name = "pluggy" }, { name = "pycparser" }, @@ -665,6 +748,7 @@ dependencies = [ { name = "referencing" }, { name = "requests" }, { name = "rpds-py" }, + { name = "scipy" }, { name = "smmap" }, { name = "typing-extensions" }, { name = "urllib3" }, @@ -694,6 +778,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.25.1" }, { name = "jsonschema-specifications", specifier = ">=2025.9.1" }, { name = "markupsafe", specifier = ">=3.0.3" }, + { name = "numpy", specifier = ">=2.5.2" }, { name = "packaging", specifier = ">=26.2" }, { name = "pluggy", specifier = ">=1.6.0" }, { name = "pycparser", specifier = ">=3.0" }, @@ -709,6 +794,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.34.2" }, { name = "rpds-py", specifier = ">=2026.5.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.16" }, + { name = "scipy", specifier = ">=1.18.0" }, { name = "smmap", specifier = ">=5.0.3" }, { name = "typing-extensions", specifier = ">=4.15.0" }, { name = "urllib3", specifier = ">=2.7.0" }, From a54102626fbe65bd27021f52dccc092192363f83 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 14 Aug 2026 13:22:52 -0400 Subject: [PATCH 03/11] docs(xtest): explain the benchmark harness in perf/README.md --- xtest/AGENTS.md | 1 + xtest/perf/README.md | 399 +++++++++++++++++++++++++++++++++++++++++ xtest/perf/__init__.py | 4 + 3 files changed, 404 insertions(+) create mode 100644 xtest/perf/README.md diff --git a/xtest/AGENTS.md b/xtest/AGENTS.md index 7b1e7b899..04f02dc12 100644 --- a/xtest/AGENTS.md +++ b/xtest/AGENTS.md @@ -15,6 +15,7 @@ fixture system. | `conftest.py` | `pytest_addoption` + the encrypt/decrypt SDK parametrization. Defines `--sdks`, `--sdks-encrypt`, `--sdks-decrypt`, `--containers`, `--no-audit-logs`. | | `fixtures/` | Module-scoped pytest fixtures: `attributes.py`, `keys.py`, `audit.py`, `assertions.py`, `kas.py`, `encryption.py`, `obligations.py`. | | `tdfs.py` | SDK abstraction layer — wraps the `cli.sh` shims under `sdk//dist//`. | +| `perf/` | Paired A/B performance regression benchmarks (opt-in via `--bench`). **Read `perf/README.md` before changing anything in here** — the design decisions fail silently when undone. | | `sdk/{go,java,js}/dist//` | SDK CLI builds. Installed by `otdf-sdk-mgr install` (see `../otdf-sdk-mgr/AGENTS.md`). | | `test.env` | Default endpoint and client-credential env vars. Source with `set -a && source test.env && set +a`. | diff --git a/xtest/perf/README.md b/xtest/perf/README.md new file mode 100644 index 000000000..9eb0c7d4b --- /dev/null +++ b/xtest/perf/README.md @@ -0,0 +1,399 @@ +# SDK performance regression benchmarks + +A paired A/B benchmark for the OpenTDF SDK CLIs. It answers one question: +**did this change make the SDK measurably and meaningfully slower?** + +Two builds — normally the newest installed release and the branch build — +are measured on the *same* runner, interleaved round by round, and only their +*ratio* is reported. Nothing is ever compared against a stored historical +number. + +It runs nightly (one runner per SDK) and on `workflow_dispatch` with +`run-benchmarks` checked. It never runs on pull requests: 30 minutes of serial +measurement is too slow for a PR gate, and a PR runner is the noisiest place to +measure. + +> **Dispatching it by hand:** set the `*-ref` inputs to `main latest`, not the +> default `main`. The nightly cron resolves `main latest` on its own, but an +> explicit `main` installs only the branch build — no release to use as a +> baseline — and every cell skips. The run fails rather than passing empty +> (see [NOTHING MEASURED](#the-verdicts)), but it will have wasted 45 minutes +> to tell you that. + +- **Section 1 — [Reading a result](#1-reading-a-result)** is for developers on + the SDKs and the platform: your build got flagged, what does that mean. +- **Section 2 — [Maintaining the harness](#2-maintaining-the-harness)** is for + whoever changes this code: how it works and why it is shaped this way. + +--- + +## 1. Reading a result + +### Where the output is + +| Artifact | Where | Contents | +| --- | --- | --- | +| Job summary | The Actions run page | The table below, plus the verdict | +| `bench-` artifact | Run artifacts | `.json` with **every raw per-round sample**, and an HTML report | +| Terminal | Job log tail | One-line summary and the JSON path | + +The JSON is the useful one. It holds each cell's full per-round vectors for both +arms, so a surprising verdict can be re-analysed offline instead of by re-running +a 30-minute job to look at the same numbers again. + +### The table + +``` +| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict | +| go-encrypt-1MiB | wall clock | 412.3 ms | 498.1 ms | 1.208x [1.171, 1.245] | <0.001 | 22 | **REGRESSION** | +``` + +- **cell** — `--`, plus `-control` for the A/A cell. + Payload sizes are 1 KiB, 1 MiB, and 32 MiB. +- **ratio** — candidate ÷ baseline. `1.208x` means the candidate took 20.8% + longer. Below 1.0 means faster. +- **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely + this run could measure; a wide interval means a noisy runner, not a big change. +- **p (BH)** — one-sided p-value, Benjamini–Hochberg adjusted across the run. +- **n** — paired rounds actually measured (20–60; the loop stops early once the + interval is narrow enough). + +### The verdicts + +**REGRESSION** — the CI lower bound exceeds the threshold (default **1.15x**, +i.e. 15% slower) *and* the adjusted p < 0.05. Both clauses are required, and +neither is redundant: the threshold alone would fire on a reproducible 0.5% +slowdown nobody cares about, and significance alone would fire on noise often +enough to be ignored within a week. This fails the job. + +**PASS** — not a regression, *and* the run had enough precision to have found +one. "We looked and found nothing" only counts when we could have found +something. + +**IMPROVED** — the same test in the other direction. Never fails anything. + +**inconclusive** — the run could not decide. Common reasons, all shown in the +note beside the verdict: +- the runner was too noisy for this cell's interval to be usable; +- the A/A noise floor was wider than the 15% effect being gated on, so a real + regression of that size could not have been distinguished from noise; +- peak RSS hit the measurement floor (see below); +- too few rounds completed inside the time budget. + +**Inconclusive is not a pass.** It means the question was not answered. If a +change you expect to be performance-sensitive comes back inconclusive on every +cell, the run told you nothing and re-running it is reasonable. + +**NOTHING MEASURED** — no cell produced a comparison at all, usually because +only one build was installed so there was no baseline to compare against. This +**fails the job**. An empty run and a clean run have the same empty list of +regressions, so without this a benchmark that had quietly stopped measuring +would keep reporting a green tick. The "Not measured" section of the report +lists the reason for each cell. + +### The A/A control + +Each SDK gets a control cell that compares the baseline build **against itself** +through the identical pipeline. Its true ratio is exactly 1.0 by construction, so +whatever it reports is the harness's own error on this runner. It does two jobs: + +- If the control *trips* — its own A/A comparison looks like a real effect — then + something is systematically biased and **the whole run stops being able to fail + the build**. Results are still reported, marked untrustworthy. +- Its interval width is the run's **noise floor**: the smallest effect this + runner could have resolved. If the floor is wider than the threshold, cells + report inconclusive rather than PASS. + +In a multi-SDK run each SDK is judged against *its own* control — go's harness +path says nothing about java's. `noise_floor_by_control` in the JSON has each +one; the top-level `noise_floor` is the worst of them. + +### Gated vs ungated metrics + +| Metric | Gated? | Why | +| --- | --- | --- | +| wall clock | yes | What users experience | +| peak RSS | yes | Regressions here are real and invisible in timing | +| CPU time | **no** | Noisiest of the three on a shared runner, and a real CPU regression shows up in wall clock anyway | + +Ungated rows are labelled `(ungated)` and reported for context only. They cannot +fail the build. + +Peak RSS additionally gets **censored** when a cell's readings sit at the +measurement floor (the RSS of the process that forked the command). Both arms +clip to the same value there, producing a `1.000x` ratio with a tight interval — +the most convincing-looking PASS the harness can emit, and completely meaningless. +Censored cells report inconclusive with the floor named in the note. + +### My build was flagged. Now what? + +1. **Read the CI column, not just the ratio.** A `1.20x [1.02, 1.41]` is a very + different claim from `1.20x [1.19, 1.21]`. +2. **Check the control row.** If the A/A cell for your SDK also looks strange, + suspect the runner before your code. +3. **Look at which cells fired.** Only the 1 KiB cells means startup cost — + process boot, package resolution, TLS handshake, token fetch. Only 32 MiB + means throughput — the crypto and IO path. Both means something structural. +4. **Reproduce locally.** The comparison is self-contained; it does not need CI. + +```bash +cd xtest && set -a && source test.env && set +a + +# whatever two builds you want, side by side under sdk//dist/ +uv run pytest --bench --sdks go \ + --bench-baseline go@v0.29.0 \ + --bench-candidate go@main \ + -v test_benchmarks.py +``` + +Useful knobs while investigating: + +| Option | Default | Use | +| --- | --- | --- | +| `--bench-threshold` | `1.15` | Smallest slowdown worth failing on | +| `--bench-min-rounds` / `--bench-max-rounds` | `20` / `60` | Rounds per cell | +| `--bench-warmup` | `5` | Discarded rounds paying one-time costs | +| `--bench-budget-seconds` | `1500` | Wall-clock allowance shared by all cells | +| `--bench-seed` | `0` | Payloads, round order, bootstrap. Fix it to reproduce | +| `--bench-out` | `test-results/benchmarks` | JSON destination | +| `--bench-no-gate` | off | Measure and report, never fail | + +A local run is noisier than CI unless the machine is otherwise idle. Close +things; the noise floor will tell you whether you succeeded. + +### What this benchmark cannot tell you + +- **Anything about absolute speed.** A number from a GitHub-hosted runner is not + comparable to a number from your laptop or from last week's runner. Only + within-run ratios mean anything. +- **Anything about trends.** There is no history and no stored baseline. Each run + is a self-contained experiment. +- **Anything about a slowdown under 15%** by default. That is the price of not + crying wolf on a shared runner. +- **Anything about your change specifically** if the baseline moved too — the + comparison is release-vs-`main`, so it catches whatever landed on `main`. + +--- + +## 2. Maintaining the harness + +### Module map + +| File | Responsibility | +| --- | --- | +| `cells.py` | The experiment matrix: payload sizes, `BenchCell`, `cells_for()`. No pytest, no `tdfs` | +| `measure.py` | Wall/CPU/RSS for one invocation, via `os.wait4` | +| `_launcher.py` | The separate process that actually forks the measured command | +| `runner.py` | The paired round loop, the stopping rule, the budget, `analyze()` | +| `stats.py` | Pure functions: log-ratios, bootstrap CI, Wilcoxon, BH, the decision rule | +| `report.py` | Session recorder, JSON artifact, step-summary markdown | +| `../fixtures/bench.py` | The pytest glue: arm selection, payloads, ciphertexts, budget | +| `../test_benchmarks.py` | One test per cell. **Records; never asserts** | +| `../conftest.py` | `--bench*` options, cell parametrization, the session-finish gate | + +Offline tests, no platform and no subprocesses needed: + +```bash +cd xtest +uv run pytest -q test_bench_stats.py test_bench_measure.py \ + test_bench_runner.py test_bench_arms.py +``` + +These run on every PR via `check.yml`, so the harness is exercised continuously +even though the benchmark itself runs nightly. + +### The design, and why + +#### Ratios within a run, never comparison against history + +CPU models vary, tenancy is shared, and steal time is unbounded on a hosted +runner. Storing a baseline and diffing against it produces false alarms until +people mute the job. Both builds are measured on the same runner and the +statistic is the within-round ratio, so runner speed is a shared factor that +divides out. + +#### Interleaved rounds, randomized within the round + +Running all of A then all of B lands every drift effect — a noisy neighbour +arriving, thermal throttling, the page cache warming — entirely on one arm, where +it reads as a difference between builds. Both arms run once per round instead. +The order *within* a round is shuffled because a fixed order is itself a +confounder: whichever arm goes second inherits the first one's cache state. + +The shuffle is seeded per cell (`f"{seed}:{cell_id}"`), so a rerun reproduces the +interleaving exactly while different cells do not share one order — which would +correlate their noise. + +#### Log-ratios + +`d_i = ln(candidate_i) - ln(baseline_i)`. Logs make ratios symmetric (a 2x +slowdown and a 2x speedup are equal and opposite) and additive, which is what +the median and the bootstrap want. Everything is exponentiated back for reporting. + +#### Stopping on precision, never on significance + +> This is the single easiest thing here to "optimize" into invalidity. + +The loop stops when the CI is narrow enough. It must never stop when the p-value +gets small. Peeking at p and stopping the moment it crosses alpha is optional +stopping: you get a fresh chance to cross the line every round and only ever stop +on the lucky side, which inflates the false-positive rate far past nominal. +Attained CI *width* is driven by the dispersion of the differences rather than +their location, so it is approximately ancillary to the effect being tested and +stopping on it does not bias the verdict. + +`_precise_enough()` therefore looks only at interval width, never at where the +interval sits. It also uses `not (width <= target)` rather than `width > target`, +because a NaN width must read as "keep going" and `NaN > target` is `False`. + +#### Both clauses of the decision rule + +A cell is a regression iff the CI lower bound exceeds `threshold` **and** the +BH-adjusted p is below alpha. Clause 1 alone fires on real-but-trivial effects +measured precisely; clause 2 alone fires on noise roughly alpha of the time per +cell, and a run has enough cells that "roughly alpha" becomes "most nights". + +#### Separate BH families + +Gated keys are corrected as their own family. Ungated metrics get a family of +their own so they still carry a reportable verdict. Adjusting the gated metrics +against metrics nobody gates on would only make a real regression harder to +confirm. Controls and censored keys are excluded from correction entirely — an +A/A cell is not a hypothesis about the candidate. + +#### One A/A control per SDK, running first + +A control measures a particular SDK's harness path. `cells_for()` emits each +SDK's control first, because a run that overruns its budget loses whatever is at +the end: losing one comparison leaves the rest trustworthy, losing the control +leaves nothing trustworthy, since without a noise floor no cell may report PASS. + +`GateResult.noise` is the *worst* control in the run, not the average. A single +tripped control means the harness may be biased on this runner, and averaging +that away with two quiet ones is exactly the reassurance the control exists to +withhold. + +#### Measurement isolation (`_launcher.py`) + +On Linux a forked child inherits the parent's resident-set accounting and +`execve` does not clear it, so `ru_maxrss` comes back as +`max(child's true peak, parent's RSS at fork time)`. Measured from a pytest +process holding numpy, scipy and a session of samples, every invocation would +report *pytest's* ~165 MiB instead of its own — a stable `1.000x` ratio that +reads as "no regression". + +`posix_spawn` and `sh -c 'exec ...'` do **not** help; both were measured and both +inherit the same floor, because an exec is too late. The only fix is to fork from +a process holding nothing, which is all `_launcher.py` is for. It reports its own +RSS as the floor alongside each reading, which is what powers censoring. + +Two things in that file look wrong and are not: +- `except BaseException` in the forked child — letting a `SystemExit` or + `KeyboardInterrupt` unwind past there would run the *parent's* atexit handlers + and flush its buffers a second time, from a process that exists only to exec. +- `os.killpg(..., SIGKILL)` on timeout — signalling the group is the point; + leaving a wedged JVM behind would hold the runner until the job timeout. + +`os.wait4` rather than `resource.getrusage(RUSAGE_CHILDREN)`, because the latter +is a process-lifetime high-water mark: once one big child has run, every later +delta reads zero. + +#### Everything except the build is pinned + +Both arms get the same plaintext, the same attribute (explicit RSA, so an arm +does not silently switch to EC), the same container, and the same target mode. +`comparability_problem()` refuses the comparison outright when the two builds +disagree on `hexless`, `hexaflexible`, or `autoconfigure` — a timing difference +there is a difference in *work*, not in speed. + +For decrypt, both arms read one ciphertext produced by the baseline. If each arm +decrypted its own output, a difference in how the two builds *write* a TDF would +show up as a difference in how fast they read one. + +#### Baselines must be final releases + +`SDK.is_released()` accepts `v0.29.0-rc.1`, and `semver()` parses it to the same +`(0, 29, 0)` as the final release — so ordering by semver alone leaves them tied +and the directory listing breaks the tie. That is a baseline nobody chose, and it +differs run to run. Baseline selection uses `is_final_release()`, which matches +only a plain `vX.Y.Z`. + +#### Payloads are seeded per payload, not per run + +`tmp_dir` persists between runs. With one RNG stream shared across the payloads, a +partially-cached set skips some `randbytes` calls and shifts the stream for every +payload after it — so a rerun measures different bytes than the run it claims to +be comparable with. Each payload derives from `f"{seed}:{label}"` instead. + +Content is random rather than repetitive because compressible input would let an +SDK that happens to compress look faster for reasons unrelated to crypto. + +#### Cells record; the session gates + +The verdict cannot be reached cell by cell — the multiplicity correction spans +the run and the A/A control can invalidate all of it at once. So +`test_sdk_performance` never asserts. `pytest_sessionfinish` runs `analyze()` +once, writes the artifacts **unconditionally and before gating** (a run about to +fail is exactly the one whose raw numbers someone wants), and only then sets the +exit status. + +A cell that cannot be measured is skipped *and* recorded as skipped, so a quiet +report is visibly quiet rather than indistinguishable from a clean one. If +*every* cell skips, `GateResult.nothing_measured` fails the run: `--bench` is an +explicit request for a measurement, and answering it with a green tick and an +empty table is the one outcome nobody inspects. + +The bench job installs `go` on every runner even when it is not the SDK under +measurement, because `otdfctl` provisions the attributes and KAS registry that +every cell needs and `conftest.py` loads it at import time. `OTDFCTL_HEADS` must +name *go's* head, not the matrix SDK's. + +#### Collection and isolation + +Benchmark cells are **deselected** without `--bench`, via +`pytest_collection_modifyitems` and the `benchmark` marker. They are not +parametrized over an empty list — `empty_parameter_set_mark` would turn that into +one *skipped* item per test, which reads as a benchmark nobody asked for. + +`--bench` refuses to run under `pytest-xdist`. Parallel workers contend for the +CPU under measurement. + +### Adding to it + +**A new payload size** — add a `Payload` to `PAYLOADS` in `cells.py`. Note that +`CONTROL_PAYLOAD = PAYLOADS[1]`, so inserting at the front moves the control. +Cell count per SDK is `1 + 2 × len(PAYLOADS)`; the 1500s budget is divided +across all of them, so adding sizes makes every cell poorer unless the budget +grows too. + +**A new metric** — add it to `METRICS` and `METRIC_LABELS` in `measure.py`, teach +`Sample.metric()` and `format_metric()` about it, and decide whether it belongs +in `BenchConfig.gated_metrics`. Default to ungated until it has shown a usable +noise floor over several nights. + +**A new operation** — extend `operation_type` and `cells_for()` in `cells.py`, +then handle it in `build_arms()` in `fixtures/bench.py`. If it needs an input +produced by the baseline, follow `CiphertextFactory`: build it once, from the +baseline only, and share it between the arms. + +**A new SDK** — nothing here needs to change; it comes from `--sdks` and the +matrix in `xtest.yml`. + +**A new comparability hazard** — add the feature name to +`_COMPARABILITY_FEATURES`. Cheap to add, and the failure it prevents (comparing +two builds doing different amounts of work) is invisible in the output. + +### Invariants — do not break these + +1. Never stop the round loop on a p-value. +2. Never compare against a stored historical number. +3. Never let a cell assert; the gate is run-level. +4. Never report PASS without a noise floor establishing the run had the power to + fail. +5. Never let the two arms differ in anything but the build. +6. Never run the measured command from a process holding memory. +7. Never run the benchmark in parallel with anything, including itself. +8. Never let a run that measured nothing report success. + +Every one of these fails *silently* and *plausibly* when broken: the numbers +still look like numbers. That is why they are written down. diff --git a/xtest/perf/__init__.py b/xtest/perf/__init__.py index 5ff6e1479..2def84a11 100644 --- a/xtest/perf/__init__.py +++ b/xtest/perf/__init__.py @@ -9,4 +9,8 @@ - ``stats``: the paired statistical comparison and its decision rule. - ``runner``: the round loop that produces paired samples. - ``report``: JSON artifacts and GitHub step-summary markdown. + +``README.md`` in this directory covers how to read a result and why the harness +is shaped the way it is. Read it before changing anything here: most of the +design decisions fail silently and plausibly when undone. """ From 3481193e5426fa89d220a8f904a8acd5a0b368f5 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 14 Aug 2026 13:22:52 -0400 Subject: [PATCH 04/11] fix(xtest): fail benchmark runs that measure nothing, and give every bench runner otdfctl Two bugs the first dispatched benchmark run exposed. A run where every cell skipped reported "No regressions" and exited 0: an empty run and a clean run have the same empty regression list, so a benchmark that has quietly stopped measuring can pass indefinitely. GateResult grows a nothing_measured property, the summary says NOTHING MEASURED instead of describing a noise floor it never established, and pytest_sessionfinish fails the run. The bench job only installed the SDK under measurement, but conftest.py loads otdfctl at import time to provision attributes and the KAS registry, so the java and js runners died during collection on a missing sdk/go/dist/main/ otdfctl.sh. Every bench runner now configures and builds go for otdfctl, and OTDFCTL_HEADS points at go's heads rather than the matrix SDK's. --- .github/workflows/xtest.yml | 40 +++++++++++++++++++++++++++++++++---- xtest/conftest.py | 7 ++++++- xtest/perf/README.md | 9 +++++++++ xtest/perf/stats.py | 21 +++++++++++++++++++ xtest/test_bench_stats.py | 21 +++++++++++++++++-- 5 files changed, 91 insertions(+), 7 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 740a3648d..5488e9274 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -37,7 +37,7 @@ on: required: false type: boolean default: false - description: "Run the SDK performance regression benchmarks (adds ~45m per SDK)" + description: "Run the SDK performance regression benchmarks (adds ~45m per SDK). Needs two builds per SDK: set the *-ref inputs to 'main latest', since a bare 'main' installs no release to use as a baseline and every cell will skip." workflow_call: inputs: platform-ref: @@ -838,8 +838,10 @@ jobs: with: node-version: "22.x" + # Not gated on matrix.sdk: every bench runner needs otdfctl now, and + # leaving these outputs empty on the java and js runners would send + # setup-cli-tool off to make its own platform checkout to build it from. - name: Capture platform otdfctl location - if: matrix.sdk == 'go' id: platform-otdfctl run: |- if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then @@ -870,8 +872,25 @@ jobs: platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + # otdfctl provisions the attributes and KAS registry every cell needs, + # whichever SDK is under measurement, and conftest.py loads it at import + # time. The go runner already has it from the step above; without this + # the java and js runners fail during collection, before a single + # measurement is taken. + - name: Configure otdfctl + id: configure-otdfctl + if: matrix.sdk != 'go' + uses: ./otdftests/xtest/setup-cli-tool + with: + path: otdftests/xtest/sdk + sdk: go + version-info: "${{ needs.resolve-versions.outputs.go }}" + platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} + platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + + # Unconditional: every bench runner builds go now, either as the SDK + # under measurement or as otdfctl. - name: Cache Go modules - if: matrix.sdk == 'go' uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 with: path: | @@ -921,6 +940,14 @@ jobs: BUF_INPUT_HTTPS_USERNAME: opentdf-bot BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + - name: Build otdfctl + if: matrix.sdk != 'go' && fromJson(steps.configure-otdfctl.outputs.heads)[0] != null + run: make + working-directory: otdftests/xtest/sdk/go + env: + BUF_INPUT_HTTPS_USERNAME: opentdf-bot + BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + ######## MEASURE ############# # --locked --no-build: install exactly what uv.lock pins, and run no # setup scripts doing it. A benchmark that measured a differently @@ -949,7 +976,12 @@ jobs: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main - OTDFCTL_HEADS: ${{ steps.configure-sdk.outputs.heads }} + # go's heads, not the matrix SDK's: conftest reads this to locate + # otdfctl under sdk/go/dist//, so pointing it at java's or + # js's head names a directory that does not exist. + OTDFCTL_HEADS: >- + ${{ matrix.sdk == 'go' && steps.configure-sdk.outputs.heads + || steps.configure-otdfctl.outputs.heads }} # The benchmark never touches the audit-log fixture; asserting on # logs would also add file IO to the measured path. DISABLE_AUDIT_ASSERTIONS: "1" diff --git a/xtest/conftest.py b/xtest/conftest.py index 903c54724..5604bf0c1 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -425,7 +425,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): if config.getoption("--bench-no-gate", default=False): return - if gate.should_fail: + # A run that measured nothing fails too, and not only one that found a + # regression. --bench is an explicit request for a measurement; answering + # it with a green tick and an empty table is the one outcome nobody + # inspects, so a benchmark that has quietly stopped measuring can survive + # indefinitely. Every reason a cell skips is already in the report. + if gate.should_fail or gate.nothing_measured: session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 9eb0c7d4b..49d933d90 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -91,6 +91,15 @@ regressions, so without this a benchmark that had quietly stopped measuring would keep reporting a green tick. The "Not measured" section of the report lists the reason for each cell. +> One cause looks like a bug and is not. If an SDK's newest release tags the same +> commit as `main` — java sat at `v0.18.0 == main == dev == 57d070b0` through +> August 2026 — then `main latest` resolves both arms to one SHA, `otdf-sdk-mgr` +> installs a single build, and every cell skips with *"no final release to compare +> against; installed: main"*. The message is true from where the harness stands, but +> the release it is looking for does exist; the two arms are just the same code. +> Check with `otdf-sdk-mgr versions resolve main latest` — one entry back +> instead of two means there is nothing to measure until `main` moves. + ### The A/A control Each SDK gets a control cell that compares the baseline build **against itself** diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py index 2f3e49255..c7bb71576 100644 --- a/xtest/perf/stats.py +++ b/xtest/perf/stats.py @@ -368,6 +368,19 @@ class GateResult: def should_fail(self) -> bool: return self.trustworthy and bool(self.regressions) + @property + def nothing_measured(self) -> bool: + """True if the run produced no comparisons at all. + + Not the same thing as "no regressions", though the two are identical + from the outside: both have an empty ``regressions`` list. A run where + every cell was skipped -- no baseline installed, an SDK that would not + build -- reports the cheerful summary of a clean one, which is how a + benchmark that quietly stopped measuring anything survives for months. + Callers gate on this separately. + """ + return not self.comparisons + def apply_multiplicity_control( comparisons: dict[str, PairedComparison], @@ -519,6 +532,14 @@ def _verdict_for( def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: + if result.nothing_measured: + # Before the noise check: with nothing measured there is no control + # either, and "the A/A control failed" would misdescribe a run that + # never got as far as running one. + return ( + "NOTHING MEASURED: no cell produced a comparison, so this run says " + "nothing about performance either way." + ) if noise.tripped: return ( f"INCONCLUSIVE: the A/A control failed its own comparison. {noise.detail}. " diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py index 20f077da0..71a94d9ae 100644 --- a/xtest/test_bench_stats.py +++ b/xtest/test_bench_stats.py @@ -338,7 +338,24 @@ def test_ungated_metric_is_reported_but_cannot_fail(self): assert g.comparisons["cpu"].verdict is Verdict.REGRESSION assert g.regressions == ["wall"], "cpu is reported but never gates" - def test_empty_run_is_not_a_failure(self): + def test_empty_run_reports_no_regressions(self): g = stats.apply_multiplicity_control({}) - assert not g.should_fail + assert not g.should_fail, "nothing measured is not a regression" assert g.regressions == [] + + def test_empty_run_says_it_measured_nothing(self): + # The dangerous case: an empty run and a clean run have the same empty + # regression list, so without this the report of a benchmark that + # skipped every cell is indistinguishable from one that passed. + g = stats.apply_multiplicity_control({}) + assert g.nothing_measured + assert "NOTHING MEASURED" in g.summary + assert "no regressions" not in g.summary.lower() + + def test_a_run_with_comparisons_measured_something(self): + rng = np.random.default_rng(33) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=33, n_resamples=RESAMPLES), control=quiet_control() + ) + assert not g.nothing_measured From 0b491d04003024baff6efdd0750e18c0f55a388f Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 20 Aug 2026 17:16:08 -0400 Subject: [PATCH 05/11] feat(xtest): benchmark two named refs via workflow_dispatch The nightly benchmark compares the newest release against the branch head, which is the right question every night and the wrong one to ask about a specific change: the baseline carries every other commit since the release. Adds bench-baseline-ref / bench-candidate-ref. Both refs are resolved and built from source side by side, and arm selection is told which is which explicitly, so neither has to be a release. The *-ref inputs are untouched and still drive the functional matrix. The bench matrix now honours focus-sdk, so investigating one SDK no longer spends 45 minutes each measuring the two nobody asked about. Bad combinations fail in resolve-versions, before a runner is spent: only one arm named, focus-sdk left at 'all', or two refs that turn out to be the same commit. otdf-sdk-mgr resolved a branch reached *by name* to a tag that kept its slashes, while the same branch reached by SHA was flattened to '--'. A dist tag is one path component -- all_versions_of() lists dist/*/ and the go Makefile finds src/*/ -- so 'feat/x' installed as a build named 'feat' with no cli.sh, which raises during collection. Flatten both paths the same way. --- .github/workflows/xtest.yml | 110 +++++++++++++++++++++-- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 9 +- otdf-sdk-mgr/tests/test_resolve.py | 19 +++- xtest/perf/README.md | 52 +++++++++++ xtest/test_bench_arms.py | 32 +++++++ 5 files changed, 214 insertions(+), 8 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 5488e9274..bda20b39e 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -38,6 +38,16 @@ on: type: boolean default: false description: "Run the SDK performance regression benchmarks (adds ~45m per SDK). Needs two builds per SDK: set the *-ref inputs to 'main latest', since a bare 'main' installs no release to use as a baseline and every cell will skip." + bench-baseline-ref: + required: false + type: string + default: "" + description: "Benchmark the ref named here against bench-candidate-ref, instead of the default newest-release-vs-branch-head comparison. Any ref otdf-sdk-mgr resolves: 'main', a branch, a tag, a SHA, 'refs/pull/N/head'. Requires bench-candidate-ref and a focus-sdk naming one SDK; ignores the *-ref inputs, which drive the functional matrix rather than this." + bench-candidate-ref: + required: false + type: string + default: "" + description: "The build under suspicion, measured against bench-baseline-ref. e.g. 'feat/DSPX-2604-createtdf-chunked'." workflow_call: inputs: platform-ref: @@ -68,6 +78,14 @@ on: required: false type: boolean default: false + bench-baseline-ref: + required: false + type: string + default: "" + bench-candidate-ref: + required: false + type: string + default: "" schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -88,6 +106,7 @@ jobs: platform-tag-list: ${{ steps.version-info.outputs.platform-tag-list }} heads: ${{ steps.version-info.outputs.platform-heads }} default-tags: ${{ steps.version-info.outputs.default-tags }} + bench-sdks: ${{ steps.bench-inputs.outputs.sdks }} go: ${{ steps.version-info.outputs.go-version-info }} java: ${{ steps.version-info.outputs.java-version-info }} js: ${{ steps.version-info.outputs.js-version-info }} @@ -107,6 +126,31 @@ jobs: echo "Invalid focus-sdk input: ${FOCUS_SDK_INPUT}. Must be one of: all, go, java, js." >> "$GITHUB_STEP_SUMMARY" exit 1 fi + # Decided here rather than in the bench job because a matrix cannot be + # narrowed from inside the job it belongs to: a bad combination would + # already have spun up three runners for 45 minutes each. + - name: Validate benchmark inputs and pick the bench matrix + id: bench-inputs + env: + FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }} + BASELINE_REF: ${{ inputs.bench-baseline-ref }} + CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + run: |- + if [[ -n "$BASELINE_REF" && -z "$CANDIDATE_REF" ]] \ + || [[ -z "$BASELINE_REF" && -n "$CANDIDATE_REF" ]]; then + echo "::error::bench-baseline-ref and bench-candidate-ref must be set together; a comparison needs both arms named." + exit 1 + fi + if [[ -n "$CANDIDATE_REF" && "$FOCUS_SDK" == "all" ]]; then + echo "::error::bench-baseline-ref/bench-candidate-ref name refs of one SDK, so focus-sdk must be go, java, or js -- not 'all'." + exit 1 + fi + if [[ "$FOCUS_SDK" == "all" ]]; then + echo 'sdks=["go","java","js"]' >> "$GITHUB_OUTPUT" + else + echo "sdks=[\"${FOCUS_SDK}\"]" >> "$GITHUB_OUTPUT" + fi + - name: Default Versions depend on context id: default-tags run: |- @@ -783,10 +827,11 @@ jobs: packages: read strategy: # One runner per SDK. Two SDKs on one runner would contend for the very - # CPU being measured. + # CPU being measured. Narrowed by focus-sdk, so investigating one SDK + # does not spend 45 minutes measuring the two nobody asked about. fail-fast: false matrix: - sdk: [go, java, js] + sdk: ${{ fromJSON(needs.resolve-versions.outputs.bench-sdks) }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -859,16 +904,57 @@ jobs: PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} ######## INSTALL BOTH ARMS OF THE COMPARISON ############# + # Two named refs instead of the default release-vs-branch pair. Resolved + # here rather than in resolve-versions because the *-ref inputs there + # drive the functional matrix, and a benchmark wants to name its two + # arms without also changing what the rest of the workflow tests. + # + # Baseline first: the tag order becomes configure-sdk's `heads` output, + # and conftest.py takes heads[0] as the otdfctl that provisions + # attributes and the KAS registry. That provisioning is not measured, + # and it should be the same build for both arms. + - name: Resolve the two benchmark arms + id: bench-arms + if: inputs.bench-candidate-ref != '' + working-directory: otdftests/otdf-sdk-mgr + env: + SDK: ${{ matrix.sdk }} + BASELINE_REF: ${{ inputs.bench-baseline-ref }} + CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + run: |- + info=$(uv run --project . otdf-sdk-mgr versions resolve \ + "$SDK" "$BASELINE_REF" "$CANDIDATE_REF") + jq . <<<"$info" + err=$(jq -r '[.[] | select(.err != null) | .err] | join("; ")' <<<"$info") + if [[ -n "$err" ]]; then + echo "::error::Could not resolve benchmark arms: $err" + exit 1 + fi + # `versions resolve` drops a ref whose SHA it has already seen, so + # two names for one commit come back as a single entry. Left alone + # that installs one build, fails arm selection in every cell, and + # spends the runner's 45 minutes arriving at NOTHING MEASURED. + if [[ "$(jq 'length' <<<"$info")" -ne 2 ]]; then + echo "::error::${BASELINE_REF} and ${CANDIDATE_REF} resolve to the same commit -- nothing to compare." + exit 1 + fi + { + echo "version-info=$(jq -c . <<<"$info")" + echo "baseline-spec=${SDK}@$(jq -r '.[0].tag' <<<"$info")" + echo "candidate-spec=${SDK}@$(jq -r '.[1].tag' <<<"$info")" + } >> "$GITHUB_OUTPUT" + # The whole design rests on this step laying down two builds side by - # side under sdk//dist/: the branch head (candidate) and the - # newest release (baseline). Arm selection picks them up from there. + # side under sdk//dist/: by default the branch head (candidate) and + # the newest release (baseline), or the two refs resolved above. Arm + # selection picks them up from there. - name: Configure ${{ matrix.sdk }} sdk id: configure-sdk uses: ./otdftests/xtest/setup-cli-tool with: path: otdftests/xtest/sdk sdk: ${{ matrix.sdk }} - version-info: "${{ needs.resolve-versions.outputs[matrix.sdk] }}" + version-info: "${{ steps.bench-arms.outputs.version-info || needs.resolve-versions.outputs[matrix.sdk] }}" platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} @@ -929,7 +1015,7 @@ jobs: fi done env: - java_version_info: ${{ needs.resolve-versions.outputs.java }} + java_version_info: ${{ steps.bench-arms.outputs.version-info || needs.resolve-versions.outputs.java }} platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} - name: Build the ${{ matrix.sdk }} cli @@ -962,9 +1048,19 @@ jobs: - name: Run performance benchmarks id: bench run: |- + # Empty unless the two arms were named explicitly, in which case + # arm selection must not fall back to "newest release vs branch + # head": neither named ref need be a release, and with two branch + # builds installed the default would pick the wrong pair or none. + arms=() + if [[ -n "$BENCH_BASELINE_SPEC" ]]; then + arms=(--bench-baseline "$BENCH_BASELINE_SPEC" + --bench-candidate "$BENCH_CANDIDATE_SPEC") + fi uv run --frozen --no-build pytest -ra -v \ --bench \ --sdks "$BENCH_SDK" \ + "${arms[@]}" \ --bench-budget-seconds 1500 \ --bench-out test-results/benchmarks \ --html "test-results/bench-${BENCH_SDK}.html" \ @@ -973,6 +1069,8 @@ jobs: working-directory: otdftests/xtest env: BENCH_SDK: ${{ matrix.sdk }} + BENCH_BASELINE_SPEC: ${{ steps.bench-arms.outputs.baseline-spec }} + BENCH_CANDIDATE_SPEC: ${{ steps.bench-arms.outputs.candidate-spec }} PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 9c50d2ce1..7ff3b6811 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -300,7 +300,14 @@ def _resolve_against( "alias": version, "head": True, "sha": sha, - "tag": version, + # Flattened the same way _classify_sha_match flattens a branch + # it reached by SHA: the tag becomes a single dist// and + # src// path component. A slash here nests those + # directories, and every consumer walks them one level deep -- + # xtest's all_versions_of() lists dist/*/ and the go Makefile + # finds src/*/, so "feat/x" is discovered as a "feat" build + # with no cli.sh in it. + "tag": version.replace("/", "--"), } if infix and version.startswith(f"{infix}/"): diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index a7c30b057..29efa6f6f 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -76,7 +76,24 @@ def test_refs_heads_non_main_branch(self): result = resolve("js", "refs/heads/release/sdk-v0.17", None) assert is_resolve_success(result) assert "head" in result and result["head"] is True - assert result["tag"] == "release/sdk-v0.17" + assert result["tag"] == "release--sdk-v0.17" + assert result["sha"] == SHA40 + + def test_branch_by_name_flattens_slashes(self): + # Same flattening the SHA path applies, and for the same reason: the + # tag is one path component under dist/ and src/. Reached by name + # rather than by SHA, which is the shape a workflow_dispatch input + # arrives in. + ls = make_ls_remote( + (SHA40, "refs/heads/feat/DSPX-2604-createtdf-chunked"), + ("d" * 40, "refs/heads/main"), + ) + with patch_git(ls): + result = resolve("go", "feat/DSPX-2604-createtdf-chunked", None) + assert is_resolve_success(result) + assert result.get("head") is True + assert result["tag"] == "feat--DSPX-2604-createtdf-chunked" + assert result["alias"] == "feat/DSPX-2604-createtdf-chunked" assert result["sha"] == SHA40 diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 49d933d90..2a61acb14 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -19,6 +19,10 @@ measure. > baseline — and every cell skips. The run fails rather than passing empty > (see [NOTHING MEASURED](#the-verdicts)), but it will have wasted 45 minutes > to tell you that. +> +> To compare **two named refs** instead — a branch against `main`, say — use +> `bench-baseline-ref` / `bench-candidate-ref` and skip all of the above; see +> [Benchmarking one branch against another](#benchmarking-one-branch-against-another). - **Section 1 — [Reading a result](#1-reading-a-result)** is for developers on the SDKs and the platform: your build got flagged, what does that mean. @@ -170,6 +174,42 @@ Useful knobs while investigating: A local run is noisier than CI unless the machine is otherwise idle. Close things; the noise floor will tell you whether you succeeded. +### Benchmarking one branch against another + +The nightly comparison is newest-release vs branch head, which is the right +question to ask every night and the wrong one to ask about a specific change: +the baseline carries every other commit that landed since the release. To +point the harness at two refs you name, dispatch X-Test with: + +| Input | Example | Meaning | +| --- | --- | --- | +| `run-benchmarks` | ✅ | Required; the bench job is off otherwise | +| `focus-sdk` | `go` | Must name one SDK — the matrix runs only this one | +| `bench-baseline-ref` | `main` | The build you are comparing *against* | +| `bench-candidate-ref` | `feat/DSPX-2604-createtdf-chunked` | The build under suspicion | + +Either ref can be anything `otdf-sdk-mgr versions resolve` accepts: a branch, a +tag, a full or short SHA, or `refs/pull/N/head`. Both are built from source and +installed side by side, and arm selection is told which is which explicitly — +so neither has to be a release, which is the whole point. + +The `*-ref` inputs are ignored by the bench job in this mode. They still drive +the functional test matrix, so a dispatch can answer "is it slower?" without +also changing what the rest of the run tests. + +Two things this mode does **not** change, both of which bound what a result +means: + +- **The server stays on `main`.** The bench job pins the platform and runs a + single KAS, whatever the refs say. A candidate whose speed depends on a + matching server change will not show it here. +- **The baseline is whatever you named.** For a stacked branch, `main` as the + baseline measures the whole stack. Name the parent branch instead to isolate + the top commit. + +It fails fast, before spending a runner, when the two refs resolve to the same +commit or when `focus-sdk` is `all`. + ### What this benchmark cannot tell you - **Anything about absolute speed.** A number from a GitHub-hosted runner is not @@ -327,6 +367,18 @@ and the directory listing breaks the tie. That is a baseline nobody chose, and i differs run to run. Baseline selection uses `is_final_release()`, which matches only a plain `vX.Y.Z`. +#### A dist tag is one path component + +`otdf-sdk-mgr` flattens `/` to `--` when it resolves a ref, so +`feat/DSPX-2604-createtdf-chunked` installs as +`dist/feat--DSPX-2604-createtdf-chunked/`. Everything downstream walks those +directories exactly one level deep — `tdfs.all_versions_of()` lists `dist/*/`, +the go `Makefile` finds `src/*/` — so a slash that survives resolution is +discovered as a build named `feat` with no `cli.sh` in it, which +`all_versions_of()` raises on before any cell runs. Branch-vs-branch dispatch +is the first thing to routinely feed it a slashed ref, and the `--bench-*` +specs name the flattened tag: `go@feat--DSPX-2604-createtdf-chunked`. + #### Payloads are seeded per payload, not per run `tmp_dir` persists between runs. With one RNG stream shared across the payloads, a diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index bf7a3194e..98f090b66 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -86,6 +86,38 @@ def test_refuses_to_compare_a_build_against_itself(self, cwd: Path): with pytest.raises(bench.ArmSelectionError, match="nothing to compare"): bench.select_arms("go", baseline_spec="go@main", candidate_spec="go@main") + def test_two_branch_builds_need_explicit_specs(self, cwd: Path): + # What a branch-vs-branch dispatch installs: two heads and no release + # at all. Named explicitly it is a fine comparison; left to the default + # there is no baseline, and "newest final release" cannot invent one. + install(cwd, "go", "main", "feat--DSPX-2604-createtdf-chunked") + baseline, candidate = bench.select_arms( + "go", + baseline_spec="go@main", + candidate_spec="go@feat--DSPX-2604-createtdf-chunked", + ) + assert baseline.version == "main" + assert candidate.version == "feat--DSPX-2604-createtdf-chunked" + with pytest.raises(bench.ArmSelectionError, match="no final go release"): + bench.select_arms("go") + + +class TestDistTagShape: + def test_a_slashed_tag_breaks_discovery(self, cwd: Path): + # Why otdf-sdk-mgr flattens '/' to '--' in a resolved ref. A branch + # installed as dist/feat/x/ is listed as a build named "feat", which + # has no cli.sh -- and this raises during collection, before any cell + # has a chance to report why. + install(cwd, "go", "feat/DSPX-2604-createtdf-chunked") + with pytest.raises(FileNotFoundError): + tdfs.all_versions_of("go") + + def test_a_flattened_tag_is_discovered(self, cwd: Path): + install(cwd, "go", "feat--DSPX-2604-createtdf-chunked") + assert [s.version for s in tdfs.all_versions_of("go")] == [ + "feat--DSPX-2604-createtdf-chunked" + ] + #: The fixture body, called directly: these tests are about the bytes it #: writes, not about pytest's fixture wiring. From c97ac8ea458f05e73e08b3fc78e1feb15a49872d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 20 Aug 2026 21:16:51 -0400 Subject: [PATCH 06/11] feat(xtest): benchmark payloads up to 1 GiB, with per-dispatch budget At 32 MiB a go encrypt spends ~450 ms on fixed startup against ~72 ms that scales with the payload, so payload work is ~14% of the cell and the 1.15x gate is wider than the whole of it: a candidate that doubled every per-segment cost would report 1.136x and PASS. Gating throughput needs a size where that ratio inverts, which at 1 GiB it does (~2.3 s payload work against the same ~450 ms fixed). Adds --bench-payloads, plus bench-payloads / bench-budget-seconds / bench-max-rounds dispatch inputs so a manual run -- which is not repeated on the nightly's schedule -- can afford sizes the nightly cannot. The scheduled run supplies no inputs, so the workflow falls back to today's 1KiB,1MiB,32MiB / 1500 s / 60 rounds rather than moving the nightly. Making 1 GiB survivable took four supporting changes: - Payload files are written in 8 MiB chunks rather than one randbytes call, so a 1 GiB payload is not first built in RAM. The chunk size is a multiple of 4 -- CPython draws a 32-bit word at a time -- so the bytes are identical to the unchunked stream and the seed contract holds. - Arm outputs are unlinked in a finally. Each cell leaves two files the size of its payload and nothing reads them again; at 1 GiB that is 2 GiB per cell that every later cell has to fit around, so the cell that fails on disk is not the one that filled it. - A disk preflight refuses the run up front. ENOSPC mid-run arrives as a non-zero exit from the CLI under measurement, which reads as 'this build is broken' rather than 'the runner is out of space'. - Cells run smallest-first within an SDK, so a run that exhausts its budget loses the most expensive cell rather than an arbitrary one. The A/A control stays pinned at 1 MiB instead of following the selection: its width is the run's noise floor and every other cell is judged against it, so it has to mean the same thing across runs. CI validates bench-payloads with perf.cells.parse_payloads under a bare python3 -- perf.cells is stdlib-only -- rather than a second copy of the grammar in bash that would drift from the one pytest enforces. --- .github/workflows/xtest.yml | 75 +++++++++++++++++++++- xtest/conftest.py | 15 ++++- xtest/fixtures/bench.py | 87 +++++++++++++++++++++++-- xtest/perf/README.md | 89 +++++++++++++++++++++++--- xtest/perf/cells.py | 100 +++++++++++++++++++++++++++-- xtest/perf/runner.py | 120 +++++++++++++++++++---------------- xtest/test_bench_arms.py | 123 +++++++++++++++++++++++++++++++++++- 7 files changed, 529 insertions(+), 80 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index bda20b39e..2d9163f77 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -48,6 +48,21 @@ on: type: string default: "" description: "The build under suspicion, measured against bench-baseline-ref. e.g. 'feat/DSPX-2604-createtdf-chunked'." + bench-payloads: + required: false + type: string + default: "" + description: "Payload sizes to benchmark, comma-separated, e.g. '1KiB,1MiB,32MiB,1GiB'. Default is 1KiB,1MiB,32MiB, at which ~86% of a go encrypt is fixed startup cost -- so the 1.15x gate is wider than the whole payload-dependent part and no throughput change can fail a cell. Add 1GiB to actually gate throughput; it needs a bench-budget-seconds to match and ~5 GiB of runner disk." + bench-budget-seconds: + required: false + type: string + default: "" + description: "Wall-clock allowance shared by every benchmark cell (default 1500). Manual runs are not on the nightly's schedule, so this is the knob to raise when adding payload sizes -- each one adds an encrypt and a decrypt cell, and the budget is divided evenly as cells start." + bench-max-rounds: + required: false + type: string + default: "" + description: "Hard cap on paired rounds per cell (default 60). Raise it together with the budget: at the default, cells routinely stop on max_rounds with budget left over, and every unspent round is interval width that could have been bought." workflow_call: inputs: platform-ref: @@ -86,6 +101,18 @@ on: required: false type: string default: "" + bench-payloads: + required: false + type: string + default: "" + bench-budget-seconds: + required: false + type: string + default: "" + bench-max-rounds: + required: false + type: string + default: "" schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -135,7 +162,21 @@ jobs: FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }} BASELINE_REF: ${{ inputs.bench-baseline-ref }} CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + BUDGET_SECONDS: ${{ inputs.bench-budget-seconds }} + MAX_ROUNDS: ${{ inputs.bench-max-rounds }} run: |- + # Only the numeric inputs are checked here. bench-payloads has a + # grammar, and a second copy of it in bash would drift from the one + # pytest enforces and start rejecting runs that would have worked; + # the bench job validates it with the real parser instead. + for pair in "bench-budget-seconds:$BUDGET_SECONDS" "bench-max-rounds:$MAX_ROUNDS"; do + name=${pair%%:*} + value=${pair#*:} + if [[ -n "$value" && ! "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::${name} must be a positive whole number, got '${value}'." + exit 1 + fi + done if [[ -n "$BASELINE_REF" && -z "$CANDIDATE_REF" ]] \ || [[ -z "$BASELINE_REF" && -n "$CANDIDATE_REF" ]]; then echo "::error::bench-baseline-ref and bench-candidate-ref must be set together; a comparison needs both arms named." @@ -839,6 +880,30 @@ jobs: path: otdftests persist-credentials: false + # Before the platform, which takes ~15 minutes to come up: a typo'd size + # is worth catching in the first thirty seconds. This calls the harness's + # own parser rather than reimplementing the grammar in bash -- perf.cells + # imports nothing outside the standard library, so a bare python3 can + # read it, and a bash copy that drifted would start refusing specs the + # run itself would have accepted. + - name: Validate benchmark payload sizes + if: inputs.bench-payloads != '' + working-directory: otdftests/xtest + env: + BENCH_PAYLOADS: ${{ inputs.bench-payloads }} + run: |- + python3 - "$BENCH_PAYLOADS" <<'PY' + import sys + + from perf.cells import parse_payloads + + try: + sizes = parse_payloads(sys.argv[1]) + except ValueError as e: + raise SystemExit(f"::error::invalid bench-payloads: {e}") + print("payload sizes:", ", ".join(p.label for p in sizes)) + PY + - name: load extra keys from file id: load-extra-keys run: |- @@ -1061,7 +1126,9 @@ jobs: --bench \ --sdks "$BENCH_SDK" \ "${arms[@]}" \ - --bench-budget-seconds 1500 \ + --bench-payloads "$BENCH_PAYLOADS" \ + --bench-budget-seconds "$BENCH_BUDGET_SECONDS" \ + --bench-max-rounds "$BENCH_MAX_ROUNDS" \ --bench-out test-results/benchmarks \ --html "test-results/bench-${BENCH_SDK}.html" \ --self-contained-html \ @@ -1071,6 +1138,12 @@ jobs: BENCH_SDK: ${{ matrix.sdk }} BENCH_BASELINE_SPEC: ${{ steps.bench-arms.outputs.baseline-spec }} BENCH_CANDIDATE_SPEC: ${{ steps.bench-arms.outputs.candidate-spec }} + # Fallbacks rather than input defaults: the scheduled nightly + # supplies no inputs at all, so `inputs.*` is empty there and these + # are what keeps its matrix and budget where they have always been. + BENCH_PAYLOADS: ${{ inputs.bench-payloads || '1KiB,1MiB,32MiB' }} + BENCH_BUDGET_SECONDS: ${{ inputs.bench-budget-seconds || '1500' }} + BENCH_MAX_ROUNDS: ${{ inputs.bench-max-rounds || '60' }} PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main diff --git a/xtest/conftest.py b/xtest/conftest.py index 5604bf0c1..da3949447 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -23,9 +23,10 @@ import pytest import tdfs +from fixtures.bench import payloads_from_options from otdfctl import OpentdfCommandLineTool from perf import report, stats -from perf.cells import cells_for +from perf.cells import DEFAULT_PAYLOAD_SPEC, cells_for logging.basicConfig(level=os.environ.get("LOGLEVEL", "DEBUG")) @@ -181,6 +182,16 @@ def _add_benchmark_options(parser: pytest.Parser): help="build under test, e.g. go@main; defaults to the installed " "unreleased build of each sdk", ) + group.addoption( + "--bench-payloads", + default=DEFAULT_PAYLOAD_SPEC, + help="comma-separated payload sizes to measure, e.g. " + "'1KiB,1MiB,32MiB,1GiB' (default: %(default)s). Sizes above the " + "default are opt-in because they are what a throughput gate actually " + "needs and what a nightly cannot afford: each one adds two cells, and " + "a run holds roughly twice the total plus the largest twice over on " + "disk", + ) group.addoption( "--bench-threshold", type=float, @@ -342,7 +353,7 @@ def _parametrize_bench_cells(metafunc: pytest.Metafunc): typing.get_args(tdfs.sdk_type) ) names = list(dict.fromkeys(s.split("@", 1)[0] for s in str(specs).split())) - cells = cells_for(names) + cells = cells_for(names, payloads_from_options(metafunc.config)) metafunc.config.stash[report.CELLS_KEY] = cells metafunc.parametrize("bench_cell", cells, ids=[c.id for c in cells]) diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py index d5194b380..e5a1b08d1 100644 --- a/xtest/fixtures/bench.py +++ b/xtest/fixtures/bench.py @@ -12,6 +12,8 @@ import os import platform import random +import shutil +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from typing import cast @@ -21,7 +23,7 @@ import abac import tdfs from perf import report -from perf.cells import PAYLOADS, BenchCell +from perf.cells import BenchCell, Payload, parse_payloads, payloads_to_generate from perf.runner import Arm, BenchConfig, Budget, Invocation @@ -133,8 +135,77 @@ def bench_config(request: pytest.FixtureRequest) -> BenchConfig: return config_from_options(request.config) +def payloads_from_options(config: pytest.Config) -> tuple[Payload, ...]: + """The run's payload set, from ``--bench-payloads``.""" + spec = cast(str, config.getoption("--bench-payloads")) + try: + return parse_payloads(spec) + except ValueError as e: + raise pytest.UsageError(f"invalid --bench-payloads: {e}") from e + + +@pytest.fixture(scope="session") +def bench_payload_set(request: pytest.FixtureRequest) -> tuple[Payload, ...]: + """Payload sizes this run measures, from --bench-payloads.""" + return payloads_from_options(request.config) + + +#: Bytes generated per ``randbytes`` call. Must stay a multiple of 4: CPython +#: draws a 32-bit word at a time, so chunking on a 4-byte boundary yields the +#: same stream as one call for the whole payload, and the promise below -- +#: that a given seed and label always produce the same bytes -- survives both +#: this constant changing and a payload growing past it. +_CHUNK_BYTES = 8 * 2**20 + +#: Free space a run keeps in hand beyond its payload arithmetic, for the +#: platform's own logs and database growth over a long benchmark. +_DISK_HEADROOM_BYTES = 2**30 + + +def write_payload(path: Path, payload: Payload, seed: int) -> None: + """Write one payload file, in chunks so a 1 GiB file is not built in RAM.""" + rng = random.Random(f"{seed}:{payload.label}") + remaining = payload.n_bytes + with path.open("wb") as f: + while remaining > 0: + n = min(remaining, _CHUNK_BYTES) + f.write(rng.randbytes(n)) + remaining -= n + + +def disk_shortfall(tmp_dir: Path, payloads: Sequence[Payload]) -> str | None: + """Return why ``payloads`` will not fit in ``tmp_dir``, or None. + + Checked up front because the alternative is finding out mid-run: ENOSPC + reaches the harness as a non-zero exit from the CLI under measurement, + which is reported as a failed measurement of that build. A run can lose + an hour before anyone notices the disk was the problem, and the report + points at the wrong thing while they look. + + The estimate is the plaintexts, plus a cached ciphertext for each (the + decrypt cells share one per size), plus the two output files the largest + cell holds while it runs. Outputs are deleted as each cell finishes, so + only one cell's worth is ever live. + """ + total = sum(p.n_bytes for p in payloads) + largest = max(p.n_bytes for p in payloads) + need = 2 * total + 2 * largest + _DISK_HEADROOM_BYTES + free = shutil.disk_usage(tmp_dir).free + if free >= need: + return None + gib = 2**30 + sizes = ", ".join(p.label for p in payloads) + return ( + f"payloads {sizes} need about {need / gib:.1f} GiB of scratch space " + f"in {tmp_dir} but only {free / gib:.1f} GiB is free; drop the largest " + "size from --bench-payloads or run somewhere with more disk" + ) + + @pytest.fixture(scope="session") -def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: +def bench_payloads( + tmp_dir: Path, bench_config: BenchConfig, bench_payload_set: tuple[Payload, ...] +) -> dict[str, Path]: """Generate one plaintext file per payload size, shared by both arms. Content is pseudo-random but seeded, so a rerun measures byte-identical @@ -147,13 +218,19 @@ def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: calls and shifts the stream for every payload after it. Deriving each payload's bytes from the seed *and* its label keeps the promise above true whether the cache is empty, full, or half there. + + The control's payload is generated whether or not it was selected -- see + :func:`perf.cells.payloads_to_generate`. """ + wanted = payloads_to_generate(bench_payload_set) + shortfall = disk_shortfall(tmp_dir, wanted) + if shortfall: + raise pytest.UsageError(shortfall) out: dict[str, Path] = {} - for payload in PAYLOADS: + for payload in wanted: path = tmp_dir / f"bench-plain-{payload.label}.bin" if not path.is_file() or path.stat().st_size != payload.n_bytes: - rng = random.Random(f"{bench_config.seed}:{payload.label}") - path.write_bytes(rng.randbytes(payload.n_bytes)) + write_payload(path, payload, bench_config.seed) out[payload.label] = path return out diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 2a61acb14..fd109e433 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -53,7 +53,9 @@ a 30-minute job to look at the same numbers again. ``` - **cell** — `--`, plus `-control` for the A/A cell. - Payload sizes are 1 KiB, 1 MiB, and 32 MiB. + Payload sizes default to 1 KiB, 1 MiB, and 32 MiB; `--bench-payloads` selects + others. See [Payload sizes and what they can gate](#payload-sizes-and-what-they-can-gate) + before reading a throughput result — at the default sizes there is not one. - **ratio** — candidate ÷ baseline. `1.208x` means the candidate took 20.8% longer. Below 1.0 means faster. - **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely @@ -145,8 +147,10 @@ Censored cells report inconclusive with the floor named in the note. 2. **Check the control row.** If the A/A cell for your SDK also looks strange, suspect the runner before your code. 3. **Look at which cells fired.** Only the 1 KiB cells means startup cost — - process boot, package resolution, TLS handshake, token fetch. Only 32 MiB - means throughput — the crypto and IO path. Both means something structural. + process boot, package resolution, TLS handshake, token fetch. The largest + cell firing on its own points at throughput — the crypto and IO path — but + only if that cell is large enough for throughput to be most of it, which at + 32 MiB it is not. Both means something structural. 4. **Reproduce locally.** The comparison is self-contained; it does not need CI. ```bash @@ -164,6 +168,7 @@ Useful knobs while investigating: | Option | Default | Use | | --- | --- | --- | | `--bench-threshold` | `1.15` | Smallest slowdown worth failing on | +| `--bench-payloads` | `1KiB,1MiB,32MiB` | Sizes to measure, e.g. `1KiB,1GiB` | | `--bench-min-rounds` / `--bench-max-rounds` | `20` / `60` | Rounds per cell | | `--bench-warmup` | `5` | Discarded rounds paying one-time costs | | `--bench-budget-seconds` | `1500` | Wall-clock allowance shared by all cells | @@ -174,6 +179,56 @@ Useful knobs while investigating: A local run is noisier than CI unless the machine is otherwise idle. Close things; the noise floor will tell you whether you succeeded. +### Payload sizes and what they can gate + +**At the default sizes this harness cannot fail a build on throughput.** Not +"is unlikely to" — cannot. On a 4-core Linux runner a go encrypt costs about +450 ms before it touches the payload: runtime start, config load, TLS +handshake, token fetch, KAS key fetch. Going from 1 KiB to 32 MiB — a 32,000x +increase in bytes — adds about 72 ms on top of that. + +| operation | 1 KiB | 1 MiB | 32 MiB | payload-dependent | +| --- | --- | --- | --- | --- | +| encrypt | 455.0 ms | 447.6 ms | 526.7 ms | ~72 ms (13.6%) | +| decrypt | 533.6 ms | 513.1 ms | 600.7 ms | ~67 ms (11.2%) | + +The gate is 1.15x of the *whole cell*, which at 32 MiB encrypt is +79 ms — +more than the entire payload-dependent portion. A candidate that doubled every +per-segment cost would come in at 1.136x and report **PASS**. The 1 MiB cells +are worse: indistinguishable from 1 KiB, so they measure startup twice. + +This is not a statistics problem. The intervals are tight and the control is +clean; the matrix is simply asking the wrong sizes. To gate throughput the +payload term has to dominate, which means going much larger: + +```bash +uv run pytest --bench --sdks go \ + --bench-payloads 1KiB,1GiB \ + --bench-budget-seconds 5400 --bench-max-rounds 200 \ + -v test_benchmarks.py +``` + +At 1 GiB the payload term is ~2.3 s against the same ~450 ms fixed cost, so it +is ~84% of the cell and a 15% gate lands inside the part being tested. + +Three things to know before adding a large size: + +- **Budget.** Each size adds an encrypt and a decrypt cell, and the budget is + divided evenly as cells start. A 1 GiB round costs ~6 s against ~1 s at 32 + MiB, so the default 1500 s will not reach `min_rounds` on both new cells. +- **Disk.** A run holds roughly twice the payload total plus the largest size + twice over. 1 GiB needs ~5 GiB free. This is checked before the first + measurement, because running out mid-run arrives as a non-zero exit from the + CLI under test and reads as "this build is broken". +- **`max_rounds` binds before the budget does.** In the run these numbers come + from, 3 of 7 cells stopped on `max_rounds` while only 418 s of 1500 s was + spent. Raising the budget alone buys nothing; raise both. + +The control stays at 1 MiB whatever you select. Its CI width is the run's noise +floor and every other cell is judged against it, so it must not move with the +matrix — otherwise two runs of the same comparison can disagree about which +cells were trustworthy for a reason unrelated to either build. + ### Benchmarking one branch against another The nightly comparison is newest-release vs branch head, which is the right @@ -187,6 +242,16 @@ point the harness at two refs you name, dispatch X-Test with: | `focus-sdk` | `go` | Must name one SDK — the matrix runs only this one | | `bench-baseline-ref` | `main` | The build you are comparing *against* | | `bench-candidate-ref` | `feat/DSPX-2604-createtdf-chunked` | The build under suspicion | +| `bench-payloads` | `1KiB,1GiB` | Sizes to measure; default `1KiB,1MiB,32MiB` | +| `bench-budget-seconds` | `5400` | Shared allowance; default `1500` | +| `bench-max-rounds` | `200` | Cap per cell; default `60` | + +The last three are why a dispatch can answer a question the nightly cannot. A +nightly runs unattended every day and has to stay inside a sensible cost; a +dispatch is asked for, once, about one thing. If the change is a throughput +claim, spend the budget — see [Payload sizes and what they can +gate](#payload-sizes-and-what-they-can-gate), because at the defaults the +answer will be **PASS** whatever the change did. Either ref can be anything `otdf-sdk-mgr versions resolve` accepts: a branch, a tag, a full or short SHA, or `refs/pull/N/head`. Both are built from source and @@ -389,6 +454,12 @@ be comparable with. Each payload derives from `f"{seed}:{label}"` instead. Content is random rather than repetitive because compressible input would let an SDK that happens to compress look faster for reasons unrelated to crypto. +Large payloads are written in chunks so a 1 GiB file is not first built as a +1 GiB `bytes` in RAM. The chunk size must stay a multiple of 4: CPython's +`randbytes` draws a 32-bit word at a time, so a 4-byte-aligned split produces +the same stream as one call would, and the seed-to-bytes promise survives both +the constant changing and a payload growing past it. + #### Cells record; the session gates The verdict cannot be reached cell by cell — the multiplicity correction spans @@ -421,11 +492,13 @@ CPU under measurement. ### Adding to it -**A new payload size** — add a `Payload` to `PAYLOADS` in `cells.py`. Note that -`CONTROL_PAYLOAD = PAYLOADS[1]`, so inserting at the front moves the control. -Cell count per SDK is `1 + 2 × len(PAYLOADS)`; the 1500s budget is divided -across all of them, so adding sizes makes every cell poorer unless the budget -grows too. +**A new payload size** — no code change: `--bench-payloads 1KiB,1GiB` (or the +`bench-payloads` dispatch input). Sizes parse as a count and a binary unit — +`B`, `KiB`, `MiB`, `GiB` — and the list is sorted ascending and deduplicated by +byte count, so `1KiB,1024B` is one cell rather than two identical ones. Changing +`DEFAULT_PAYLOAD_SPEC` in `cells.py` changes what the nightly measures; think +about the budget first. Cell count per SDK is `1 + 2 × len(payloads)`, and the +budget is divided evenly across all of them. **A new metric** — add it to `METRICS` and `METRIC_LABELS` in `measure.py`, teach `Sample.metric()` and `format_metric()` about it, and decide whether it belongs diff --git a/xtest/perf/cells.py b/xtest/perf/cells.py index 4a1d4ba47..f8a34413f 100644 --- a/xtest/perf/cells.py +++ b/xtest/perf/cells.py @@ -6,6 +6,8 @@ from __future__ import annotations +import re +from collections.abc import Sequence from dataclasses import dataclass from typing import Literal @@ -22,22 +24,100 @@ class Payload: 32 MiB the crypto and IO dominate and a startup regression is invisible. A benchmark at one size only will miss half the regressions it claims to cover. + + "Dominate" is relative, and at 32 MiB it is not yet true. On a 4-core + Linux runner a go encrypt costs ~450 ms of fixed startup against ~72 ms + that scales with the payload, so payload work is ~14% of the cell and the + default 1.15x gate is wider than the whole of it -- a candidate that + doubled every per-segment cost would still report PASS. Gating throughput + needs a size where the ratio inverts, which is what ``--bench-payloads`` + is for: at 1 GiB the payload term is ~2.3 s against the same ~450 ms. """ label: str n_bytes: int -PAYLOADS: tuple[Payload, ...] = ( - Payload("1KiB", 1024), - Payload("1MiB", 2**20), - Payload("32MiB", 32 * 2**20), +#: Binary units only. A label is a filename and a cell id, and "1MB" sitting +#: next to "1MiB" in a report is a misreading waiting to happen. +_UNITS: tuple[tuple[str, int], ...] = ( + ("B", 1), + ("KiB", 2**10), + ("MiB", 2**20), + ("GiB", 2**30), ) +_SIZE_RE = re.compile(r"^\s*(\d+)\s*([a-z]+)\s*$", re.IGNORECASE) + + +def parse_payload(spec: str) -> Payload: + """Parse one size spec, e.g. ``"32MiB"``, into a :class:`Payload`. + + The unit is matched case-insensitively but the label is rebuilt from the + canonical spelling, so ``"1gib"`` and ``"1GiB"`` name the same cell rather + than two cells that measure the same thing under different ids. + """ + match = _SIZE_RE.match(spec) + units = {name.lower(): (name, mult) for name, mult in _UNITS} + if match is None or match.group(2).lower() not in units: + raise ValueError( + f"{spec!r} is not a payload size; expected a count and one of " + f"{', '.join(name for name, _ in _UNITS)}, e.g. '32MiB'" + ) + count = int(match.group(1)) + name, multiplier = units[match.group(2).lower()] + if count <= 0: + raise ValueError(f"{spec!r} is not a payload size; it must be above zero") + return Payload(f"{count}{name}", count * multiplier) + + +def parse_payloads(spec: str) -> tuple[Payload, ...]: + """Parse a comma-separated size list into the run's payload set. + + Sorted ascending and deduplicated *by byte count*, not by label: ``1024B`` + and ``1KiB`` are one size written two ways, and admitting both would run + two identically-sized cells whose only difference is the id in the report. + """ + by_size: dict[int, Payload] = {} + for part in spec.split(","): + if not part.strip(): + continue + payload = parse_payload(part) + by_size.setdefault(payload.n_bytes, payload) + if not by_size: + raise ValueError("no payload sizes given") + return tuple(by_size[n] for n in sorted(by_size)) + + +#: What a run measures unless ``--bench-payloads`` says otherwise. Anything +#: larger is opt-in: a 1 GiB cell costs minutes of budget and gigabytes of +#: scratch disk, which a nightly should not spend without being asked. +DEFAULT_PAYLOAD_SPEC = "1KiB,1MiB,32MiB" + +PAYLOADS: tuple[Payload, ...] = parse_payloads(DEFAULT_PAYLOAD_SPEC) + #: Payload used for the A/A control. Mid-size: large enough that startup noise #: does not dominate it, small enough that the control is not a big slice of #: the budget. -CONTROL_PAYLOAD = PAYLOADS[1] +#: +#: Fixed rather than picked out of the selected set, because the control's +#: reported width *is* the run's noise floor and every other cell is judged +#: against it. Letting it follow ``--bench-payloads`` would move the floor +#: whenever the matrix changed, so two runs of the same comparison could +#: disagree about which cells were trustworthy for a reason that has nothing +#: to do with either build. +CONTROL_PAYLOAD = Payload("1MiB", 2**20) + + +def payloads_to_generate(payloads: Sequence[Payload]) -> tuple[Payload, ...]: + """Every payload file a run needs, including the control's. + + The control's size need not be in the selected set -- ``--bench-payloads + 1GiB`` is a legitimate ask -- but its file is still required, and a + missing one surfaces as a KeyError deep in arm construction. + """ + by_label = {p.label: p for p in (*payloads, CONTROL_PAYLOAD)} + return tuple(sorted(by_label.values(), key=lambda p: p.n_bytes)) @dataclass(frozen=True, slots=True) @@ -61,7 +141,9 @@ def __str__(self) -> str: return self.id -def cells_for(sdks: list[str]) -> list[BenchCell]: +def cells_for( + sdks: list[str], payloads: Sequence[Payload] = PAYLOADS +) -> list[BenchCell]: """Build the full cell list for a run, each SDK's control cell first. One control per SDK rather than one per run: a control measures a @@ -72,13 +154,17 @@ def cells_for(sdks: list[str]) -> list[BenchCell]: whatever is at the end. Losing one comparison leaves the rest trustworthy; losing the control leaves nothing trustworthy at all, since without a noise floor no cell may report PASS. + + Within an SDK the payloads run smallest first, so that when the budget + does run out it is the most expensive cell that is lost rather than an + arbitrary one. """ cells: list[BenchCell] = [] for sdk in sdks: cells.append(BenchCell(sdk, "encrypt", CONTROL_PAYLOAD, control=True)) cells += [ BenchCell(sdk, op, payload) + for payload in payloads for op in ("encrypt", "decrypt") - for payload in PAYLOADS ] return cells diff --git a/xtest/perf/runner.py b/xtest/perf/runner.py index af69ff68b..ba8f34a00 100644 --- a/xtest/perf/runner.py +++ b/xtest/perf/runner.py @@ -261,66 +261,76 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: for metric in METRICS: into[arm.name][metric].append(sample.metric(metric)) - for i in range(config.warmup): - # Warm-up rounds pay the one-time costs -- page cache, `go build` - # cache, npx package resolution, JIT warm-up -- that would otherwise - # land unevenly and show up as a difference between builds. Their - # samples are collected into a throwaway dict and dropped. - # - # The deadline is checked here too, and not only in the measured loop - # below. The budget's end is absolute, so warm-ups that overrun it - # spend the *following* cells' time and then reach the measured loop - # with nothing left -- paying the full cost of the cell and producing - # no data. Better to give up here and say why. - if deadline is not None and clock() >= deadline: - raise BudgetExhausted( - f"{cell_id}: budget ran out after {i} of {config.warmup} " - f"warm-up rounds ({clock() - started:.0f}s), " - "before any measurement began" - ) - one_round(_empty_samples()) - - stopped_because = "max_rounds" - for _ in range(config.max_rounds): - round_start = clock() - if deadline is not None and round_start >= deadline: - stopped_because = "budget" - break - if deadline is not None and round_durations: - # Do not start a round we cannot finish: a half-measured round is - # unpaired data, and unpaired data is exactly what this design - # exists to avoid. - expected = float(np.median(round_durations)) - if round_start + expected > deadline: + try: + for i in range(config.warmup): + # Warm-up rounds pay the one-time costs -- page cache, `go build` + # cache, npx package resolution, JIT warm-up -- that would + # otherwise land unevenly and show up as a difference between + # builds. Their samples are collected into a throwaway dict and + # dropped. + # + # The deadline is checked here too, and not only in the measured + # loop below. The budget's end is absolute, so warm-ups that + # overrun it spend the *following* cells' time and then reach the + # measured loop with nothing left -- paying the full cost of the + # cell and producing no data. Better to give up here and say why. + if deadline is not None and clock() >= deadline: + raise BudgetExhausted( + f"{cell_id}: budget ran out after {i} of {config.warmup} " + f"warm-up rounds ({clock() - started:.0f}s), " + "before any measurement began" + ) + one_round(_empty_samples()) + + stopped_because = "max_rounds" + for _ in range(config.max_rounds): + round_start = clock() + if deadline is not None and round_start >= deadline: stopped_because = "budget" break - one_round(samples) - round_durations.append(clock() - round_start) + if deadline is not None and round_durations: + # Do not start a round we cannot finish: a half-measured round + # is unpaired data, and unpaired data is exactly what this + # design exists to avoid. + expected = float(np.median(round_durations)) + if round_start + expected > deadline: + stopped_because = "budget" + break + one_round(samples) + round_durations.append(clock() - round_start) + + n = len(samples["baseline"]["wall"]) + if n >= config.min_rounds and _precise_enough(samples, config): + stopped_because = "precision" + break + elapsed = clock() - started n = len(samples["baseline"]["wall"]) - if n >= config.min_rounds and _precise_enough(samples, config): - stopped_because = "precision" - break - - elapsed = clock() - started - n = len(samples["baseline"]["wall"]) - if n < stats.MIN_USABLE_ROUNDS: - raise BudgetExhausted( - f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " - f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + if n < stats.MIN_USABLE_ROUNDS: + raise BudgetExhausted( + f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " + f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + ) + return CellResult( + cell_id=cell_id, + baseline_label=baseline.label, + candidate_label=candidate.label, + samples=samples, + n_warmup=config.warmup, + elapsed_s=elapsed, + stopped_because=stopped_because, + control=control, + sdk=sdk, + rss_floor_bytes=rss_floor, ) - return CellResult( - cell_id=cell_id, - baseline_label=baseline.label, - candidate_label=candidate.label, - samples=samples, - n_warmup=config.warmup, - elapsed_s=elapsed, - stopped_because=stopped_because, - control=control, - sdk=sdk, - rss_floor_bytes=rss_floor, - ) + finally: + # Each arm leaves behind an output the size of the payload, and + # nothing reads it once the cell is done. Keeping them costs 2 GiB per + # 1 GiB cell, which every later cell then has to fit around -- so the + # cell that fails on disk is not the one that filled it. + for arm in arms: + if arm.invocation.output is not None: + arm.invocation.output.unlink(missing_ok=True) def _precise_enough( diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index 98f090b66..97f654350 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -10,13 +10,22 @@ ``tmp_path``. """ +from collections.abc import Sequence from pathlib import Path import pytest import tdfs from fixtures import bench -from perf.cells import PAYLOADS +from perf.cells import ( + CONTROL_PAYLOAD, + DEFAULT_PAYLOAD_SPEC, + PAYLOADS, + Payload, + cells_for, + parse_payload, + parse_payloads, +) from perf.runner import BenchConfig @@ -119,9 +128,88 @@ def test_a_flattened_tag_is_discovered(self, cwd: Path): ] +class TestPayloadSpec: + @pytest.mark.parametrize( + ("spec", "n_bytes"), + [ + ("512B", 512), + ("1KiB", 1024), + ("32MiB", 32 * 2**20), + ("1GiB", 2**30), + ("4GiB", 4 * 2**30), + ], + ) + def test_sizes_parse(self, spec: str, n_bytes: int): + assert parse_payload(spec).n_bytes == n_bytes + + def test_the_label_is_canonical_regardless_of_case(self): + # The label is a filename and a cell id. '1gib' and '1GiB' naming two + # cells would measure one size twice and report it as two results. + assert parse_payload("1gib").label == "1GiB" + assert parse_payload(" 1 GIB ").label == "1GiB" + + @pytest.mark.parametrize( + "spec", ["", "1", "MiB", "1MB", "1.5GiB", "-1GiB", "0GiB", "1GiB extra"] + ) + def test_junk_is_refused(self, spec: str): + with pytest.raises(ValueError): + parse_payload(spec) + + def test_a_list_is_sorted_ascending(self): + labels = [p.label for p in parse_payloads("1GiB,1KiB,32MiB")] + assert labels == ["1KiB", "32MiB", "1GiB"] + + def test_one_size_written_two_ways_is_one_payload(self): + # Otherwise the run pays for two identical cells and reports them as + # independent results, which the multiplicity correction then treats + # as two tests. + assert [p.label for p in parse_payloads("1KiB,1024B")] == ["1KiB"] + + def test_an_empty_list_is_refused(self): + with pytest.raises(ValueError, match="no payload sizes"): + parse_payloads(" , ") + + +class TestCellMatrix: + def test_the_default_matrix_is_unchanged(self): + ids = [c.id for c in cells_for(["go"], parse_payloads(DEFAULT_PAYLOAD_SPEC))] + assert ids == [ + "go-encrypt-1MiB-control", + "go-encrypt-1KiB", + "go-decrypt-1KiB", + "go-encrypt-1MiB", + "go-decrypt-1MiB", + "go-encrypt-32MiB", + "go-decrypt-32MiB", + ] + + def test_the_control_comes_first_and_the_biggest_pair_last(self): + # The budget is spent in cell order, so whatever is last is what a + # short run loses. Losing the control invalidates every other cell; + # losing the largest pair costs the most expensive measurement but + # leaves the rest readable. + cells = cells_for(["go"], parse_payloads("1KiB,1GiB")) + assert cells[0].control + assert [c.id for c in cells[-2:]] == ["go-encrypt-1GiB", "go-decrypt-1GiB"] + + def test_the_control_size_does_not_follow_the_selection(self): + # The control's CI width is the run's noise floor and every cell is + # judged against it. If it moved with --bench-payloads, two runs of + # the same comparison could disagree on which cells are trustworthy. + for spec in ("1KiB", "1GiB", DEFAULT_PAYLOAD_SPEC): + control = next(c for c in cells_for(["go"], parse_payloads(spec))) + assert control.payload == CONTROL_PAYLOAD + + #: The fixture body, called directly: these tests are about the bytes it #: writes, not about pytest's fixture wiring. -make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] +_make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] + + +def make_payloads( + tmp_path: Path, config: BenchConfig, payloads: Sequence[Payload] = PAYLOADS +) -> dict[str, Path]: + return _make_payloads(tmp_path, config, tuple(payloads)) class TestPayloads: @@ -156,6 +244,37 @@ def test_a_truncated_cache_entry_is_regenerated(self, tmp_path: Path): second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) assert first == second + def test_the_controls_payload_is_generated_even_when_not_selected( + self, tmp_path: Path + ): + # --bench-payloads 1GiB is a legitimate ask, and the A/A control still + # needs its own file. Without it the control cell dies on a KeyError + # in arm construction -- and a run with no control can pass nothing. + out = make_payloads(tmp_path, BenchConfig(seed=1), [Payload("4KiB", 4096)]) + assert out[CONTROL_PAYLOAD.label].stat().st_size == CONTROL_PAYLOAD.n_bytes + + def test_chunking_does_not_change_the_bytes( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + # A 1 GiB payload is written in chunks rather than built in RAM. The + # chunk size must not be part of the seed contract: a run compared + # against an earlier one has to measure the same bytes, and the check + # is cheap next to the cost of discovering otherwise. + payload = Payload("40KiB", 40 * 1024) + whole = subdir(tmp_path, "whole") + bench.write_payload(whole / "p.bin", payload, seed=7) + monkeypatch.setattr(bench, "_CHUNK_BYTES", 4096) + chunked = subdir(tmp_path, "chunked") + bench.write_payload(chunked / "p.bin", payload, seed=7) + assert (whole / "p.bin").read_bytes() == (chunked / "p.bin").read_bytes() + + def test_a_payload_too_big_for_the_disk_is_refused_up_front(self, tmp_path: Path): + # Running out of disk mid-benchmark surfaces as a non-zero exit from + # the CLI under measurement, which reads as "this build is broken". + huge = Payload("1024GiB", 1024 * 2**30) + assert bench.disk_shortfall(tmp_path, [huge]) is not None + assert bench.disk_shortfall(tmp_path, [Payload("1KiB", 1024)]) is None + def read_all(paths: dict[str, Path]) -> dict[str, bytes]: return {label: p.read_bytes() for label, p in paths.items()} From 649459dd5d4d574c275a58f5a02eaeb92f7377b2 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 21 Aug 2026 13:16:15 -0400 Subject: [PATCH 07/11] ci(xtest): give the bench job time to spend its budget timeout-minutes was 45 against a default bench-budget-seconds of 1500. That only ever worked because setup was ~3 minutes with a warm Go module cache; a cold cache took 19, and 19 + 25 does not fit in 45. A job killed mid-measurement loses the report entirely, which is strictly worse than one that collects fewer rounds -- the budget already handles the latter. Raise it to 90 so the backstop is for a hung job rather than a working one, which is also what a dispatch asking for 1 GiB payloads needs. --- .github/workflows/xtest.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 2d9163f77..206452ac5 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -854,7 +854,14 @@ jobs: # Never runs on pull requests: 30 minutes of serial measurement is too slow # for a PR gate, and a PR runner is the noisiest place to measure. bench: - timeout-minutes: 45 + # Has to cover setup plus the whole of bench-budget-seconds, and setup is + # not a small constant: a warm Go module cache builds both arms in ~3 + # minutes, a cold one took 19. At 45 this job could not even finish its + # own default 1500s budget after a cold start -- it would be killed + # mid-measurement, which loses the report entirely rather than reporting + # fewer rounds. The budget is the knob that bounds the run; this is only + # the backstop for a hung one. + timeout-minutes: 90 runs-on: ubuntu-latest needs: resolve-versions # Nightly cron only, not the Mon/Wed or weekly ones: three runs a week of From 86a4164451acd78a5451711f1ee9fab3b7a1e089 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 26 Aug 2026 09:08:58 -0400 Subject: [PATCH 08/11] feat(xtest): compare up to four benchmark arms in one run The bench harness measured exactly two builds, which makes a bake-off between two implementations of the same feature unanswerable: two dispatches put the candidates on different runners, and cross-runner timings share no denominator. So a cell now measures K arms per round, 2 <= K <= 4, and every pairwise contrast is a within-run ratio. Verdicts split in two. A contrast against the reference (arms[0]) keeps the one-sided REGRESSION rule and can fail the build. Every other pair is judged against the equivalence band as FASTER/SLOWER/TIED, reported and ranked but never gated -- a bake-off ranks, it does not gate. TIED is a real answer and must not be reported as PASS, which is a one-sided claim. Three BH families now: gated, symmetric, rest. The A/A control grows to K arms. In a three-arm round the third invocation is two commands after the first, so it carries more drift than any two-arm pair does; a cheap two-arm control would understate the noise of exactly the contrasts being judged. A K-arm round costs K invocations, so a fixed budget buys 2/K as many rounds and every interval widens by ~sqrt(K/2). The default budget therefore scales by K/2, and a run that still could not resolve its gated contrasts says so instead of returning a quiet wall of INCONCLUSIVE. Workflow: new bench-refs input (2-4, first is the reference; the cap is setup-cli-tool's four install slots). bench-baseline-ref and bench-candidate-ref stay as deprecated aliases folded into it. The nightly cron passes no inputs, so it still runs two arms on 1500s. timeout-minutes 90 -> 240, which 90 could not cover once the budget scales. --- .github/workflows/check.yml | 2 +- .github/workflows/xtest.yml | 153 ++++++++++------ xtest/conftest.py | 47 +++-- xtest/fixtures/bench.py | 337 +++++++++++++++++++++++++----------- xtest/perf/README.md | 278 ++++++++++++++++++++++------- xtest/perf/report.py | 258 ++++++++++++++++++++++++--- xtest/perf/runner.py | 275 +++++++++++++++++++++-------- xtest/perf/stats.py | 194 +++++++++++++++++++-- xtest/test_bench_arms.py | 267 ++++++++++++++++++++++++++-- xtest/test_bench_report.py | 195 +++++++++++++++++++++ xtest/test_bench_runner.py | 306 +++++++++++++++++++++++++++----- xtest/test_bench_stats.py | 161 +++++++++++++++++ xtest/test_benchmarks.py | 10 +- 13 files changed, 2090 insertions(+), 393 deletions(-) create mode 100644 xtest/test_bench_report.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 7e42f1435..77950d388 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -45,7 +45,7 @@ jobs: run: >- uv run --frozen --no-build pytest --no-header -q test_bench_stats.py test_bench_measure.py test_bench_runner.py - test_bench_arms.py test_sdk_commands.py + test_bench_arms.py test_bench_report.py test_sdk_commands.py working-directory: xtest - name: Lint and test otdf-local run: | diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 206452ac5..568ce9f3e 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -38,16 +38,21 @@ on: type: boolean default: false description: "Run the SDK performance regression benchmarks (adds ~45m per SDK). Needs two builds per SDK: set the *-ref inputs to 'main latest', since a bare 'main' installs no release to use as a baseline and every cell will skip." + bench-refs: + required: false + type: string + default: "" + description: "Benchmark these refs against each other instead of the default newest-release-vs-branch-head pair. Comma-separated, 2 to 4 entries, first is the reference every other arm is gated against: 'main,fix/streaming-writer,DSPX-4499-streaming-codec'. Any ref otdf-sdk-mgr resolves works: a branch, a tag, a SHA, 'refs/pull/N/head'. The 4-arm ceiling is setup-cli-tool's -- it installs at most four builds side by side. All arms are measured in the same rounds on this one runner, so every pair is comparable, including two candidates against each other. For a bake-off between two implementations of the same feature, make the reference their shared parent rather than 'main' if they are stacked: 'main' then measures each candidate's whole stack, though the head-to-head that decides the bake-off is unaffected either way. Requires a focus-sdk naming one SDK; ignores the *-ref inputs, which drive the functional matrix rather than this." bench-baseline-ref: required: false type: string default: "" - description: "Benchmark the ref named here against bench-candidate-ref, instead of the default newest-release-vs-branch-head comparison. Any ref otdf-sdk-mgr resolves: 'main', a branch, a tag, a SHA, 'refs/pull/N/head'. Requires bench-candidate-ref and a focus-sdk naming one SDK; ignores the *-ref inputs, which drive the functional matrix rather than this." + description: "Deprecated two-arm spelling of bench-refs; 'X' here and 'Y' in bench-candidate-ref means bench-refs='X,Y'. Setting both forms is an error." bench-candidate-ref: required: false type: string default: "" - description: "The build under suspicion, measured against bench-baseline-ref. e.g. 'feat/DSPX-2604-createtdf-chunked'." + description: "Deprecated: the second arm of the bench-baseline-ref pair. e.g. 'feat/DSPX-2604-createtdf-chunked'." bench-payloads: required: false type: string @@ -57,7 +62,7 @@ on: required: false type: string default: "" - description: "Wall-clock allowance shared by every benchmark cell (default 1500). Manual runs are not on the nightly's schedule, so this is the knob to raise when adding payload sizes -- each one adds an encrypt and a decrypt cell, and the budget is divided evenly as cells start." + description: "Wall-clock allowance shared by every benchmark cell (default 1500 for two arms, scaled by K/2 for K arms). Manual runs are not on the nightly's schedule, so this is the knob to raise when adding payload sizes -- each one adds an encrypt and a decrypt cell, and the budget is divided evenly as cells start. A K-arm round costs K invocations rather than 2, so at a fixed budget every interval widens by ~sqrt(K/2); the scaled default buys that back." bench-max-rounds: required: false type: string @@ -93,6 +98,10 @@ on: required: false type: boolean default: false + bench-refs: + required: false + type: string + default: "" bench-baseline-ref: required: false type: string @@ -134,6 +143,10 @@ jobs: heads: ${{ steps.version-info.outputs.platform-heads }} default-tags: ${{ steps.version-info.outputs.default-tags }} bench-sdks: ${{ steps.bench-inputs.outputs.sdks }} + # Normalized here so the bench job never sees the deprecated pair, and + # so the arm count that scales the budget is counted once. + bench-refs: ${{ steps.bench-inputs.outputs.refs }} + bench-budget-seconds: ${{ steps.bench-inputs.outputs.budget-seconds }} go: ${{ steps.version-info.outputs.go-version-info }} java: ${{ steps.version-info.outputs.java-version-info }} js: ${{ steps.version-info.outputs.js-version-info }} @@ -160,6 +173,7 @@ jobs: id: bench-inputs env: FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }} + BENCH_REFS: ${{ inputs.bench-refs }} BASELINE_REF: ${{ inputs.bench-baseline-ref }} CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} BUDGET_SECONDS: ${{ inputs.bench-budget-seconds }} @@ -177,15 +191,51 @@ jobs: exit 1 fi done - if [[ -n "$BASELINE_REF" && -z "$CANDIDATE_REF" ]] \ - || [[ -z "$BASELINE_REF" && -n "$CANDIDATE_REF" ]]; then - echo "::error::bench-baseline-ref and bench-candidate-ref must be set together; a comparison needs both arms named." - exit 1 + # Fold the deprecated pair into bench-refs, so everything downstream + # of this step deals with one list and one arm count. + refs=$BENCH_REFS + if [[ -n "$BASELINE_REF" || -n "$CANDIDATE_REF" ]]; then + if [[ -n "$refs" ]]; then + echo "::error::bench-refs replaces bench-baseline-ref/bench-candidate-ref -- give one form or the other, not both." + exit 1 + fi + if [[ -z "$BASELINE_REF" || -z "$CANDIDATE_REF" ]]; then + echo "::error::bench-baseline-ref and bench-candidate-ref must be set together; a comparison needs both arms named." + exit 1 + fi + refs="${BASELINE_REF},${CANDIDATE_REF}" fi - if [[ -n "$CANDIDATE_REF" && "$FOCUS_SDK" == "all" ]]; then - echo "::error::bench-baseline-ref/bench-candidate-ref name refs of one SDK, so focus-sdk must be go, java, or js -- not 'all'." - exit 1 + # Two arms unless told otherwise: with no refs at all the bench job + # falls back to newest-release-vs-branch-head, which is a pair. + n_arms=2 + if [[ -n "$refs" ]]; then + read -r -a arms <<<"${refs//,/ }" + n_arms=${#arms[@]} + # 4 is setup-cli-tool's ceiling (slots a/b/c/d); a 5th arm would + # be silently dropped there and then missing from every round. + if ((n_arms < 2 || n_arms > 4)); then + echo "::error::bench-refs needs 2 to 4 refs, got ${n_arms}: '${refs}'. The 4-arm cap is setup-cli-tool's, which installs at most four builds side by side." + exit 1 + fi + if [[ "$(printf '%s\n' "${arms[@]}" | sort -u | wc -l)" -ne "$n_arms" ]]; then + echo "::error::bench-refs names the same ref twice: '${refs}'. Every arm has to be a distinct build." + exit 1 + fi + if [[ "$FOCUS_SDK" == "all" ]]; then + echo "::error::bench-refs names refs of one SDK, so focus-sdk must be go, java, or js -- not 'all'." + exit 1 + fi + printf -v refs '%s,' "${arms[@]}" + refs=${refs%,} fi + # A K-arm round costs K invocations, so a fixed budget buys K/2 as + # many rounds and every interval widens by ~sqrt(K/2). Scaling the + # default keeps a 3-arm run as precise as the 2-arm one it replaces. + budget=${BUDGET_SECONDS:-$((1500 * n_arms / 2))} + { + echo "refs=${refs}" + echo "budget-seconds=${budget}" + } >> "$GITHUB_OUTPUT" if [[ "$FOCUS_SDK" == "all" ]]; then echo 'sdks=["go","java","js"]' >> "$GITHUB_OUTPUT" else @@ -842,15 +892,20 @@ jobs: ${{ steps.kas-km2.outputs.log-file }} if-no-files-found: ignore - # Paired A/B performance regression benchmark. + # Paired performance regression benchmark: two arms by default, up to four + # with bench-refs. # # Absolute timings from a GitHub-hosted runner are not comparable to # timings from any other runner -- CPU model, tenancy, and steal time all # vary more than any regression worth catching. So nothing is compared to - # history. Instead both builds under comparison run on *this* runner, in - # the same interleaved round, and only their ratio is reported. Runner + # history. Instead every build under comparison runs on *this* runner, in + # the same interleaved round, and only their ratios are reported. Runner # speed divides out. # + # That is also why a bake-off has to be one job rather than two dispatches: + # two runs put the candidates on different runners, so their ratios share + # no denominator and the comparison is invalid by the premise above. + # # Never runs on pull requests: 30 minutes of serial measurement is too slow # for a PR gate, and a PR runner is the noisiest place to measure. bench: @@ -860,8 +915,10 @@ jobs: # own default 1500s budget after a cold start -- it would be killed # mid-measurement, which loses the report entirely rather than reporting # fewer rounds. The budget is the knob that bounds the run; this is only - # the backstop for a hung one. - timeout-minutes: 90 + # the backstop for a hung one. 240 rather than 90 because a K-arm run + # scales its budget by K/2 and adds a build per arm: three arms at 1 GiB + # want 8100s = 135 minutes of measurement before any setup at all. + timeout-minutes: 240 runs-on: ubuntu-latest needs: resolve-versions # Nightly cron only, not the Mon/Wed or weekly ones: three runs a week of @@ -975,27 +1032,27 @@ jobs: env: PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - ######## INSTALL BOTH ARMS OF THE COMPARISON ############# - # Two named refs instead of the default release-vs-branch pair. Resolved + ######## INSTALL EVERY ARM OF THE COMPARISON ############# + # Named refs instead of the default release-vs-branch pair. Resolved # here rather than in resolve-versions because the *-ref inputs there - # drive the functional matrix, and a benchmark wants to name its two + # drive the functional matrix, and a benchmark wants to name its own # arms without also changing what the rest of the workflow tests. # - # Baseline first: the tag order becomes configure-sdk's `heads` output, - # and conftest.py takes heads[0] as the otdfctl that provisions - # attributes and the KAS registry. That provisioning is not measured, - # and it should be the same build for both arms. - - name: Resolve the two benchmark arms + # Reference first, and input order preserved throughout: the tag order + # becomes configure-sdk's `heads` output, and conftest.py takes heads[0] + # as the otdfctl that provisions attributes and the KAS registry. That + # provisioning is not measured, and it should be the same build for + # every arm. + - name: Resolve the benchmark arms id: bench-arms - if: inputs.bench-candidate-ref != '' + if: needs.resolve-versions.outputs.bench-refs != '' working-directory: otdftests/otdf-sdk-mgr env: SDK: ${{ matrix.sdk }} - BASELINE_REF: ${{ inputs.bench-baseline-ref }} - CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + BENCH_REFS: ${{ needs.resolve-versions.outputs.bench-refs }} run: |- - info=$(uv run --project . otdf-sdk-mgr versions resolve \ - "$SDK" "$BASELINE_REF" "$CANDIDATE_REF") + read -r -a refs <<<"${BENCH_REFS//,/ }" + info=$(uv run --project . otdf-sdk-mgr versions resolve "$SDK" "${refs[@]}") jq . <<<"$info" err=$(jq -r '[.[] | select(.err != null) | .err] | join("; ")' <<<"$info") if [[ -n "$err" ]]; then @@ -1003,23 +1060,22 @@ jobs: exit 1 fi # `versions resolve` drops a ref whose SHA it has already seen, so - # two names for one commit come back as a single entry. Left alone - # that installs one build, fails arm selection in every cell, and - # spends the runner's 45 minutes arriving at NOTHING MEASURED. - if [[ "$(jq 'length' <<<"$info")" -ne 2 ]]; then - echo "::error::${BASELINE_REF} and ${CANDIDATE_REF} resolve to the same commit -- nothing to compare." + # two names for one commit come back as one entry. Left alone that + # installs fewer builds than there are arms, fails arm selection in + # every cell, and spends the runner arriving at NOTHING MEASURED. + if [[ "$(jq 'length' <<<"$info")" -ne "${#refs[@]}" ]]; then + echo "::error::${BENCH_REFS} does not name ${#refs[@]} distinct commits -- two of these refs are the same build, so there is nothing to compare between them." exit 1 fi { echo "version-info=$(jq -c . <<<"$info")" - echo "baseline-spec=${SDK}@$(jq -r '.[0].tag' <<<"$info")" - echo "candidate-spec=${SDK}@$(jq -r '.[1].tag' <<<"$info")" + echo "refs-spec=$(jq -r --arg sdk "$SDK" '[.[] | "\($sdk)@\(.tag)"] | join(",")' <<<"$info")" } >> "$GITHUB_OUTPUT" - # The whole design rests on this step laying down two builds side by - # side under sdk//dist/: by default the branch head (candidate) and - # the newest release (baseline), or the two refs resolved above. Arm - # selection picks them up from there. + # The whole design rests on this step laying down every arm side by side + # under sdk//dist/: by default the branch head (candidate) and the + # newest release (baseline), or the refs resolved above. Arm selection + # picks them up from there. - name: Configure ${{ matrix.sdk }} sdk id: configure-sdk uses: ./otdftests/xtest/setup-cli-tool @@ -1120,14 +1176,13 @@ jobs: - name: Run performance benchmarks id: bench run: |- - # Empty unless the two arms were named explicitly, in which case - # arm selection must not fall back to "newest release vs branch - # head": neither named ref need be a release, and with two branch - # builds installed the default would pick the wrong pair or none. + # Empty unless the arms were named explicitly, in which case arm + # selection must not fall back to "newest release vs branch head": + # no named ref need be a release, and with several branch builds + # installed the default would pick the wrong pair or none. arms=() - if [[ -n "$BENCH_BASELINE_SPEC" ]]; then - arms=(--bench-baseline "$BENCH_BASELINE_SPEC" - --bench-candidate "$BENCH_CANDIDATE_SPEC") + if [[ -n "$BENCH_REFS_SPEC" ]]; then + arms=(--bench-refs "$BENCH_REFS_SPEC") fi uv run --frozen --no-build pytest -ra -v \ --bench \ @@ -1143,13 +1198,13 @@ jobs: working-directory: otdftests/xtest env: BENCH_SDK: ${{ matrix.sdk }} - BENCH_BASELINE_SPEC: ${{ steps.bench-arms.outputs.baseline-spec }} - BENCH_CANDIDATE_SPEC: ${{ steps.bench-arms.outputs.candidate-spec }} + BENCH_REFS_SPEC: ${{ steps.bench-arms.outputs.refs-spec }} # Fallbacks rather than input defaults: the scheduled nightly # supplies no inputs at all, so `inputs.*` is empty there and these # are what keeps its matrix and budget where they have always been. BENCH_PAYLOADS: ${{ inputs.bench-payloads || '1KiB,1MiB,32MiB' }} - BENCH_BUDGET_SECONDS: ${{ inputs.bench-budget-seconds || '1500' }} + # Already defaulted (and scaled by arm count) in resolve-versions. + BENCH_BUDGET_SECONDS: ${{ needs.resolve-versions.outputs.bench-budget-seconds }} BENCH_MAX_ROUNDS: ${{ inputs.bench-max-rounds || '60' }} PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" diff --git a/xtest/conftest.py b/xtest/conftest.py index da3949447..4c9896086 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -23,7 +23,7 @@ import pytest import tdfs -from fixtures.bench import payloads_from_options +from fixtures.bench import MAX_ARMS, payloads_from_options from otdfctl import OpentdfCommandLineTool from perf import report, stats from perf.cells import DEFAULT_PAYLOAD_SPEC, cells_for @@ -172,15 +172,24 @@ def _add_benchmark_options(parser: pytest.Parser): help="run the performance regression benchmarks (they are long, so they " "are opt-in and collect nothing otherwise)", ) + group.addoption( + "--bench-refs", + help=f"builds to compare, comma- or space-separated, e.g. " + f"'go@main,go@my-branch'. The first is the reference: every gated " + f"contrast is taken against it, and the rest are ranked head-to-head " + f"as a bake-off. 2 to {MAX_ARMS} entries (the ceiling is how " + f"many builds setup-cli-tool can install side by side). Defaults to " + f"the newest installed release against the branch build", + ) group.addoption( "--bench-baseline", - help="build to compare against, e.g. go@v0.29.0; defaults to the newest " - "installed release of each sdk", + help="two-arm shorthand for the reference half of --bench-refs, e.g. " + "go@v0.29.0; must be given with --bench-candidate", ) group.addoption( "--bench-candidate", - help="build under test, e.g. go@main; defaults to the installed " - "unreleased build of each sdk", + help="two-arm shorthand for the candidate half of --bench-refs, e.g. " + "go@main; must be given with --bench-baseline", ) group.addoption( "--bench-payloads", @@ -189,8 +198,8 @@ def _add_benchmark_options(parser: pytest.Parser): "'1KiB,1MiB,32MiB,1GiB' (default: %(default)s). Sizes above the " "default are opt-in because they are what a throughput gate actually " "needs and what a nightly cannot afford: each one adds two cells, and " - "a run holds roughly twice the total plus the largest twice over on " - "disk", + "a run holds roughly twice the total plus one live output per arm of " + "the largest on disk", ) group.addoption( "--bench-threshold", @@ -222,8 +231,11 @@ def _add_benchmark_options(parser: pytest.Parser): group.addoption( "--bench-budget-seconds", type=float, - default=1500.0, - help="wall-clock allowance shared by every cell (default: %(default)s)", + default=None, + help="wall-clock allowance shared by every cell. Defaults to " + "1500s scaled by (arms / 2), because a K-arm round costs K " + "invocations and holding the same precision costs proportionally " + "more time", ) group.addoption( "--bench-seed", @@ -348,7 +360,7 @@ def _parametrize_bench_cells(metafunc: pytest.Metafunc): return # --sdks may be version-qualified (go@main); benchmark arms come from - # --bench-baseline/--bench-candidate instead, so only the name matters. + # --bench-refs instead, so only the name matters here. specs = metafunc.config.getoption("--sdks") or " ".join( typing.get_args(tdfs.sdk_type) ) @@ -372,6 +384,11 @@ def pytest_configure(config: pytest.Config): "--bench cannot run under pytest-xdist: parallel workers compete " "for the CPU being measured. Drop -n / --dist." ) + # Resolve the arm specs now so a malformed --bench-refs is a usage error + # before anything is installed or measured, not an hour into the run. + from fixtures import bench + + bench.arm_specs_from_options(config) def pytest_collection_modifyitems( @@ -432,6 +449,16 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): if reporter is not None: reporter.write_sep("=", "benchmark results") reporter.write_line(gate.summary) + # Repeated on the terminal as well as in the step summary: "the run + # was too short for the number of arms you asked for" is the one + # finding a reader is most likely to mistake for a real result. + underpowered = report.underpowered_warning(recorder, bench_config, gate) + if underpowered: + reporter.write_line(underpowered) + for bake_off in report.bake_offs(recorder, bench_config, gate): + reporter.write_line( + f"{bake_off.cell_id} [{bake_off.metric}]: {bake_off.detail}" + ) reporter.write_line(f"raw samples and statistics: {json_path}") if config.getoption("--bench-no-gate", default=False): diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py index e5a1b08d1..064b9e419 100644 --- a/xtest/fixtures/bench.py +++ b/xtest/fixtures/bench.py @@ -4,7 +4,7 @@ time budget all live here. The measurement loop itself is in ``perf/runner.py`` and the statistics in ``perf/stats.py``; this module is the glue that turns pytest's world (options, fixtures, SDK discovery) into the runner's world -(two arms and a config). +(K arms and a config). """ from __future__ import annotations @@ -12,6 +12,7 @@ import os import platform import random +import re import shutil from collections.abc import Sequence from dataclasses import dataclass @@ -26,28 +27,59 @@ from perf.cells import BenchCell, Payload, parse_payloads, payloads_to_generate from perf.runner import Arm, BenchConfig, Budget, Invocation +#: Most builds one run can compare. The ceiling comes from +#: ``xtest/setup-cli-tool/action.yaml``, which installs into four fixed slots +#: (a/b/c/d) and refuses a fifth. Raising it here without raising it there +#: gives a run that resolves five refs and then measures four of them. +MAX_ARMS = 4 + class ArmSelectionError(Exception): - """The two builds a comparison needs are not both installed.""" + """The builds a comparison needs are not all installed.""" + + +def parse_refs(spec: str) -> tuple[str, ...]: + """Split a ``--bench-refs`` value into build specs, first = reference. + + Commas or whitespace, so both the shell-friendly + ``go@main,go@my-branch`` and a quoted space-separated list work. + + Raises: + ValueError: if the result is not between 2 and :data:`MAX_ARMS` + entries, or if a build is named twice. + """ + refs = tuple(p for p in re.split(r"[,\s]+", spec.strip()) if p) + if not 2 <= len(refs) <= MAX_ARMS: + raise ValueError( + f"need 2 to {MAX_ARMS} refs, got {len(refs)}: {spec!r}. The first " + "is the reference every gated contrast is taken against." + ) + if len(set(refs)) != len(refs): + # Two arms running the same build is what the A/A control cell is for, + # and it is added automatically. Asking for it here would spend a slot + # measuring a comparison the run already makes. + raise ValueError(f"duplicate refs in {spec!r}; each arm needs a distinct build") + return refs def select_arms( sdk: str, - *, - baseline_spec: str | None = None, - candidate_spec: str | None = None, -) -> tuple[tdfs.SDK, tdfs.SDK]: - """Pick (baseline, candidate) builds for one SDK. + specs: Sequence[str] | None = None, +) -> tuple[tdfs.SDK, ...]: + """Pick the builds to compare for one SDK, reference first. - By default the candidate is the branch build (``main``) and the baseline - is the newest installed release, which is exactly what the CI setup action - lays down side by side. Explicit specs override either side, for - reproducing a comparison or for pinning a specific release. + With no specs the run is the nightly two-arm comparison: the reference is + the newest installed final release and the candidate is the branch build + (``main``), which is exactly what the CI setup action lays down side by + side. Explicit specs name the arms instead, for reproducing a comparison, + pinning a specific release, or running a bake-off between several + candidates. Raises: - ArmSelectionError: if either side is missing or the two resolve to the - same build (a comparison of a build against itself is only - meaningful as the explicit A/A control). + ArmSelectionError: if any named build is missing, if the default pair + cannot be found, or if two arms resolve to the same build (a + comparison of a build against itself is only meaningful as the + explicit A/A control). """ installed = tdfs.all_versions_of(sdk) # pyright: ignore[reportArgumentType] if not installed: @@ -65,8 +97,11 @@ def resolve(spec: str, role: str) -> tdfs.SDK: ) return matches[0] - if candidate_spec: - candidate = resolve(candidate_spec, "candidate") + if specs: + arms = tuple( + resolve(spec, "reference" if i == 0 else f"arm {i + 1}") + for i, spec in enumerate(specs) + ) else: heads = [s for s in installed if not s.is_released()] if not heads: @@ -76,10 +111,6 @@ def resolve(spec: str, role: str) -> tdfs.SDK: ) # Prefer 'main' when several branch builds are present. candidate = next((s for s in heads if s.version == "main"), heads[0]) - - if baseline_spec: - baseline = resolve(baseline_spec, "baseline") - else: # Final releases only. A release candidate parses to the same semver # as its final release, so including them leaves `max` breaking a tie # on whatever order the directory listing happened to produce -- and a @@ -91,44 +122,103 @@ def resolve(spec: str, role: str) -> tdfs.SDK: f"not count); installed: " f"{', '.join(sorted(s.version for s in installed))}" ) - baseline = max(releases, key=lambda s: s.semver() or (0, 0, 0)) + arms = (max(releases, key=lambda s: s.semver() or (0, 0, 0)), candidate) - if baseline == candidate: - raise ArmSelectionError( - f"baseline and candidate are both {baseline}; nothing to compare" - ) - return baseline, candidate + if len(set(arms)) != len(arms): + names = ", ".join(str(a) for a in arms) + raise ArmSelectionError(f"arms resolved to the same build: {names}") + return arms # --- Session-scoped configuration ------------------------------------------- +def arm_specs_from_options(config: pytest.Config) -> tuple[str, ...] | None: + """The run's build specs, reference first, or None for the default pair. + + ``--bench-refs`` is the K-arm form. ``--bench-baseline`` / + ``--bench-candidate`` are the two-arm shorthand it grew out of; they are + still accepted because the shape reads better for the common case, but + mixing the two forms is an error rather than a merge -- there is no + reading of ``--bench-refs a,b --bench-candidate c`` that is not a mistake. + """ + refs = cast(str | None, config.getoption("--bench-refs")) + baseline = cast(str | None, config.getoption("--bench-baseline")) + candidate = cast(str | None, config.getoption("--bench-candidate")) + if refs and (baseline or candidate): + raise pytest.UsageError( + "--bench-refs cannot be combined with --bench-baseline or " + "--bench-candidate; --bench-refs supersedes both" + ) + if refs: + try: + return parse_refs(refs) + except ValueError as e: + raise pytest.UsageError(f"invalid --bench-refs: {e}") from e + if baseline and candidate: + return (baseline, candidate) + if baseline or candidate: + # Half a pair cannot be resolved: the unnamed side would fall back to + # a default chosen for a different question, and nothing in the report + # would say the comparison was not the one that was asked for. + raise pytest.UsageError( + "--bench-baseline and --bench-candidate must be given together" + ) + return None + + +def arm_count(config: pytest.Config) -> int: + """How many arms this run will measure per cell.""" + specs = arm_specs_from_options(config) + return len(specs) if specs else 2 + + def config_from_options(config: pytest.Config) -> BenchConfig: """Build a :class:`BenchConfig` from the ``--bench-*`` options. - Every option has a default, so ``getoption`` never returns None here; the - casts are for the type checker, which cannot see the parser setup. + Every option except the budget has a default, so ``getoption`` never + returns None here; the casts are for the type checker, which cannot see + the parser setup. """ def as_int(name: str) -> int: return int(cast(int, config.getoption(name))) - def as_float(name: str) -> float: - return float(cast(float, config.getoption(name))) - + budget = cast(float | None, config.getoption("--bench-budget-seconds")) try: return BenchConfig( min_rounds=as_int("--bench-min-rounds"), max_rounds=as_int("--bench-max-rounds"), warmup=as_int("--bench-warmup"), - budget_seconds=as_float("--bench-budget-seconds"), + budget_seconds=( + float(budget) + if budget is not None + else default_budget_seconds(arm_count(config)) + ), seed=as_int("--bench-seed"), - threshold=as_float("--bench-threshold"), + threshold=float(cast(float, config.getoption("--bench-threshold"))), ) except ValueError as e: raise pytest.UsageError(f"invalid benchmark options: {e}") from e +def default_budget_seconds(n_arms: int) -> float: + """The default time allowance for a K-arm run. + + A round costs one invocation per arm, so at a fixed budget the attained + round count falls as ``2/K`` and every interval widens as ``sqrt(K/2)``. + Scaling the default by ``K/2`` keeps a three-arm run about as precise as + the two-arm run the number was chosen for, instead of quietly trading + precision for arms and reporting the difference as INCONCLUSIVE. + + An explicit ``--bench-budget-seconds`` is taken as given; someone who + named a number has already decided what they are willing to spend. + """ + # `BenchConfig` has slots, so the class attribute is a slot descriptor + # rather than the default; an instance is how you read one back. + return BenchConfig().budget_seconds * n_arms / 2 + + @pytest.fixture(scope="session") def bench_config(request: pytest.FixtureRequest) -> BenchConfig: """Round-loop and analysis settings, from the --bench-* options.""" @@ -173,7 +263,9 @@ def write_payload(path: Path, payload: Payload, seed: int) -> None: remaining -= n -def disk_shortfall(tmp_dir: Path, payloads: Sequence[Payload]) -> str | None: +def disk_shortfall( + tmp_dir: Path, payloads: Sequence[Payload], n_arms: int = 2 +) -> str | None: """Return why ``payloads`` will not fit in ``tmp_dir``, or None. Checked up front because the alternative is finding out mid-run: ENOSPC @@ -183,13 +275,15 @@ def disk_shortfall(tmp_dir: Path, payloads: Sequence[Payload]) -> str | None: points at the wrong thing while they look. The estimate is the plaintexts, plus a cached ciphertext for each (the - decrypt cells share one per size), plus the two output files the largest - cell holds while it runs. Outputs are deleted as each cell finishes, so - only one cell's worth is ever live. + decrypt cells share one per size), plus one live output per arm in the + largest cell. Outputs are deleted as each cell finishes, so only one + cell's worth is ever live -- but that cell holds K of them, not two, and + at 1 GiB payloads the difference between K and 2 is the whole margin on a + GitHub runner. """ total = sum(p.n_bytes for p in payloads) largest = max(p.n_bytes for p in payloads) - need = 2 * total + 2 * largest + _DISK_HEADROOM_BYTES + need = 2 * total + n_arms * largest + _DISK_HEADROOM_BYTES free = shutil.disk_usage(tmp_dir).free if free >= need: return None @@ -204,9 +298,12 @@ def disk_shortfall(tmp_dir: Path, payloads: Sequence[Payload]) -> str | None: @pytest.fixture(scope="session") def bench_payloads( - tmp_dir: Path, bench_config: BenchConfig, bench_payload_set: tuple[Payload, ...] + request: pytest.FixtureRequest, + tmp_dir: Path, + bench_config: BenchConfig, + bench_payload_set: tuple[Payload, ...], ) -> dict[str, Path]: - """Generate one plaintext file per payload size, shared by both arms. + """Generate one plaintext file per payload size, shared by every arm. Content is pseudo-random but seeded, so a rerun measures byte-identical input. Random rather than repetitive because compressible input would let @@ -223,7 +320,7 @@ def bench_payloads( :func:`perf.cells.payloads_to_generate`. """ wanted = payloads_to_generate(bench_payload_set) - shortfall = disk_shortfall(tmp_dir, wanted) + shortfall = disk_shortfall(tmp_dir, wanted, arm_count(request.config)) if shortfall: raise pytest.UsageError(shortfall) out: dict[str, Path] = {} @@ -263,52 +360,70 @@ def _selected_cells(config: pytest.Config) -> list[BenchCell]: @dataclass(frozen=True, slots=True) class BenchArms: - baseline: tdfs.SDK - candidate: tdfs.SDK + """The builds one SDK's cells compare, reference first.""" + + arms: tuple[tdfs.SDK, ...] + + @property + def reference(self) -> tdfs.SDK: + """The build every gated contrast is taken against.""" + return self.arms[0] + + @property + def candidates(self) -> tuple[tdfs.SDK, ...]: + """Everything else -- one arm in a regression run, more in a bake-off.""" + return self.arms[1:] class ArmResolver: - """Resolves and memoizes the two builds to compare, per SDK. + """Resolves and memoizes the builds to compare, per SDK. Resolution is lazy so that a missing build skips one SDK's cells with a readable reason instead of erroring out every cell in the module. """ - def __init__(self, baseline_spec: str | None, candidate_spec: str | None) -> None: - self._baseline_spec = baseline_spec - self._candidate_spec = candidate_spec + def __init__(self, specs: Sequence[str] | None) -> None: + self._specs = tuple(specs) if specs else None self._cache: dict[str, BenchArms] = {} def __call__(self, sdk: str) -> BenchArms: cached = self._cache.get(sdk) if cached is None: - baseline, candidate = select_arms( - sdk, - baseline_spec=_spec_for(self._baseline_spec, sdk), - candidate_spec=_spec_for(self._candidate_spec, sdk), + cached = self._cache[sdk] = BenchArms( + select_arms(sdk, _specs_for(self._specs, sdk)) ) - cached = self._cache[sdk] = BenchArms(baseline, candidate) return cached @pytest.fixture(scope="module") def bench_arms(request: pytest.FixtureRequest) -> ArmResolver: - """Resolver for the (baseline, candidate) pair of any SDK in the run.""" - return ArmResolver( - cast(str | None, request.config.getoption("--bench-baseline")), - cast(str | None, request.config.getoption("--bench-candidate")), - ) + """Resolver for the arms of any SDK in the run, reference first.""" + return ArmResolver(arm_specs_from_options(request.config)) -def _spec_for(spec: str | None, sdk: str) -> str | None: - """Return ``spec`` only if it names this SDK, so one flag can cover a run.""" - if not spec: +def _specs_for(specs: Sequence[str] | None, sdk: str) -> tuple[str, ...] | None: + """Return ``specs`` only if they name this SDK, so one flag covers a run. + + A run that measures several SDKs but names arms for one of them lets the + others fall back to their default pair. Specs that name a *mix* of SDKs + are an error: the arms of a cell are all one SDK by construction, so there + is nothing a mixed list could mean. + """ + if not specs: + return None + named = tuple(s for s in specs if s.split("@", 1)[0] == sdk) + if not named: return None - return spec if spec.split("@", 1)[0] == sdk else None + if len(named) != len(specs): + raise ArmSelectionError( + f"benchmark refs name more than one SDK ({', '.join(specs)}); " + "the arms of a comparison must all be builds of the same SDK" + ) + return named #: Features whose presence changes what an encrypt or decrypt actually *does*. -#: If the two arms disagree on one of these they are not performing the same +#: If two arms disagree on one of these they are not performing the same #: operation, and a timing difference between them is a difference in work, #: not in speed. _COMPARABILITY_FEATURES: tuple[tdfs.feature_type, ...] = ( @@ -319,13 +434,25 @@ def _spec_for(spec: str | None, sdk: str) -> str | None: def comparability_problem(arms: BenchArms) -> str | None: - """Return why these two builds cannot be fairly compared, or None.""" + """Return why these builds cannot be fairly compared, or None. + + Every arm is checked against the reference rather than only pairwise + neighbours: the reference is what all the gated contrasts are taken + against, so a candidate that disagrees with it invalidates its own gate + whatever the other candidates do. + """ for feature in _COMPARABILITY_FEATURES: - if arms.baseline.supports(feature) != arms.candidate.supports(feature): + ref_has = arms.reference.supports(feature) + for arm in arms.candidates: + if arm.supports(feature) == ref_has: + continue supporter, other = ( - (arms.baseline, arms.candidate) - if arms.baseline.supports(feature) - else (arms.candidate, arms.baseline) + (arm, arms.reference) + if not ref_has + else ( + arms.reference, + arm, + ) ) return ( f"{supporter} supports [{feature}] and {other} does not, so the " @@ -335,28 +462,25 @@ def comparability_problem(arms: BenchArms) -> str | None: def pinned_target_mode(arms: BenchArms) -> tdfs.container_version | None: - """Pick one container version both arms emit, or None for their default. + """Pick one container version every arm emits, or None for their default. - Letting each arm choose its own target would compare two output formats. - ``None`` is only returned when neither arm can be told which to use, in + Letting each arm choose its own target would compare output formats. + ``None`` is only returned when the arms cannot be told which to use, in which case :func:`comparability_problem` has already established that they agree on the relevant features and will pick the same one. """ - if not ( - arms.baseline.supports("hexaflexible") - and arms.candidate.supports("hexaflexible") - ): + if not all(a.supports("hexaflexible") for a in arms.arms): return None - if arms.baseline.supports("hexless") and arms.candidate.supports("hexless"): + if all(a.supports("hexless") for a in arms.arms): return "4.3.0" return "4.2.2" class CiphertextFactory: - """Baseline-produced ciphertexts for the decrypt cells, made on demand. + """Reference-produced ciphertexts for the decrypt cells, made on demand. - Both arms of a decrypt comparison must read the *same* file. If each arm - decrypted its own output, a difference in how the two builds *write* a TDF + Every arm of a decrypt comparison must read the *same* file. If each arm + decrypted its own output, a difference in how the builds *write* a TDF would show up as a difference in how fast they read one. """ @@ -372,12 +496,13 @@ def __init__( self._cache: dict[tuple[str, str], Path] = {} def __call__(self, arms: BenchArms, payload_label: str) -> Path: - key = (str(arms.baseline), payload_label) + reference = arms.reference + key = (str(reference), payload_label) cached = self._cache.get(key) if cached is not None: return cached - ct_file = self._tmp_dir / f"bench-ct-{arms.baseline}-{payload_label}.tdf" - arms.baseline.encrypt( + ct_file = self._tmp_dir / f"bench-ct-{reference}-{payload_label}.tdf" + reference.encrypt( self._payloads[payload_label], ct_file, container="ztdf", @@ -413,28 +538,30 @@ def build_arms( ct_file: Path | None, tmp_dir: Path, attr_values: list[str], -) -> tuple[Arm, Arm]: - """Turn a cell plus its two builds into two ready-to-run invocations. +) -> tuple[Arm, ...]: + """Turn a cell plus its builds into one ready-to-run invocation each. Everything that is not the build under test is pinned identically across - the arms: same plaintext, same attribute (so both wrap with RSA), same - container, same target mode. A functional difference between the builds - that changed any of these would otherwise show up as a speed difference. - - For decrypt, both arms read the *same* ``ct_file``, produced once by the - baseline. Letting each arm decrypt its own output would compare the cost - of reading two different files. - - In a control cell both arms are the baseline build, so the pair differs - only in the output path -- exactly the harness overhead the A/A cell - exists to measure. + the arms: same plaintext, same attribute (so every arm wraps with RSA), + same container, same target mode. A functional difference between the + builds that changed any of these would otherwise show up as a speed + difference. + + For decrypt, every arm reads the *same* ``ct_file``, produced once by the + reference. Letting each arm decrypt its own output would compare the cost + of reading different files. + + In a control cell every arm is the reference build, so they differ only in + the output path -- exactly the harness overhead the A/A cell exists to + measure. It is built with as many arms as the real cells have rather than + a cheap pair, because in a K-arm round the last arm runs K-1 invocations + after the first and carries more drift than an adjacent pair does; a + two-arm control would understate the noise of the contrasts being judged. """ - baseline_sdk = arms.baseline - candidate_sdk = arms.baseline if cell.control else arms.candidate target_mode = pinned_target_mode(arms) - def invocation(sdk: tdfs.SDK, role: str) -> Invocation: - out = tmp_dir / f"bench-{cell.id}-{role}" + def invocation(sdk: tdfs.SDK, arm_id: str) -> Invocation: + out = tmp_dir / f"bench-{cell.id}-{arm_id}" if cell.operation == "encrypt": out = out.with_suffix(".tdf") argv, env = sdk.encrypt_command( @@ -451,9 +578,17 @@ def invocation(sdk: tdfs.SDK, role: str) -> Invocation: argv, env = sdk.decrypt_command(ct_file, out, container="ztdf") return Invocation(argv, env, out) - return ( - Arm("baseline", str(baseline_sdk), invocation(baseline_sdk, "baseline")), - Arm("candidate", str(candidate_sdk), invocation(candidate_sdk, "candidate")), + if cell.control: + # Same build K times. The ids have to differ -- they key the sample + # vectors -- so they are numbered rather than named after the version. + builds = [ + (f"{arms.reference.version}#{i + 1}", arms.reference) + for i in range(len(arms.arms)) + ] + else: + builds = [(sdk.version, sdk) for sdk in arms.arms] + return tuple( + Arm(arm_id, str(sdk), invocation(sdk, arm_id)) for arm_id, sdk in builds ) diff --git a/xtest/perf/README.md b/xtest/perf/README.md index fd109e433..8c2574864 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -1,12 +1,14 @@ # SDK performance regression benchmarks -A paired A/B benchmark for the OpenTDF SDK CLIs. It answers one question: -**did this change make the SDK measurably and meaningfully slower?** +A paired benchmark for the OpenTDF SDK CLIs. It answers one question: +**did this change make the SDK measurably and meaningfully slower?** — and, +with more than two arms, the follow-up: **which of these implementations is +faster?** -Two builds — normally the newest installed release and the branch build — -are measured on the *same* runner, interleaved round by round, and only their -*ratio* is reported. Nothing is ever compared against a stored historical -number. +Two to four builds — normally the newest installed release and the branch +build — are measured on the *same* runner, interleaved round by round, and only +their *ratios* are reported. Nothing is ever compared against a stored +historical number. It runs nightly (one runner per SDK) and on `workflow_dispatch` with `run-benchmarks` checked. It never runs on pull requests: 30 minutes of serial @@ -20,9 +22,10 @@ measure. > (see [NOTHING MEASURED](#the-verdicts)), but it will have wasted 45 minutes > to tell you that. > -> To compare **two named refs** instead — a branch against `main`, say — use -> `bench-baseline-ref` / `bench-candidate-ref` and skip all of the above; see -> [Benchmarking one branch against another](#benchmarking-one-branch-against-another). +> To compare **refs you name** instead — a branch against `main`, or two +> competing implementations against their shared parent — use `bench-refs` and +> skip all of the above; see [Benchmarking named refs against each +> other](#benchmarking-named-refs-against-each-other). - **Section 1 — [Reading a result](#1-reading-a-result)** is for developers on the SDKs and the platform: your build got flagged, what does that mean. @@ -41,23 +44,31 @@ measure. | `bench-` artifact | Run artifacts | `.json` with **every raw per-round sample**, and an HTML report | | Terminal | Job log tail | One-line summary and the JSON path | -The JSON is the useful one. It holds each cell's full per-round vectors for both -arms, so a surprising verdict can be re-analysed offline instead of by re-running -a 30-minute job to look at the same numbers again. +The JSON is the useful one. It holds each cell's full per-round vectors for +every arm, so a surprising verdict can be re-analysed offline instead of by +re-running a 30-minute job to look at the same numbers again. It is `"schema": +2`: each cell carries `arms`, `reference`, and `contrasts` keyed `"_vs_"`. +`baseline` and `candidate` are still there for readers that predate the K-arm +schema, but past two arms they name only the reference and the *first* +candidate — use `arms` and `contrasts`. ### The table ``` -| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict | -| go-encrypt-1MiB | wall clock | 412.3 ms | 498.1 ms | 1.208x [1.171, 1.245] | <0.001 | 22 | **REGRESSION** | +| cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict | +| go-encrypt-1MiB | `cand` vs `main` | wall clock | 412.3 ms | 498.1 ms | 1.208x [1.171, 1.245] | <0.001 | 22 | **REGRESSION** | ``` - **cell** — `--`, plus `-control` for the A/A cell. Payload sizes default to 1 KiB, 1 MiB, and 32 MiB; `--bench-payloads` selects others. See [Payload sizes and what they can gate](#payload-sizes-and-what-they-can-gate) before reading a throughput result — at the default sizes there is not one. -- **ratio** — candidate ÷ baseline. `1.208x` means the candidate took 20.8% - longer. Below 1.0 means faster. +- **contrast** — `b` vs `a`, the two arms this row compares. A two-arm run has + one row per cell and metric; a K-arm run has one per *pair*, so three arms + give three rows. Only the rows whose `a` is the **reference** (the first arm) + can fail the build; see [Bake-offs](#bake-offs-more-than-two-arms). +- **ratio** — `b` ÷ `a`. `1.208x` means `b` took 20.8% longer. Below 1.0 means + `b` is faster. - **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely this run could measure; a wide interval means a noisy runner, not a big change. - **p (BH)** — one-sided p-value, Benjamini–Hochberg adjusted across the run. @@ -90,6 +101,21 @@ note beside the verdict: change you expect to be performance-sensitive comes back inconclusive on every cell, the run told you nothing and re-running it is reasonable. +A head-to-head between two arms that are *not* the reference gets a different +vocabulary, because the question has no privileged direction — neither arm is +the incumbent — and it can never fail the build: + +**FASTER** / **SLOWER** — the whole CI sits outside the equivalence band +`[1/1.15, 1.15]` on one side. `b` is meaningfully faster (or slower) than `a`. + +**TIED** — the whole CI sits *inside* the band. This is a real answer, not the +absence of one: the two implementations are indistinguishable at 15%, and the +choice between them should be made on something other than speed. It is +reported as TIED rather than PASS because PASS is a one-sided claim. + +**inconclusive** — the CI straddles a band edge, so the run cannot say which of +the three it is. + **NOTHING MEASURED** — no cell produced a comparison at all, usually because only one build was installed so there was no baseline to compare against. This **fails the job**. An empty run and a clean run have the same empty list of @@ -108,9 +134,11 @@ lists the reason for each cell. ### The A/A control -Each SDK gets a control cell that compares the baseline build **against itself** -through the identical pipeline. Its true ratio is exactly 1.0 by construction, so -whatever it reports is the harness's own error on this runner. It does two jobs: +Each SDK gets a control cell that runs the reference build **against itself**, +as many arms as the real cells have, through the identical pipeline. Every one +of its C(K,2) contrasts has a true ratio of exactly 1.0 by construction, so +whatever they report is the harness's own error on this runner. It does two +jobs: - If the control *trips* — its own A/A comparison looks like a real effect — then something is systematically biased and **the whole run stops being able to fail @@ -119,6 +147,12 @@ whatever it reports is the harness's own error on this runner. It does two jobs: runner could have resolved. If the floor is wider than the threshold, cells report inconclusive rather than PASS. +The floor is the **worst** of the control's pairwise contrasts, and at K > 2 it +has to be: in a three-arm round the third invocation happens two commands after +the first, so that pair carries more drift than an adjacent one does. A cheap +two-arm control alongside three-arm cells would understate the noise of exactly +the contrasts being judged. + In a multi-SDK run each SDK is judged against *its own* control — go's harness path says nothing about java's. `noise_floor_by_control` in the JSON has each one; the top-level `noise_floor` is the worst of them. @@ -135,8 +169,8 @@ Ungated rows are labelled `(ungated)` and reported for context only. They cannot fail the build. Peak RSS additionally gets **censored** when a cell's readings sit at the -measurement floor (the RSS of the process that forked the command). Both arms -clip to the same value there, producing a `1.000x` ratio with a tight interval — +measurement floor (the RSS of the process that forked the command). Every arm +clips to the same value there, producing a `1.000x` ratio with a tight interval — the most convincing-looking PASS the harness can emit, and completely meaningless. Censored cells report inconclusive with the floor named in the note. @@ -156,10 +190,9 @@ Censored cells report inconclusive with the floor named in the note. ```bash cd xtest && set -a && source test.env && set +a -# whatever two builds you want, side by side under sdk//dist/ +# whatever builds you want, side by side under sdk//dist/ uv run pytest --bench --sdks go \ - --bench-baseline go@v0.29.0 \ - --bench-candidate go@main \ + --bench-refs "go@v0.29.0,go@main" \ -v test_benchmarks.py ``` @@ -167,15 +200,19 @@ Useful knobs while investigating: | Option | Default | Use | | --- | --- | --- | +| `--bench-refs` | newest release, branch head | 2–4 build specs, first is the reference | | `--bench-threshold` | `1.15` | Smallest slowdown worth failing on | | `--bench-payloads` | `1KiB,1MiB,32MiB` | Sizes to measure, e.g. `1KiB,1GiB` | | `--bench-min-rounds` / `--bench-max-rounds` | `20` / `60` | Rounds per cell | | `--bench-warmup` | `5` | Discarded rounds paying one-time costs | -| `--bench-budget-seconds` | `1500` | Wall-clock allowance shared by all cells | +| `--bench-budget-seconds` | `1500` × K/2 | Wall-clock allowance shared by all cells | | `--bench-seed` | `0` | Payloads, round order, bootstrap. Fix it to reproduce | | `--bench-out` | `test-results/benchmarks` | JSON destination | | `--bench-no-gate` | off | Measure and report, never fail | +`--bench-baseline` / `--bench-candidate` still work as the two-arm spelling of +`--bench-refs`; giving both forms is a usage error. + A local run is noisier than CI unless the machine is otherwise idle. Close things; the noise floor will tell you whether you succeeded. @@ -216,8 +253,9 @@ Three things to know before adding a large size: - **Budget.** Each size adds an encrypt and a decrypt cell, and the budget is divided evenly as cells start. A 1 GiB round costs ~6 s against ~1 s at 32 MiB, so the default 1500 s will not reach `min_rounds` on both new cells. -- **Disk.** A run holds roughly twice the payload total plus the largest size - twice over. 1 GiB needs ~5 GiB free. This is checked before the first +- **Disk.** A run holds roughly twice the payload total plus one live output per + arm at the largest size — `2 × total + K × largest` plus headroom. Two arms at + 1 GiB needs ~5 GiB free, three needs ~6. This is checked before the first measurement, because running out mid-run arrives as a non-zero exit from the CLI under test and reads as "this build is broken". - **`max_rounds` binds before the budget does.** In the run these numbers come @@ -229,21 +267,20 @@ floor and every other cell is judged against it, so it must not move with the matrix — otherwise two runs of the same comparison can disagree about which cells were trustworthy for a reason unrelated to either build. -### Benchmarking one branch against another +### Benchmarking named refs against each other The nightly comparison is newest-release vs branch head, which is the right question to ask every night and the wrong one to ask about a specific change: the baseline carries every other commit that landed since the release. To -point the harness at two refs you name, dispatch X-Test with: +point the harness at refs you name, dispatch X-Test with: | Input | Example | Meaning | | --- | --- | --- | | `run-benchmarks` | ✅ | Required; the bench job is off otherwise | | `focus-sdk` | `go` | Must name one SDK — the matrix runs only this one | -| `bench-baseline-ref` | `main` | The build you are comparing *against* | -| `bench-candidate-ref` | `feat/DSPX-2604-createtdf-chunked` | The build under suspicion | +| `bench-refs` | `main,feat/DSPX-2604-createtdf-chunked` | 2–4 refs; **the first is the reference** | | `bench-payloads` | `1KiB,1GiB` | Sizes to measure; default `1KiB,1MiB,32MiB` | -| `bench-budget-seconds` | `5400` | Shared allowance; default `1500` | +| `bench-budget-seconds` | `5400` | Shared allowance; default `1500` × K/2 | | `bench-max-rounds` | `200` | Cap per cell; default `60` | The last three are why a dispatch can answer a question the nightly cannot. A @@ -253,10 +290,10 @@ claim, spend the budget — see [Payload sizes and what they can gate](#payload-sizes-and-what-they-can-gate), because at the defaults the answer will be **PASS** whatever the change did. -Either ref can be anything `otdf-sdk-mgr versions resolve` accepts: a branch, a -tag, a full or short SHA, or `refs/pull/N/head`. Both are built from source and +Any ref `otdf-sdk-mgr versions resolve` accepts works: a branch, a tag, a full +or short SHA, or `refs/pull/N/head`. All of them are built from source and installed side by side, and arm selection is told which is which explicitly — -so neither has to be a release, which is the whole point. +so none has to be a release, which is the whole point. The `*-ref` inputs are ignored by the bench job in this mode. They still drive the functional test matrix, so a dispatch can answer "is it slower?" without @@ -268,12 +305,71 @@ means: - **The server stays on `main`.** The bench job pins the platform and runs a single KAS, whatever the refs say. A candidate whose speed depends on a matching server change will not show it here. -- **The baseline is whatever you named.** For a stacked branch, `main` as the - baseline measures the whole stack. Name the parent branch instead to isolate +- **The reference is whatever you named.** For a stacked branch, `main` as the + reference measures the whole stack. Name the parent branch instead to isolate the top commit. -It fails fast, before spending a runner, when the two refs resolve to the same -commit or when `focus-sdk` is `all`. +It fails fast, before spending a runner, when two refs resolve to the same +commit, when there are fewer than 2 or more than 4 of them, or when `focus-sdk` +is `all`. + +`bench-baseline-ref` / `bench-candidate-ref` remain as the deprecated two-arm +spelling; setting both forms is an error. + +### Bake-offs: more than two arms + +Two implementations of the same feature, and the only question that matters is +which one to merge. This **cannot** be answered with two dispatches: the two +candidates would land on different runners, their ratios would share no +denominator, and comparing them would violate the premise the whole harness +rests on ([Ratios within a run](#ratios-within-a-run-never-comparison-against-history)). + +Name them all in one dispatch instead. Every arm is then measured in the *same* +round on the *same* runner, so every pair is a valid within-run ratio — +including the two candidates against each other, where neither side is the +reference: + +``` +bench-refs: main,fix/otdfctl-streaming-encrypt-writer,DSPX-4499-streaming-codec +``` + +- **The reference is `bench-refs[0]`.** Only contrasts against it are gated; + every other pair is judged symmetrically, reported, ranked, and can never + fail the build (invariant 9). A bake-off ranks, it does not gate. +- **Pick the reference deliberately.** If the two candidates are stacked on a + shared parent, `main` as the reference measures each candidate's whole stack — + the vs-reference numbers then answer "how much did this branch cost overall", + not "what did this implementation do". The head-to-head that decides the + bake-off is unaffected either way, so `main` is a fine default and the parent + is the sharper one. +- **Four arms is the ceiling.** `xtest/setup-cli-tool` installs at most four + builds side by side (slots a/b/c/d). A fifth would be dropped there and then + be missing from every round. + +The summary gains a **Bake-off** block: the candidates ranked per metric, with +the head-to-head contrast and its verdict. It names a winner only when that +head-to-head is FASTER — a TIED top pair reports "no measurable difference +between A and B", which is an answer, and an unresolved one says it cannot +separate them rather than pointing at whichever point estimate landed lower. + +#### Budget: a K-arm round costs K invocations + +At a fixed budget, K arms buy `2/K` as many rounds as two arms would, and CI +width scales as `1/sqrt(n)` — so **every interval widens by ~`sqrt(K/2)`**. +Three arms on a two-arm budget is how a run comes back as a wall of +inconclusive after burning the whole runner. + +So the default budget scales with the arm count: `1500 × K/2`, applied both by +the workflow and by the pytest fixture when `--bench-budget-seconds` is not +given explicitly. An explicit value is taken as given — someone who names a +budget has already decided what to spend. If the attained rounds still leave +the gated contrasts unresolved, the report says so in an "Underpowered" warning +naming the arm count, rather than leaving the reader to infer that time was the +missing ingredient. + +`max_rounds` binds before the budget does at its default of 60; raise both. +A 3-arm 1 GiB run wants roughly `--bench-payloads 1KiB,1GiB +--bench-budget-seconds 8100 --bench-max-rounds 200`. ### What this benchmark cannot tell you @@ -310,7 +406,7 @@ Offline tests, no platform and no subprocesses needed: ```bash cd xtest uv run pytest -q test_bench_stats.py test_bench_measure.py \ - test_bench_runner.py test_bench_arms.py + test_bench_runner.py test_bench_arms.py test_bench_report.py ``` These run on every PR via `check.yml`, so the harness is exercised continuously @@ -322,25 +418,36 @@ even though the benchmark itself runs nightly. CPU models vary, tenancy is shared, and steal time is unbounded on a hosted runner. Storing a baseline and diffing against it produces false alarms until -people mute the job. Both builds are measured on the same runner and the +people mute the job. Every build is measured on the same runner and the statistic is the within-round ratio, so runner speed is a shared factor that divides out. +The same premise is what forces a bake-off into one job: results from two +dispatches have two different shared factors, and dividing one by the other +does not cancel anything. + #### Interleaved rounds, randomized within the round Running all of A then all of B lands every drift effect — a noisy neighbour arriving, thermal throttling, the page cache warming — entirely on one arm, where -it reads as a difference between builds. Both arms run once per round instead. +it reads as a difference between builds. Every arm runs once per round instead. The order *within* a round is shuffled because a fixed order is itself a confounder: whichever arm goes second inherits the first one's cache state. +At K arms the shuffle matters more, not less — there are K positions to be +last in, and an unshuffled third slot would be a systematic penalty. The shuffle is seeded per cell (`f"{seed}:{cell_id}"`), so a rerun reproduces the interleaving exactly while different cells do not share one order — which would correlate their noise. +The stopping rule reads *every* gated contrast, not the first: with K-1 +candidates against the reference, one of them converging says nothing about the +others, and stopping there would leave the rest reported at whatever width they +happened to have reached. + #### Log-ratios -`d_i = ln(candidate_i) - ln(baseline_i)`. Logs make ratios symmetric (a 2x +`d_i = ln(b_i) - ln(a_i)`. Logs make ratios symmetric (a 2x slowdown and a 2x speedup are equal and opposite) and additive, which is what the median and the bootstrap want. Everything is exponentiated back for reporting. @@ -367,21 +474,51 @@ BH-adjusted p is below alpha. Clause 1 alone fires on real-but-trivial effects measured precisely; clause 2 alone fires on noise roughly alpha of the time per cell, and a run has enough cells that "roughly alpha" becomes "most nights". -#### Separate BH families +#### The symmetric rule for head-to-heads + +A vs-reference contrast asks a one-sided question: did the candidate get +slower? A head-to-head between two candidates has no incumbent, so it gets an +equivalence-band rule against `[1/threshold, threshold]` instead — CI wholly +above the band is SLOWER, wholly below is FASTER, wholly inside is TIED, and +anything straddling an edge is inconclusive. -Gated keys are corrected as their own family. Ungated metrics get a family of -their own so they still carry a reportable verdict. Adjusting the gated metrics -against metrics nobody gates on would only make a real regression harder to -confirm. Controls and censored keys are excluded from correction entirely — an -A/A cell is not a hypothesis about the candidate. +The TIED arm of that is the interesting one. A CI-inside-band test at 95% is +TOST at 2.5% per side, so declaring TIED is *conservative*: it is harder to +claim equivalence than the nominal alpha suggests, which is the right direction +for a claim that will be used to stop looking. Reusing PASS here would be +wrong — PASS says "not slower", which is not the same as "the same". -#### One A/A control per SDK, running first +Invariant 4 still applies: a symmetric verdict, like a gated one, needs a noise +floor narrower than the band before it may say anything but inconclusive. + +#### Three separate BH families + +Gated keys — non-reference arm vs the reference, on a gated metric — are +corrected as their own family. Head-to-head contrasts get a second family, and +ungated metrics a third, so both still carry a reportable verdict. Adjusting +the gated metrics against metrics nobody gates on would only make a real +regression harder to confirm, and the same argument covers the bake-off: it is +a question of interest, not a build gate, so it must not dilute the gate +either. A key that is somehow in both the gated and symmetric sets is treated +as gated, because the one-sided rule is the one that can turn the build red. + +Controls and censored keys are excluded from correction entirely — an A/A cell +is not a hypothesis about the candidate. + +#### One A/A control per SDK, running first, with as many arms as the run A control measures a particular SDK's harness path. `cells_for()` emits each SDK's control first, because a run that overruns its budget loses whatever is at the end: losing one comparison leaves the rest trustworthy, losing the control leaves nothing trustworthy, since without a noise floor no cell may report PASS. +It runs K copies of the reference build — same binary, distinct output paths — +so it produces C(K,2) contrasts, and the floor is the worst of them. Keeping a +cheap two-arm control while the real cells run three would measure the noise of +a different experiment: the gap between the first and third invocation of a +round is not the gap between the first and second, and it is the widest pairs +that decide whether a run had the power to fail. + `GateResult.noise` is the *worst* control in the run, not the average. A single tripped control means the harness may be biased on this runner, and averaging that away with two quiet ones is exactly the reassurance the control exists to @@ -414,23 +551,28 @@ delta reads zero. #### Everything except the build is pinned -Both arms get the same plaintext, the same attribute (explicit RSA, so an arm +Every arm gets the same plaintext, the same attribute (explicit RSA, so an arm does not silently switch to EC), the same container, and the same target mode. -`comparability_problem()` refuses the comparison outright when the two builds -disagree on `hexless`, `hexaflexible`, or `autoconfigure` — a timing difference -there is a difference in *work*, not in speed. +`comparability_problem()` refuses the comparison outright when any arm +disagrees with the reference on `hexless`, `hexaflexible`, or `autoconfigure` — +a timing difference there is a difference in *work*, not in speed. Pinned +target mode likewise requires *all* arms to support the feature, not a +majority; one arm falling back would be measuring a different format. -For decrypt, both arms read one ciphertext produced by the baseline. If each arm -decrypted its own output, a difference in how the two builds *write* a TDF would -show up as a difference in how fast they read one. +For decrypt, every arm reads one ciphertext produced by the reference. If each +arm decrypted its own output, a difference in how the builds *write* a TDF +would show up as a difference in how fast they read one. -#### Baselines must be final releases +#### The default reference must be a final release +When no refs are named, the reference is the newest installed release. `SDK.is_released()` accepts `v0.29.0-rc.1`, and `semver()` parses it to the same `(0, 29, 0)` as the final release — so ordering by semver alone leaves them tied -and the directory listing breaks the tie. That is a baseline nobody chose, and it -differs run to run. Baseline selection uses `is_final_release()`, which matches -only a plain `vX.Y.Z`. +and the directory listing breaks the tie. That is a reference nobody chose, and +it differs run to run. Default selection uses `is_final_release()`, which +matches only a plain `vX.Y.Z`. With explicit refs the question does not arise: +the reference is `bench-refs[0]`, released or not, which is the point of naming +them. #### A dist tag is one path component @@ -441,8 +583,10 @@ directories exactly one level deep — `tdfs.all_versions_of()` lists `dist/*/`, the go `Makefile` finds `src/*/` — so a slash that survives resolution is discovered as a build named `feat` with no `cli.sh` in it, which `all_versions_of()` raises on before any cell runs. Branch-vs-branch dispatch -is the first thing to routinely feed it a slashed ref, and the `--bench-*` -specs name the flattened tag: `go@feat--DSPX-2604-createtdf-chunked`. +is the first thing to routinely feed it a slashed ref, and `--bench-refs` names +the flattened tag: `go@feat--DSPX-2604-createtdf-chunked`. The workflow input +`bench-refs` takes the *unflattened* ref, because it hands it to +`versions resolve`, which is what does the flattening. #### Payloads are seeded per payload, not per run @@ -507,8 +651,8 @@ noise floor over several nights. **A new operation** — extend `operation_type` and `cells_for()` in `cells.py`, then handle it in `build_arms()` in `fixtures/bench.py`. If it needs an input -produced by the baseline, follow `CiphertextFactory`: build it once, from the -baseline only, and share it between the arms. +produced by another arm, follow `CiphertextFactory`: build it once, from the +reference only, and share it across every arm. **A new SDK** — nothing here needs to change; it comes from `--sdks` and the matrix in `xtest.yml`. @@ -524,10 +668,12 @@ two builds doing different amounts of work) is invisible in the output. 3. Never let a cell assert; the gate is run-level. 4. Never report PASS without a noise floor establishing the run had the power to fail. -5. Never let the two arms differ in anything but the build. +5. Never let the arms differ in anything but the build. 6. Never run the measured command from a process holding memory. 7. Never run the benchmark in parallel with anything, including itself. 8. Never let a run that measured nothing report success. +9. Never gate a contrast that does not involve the reference. A bake-off ranks; + it does not fail the build. Every one of these fails *silently* and *plausibly* when broken: the numbers still look like numbers. That is why they are written down. diff --git a/xtest/perf/report.py b/xtest/perf/report.py index a100044ae..1068e589f 100644 --- a/xtest/perf/report.py +++ b/xtest/perf/report.py @@ -29,7 +29,7 @@ from perf import stats from perf.cells import BenchCell from perf.measure import METRIC_LABELS, METRICS, format_metric -from perf.runner import BenchConfig, CellResult, analyze +from perf.runner import BenchConfig, CellResult, analyze, contrast_key #: Cells the session intends to run. Set by the conftest parametrizer, read by #: the budget and arm-resolution fixtures. @@ -69,12 +69,132 @@ def gate(self, config: BenchConfig) -> stats.GateResult: return analyze(self.results, config) +@dataclass(frozen=True, slots=True) +class BakeOff: + """One cell's head-to-head ranking of the non-reference arms. + + The gate answers "did anything get slower than the reference". A bake-off + answers a different question -- "of these candidate implementations, which + should we merge" -- and it is deliberately kept out of the gate: ranking + two candidates against each other says nothing about whether either is a + regression, and a build must not go red because the runner-up lost. + """ + + cell_id: str + metric: str + #: Candidate arm ids, best first, ordered by ratio against the reference. + order: list[str] + #: The contrast between the top two candidates, as configured order. + head_to_head: str + verdict: stats.Verdict + #: The winning arm id, or None when the top two could not be separated. + winner: str | None + detail: str + + +def bake_offs( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> list[BakeOff]: + """Rank the candidates in every cell that ran more than one of them. + + Empty for a two-arm run, which has nothing to rank: there is one candidate + and the gate has already said everything there is to say about it. + + A winner is named only when the top pair's own contrast came back FASTER + or SLOWER. A TIED top pair reports that the two are indistinguishable, + which is a real answer and frequently the correct one -- picking the arm + whose point estimate happened to land lower would be reading noise as a + result. + """ + out: list[BakeOff] = [] + for r in recorder.results: + candidates = [a for a in r.arm_ids if a != r.reference] + if r.control or len(candidates) < 2: + continue + for metric in config.gated_metrics: + ranked = _rank_candidates(r, candidates, metric, gate) + if len(ranked) < 2: + continue + # Head-to-head keys exist in configured order only, so recover + # that order for the top two rather than assuming the ranking's. + top = [a for a in candidates if a in ranked[:2]] + key = contrast_key(r.cell_id, top[0], top[1], metric) + c = gate.comparisons.get(key) + if c is None: + continue + winner = { + stats.Verdict.FASTER: top[1], + stats.Verdict.SLOWER: top[0], + }.get(c.verdict) + out.append( + BakeOff( + cell_id=r.cell_id, + metric=metric, + order=ranked, + head_to_head=f"{top[1]}_vs_{top[0]}", + verdict=c.verdict, + winner=winner, + detail=_bake_off_detail(c, top, winner), + ) + ) + return out + + +def _rank_candidates( + r: CellResult, candidates: list[str], metric: str, gate: stats.GateResult +) -> list[str]: + """Candidate ids ordered by their ratio against the reference, best first. + + Candidates whose reference contrast produced no usable ratio are dropped: + an unmeasurable arm has no place in a ranking, and sorting NaN would put + it wherever the sort happened to leave it. + """ + ratios: dict[str, float] = {} + for arm in candidates: + c = gate.comparisons.get(contrast_key(r.cell_id, r.reference, arm, metric)) + if c is not None and math.isfinite(c.ratio): + ratios[arm] = c.ratio + return sorted(ratios, key=lambda a: ratios[a]) + + +def _bake_off_detail( + c: stats.PairedComparison, top: list[str], winner: str | None +) -> str: + interval = ( + f"{c.ratio:.3f}x [{c.ci_low:.3f}, {c.ci_high:.3f}]" + if math.isfinite(c.ci_low) and math.isfinite(c.ci_high) + else "no usable interval" + ) + pair = f"`{top[1]}` vs `{top[0]}` {interval}" + if winner is not None: + return f"`{winner}` wins: {pair}" + if c.verdict is stats.Verdict.TIED: + return f"no measurable difference between `{top[0]}` and `{top[1]}`: {pair}" + return f"cannot separate `{top[0]}` and `{top[1]}`: {pair}" + + +def _bake_off_dict(b: BakeOff) -> dict[str, object]: + return { + "cell": b.cell_id, + "metric": b.metric, + "order": b.order, + "head_to_head": b.head_to_head, + "verdict": str(b.verdict), + "winner": b.winner, + "detail": b.detail, + } + + def to_dict( recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult ) -> dict[str, object]: """Serialize a whole run, raw samples included.""" return { - "schema": 1, + # 2: cells hold K arms. `samples` is keyed by arm id rather than by + # `"baseline"`/`"candidate"`, and per-metric statistics moved under + # `contrasts["_vs_"]`. The `baseline`/`candidate` labels stay for + # a two-arm run so existing readers keep working. + "schema": 2, "metadata": recorder.metadata, "config": { "min_rounds": config.min_rounds, @@ -97,11 +217,19 @@ def to_dict( "trustworthy": gate.trustworthy, "regressions": gate.regressions, "improvements": gate.improvements, + "ranked": gate.ranked, + "bake_off": [_bake_off_dict(b) for b in bake_offs(recorder, config, gate)], "summary": gate.summary, "skipped": recorder.skipped, "cells": [ { "id": r.cell_id, + "arms": list(r.arm_ids), + "arm_labels": r.arm_labels, + "reference": r.reference, + # Kept for two-arm readers that predate the K-arm schema; at + # K > 2 `arms`/`arm_labels` are the complete picture and these + # name only the first candidate. "baseline": r.baseline_label, "candidate": r.candidate_label, "control": r.control, @@ -111,10 +239,13 @@ def to_dict( "stopped_because": r.stopped_because, "rss_floor_bytes": r.rss_floor_bytes, "samples": r.samples, - "metrics": { - m: _comparison_dict(gate.comparisons[f"{r.cell_id}/{m}"]) - for m in METRICS - if f"{r.cell_id}/{m}" in gate.comparisons + "contrasts": { + f"{b}_vs_{a}": { + m: _comparison_dict(gate.comparisons[key]) + for m in METRICS + if (key := contrast_key(r.cell_id, a, b, m)) in gate.comparisons + } + for a, b in r.contrast_pairs() }, } for r in recorder.results @@ -167,22 +298,76 @@ def write_json( return path +def underpowered_warning( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> str | None: + """Say so when the run did not buy the precision it was asked for. + + A K-arm round costs K invocations, so at a fixed time budget the round + count falls as arms are added and every interval widens by roughly + ``sqrt(K/2)``. Asking for three arms on a two-arm budget therefore comes + back as a wall of INCONCLUSIVE after burning the whole runner, with + nothing in the output saying that more time was the missing ingredient. + This says it, and says how much more. + + Returns None when every contrast reached the precision target. + """ + widest = 0.0 + rounds = 0 + for r in recorder.results: + if r.control: + continue + for a, b in r.contrast_pairs(): + for metric in config.gated_metrics: + c = gate.comparisons.get(contrast_key(r.cell_id, a, b, metric)) + if c is None or not math.isfinite(c.ci_half_width_log): + continue + if c.ci_half_width_log > widest: + widest, rounds = c.ci_half_width_log, c.n_rounds + + target = config.target_half_width_log + if widest <= target or target <= 0: + return None + + n_arms = max((len(r.arm_ids) for r in recorder.results), default=2) + # Interval width falls as 1/sqrt(n), so closing a factor-f gap costs f^2 + # times the rounds -- and, at a fixed per-round cost, f^2 times the budget. + shortfall = (widest / target) ** 2 + return ( + f"Underpowered: the widest contrast reached " + f"+/-{(math.exp(widest) - 1) * 100:.1f}% after {rounds} rounds, against " + f"a +/-{(math.exp(target) - 1) * 100:.1f}% target. A {n_arms}-arm round " + f"costs {n_arms} invocations; holding precision needs about " + f"{shortfall:.1f}x the rounds, so roughly " + f"{config.budget_seconds * shortfall:.0f}s of budget (currently " + f"{config.budget_seconds:.0f}s) and a max-rounds ceiling above " + f"{math.ceil(rounds * shortfall)}. Contrasts the interval could not " + f"separate are reported INCONCLUSIVE rather than as no difference." + ) + + def markdown( recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult ) -> str: """Render the run as a GitHub step summary.""" threshold_pct = (config.threshold - 1) * 100 + n_arms = max((len(r.arm_ids) for r in recorder.results), default=2) lines = [ "## SDK performance regression benchmark", "", - f"Paired A/B on one runner. A cell fails only if the 95% CI lower " - f"bound exceeds **{config.threshold:.2f}x** (+{threshold_pct:.0f}%) " + f"Paired {n_arms}-arm comparison, all arms in the same rounds on one " + f"runner. A contrast against the reference fails only if the 95% CI " + f"lower bound exceeds **{config.threshold:.2f}x** (+{threshold_pct:.0f}%) " f"*and* the BH-adjusted p < {stats.DEFAULT_ALPHA}.", "", f"**{gate.summary}**", "", ] + warning = underpowered_warning(recorder, config, gate) + if warning: + lines += ["> [!WARNING]", f"> {warning}", ""] + noise = gate.noise if noise is not None and noise.detail: lines += [f"> {noise.detail}", ""] @@ -194,23 +379,46 @@ def markdown( ] lines += [ - "| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict |", - "| --- | --- | --- | --- | --- | --- | --- | --- |", + "| cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] for result in recorder.results: - for metric in METRICS: - key = f"{result.cell_id}/{metric}" - c = gate.comparisons.get(key) - if c is None: - continue - gated = metric in config.gated_metrics and not result.control - label = METRIC_LABELS[metric][0] + ("" if gated else " (ungated)") + for a, b in result.contrast_pairs(): + head_to_head = result.reference not in (a, b) + for metric in METRICS: + c = gate.comparisons.get(contrast_key(result.cell_id, a, b, metric)) + if c is None: + continue + gated = ( + metric in config.gated_metrics + and not result.control + and not head_to_head + ) + label = METRIC_LABELS[metric][0] + ("" if gated else " (ungated)") + lines.append( + f"| {result.cell_id} | `{b}` vs `{a}` | {label} " + f"| {format_metric(metric, c.baseline_median)} " + f"| {format_metric(metric, c.candidate_median)} " + f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " + f"| {_verdict_cell(c)} |" + ) + + rankings = bake_offs(recorder, config, gate) + if rankings: + lines += [ + "", + "### Bake-off", + "", + "Head-to-head between candidates, measured in the same rounds as " + "everything else. Ranking only -- these contrasts never fail the " + "build.", + "", + ] + for bo in rankings: + order = " < ".join(f"`{a}`" for a in bo.order) lines.append( - f"| {result.cell_id} | {label} " - f"| {format_metric(metric, c.baseline_median)} " - f"| {format_metric(metric, c.candidate_median)} " - f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " - f"| {_verdict_cell(c)} |" + f"- **{bo.cell_id}** ({METRIC_LABELS[bo.metric][0]}): " + f"{order} -- {bo.detail}" ) if recorder.skipped: @@ -246,6 +454,12 @@ def _p_cell(c: stats.PairedComparison) -> str: stats.Verdict.REGRESSION: "**REGRESSION**", stats.Verdict.IMPROVED: "IMPROVED", stats.Verdict.INCONCLUSIVE: "inconclusive", + # Head-to-head vocabulary. Not bolded: none of these can fail the build, + # and a SLOWER that looked like a REGRESSION would invite someone to treat + # it as one. + stats.Verdict.FASTER: "faster", + stats.Verdict.SLOWER: "slower", + stats.Verdict.TIED: "tied", } diff --git a/xtest/perf/runner.py b/xtest/perf/runner.py index ba8f34a00..ef8b4e3d7 100644 --- a/xtest/perf/runner.py +++ b/xtest/perf/runner.py @@ -1,20 +1,29 @@ """The paired round loop that produces comparable samples for one cell. -A *cell* is one operation at one payload size, measured for two SDK builds. -The loop runs both arms once per round, in a randomized order, until it has -either enough precision or no more time. +A *cell* is one operation at one payload size, measured for K SDK builds +(2 to 4). The loop runs every arm once per round, in a randomized order, until +it has either enough precision or no more time. Why rounds rather than "run A 30 times, then B 30 times" -------------------------------------------------------- A shared runner drifts: a noisy neighbour arrives, the CPU thermally throttles, the page cache warms. Run all of A and then all of B and every one of those effects lands entirely on one arm and shows up as a difference between builds. -Interleaving means both arms see the same conditions within a round, and the +Interleaving means every arm sees the same conditions within a round, and the per-round ratio differences it out. The order *within* a round is randomized because a fixed order is itself a confounder -- whichever arm runs second inherits the first one's cache state. +Why every arm shares a round, and why that is the whole point at K > 2 +---------------------------------------------------------------------- +Because all K arms are measured in the same round on the same runner, *every* +pairwise contrast is a within-run ratio -- not just each candidate against the +reference, but candidate-against-candidate too. That is what makes a bake-off +between two competing implementations answerable at all. Running them as two +separate 2-arm jobs puts them on different runners, and this harness's founding +premise is that timings from different runners are not comparable. + Why stopping on precision and not on significance ------------------------------------------------- The loop stops when the confidence interval is narrow enough, never when the @@ -124,11 +133,17 @@ def child_env(self) -> dict[str, str]: @dataclass(frozen=True, slots=True) class Arm: - """One side of a comparison.""" + """One build participating in a cell. + + At K > 2 there is no such thing as "the candidate", so arms are keyed by + identity rather than by role: ``name`` is the arm id, and which arm is the + reference is a property of the cell, not of the arm. + """ - #: ``"baseline"`` or ``"candidate"``; identifies the role, not the build. + #: Arm id, unique within a cell -- the flattened dist tag, e.g. ``"main"`` + #: or ``"fix--otdfctl-streaming-encrypt-writer"``. name: str - #: The build under this role, e.g. ``"go@v0.29.0"``. + #: The build this arm runs, e.g. ``"go@v0.29.0"``. label: str invocation: Invocation @@ -143,14 +158,18 @@ class CellResult: """ cell_id: str - baseline_label: str - candidate_label: str - #: ``samples[arm_name][metric]`` is the per-round vector, warm-up excluded. + #: Arm ids in the order they were configured. ``arm_ids[0]`` is the + #: reference by convention, and ``reference`` names it explicitly. + arm_ids: tuple[str, ...] + #: Arm id -> the build it ran, e.g. ``"go@v0.29.0"``. + arm_labels: dict[str, str] + reference: str + #: ``samples[arm_id][metric]`` is the per-round vector, warm-up excluded. samples: dict[str, dict[str, list[float]]] n_warmup: int elapsed_s: float stopped_because: str - #: True for the A/A control, where both arms are the same build. + #: True for the A/A control, where every arm is the same build. control: bool = False #: Which SDK's control cell assesses this cell's noise floor. A run may #: measure several SDKs, and each has its own harness path and its own @@ -160,12 +179,32 @@ class CellResult: #: that forked each invocation. See :mod:`perf._launcher`. rss_floor_bytes: int = 0 + @property + def baseline_label(self) -> str: + """The reference build's label. + + Kept alongside :attr:`candidate_label` so that the two-arm shape of the + JSON artifact -- which predates K arms and which people have scripts + pointed at -- still reads correctly for a two-arm run. + """ + return self.arm_labels[self.reference] + + @property + def candidate_label(self) -> str: + """The second arm's build label; see :attr:`baseline_label`. + + At K > 2 this is one candidate among several and says nothing about the + rest, which is why the artifact also carries the full ``arm_labels``. + """ + others = [a for a in self.arm_ids if a != self.reference] + return self.arm_labels[others[0]] if others else self.baseline_label + @property def rss_censored_reason(self) -> str | None: """Why this cell's peak RSS cannot be compared, or None if it can. A command whose peak sits at the floor was not measured, it was - clipped, and both arms clip to the same value. The resulting ratio is + clipped, and every arm clips to the same value. The resulting ratio is 1.000 with a tight interval, which is the most convincing-looking PASS the harness can emit and means nothing at all. """ @@ -176,7 +215,7 @@ def rss_censored_reason(self) -> str | None: return None return ( f"peak rss reaches the {self.rss_floor_bytes / 2**20:.0f} MiB " - "measurement floor, so the two arms are not distinguishable" + "measurement floor, so the arms are not distinguishable" ) @property @@ -184,31 +223,54 @@ def n_rounds(self) -> int: first = next(iter(self.samples.values()), {}) return len(next(iter(first.values()), [])) - def metric_pair(self, metric: str) -> tuple[list[float], list[float]]: - """Return ``(baseline, candidate)`` vectors for one metric.""" - return self.samples["baseline"][metric], self.samples["candidate"][metric] + def contrast( + self, a: str, b: str, metric: str, config: BenchConfig + ) -> stats.PairedComparison: + """Compare arm ``b`` against arm ``a`` -- a ratio of b over a. - def compare(self, metric: str, config: BenchConfig) -> stats.PairedComparison: - baseline, candidate = self.metric_pair(metric) + The argument order matches the reading of the result: + ``contrast(reference, candidate, ...)`` answers "how much slower is the + candidate than the reference", which is the direction every ratio in + the report is quoted in. + """ return stats.compare( - baseline, - candidate, + self.samples[a][metric], + self.samples[b][metric], confidence=config.confidence, seed=config.seed, n_resamples=config.n_resamples, ) + def contrast_pairs(self) -> list[tuple[str, str]]: + """Every ordered ``(a, b)`` pair to report, reference contrasts first. + + Reference contrasts are quoted as ``(reference, candidate)`` so they + read as "candidate vs reference". Head-to-head pairs are emitted in + configured order, once each -- a pair and its inverse are the same + measurement read two ways, and reporting both would double-count it in + the multiplicity correction. + """ + others = [a for a in self.arm_ids if a != self.reference] + pairs = [(self.reference, b) for b in others] + pairs += [(a, b) for i, a in enumerate(others) for b in others[i + 1 :]] + return pairs + + +def contrast_key(cell_id: str, a: str, b: str, metric: str) -> str: + """The stable key for one contrast, as used throughout the analysis.""" + return f"{cell_id}/{b}_vs_{a}/{metric}" + -def _empty_samples() -> dict[str, dict[str, list[float]]]: - return {arm: {m: [] for m in METRICS} for arm in ("baseline", "candidate")} +def _empty_samples(arm_ids: Sequence[str]) -> dict[str, dict[str, list[float]]]: + return {arm: {m: [] for m in METRICS} for arm in arm_ids} def run_cell( cell_id: str, - baseline: Arm, - candidate: Arm, + arms: Sequence[Arm], config: BenchConfig, *, + reference: str | None = None, deadline: float | None = None, control: bool = False, sdk: str = "", @@ -220,9 +282,11 @@ def run_cell( Args: cell_id: stable identifier, also the per-cell RNG seed material so that two cells do not share an interleaving order. - baseline: the arm the candidate is compared against. - candidate: the arm under test. For an A/A control this is the same - build as ``baseline``, running through the identical path. + arms: the 2 to 4 builds to measure, all in every round. For an A/A + control these are the same build running through identical paths. + config: round-loop knobs. + reference: id of the arm every gated contrast is taken against; + defaults to ``arms[0]``. deadline: absolute ``clock()`` value past which no new round starts. control: records that this is the A/A cell; does not change the loop. sdk: which SDK this cell belongs to, so that the analysis can pair it @@ -231,16 +295,28 @@ def run_cell( run: injectable measurement function, for testing the loop itself. Raises: + ValueError: if fewer than two arms were given, or their ids collide. MeasurementError: if any invocation fails. A benchmark over an operation that errors out is measuring the error path. BudgetExhausted: if the deadline passed before ``min_rounds``, or during warm-up. """ - arms = (baseline, candidate) + arms = tuple(arms) + if len(arms) < 2: + raise ValueError(f"{cell_id}: a cell needs at least two arms to compare") + arm_ids = tuple(a.name for a in arms) + if len(set(arm_ids)) != len(arm_ids): + # Ids key the sample vectors, so a collision would silently interleave + # two builds' measurements into one arm and compare it with itself. + raise ValueError(f"{cell_id}: arm ids must be unique, got {list(arm_ids)}") + reference = arm_ids[0] if reference is None else reference + if reference not in arm_ids: + raise ValueError(f"{cell_id}: reference {reference!r} is not one of the arms") + # Seeded per cell so a rerun reproduces the interleaving exactly, but the # cells do not all share one order (which would correlate their noise). rng = random.Random(f"{config.seed}:{cell_id}") - samples = _empty_samples() + samples = _empty_samples(arm_ids) round_durations: list[float] = [] rss_floor = 0 started = clock() @@ -280,7 +356,7 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: f"warm-up rounds ({clock() - started:.0f}s), " "before any measurement began" ) - one_round(_empty_samples()) + one_round(_empty_samples(arm_ids)) stopped_because = "max_rounds" for _ in range(config.max_rounds): @@ -299,13 +375,15 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: one_round(samples) round_durations.append(clock() - round_start) - n = len(samples["baseline"]["wall"]) - if n >= config.min_rounds and _precise_enough(samples, config): + n = len(samples[reference]["wall"]) + if n >= config.min_rounds and _precise_enough( + samples, config, reference=reference + ): stopped_because = "precision" break elapsed = clock() - started - n = len(samples["baseline"]["wall"]) + n = len(samples[reference]["wall"]) if n < stats.MIN_USABLE_ROUNDS: raise BudgetExhausted( f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " @@ -313,8 +391,9 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: ) return CellResult( cell_id=cell_id, - baseline_label=baseline.label, - candidate_label=candidate.label, + arm_ids=arm_ids, + arm_labels={a.name: a.label for a in arms}, + reference=reference, samples=samples, n_warmup=config.warmup, elapsed_s=elapsed, @@ -325,7 +404,7 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: ) finally: # Each arm leaves behind an output the size of the payload, and - # nothing reads it once the cell is done. Keeping them costs 2 GiB per + # nothing reads it once the cell is done. Keeping them costs K GiB per # 1 GiB cell, which every later cell then has to fit around -- so the # cell that fails on disk is not the one that filled it. for arm in arms: @@ -334,26 +413,38 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: def _precise_enough( - samples: dict[str, dict[str, list[float]]], config: BenchConfig + samples: dict[str, dict[str, list[float]]], + config: BenchConfig, + *, + reference: str, ) -> bool: - """True once every gated metric's CI is narrow enough to decide on. + """True once every gated contrast's CI is narrow enough to decide on. + + Every non-reference arm must be resolved on every gated metric, not just + the first one: stopping as soon as *some* contrast is precise would leave + the rest INCONCLUSIVE while the budget was still there to spend on them, + and at K arms the slowest contrast to converge is the one that matters. Deliberately looks only at interval *width*, never at where the interval sits or at any p-value -- see the module docstring. """ for metric in config.gated_metrics: - c = stats.compare( - samples["baseline"][metric], - samples["candidate"][metric], - confidence=config.confidence, - seed=config.seed, - n_resamples=_INTERIM_RESAMPLES, - ) - # `not (a <= b)` rather than `a > b`, which is not the same thing when - # a is NaN: an unusable interval must read as "keep going", and - # `NaN > b` is False, which would end the loop and call it precise. - if not c.ci_half_width_log <= config.target_half_width_log: # NOSONAR - return False + for arm, vectors in samples.items(): + if arm == reference: + continue + c = stats.compare( + samples[reference][metric], + vectors[metric], + confidence=config.confidence, + seed=config.seed, + n_resamples=_INTERIM_RESAMPLES, + ) + # `not (a <= b)` rather than `a > b`, which is not the same thing + # when a is NaN: an unusable interval must read as "keep going", + # and `NaN > b` is False, which would end the loop and call it + # precise. + if not c.ci_half_width_log <= config.target_half_width_log: # NOSONAR + return False return True @@ -393,12 +484,24 @@ def remaining_s(self) -> float: def analyze(results: Sequence[CellResult], config: BenchConfig) -> stats.GateResult: """Turn every cell's raw samples into one gate decision. - ``GateResult.comparisons`` is keyed by ``"/"`` and holds - the *finalized* comparisons -- the ones carrying adjusted p-values and - verdicts. Ungated metrics are included so they appear in the report, but - they cannot fail the build, and they are corrected separately from the - gated ones: adjusting across tests nobody gates on only makes a real - regression harder to confirm. + ``GateResult.comparisons`` is keyed by ``"/_vs_/"`` + and holds the *finalized* comparisons -- the ones carrying adjusted + p-values and verdicts. Every pairwise contrast in every cell is here, but + they fall into three groups that are judged and corrected separately: + + - **Gated**: a non-reference arm against the reference, on a gated metric. + One-sided; only these can fail the build. At K arms there are K-1 of + them per cell and metric rather than one. + - **Symmetric**: a head-to-head between two non-reference arms -- the + bake-off question. Judged two-sided against an equivalence band, ranked + and reported, never gated. See invariant #9 in ``README.md``. + - **Ungated**: everything else, reported for context only. + + They get separate BH families for the reason documented in + :func:`stats.apply_multiplicity_control`: adjusting the gate against tests + nobody gates on only makes a real regression harder to confirm. A bake-off + is a question of interest, not a build gate, so it must not dilute the + gate either. Each SDK's cells are paired with *that SDK's* control. A run measuring go and java has two harness paths and two noise floors, and judging java's @@ -407,32 +510,60 @@ def analyze(results: Sequence[CellResult], config: BenchConfig) -> stats.GateRes """ comparisons: dict[str, stats.PairedComparison] = {} gated: set[str] = set() + symmetric: set[str] = set() censored: dict[str, str] = {} control_keys: set[str] = set() controls: dict[str, str] = {} - control_for_sdk = { - r.sdk: f"{r.cell_id}/{_CONTROL_METRIC}" for r in results if r.control - } for result in results: floored = result.rss_censored_reason - for metric in METRICS: - key = f"{result.cell_id}/{metric}" - comparisons[key] = result.compare(metric, config) - control_key = control_for_sdk.get(result.sdk) - if control_key is not None: - controls[key] = control_key - if metric == "rss" and floored is not None: - censored[key] = floored - continue - if result.control: - control_keys.add(key) - elif metric in config.gated_metrics: - gated.add(key) + for a, b in result.contrast_pairs(): + head_to_head = result.reference not in (a, b) + for metric in METRICS: + key = contrast_key(result.cell_id, a, b, metric) + comparisons[key] = result.contrast(a, b, metric, config) + if metric == "rss" and floored is not None: + censored[key] = floored + continue + if result.control: + control_keys.add(key) + elif head_to_head: + # Every head-to-head is judged symmetrically, on gated and + # ungated metrics alike: "which arm is faster" has no + # privileged direction, and the one-sided PASS/REGRESSION + # vocabulary would misdescribe it. None of them gate, so + # sharing one BH family costs the gate nothing -- the + # separation that matters is keeping them *out* of it. + symmetric.add(key) + elif metric in config.gated_metrics: + gated.add(key) + + # Pick each SDK's noise floor only once every control contrast exists: a + # K-arm control produces C(K,2) of them and the worst one is the floor. + assessor_for_sdk = { + r.sdk: stats.worst_control_key( + comparisons, + [ + contrast_key(r.cell_id, a, b, _CONTROL_METRIC) + for a, b in r.contrast_pairs() + ], + threshold=config.threshold, + ) + for r in results + if r.control + } + for result in results: + assessor = assessor_for_sdk.get(result.sdk) + if assessor is None: + continue + for a, b in result.contrast_pairs(): + for metric in METRICS: + controls[contrast_key(result.cell_id, a, b, metric)] = assessor return stats.apply_multiplicity_control( comparisons, gated=gated, + symmetric=symmetric, controls=controls, control_keys=control_keys, censored=censored, diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py index c7bb71576..c70498557 100644 --- a/xtest/perf/stats.py +++ b/xtest/perf/stats.py @@ -1,4 +1,4 @@ -"""Paired statistical comparison of two SDK builds. +"""Paired statistical comparison of SDK builds. Why this shape -------------- @@ -38,6 +38,24 @@ Together they answer the only question worth gating on: is the slowdown both real and large enough to care about? + +Head-to-head contrasts +---------------------- +A run may measure more than two arms. Every arm runs once per round, so any +*pair* of them is a valid within-round comparison -- which is what makes a +bake-off between two competing implementations possible at all, and what two +separate two-arm runs on two different runners could never give you. + +Contrasts against the run's designated reference keep the rule above and can +fail the build. Contrasts between two non-reference arms are judged by +:func:`_symmetric_verdict_for` instead, which reads the same threshold as a +two-sided equivalence band and returns FASTER, SLOWER, or TIED. They never +gate: a bake-off ranks candidates, it does not decide whether the build is +broken, and there is no incumbent for "regression" to be relative to. + +The three families -- gated, head-to-head, and ungated -- are BH-corrected +separately, so adding candidates to a bake-off does not cost the regression +gate any power. """ from __future__ import annotations @@ -67,13 +85,28 @@ class Verdict(StrEnum): - """Outcome for a single comparison cell.""" + """Outcome for a single comparison. + + The first four are the *gated* vocabulary, used for a contrast against the + run's reference build: the question is one-sided ("did the candidate get + slower?") and only REGRESSION can turn the build red. + + The last three are the *symmetric* vocabulary, used for a head-to-head + between two non-reference arms in a bake-off. There the question has no + privileged direction -- neither arm is the incumbent -- and "no measurable + difference" is a real answer rather than the absence of one, so TIED exists + instead of reusing PASS. + """ PASS = "PASS" REGRESSION = "REGRESSION" IMPROVED = "IMPROVED" INCONCLUSIVE = "INCONCLUSIVE" + FASTER = "FASTER" + SLOWER = "SLOWER" + TIED = "TIED" + @dataclass(frozen=True, slots=True) class PairedComparison: @@ -312,6 +345,13 @@ def assess_noise_floor( ) +def _noise_rank(n: NoiseFloor) -> tuple[bool, bool, float]: + """Order noise floors from most to least reassuring.""" + # A NaN width is an unusable interval, which is worse than any real one. + width = n.width_ratio if math.isfinite(n.width_ratio) else math.inf + return (n.tripped, n.underpowered, width) + + def _worst_noise(noises: Iterable[NoiseFloor]) -> NoiseFloor | None: """The least reassuring control in the run, or None if there were none. @@ -319,13 +359,31 @@ def _worst_noise(noises: Iterable[NoiseFloor]) -> NoiseFloor | None: harness may be biased on this runner, and averaging that away with two quiet ones is exactly the reassurance the control exists to withhold. """ + return max(noises, key=_noise_rank, default=None) + + +def worst_control_key( + comparisons: Mapping[str, PairedComparison], + keys: Sequence[str], + *, + threshold: float = DEFAULT_THRESHOLD, +) -> str | None: + """Which of several A/A contrasts should stand as the noise floor. - def rank(n: NoiseFloor) -> tuple[bool, bool, float]: - # A NaN width is an unusable interval, which is worse than any real one. - width = n.width_ratio if math.isfinite(n.width_ratio) else math.inf - return (n.tripped, n.underpowered, width) + A K-arm control cell yields C(K,2) A/A contrasts rather than one, and they + are not interchangeable: arm 3 runs two invocations after arm 1, so it + carries more within-round drift than an adjacent pair does. Taking the + worst of them keeps the floor honest for the widest-spaced contrast the + run actually judges. Taking whichever came first would let dict ordering + decide how noisy the run is allowed to look. - return max(noises, key=rank, default=None) + Ties break on input order, so the choice is reproducible across runs. + """ + ranked = [ + (_noise_rank(assess_noise_floor(comparisons.get(k), threshold=threshold)), i, k) + for i, k in enumerate(keys) + ] + return max(ranked, key=lambda r: (r[0], -r[1]))[2] if ranked else None def benjamini_hochberg(p_values: Sequence[float]) -> list[float]: @@ -359,6 +417,12 @@ class GateResult: #: Keys of cells that are confirmed regressions on a gated metric. regressions: list[str] = field(default_factory=list) improvements: list[str] = field(default_factory=list) + #: Head-to-head keys that came back FASTER or SLOWER -- a bake-off contrast + #: between two non-reference arms that the run was able to decide. Never + #: gates anything; this is the material a caller ranks arms from. Naming a + #: winner needs to know which arm is which, and a key is opaque here, so + #: that lives in the reporting layer. + ranked: list[str] = field(default_factory=list) #: True if the run may fail the build. False when the A/A control tripped: #: we still report, but a gate we cannot trust must not turn the build red. trustworthy: bool = True @@ -386,6 +450,7 @@ def apply_multiplicity_control( comparisons: dict[str, PairedComparison], *, gated: set[str] | None = None, + symmetric: set[str] | None = None, controls: Mapping[str, str] | None = None, control_keys: set[str] | None = None, censored: dict[str, str] | None = None, @@ -399,6 +464,11 @@ def apply_multiplicity_control( gated: keys allowed to fail the build. Keys outside this set are still given a verdict and reported, but never counted as a regression. ``None`` means every key is gated. + symmetric: keys judged by the two-sided FASTER/SLOWER/TIED rule instead + of the one-sided regression rule -- head-to-head contrasts between + two arms neither of which is the run's reference. They are reported + and ranked but never gate, so a bake-off cannot turn the build red + on the strength of a comparison that has no incumbent. controls: comparison key -> the A/A control key that assesses *its* noise floor. A run measuring several SDKs has one control each, and a cell judged against another SDK's control is judged against @@ -421,6 +491,7 @@ def apply_multiplicity_control( """ keys = list(comparisons) controls = controls or {} + symmetric = symmetric or set() # The keys actually doing the assessing: one metric of one control cell # per SDK. A control cell's other metrics are still control keys -- kept # out of the gate -- but they are not anybody's noise floor. @@ -444,11 +515,22 @@ def apply_multiplicity_control( # reason: adjusting them against metrics nobody gates on only makes a real # regression harder to confirm. Ungated metrics still get a family of # their own so that they carry a reportable verdict. + # + # Head-to-head contrasts get a third family on the same argument. A + # bake-off between two candidates is a question of interest, not a build + # gate, and correcting the gate against it would cost the gate power for + # tests that cannot fail the build -- which is the exact trade the + # gated/ungated split already refuses to make. adjustable = [k for k in keys if k not in control_keys and k not in censored] gated_family = [k for k in adjustable if gated is None or k in gated] - rest = [k for k in adjustable if k not in set(gated_family)] + gated_set = set(gated_family) + # Gated wins a tie. A key that is somehow both is a vs-reference contrast, + # and the one-sided rule is the one that can fail the build. + symmetric_family = [k for k in adjustable if k not in gated_set and k in symmetric] + symmetric_set = set(symmetric_family) + rest = [k for k in adjustable if k not in gated_set and k not in symmetric_set] p_adj: dict[str, float] = {} - for family in (gated_family, rest): + for family in (gated_family, symmetric_family, rest): p_adj.update( zip( family, @@ -461,16 +543,33 @@ def apply_multiplicity_control( for key in keys: c = comparisons[key] pa = p_adj.get(key) + key_noise = noise_by_control.get(controls.get(key, ""), uncontrolled) if key in censored: verdict, note = Verdict.INCONCLUSIVE, censored[key] + elif key in control_keys: + # A control's own contrast is reported in the gated vocabulary + # whatever kind of pair it is: its job is to say what the harness's + # error looks like in the same terms the gate uses. + verdict, note = _verdict_for( + c, + pa, + threshold=threshold, + alpha=alpha, + noise=key_noise, + is_control=True, + ) + elif key in symmetric_set: + verdict, note = _symmetric_verdict_for( + c, pa, threshold=threshold, alpha=alpha, noise=key_noise + ) else: verdict, note = _verdict_for( c, pa, threshold=threshold, alpha=alpha, - noise=noise_by_control.get(controls.get(key, ""), uncontrolled), - is_control=key in control_keys, + noise=key_noise, + is_control=False, ) result.comparisons[key] = PairedComparison( n_rounds=c.n_rounds, @@ -490,6 +589,8 @@ def apply_multiplicity_control( result.regressions.append(key) elif verdict is Verdict.IMPROVED: result.improvements.append(key) + elif verdict in (Verdict.FASTER, Verdict.SLOWER): + result.ranked.append(key) result.trustworthy = not noise.tripped result.summary = _summarize(result, noise, threshold) @@ -531,6 +632,67 @@ def _verdict_for( return Verdict.PASS, "" +def _symmetric_verdict_for( + c: PairedComparison, + p_adjusted: float | None, + *, + threshold: float, + alpha: float, + noise: NoiseFloor, +) -> tuple[Verdict, str]: + """Rank two arms against each other, with no privileged direction. + + Used for a head-to-head between two candidates in a bake-off, where the + one-sided regression rule does not apply: neither arm is the incumbent, so + there is no "did it get worse" to ask. + + The band is ``[1/threshold, threshold]`` -- the same effect size the gate + cares about, read in both directions: + + - interval entirely above the band -> SLOWER + - interval entirely below the band -> FASTER + - interval entirely *inside* the band -> TIED, meaning any real difference + is smaller than the effect anybody has claimed to care about. This is a + positive finding and the most likely honest answer for two + implementations of the same idea, which is why it is not folded into + PASS: PASS is the one-sided claim "did not regress", and reporting a + bake-off that way would let a slower arm read as a clean result. + - anything straddling a band edge -> INCONCLUSIVE, the run could not rank + them. + + A CI-inside-band test at 95% is TOST at 2.5% rather than the nominal 5%, + so TIED is the conservative call: harder to earn than the equivalence test + it stands in for, never easier. + """ + if c.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite(c.ci_low): + return Verdict.INCONCLUSIVE, c.note or "no usable interval" + + # Same precondition as the gated rule: without a noise floor establishing + # that an effect of this size was resolvable, neither a ranking nor a tie + # is a statement about the arms. Invariant 4 covers TIED too -- a tie + # nobody had the power to distinguish from a difference is not a tie. + if noise.underpowered: + return Verdict.INCONCLUSIVE, noise.detail + + p = p_adjusted + if p is None or not math.isfinite(p): + return Verdict.INCONCLUSIVE, "no p-value" + + # Direction clauses mirror REGRESSION and IMPROVED exactly, including the + # upper-tail read of the one-sided p for the faster direction. + if c.ci_low > threshold and p < alpha: + return Verdict.SLOWER, "" + if c.ci_high < 1 / threshold and p > 1 - alpha: + return Verdict.FASTER, "" + if c.ci_low > 1 / threshold and c.ci_high < threshold: + return Verdict.TIED, "" + return ( + Verdict.INCONCLUSIVE, + f"interval [{c.ci_low:.3f}, {c.ci_high:.3f}] straddles the " + f"+/-{(threshold - 1) * 100:.0f}% band; the two arms cannot be ranked", + ) + + def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: if result.nothing_measured: # Before the noise check: with nothing measured there is no control @@ -550,6 +712,14 @@ def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: f"{len(result.regressions)} confirmed regression(s) past the " f"{(threshold - 1) * 100:.0f}% threshold: {', '.join(result.regressions)}" ) + # Appended rather than returned on its own: a bake-off still has a gate + # running against the reference, and "which candidate won" must not + # displace "did either of them regress". + head_to_head = ( + f" {len(result.ranked)} head-to-head contrast(s) decided." + if result.ranked + else "" + ) inconclusive = [ k for k, c in result.comparisons.items() if c.verdict is Verdict.INCONCLUSIVE ] @@ -557,9 +727,11 @@ def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: return ( f"No confirmed regressions. {len(inconclusive)} cell(s) INCONCLUSIVE " f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + f"{head_to_head}" ) return ( f"No regressions. All cells resolved within the " f"{(threshold - 1) * 100:.0f}% threshold " f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + f"{head_to_head}" ) diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index 97f654350..a35229126 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -10,8 +10,11 @@ ``tmp_path``. """ +import shutil from collections.abc import Sequence from pathlib import Path +from types import SimpleNamespace +from typing import cast import pytest @@ -44,6 +47,27 @@ def cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return tmp_path +class FakeConfig: + """A ``pytest.Config`` stub answering the ``--bench-*`` options named.""" + + def __init__(self, **opts: str | None) -> None: + self._opts = {name.replace("_", "-"): value for name, value in opts.items()} + + def getoption(self, name: str) -> str | None: + return self._opts.get(name.removeprefix("--")) + + +class FakeRequest: + """Just enough of ``FixtureRequest``: the fixtures only read ``config``.""" + + def __init__(self, config: FakeConfig) -> None: + self.config = config + + +def options(**opts: str | None) -> pytest.Config: + return cast(pytest.Config, FakeConfig(**opts)) + + class TestFinalRelease: @pytest.mark.parametrize("version", ["v0.29.0", "0.29.0"]) def test_accepts_a_plain_tag(self, cwd: Path, version: str): @@ -85,15 +109,13 @@ def test_no_branch_build_is_a_clear_refusal(self, cwd: Path): def test_explicit_specs_win(self, cwd: Path): install(cwd, "go", "main", "v0.28.0", "v0.29.0") - baseline, candidate = bench.select_arms( - "go", baseline_spec="go@v0.28.0", candidate_spec="go@v0.29.0" - ) + baseline, candidate = bench.select_arms("go", ["go@v0.28.0", "go@v0.29.0"]) assert (baseline.version, candidate.version) == ("v0.28.0", "v0.29.0") def test_refuses_to_compare_a_build_against_itself(self, cwd: Path): install(cwd, "go", "main", "v0.29.0") - with pytest.raises(bench.ArmSelectionError, match="nothing to compare"): - bench.select_arms("go", baseline_spec="go@main", candidate_spec="go@main") + with pytest.raises(bench.ArmSelectionError, match="same build"): + bench.select_arms("go", ["go@main", "go@main"]) def test_two_branch_builds_need_explicit_specs(self, cwd: Path): # What a branch-vs-branch dispatch installs: two heads and no release @@ -101,15 +123,120 @@ def test_two_branch_builds_need_explicit_specs(self, cwd: Path): # there is no baseline, and "newest final release" cannot invent one. install(cwd, "go", "main", "feat--DSPX-2604-createtdf-chunked") baseline, candidate = bench.select_arms( - "go", - baseline_spec="go@main", - candidate_spec="go@feat--DSPX-2604-createtdf-chunked", + "go", ["go@main", "go@feat--DSPX-2604-createtdf-chunked"] ) assert baseline.version == "main" assert candidate.version == "feat--DSPX-2604-createtdf-chunked" with pytest.raises(bench.ArmSelectionError, match="no final go release"): bench.select_arms("go") + def test_the_first_spec_is_the_reference(self, cwd: Path): + # Order is the whole interface: every gated contrast is taken against + # arms[0], so reversing the list reverses which build is on trial. + install(cwd, "go", "main", "a--impl", "b--impl") + arms = bench.select_arms("go", ["go@main", "go@a--impl", "go@b--impl"]) + assert [a.version for a in arms] == ["main", "a--impl", "b--impl"] + assert bench.BenchArms(arms).reference.version == "main" + assert [a.version for a in bench.BenchArms(arms).candidates] == [ + "a--impl", + "b--impl", + ] + + def test_a_missing_arm_names_which_one(self, cwd: Path): + install(cwd, "go", "main", "a--impl") + with pytest.raises(bench.ArmSelectionError, match="arm 3"): + bench.select_arms("go", ["go@main", "go@a--impl", "go@nope"]) + + +class TestRefSpecParsing: + def test_commas_or_whitespace_both_work(self): + assert bench.parse_refs("go@main,go@a") == ("go@main", "go@a") + assert bench.parse_refs("go@main go@a") == ("go@main", "go@a") + + def test_a_single_ref_is_refused(self): + # One arm is not a comparison, and the harness reports only ratios. + with pytest.raises(ValueError, match="need 2 to 4 refs"): + bench.parse_refs("go@main") + + def test_more_than_four_is_refused(self): + # setup-cli-tool installs four builds side by side; a fifth would be + # silently absent at measurement time. + with pytest.raises(ValueError, match="need 2 to 4 refs"): + bench.parse_refs("go@a,go@b,go@c,go@d,go@e") + + def test_a_repeated_ref_is_refused(self): + with pytest.raises(ValueError, match="duplicate"): + bench.parse_refs("go@main,go@main") + + +class TestSpecsForSdk: + def test_specs_naming_another_sdk_fall_back_to_the_default(self): + # A run measuring go and java with refs for go only: java still gets + # its own default pair rather than an error or go's builds. + assert bench._specs_for(("go@main", "go@a"), "java") is None + + def test_a_mixed_sdk_list_is_an_error(self): + with pytest.raises(bench.ArmSelectionError, match="more than one SDK"): + bench._specs_for(("go@main", "java@main"), "go") + + +class TestArmOptions: + def test_no_options_means_the_default_pair(self): + # The nightly passes nothing at all, and must keep getting the + # (newest release, branch head) comparison it has always run. + assert bench.arm_specs_from_options(options()) is None + assert bench.arm_count(options()) == 2 + + def test_the_two_arm_shorthand_still_works(self): + cfg = options(bench_baseline="go@v0.29.0", bench_candidate="go@main") + assert bench.arm_specs_from_options(cfg) == ("go@v0.29.0", "go@main") + + def test_half_a_pair_is_a_usage_error(self): + with pytest.raises(pytest.UsageError, match="together"): + bench.arm_specs_from_options(options(bench_baseline="go@main")) + + def test_mixing_the_two_forms_is_a_usage_error(self): + # There is no reading of "--bench-refs a,b --bench-candidate c" that + # is not a mistake, and silently picking one would measure something + # nobody asked for. + cfg = options(bench_refs="go@a,go@b", bench_candidate="go@c") + with pytest.raises(pytest.UsageError, match="cannot be combined"): + bench.arm_specs_from_options(cfg) + + def test_a_malformed_refs_list_is_a_usage_error(self): + with pytest.raises(pytest.UsageError, match="invalid --bench-refs"): + bench.arm_specs_from_options(options(bench_refs="go@main")) + + def test_the_arm_count_follows_the_refs(self): + assert bench.arm_count(options(bench_refs="go@a,go@b,go@c")) == 3 + + +class TestDefaultBudget: + def test_two_arms_keep_the_number_the_default_was_chosen_for(self): + assert bench.default_budget_seconds(2) == BenchConfig().budget_seconds + + def test_the_budget_scales_with_the_arm_count(self): + # A round costs one invocation per arm, so at a fixed budget the round + # count falls as 2/K and every interval widens as sqrt(K/2). Scaling + # the default by K/2 buys back the precision instead of quietly + # trading it for arms and reporting the loss as INCONCLUSIVE. + base = BenchConfig().budget_seconds + assert bench.default_budget_seconds(3) == base * 1.5 + assert bench.default_budget_seconds(4) == base * 2 + + def test_an_explicit_budget_is_taken_as_given(self): + cfg = FakeConfig( + bench_refs="go@a,go@b,go@c", + bench_budget_seconds="900", + bench_min_rounds="20", + bench_max_rounds="60", + bench_warmup="5", + bench_seed="1", + bench_threshold="1.15", + ) + built = bench.config_from_options(cast(pytest.Config, cfg)) + assert built.budget_seconds == 900.0, "a named number is not scaled" + class TestDistTagShape: def test_a_slashed_tag_breaks_discovery(self, cwd: Path): @@ -201,15 +328,120 @@ def test_the_control_size_does_not_follow_the_selection(self): assert control.payload == CONTROL_PAYLOAD +def stub_sdk(cwd: Path, version: str, **features: bool) -> tdfs.SDK: + """An installed stub build whose feature support is stated, not inferred. + + Real support is derived from the version string, which would make these + tests assertions about the version table rather than about arm selection. + """ + install(cwd, "go", version) + sdk = tdfs.SDK("go", version) + sdk._supports.update(cast(dict[tdfs.feature_type, bool], features)) + return sdk + + +#: Every comparability feature present, which is the uninteresting case. +COMPARABLE = {"hexless": True, "hexaflexible": True, "autoconfigure": True} + + +class TestComparability: + def test_matching_arms_are_comparable(self, cwd: Path): + arms = bench.BenchArms( + ( + stub_sdk(cwd, "main", **COMPARABLE), + stub_sdk(cwd, "a--impl", **COMPARABLE), + stub_sdk(cwd, "b--impl", **COMPARABLE), + ) + ) + assert bench.comparability_problem(arms) is None + + def test_every_candidate_is_checked_against_the_reference(self, cwd: Path): + # Checking only adjacent pairs would clear a third arm that disagrees + # with the reference, and its gated contrast is taken against exactly + # that reference -- so it would be timing different work. + odd_one_out = bench.BenchArms( + ( + stub_sdk(cwd, "main", **COMPARABLE), + stub_sdk(cwd, "a--impl", **COMPARABLE), + stub_sdk(cwd, "b--impl", **(COMPARABLE | {"autoconfigure": False})), + ) + ) + problem = bench.comparability_problem(odd_one_out) + assert problem is not None + assert "b--impl" in problem and "autoconfigure" in problem + + def test_the_target_mode_is_pinned_only_when_every_arm_can_be_told(self, cwd: Path): + # Letting one arm choose its own container version would compare + # output formats rather than speed. + all_new = ( + stub_sdk(cwd, "main", **COMPARABLE), + stub_sdk(cwd, "a--impl", **COMPARABLE), + ) + assert bench.pinned_target_mode(bench.BenchArms(all_new)) == "4.3.0" + one_old = all_new + ( + stub_sdk(cwd, "b--impl", **(COMPARABLE | {"hexaflexible": False})), + ) + assert bench.pinned_target_mode(bench.BenchArms(one_old)) is None + + +class TestBuildArms: + def cell_arms(self, cwd: Path, control: bool, n: int): + arms = bench.BenchArms( + tuple( + stub_sdk(cwd, v, **COMPARABLE) + for v in ("main", "a--impl", "b--impl", "c--impl")[:n] + ) + ) + cells = cells_for(["go"], parse_payloads("1KiB")) + cell = next( + c for c in cells if c.control is control and c.operation == "encrypt" + ) + pt = cwd / "plain.bin" + pt.write_bytes(b"x" * 1024) + return bench.build_arms( + cell, + arms, + pt_file=pt, + ct_file=None, + tmp_dir=cwd, + attr_values=[], + ) + + def test_one_arm_per_build(self, cwd: Path): + built = self.cell_arms(cwd, control=False, n=3) + assert [a.name for a in built] == ["main", "a--impl", "b--impl"] + + def test_every_arm_writes_its_own_output(self, cwd: Path): + # Sharing an output path would have the arms overwrite each other + # mid-round, and the second one would be measured deleting the first. + built = self.cell_arms(cwd, control=False, n=3) + assert len({a.invocation.output for a in built}) == 3 + + def test_the_control_is_k_copies_of_the_reference(self, cwd: Path): + # Not a cheap pair: in a K-arm round the last arm runs K-1 invocations + # after the first, so a two-arm control would measure less drift than + # the contrasts it is the noise floor for. + built = self.cell_arms(cwd, control=True, n=3) + assert len(built) == 3 + assert {a.label for a in built} == {"go@main"} + assert len({a.name for a in built}) == 3, "ids key the sample vectors" + assert len({a.invocation.output for a in built}) == 3 + + #: The fixture body, called directly: these tests are about the bytes it #: writes, not about pytest's fixture wiring. _make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] def make_payloads( - tmp_path: Path, config: BenchConfig, payloads: Sequence[Payload] = PAYLOADS + tmp_path: Path, + config: BenchConfig, + payloads: Sequence[Payload] = PAYLOADS, + *, + refs: str | None = None, ) -> dict[str, Path]: - return _make_payloads(tmp_path, config, tuple(payloads)) + request = cast(pytest.FixtureRequest, FakeRequest(FakeConfig(bench_refs=refs))) + return _make_payloads(request, tmp_path, config, tuple(payloads)) class TestPayloads: @@ -275,6 +507,21 @@ def test_a_payload_too_big_for_the_disk_is_refused_up_front(self, tmp_path: Path assert bench.disk_shortfall(tmp_path, [huge]) is not None assert bench.disk_shortfall(tmp_path, [Payload("1KiB", 1024)]) is None + def test_the_disk_estimate_grows_with_the_arm_count( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + # A cell holds one live output per arm, so a 1 GiB four-arm run needs + # two more GiB than the two-arm arithmetic accounts for -- which on a + # GitHub runner is the whole margin. + gib = 2**30 + payloads = [Payload("1GiB", gib)] + free = 2 * gib + 3 * gib + bench._DISK_HEADROOM_BYTES # exactly three arms + monkeypatch.setattr( + shutil, "disk_usage", lambda p: SimpleNamespace(total=0, used=0, free=free) + ) + assert bench.disk_shortfall(tmp_path, payloads, 3) is None + assert bench.disk_shortfall(tmp_path, payloads, 4) is not None + def read_all(paths: dict[str, Path]) -> dict[str, bytes]: return {label: p.read_bytes() for label, p in paths.items()} diff --git a/xtest/test_bench_report.py b/xtest/test_bench_report.py new file mode 100644 index 000000000..d80b7cb70 --- /dev/null +++ b/xtest/test_bench_report.py @@ -0,0 +1,195 @@ +"""Tests for what a benchmark run publishes: the JSON artifact and the summary. + +The report is where a K-arm run stops being a pile of ratios and starts being +an answer, so the things worth pinning down are the ones a reader would act on +without checking: which arm won a bake-off, whether a tie is reported as a tie, +and whether a run that could not resolve anything says so instead of returning +a quiet page of INCONCLUSIVE. + +Measurement is simulated exactly as in ``test_bench_runner``; nothing here +touches a platform or a subprocess. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path + +from perf import report, stats +from perf.runner import BenchConfig +from test_bench_runner import REF, config, run + + +def recorder( + costs: Mapping[str, float], + *, + cfg: BenchConfig | None = None, + noise: float = 0.05, + control: bool = True, +) -> report.BenchmarkRecorder: + """A recorder holding one measured cell and, by default, its A/A control.""" + cfg = cfg or config(max_rounds=40) + rec = report.BenchmarkRecorder() + if control: + aa, _ = run( + dict.fromkeys(costs, 1.0), + cfg=cfg, + noise=noise, + seed=11, + cell_id="aa", + control=True, + sdk="go", + ) + rec.record(aa) + measured, _ = run(costs, cfg=cfg, noise=noise, seed=12, cell_id="encrypt", sdk="go") + rec.record(measured) + return rec + + +def bake_offs( + costs: Mapping[str, float], *, cfg: BenchConfig | None = None, noise: float = 0.05 +) -> tuple[list[report.BakeOff], BenchConfig]: + cfg = cfg or config(max_rounds=40) + rec = recorder(costs, cfg=cfg, noise=noise) + return report.bake_offs(rec, cfg, rec.gate(cfg)), cfg + + +class TestBakeOff: + def test_a_two_arm_run_has_nothing_to_rank(self): + # One candidate is not a bake-off, and the gate has already said + # everything there is to say about it. + offs, _ = bake_offs({REF: 1.0, "cand": 1.3}) + assert offs == [] + + def test_the_control_is_never_ranked(self): + offs, _ = bake_offs({REF: 1.0, "a": 1.0, "b": 1.3}) + assert offs and all(b.cell_id == "encrypt" for b in offs) + + def test_the_faster_candidate_wins(self): + offs, _ = bake_offs({REF: 1.0, "slow": 1.4, "quick": 1.0}, noise=0.02) + wall = next(b for b in offs if b.metric == "wall") + assert wall.winner == "quick" + assert wall.order == ["quick", "slow"] + assert "`quick` wins" in wall.detail + + def test_a_tie_refuses_to_name_a_winner(self): + # Picking whichever point estimate landed lower would be reporting + # noise as a decision, and a merge would be made on it. + offs, _ = bake_offs({REF: 1.0, "a": 1.0, "b": 1.0}, noise=0.01) + wall = next(b for b in offs if b.metric == "wall") + assert wall.verdict is stats.Verdict.TIED + assert wall.winner is None + assert "no measurable difference" in wall.detail + + def test_an_unresolvable_pair_says_so_rather_than_guessing(self): + offs, _ = bake_offs( + {REF: 1.0, "a": 1.0, "b": 1.15}, + cfg=config(max_rounds=20), + noise=0.35, + ) + wall = next(b for b in offs if b.metric == "wall") + assert wall.winner is None + assert "cannot separate" in wall.detail + + def test_the_ranking_covers_every_gated_metric(self): + offs, cfg = bake_offs({REF: 1.0, "a": 1.0, "b": 1.4}, noise=0.02) + assert {b.metric for b in offs} == set(cfg.gated_metrics) + + +class TestUnderpoweredWarning: + def test_a_precise_run_says_nothing(self): + cfg = config(max_rounds=60) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.0}, cfg=cfg, noise=0.005) + assert report.underpowered_warning(rec, cfg, rec.gate(cfg)) is None + + def test_a_run_that_could_not_resolve_anything_asks_for_more_budget(self): + # Otherwise three arms on a two-arm budget comes back as a wall of + # INCONCLUSIVE after burning the whole runner, with nothing in the + # output saying that time was the missing ingredient. + cfg = config(max_rounds=20) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.0}, cfg=cfg, noise=0.4) + warning = report.underpowered_warning(rec, cfg, rec.gate(cfg)) + assert warning is not None + assert "3-arm round costs 3 invocations" in warning + assert "INCONCLUSIVE" in warning + + def test_the_warning_reaches_the_summary(self): + cfg = config(max_rounds=20) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.0}, cfg=cfg, noise=0.4) + md = report.markdown(rec, cfg, rec.gate(cfg)) + assert "[!WARNING]" in md and "Underpowered" in md + + +class TestJsonArtifact: + def artifact(self, tmp_path: Path, costs: Mapping[str, float]) -> dict: + cfg = config(max_rounds=40) + rec = recorder(costs, cfg=cfg, noise=0.02) + path = report.write_json(tmp_path / "bench.json", rec, cfg, rec.gate(cfg)) + return json.loads(path.read_text()) + + def test_arms_and_the_reference_are_recorded(self, tmp_path: Path): + # Which arm the ratios are taken against is not recoverable from the + # numbers, and every contrast in the file is meaningless without it. + doc = self.artifact(tmp_path, {REF: 1.0, "a": 1.0, "b": 1.3}) + cell = next(c for c in doc["cells"] if c["id"] == "encrypt") + assert cell["arms"] == [REF, "a", "b"] + assert cell["reference"] == REF + + def test_every_pair_appears_once(self, tmp_path: Path): + doc = self.artifact(tmp_path, {REF: 1.0, "a": 1.0, "b": 1.3}) + cell = next(c for c in doc["cells"] if c["id"] == "encrypt") + assert set(cell["contrasts"]) == {f"a_vs_{REF}", f"b_vs_{REF}", "b_vs_a"} + + def test_two_arm_readers_still_find_a_baseline_and_candidate(self, tmp_path: Path): + doc = self.artifact(tmp_path, {REF: 1.0, "cand": 1.3}) + cell = next(c for c in doc["cells"] if c["id"] == "encrypt") + assert doc["schema"] == 2 + assert cell["baseline"] == f"sdk@{REF}" + assert cell["candidate"] == "sdk@cand" + + def test_raw_samples_survive_for_every_arm(self, tmp_path: Path): + # Re-analysing a surprising result offline is the difference between + # understanding a red build and re-running the whole job to see the + # same numbers again. + doc = self.artifact(tmp_path, {REF: 1.0, "a": 1.0, "b": 1.3}) + cell = next(c for c in doc["cells"] if c["id"] == "encrypt") + assert set(cell["samples"]) == {REF, "a", "b"} + for arm in cell["samples"].values(): + assert len(arm["wall"]) == cell["n_rounds"] + + def test_the_bake_off_is_in_the_artifact(self, tmp_path: Path): + doc = self.artifact(tmp_path, {REF: 1.0, "slow": 1.4, "quick": 1.0}) + wall = next(b for b in doc["bake_off"] if b["metric"] == "wall") + assert wall["winner"] == "quick" + + def test_the_file_is_valid_json_despite_nan(self, tmp_path: Path): + # A cell with no usable interval produces NaN, which `json.dumps` + # would happily write as a bare `NaN` that no strict parser accepts. + cfg = config(min_rounds=stats.MIN_USABLE_ROUNDS, max_rounds=40) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.0}, cfg=cfg, control=False) + path = report.write_json(tmp_path / "b.json", rec, cfg, rec.gate(cfg)) + json.loads(path.read_text()) # strict by default: no NaN accepted + + +class TestMarkdown: + def test_the_header_names_the_arm_count(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.0}, cfg=cfg, noise=0.02) + md = report.markdown(rec, cfg, rec.gate(cfg)) + assert "3-arm" in md + + def test_the_table_names_both_sides_of_each_contrast(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "a": 1.0, "b": 1.3}, cfg=cfg, noise=0.02) + md = report.markdown(rec, cfg, rec.gate(cfg)) + assert "| contrast |" in md + assert "| `b` vs `a` |" in md, "the head-to-head is the point of a bake-off" + assert "| `a` vs `base` |" in md + + def test_a_bake_off_section_appears_only_with_candidates_to_rank(self): + cfg = config(max_rounds=40) + two = recorder({REF: 1.0, "cand": 1.3}, cfg=cfg, noise=0.02) + assert "### Bake-off" not in report.markdown(two, cfg, two.gate(cfg)) + three = recorder({REF: 1.0, "a": 1.0, "b": 1.3}, cfg=cfg, noise=0.02) + assert "### Bake-off" in report.markdown(three, cfg, three.gate(cfg)) diff --git a/xtest/test_bench_runner.py b/xtest/test_bench_runner.py index 3b9e9f4f1..09975349b 100644 --- a/xtest/test_bench_runner.py +++ b/xtest/test_bench_runner.py @@ -13,7 +13,7 @@ import math import random -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path import pytest @@ -27,6 +27,7 @@ BudgetExhausted, Invocation, analyze, + contrast_key, run_cell, ) @@ -34,14 +35,18 @@ BASELINE_RSS = 100_000_000 BASELINE_CPU = 0.8 +#: Arm id of the reference in every cell built here. +REF = "base" -def arm(role: str, key: str, output: Path | None = None) -> Arm: - """An arm whose argv is a single token, so ``FakeRuns`` can recognize it. - ``role`` is what the runner keys samples by ("baseline"/"candidate"); - ``key`` is the stand-in for the build. - """ - return Arm(role, f"sdk@{key}", Invocation([key], {}, output)) +def arm(name: str, output: Path | None = None) -> Arm: + """An arm whose argv is its own id, so ``FakeRuns`` can recognize it.""" + return Arm(name, f"sdk@{name}", Invocation([name], {}, output)) + + +def key(cell_id: str, metric: str, *, a: str = REF, b: str = "cand") -> str: + """The contrast key for ``b`` against ``a`` -- the default vs-reference.""" + return contrast_key(cell_id, a, b, metric) def config(**overrides: object) -> BenchConfig: @@ -108,7 +113,7 @@ def clock_from(runs: FakeRuns) -> Callable[[], float]: def run( - ratio: float, + ratios: float | Mapping[str, float], *, cfg: BenchConfig | None = None, noise: float = 0.05, @@ -118,14 +123,21 @@ def run( sdk: str = "", rss_floor: int = 0, ): - """Run one cell where the candidate costs ``ratio`` times the baseline.""" - runs = FakeRuns( - {"base": 1.0, "cand": ratio}, noise=noise, seed=seed, rss_floor=rss_floor + """Run one cell whose arms cost ``ratios`` times the reference. + + A bare float is the two-arm shorthand: a reference at 1.0 and one + candidate at that ratio. A mapping names the arms, and the first key is + the reference -- which is how a bake-off is set up here. + """ + costs = ( + {REF: 1.0, "cand": float(ratios)} + if isinstance(ratios, (int, float)) + else dict(ratios) ) + runs = FakeRuns(costs, noise=noise, seed=seed, rss_floor=rss_floor) result = run_cell( cell_id, - arm("baseline", "base"), - arm("candidate", "cand"), + [arm(name) for name in costs], cfg or config(), control=control, sdk=sdk, @@ -138,19 +150,47 @@ def run( class TestRoundLoop: def test_arms_are_paired_every_round(self): _, runs = run(1.0) - assert runs.calls.count("base") == runs.calls.count("cand") + assert runs.calls.count(REF) == runs.calls.count("cand") # Every consecutive pair holds one of each: that is what pairing means. pairs = [set(runs.calls[i : i + 2]) for i in range(0, len(runs.calls), 2)] - assert all(p == {"base", "cand"} for p in pairs) + assert all(p == {REF, "cand"} for p in pairs) + + def test_every_arm_runs_once_per_round_at_three_arms(self): + # The whole reason a bake-off is answerable: all three arms share a + # round, so the a-vs-b contrast is a within-round ratio like any other. + _, runs = run({REF: 1.0, "a": 1.1, "b": 1.2}) + rounds = [set(runs.calls[i : i + 3]) for i in range(0, len(runs.calls), 3)] + assert all(r == {REF, "a", "b"} for r in rounds) def test_order_within_rounds_is_shuffled(self): _, runs = run(1.0) firsts = runs.calls[::2] - assert "base" in firsts and "cand" in firsts, ( + assert REF in firsts and "cand" in firsts, ( "a fixed within-round order lets the second arm inherit the first " "one's cache state" ) + def test_all_three_arms_take_turns_going_first(self): + _, runs = run({REF: 1.0, "a": 1.0, "b": 1.0}) + assert set(runs.calls[::3]) == {REF, "a", "b"}, ( + "an arm pinned to one slot in the round inherits the same cache " + "state every time, which is a confounder and not a measurement" + ) + + def test_rejects_a_single_arm(self): + with pytest.raises(ValueError, match="at least two arms"): + run_cell("cell", [arm(REF)], config()) + + def test_rejects_colliding_arm_ids(self): + # Ids key the sample vectors, so a collision would interleave two + # builds' measurements into one arm and compare it with itself. + with pytest.raises(ValueError, match="unique"): + run_cell("cell", [arm(REF), arm(REF)], config()) + + def test_rejects_a_reference_that_is_not_an_arm(self): + with pytest.raises(ValueError, match="reference"): + run_cell("cell", [arm(REF), arm("cand")], config(), reference="other") + def test_warmup_rounds_are_discarded(self): cfg = config(warmup=3, max_rounds=stats.MIN_USABLE_ROUNDS) result, runs = run(1.0, cfg=cfg) @@ -173,10 +213,10 @@ def test_output_is_removed_before_each_run(self, tmp_path: Path): out = tmp_path / "out.tdf" out.write_bytes(b"stale") seen: list[bool] = [] - runs = FakeRuns({"base": 1.0, "cand": 1.0}) + runs = FakeRuns({REF: 1.0, "cand": 1.0}) def observe(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: - if argv[0] == "base": + if argv[0] == REF: # What the arm that owns this output sees when it starts. seen.append(out.exists()) out.write_bytes(b"produced") @@ -184,8 +224,7 @@ def observe(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: run_cell( "cell", - arm("baseline", "base", out), - arm("candidate", "cand"), + [arm(REF, out), arm("cand")], config(max_rounds=stats.MIN_USABLE_ROUNDS), clock=clock_from(runs), run=observe, @@ -194,10 +233,32 @@ def observe(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: def test_samples_are_collected_for_every_metric(self): result, _ = run(1.0) - for name in ("baseline", "candidate"): + for name in (REF, "cand"): for metric in ("wall", "cpu", "rss"): assert len(result.samples[name][metric]) == result.n_rounds + def test_records_its_arms_and_which_one_is_the_reference(self): + result, _ = run({REF: 1.0, "a": 1.0, "b": 1.0}) + assert result.arm_ids == (REF, "a", "b") + assert result.reference == REF + assert result.arm_labels == {REF: "sdk@base", "a": "sdk@a", "b": "sdk@b"} + + def test_contrast_pairs_cover_every_pair_once(self): + result, _ = run({REF: 1.0, "a": 1.0, "b": 1.0}) + # Reference contrasts first, then the head-to-head. A pair and its + # inverse are the same measurement, so only one of each appears. + assert result.contrast_pairs() == [(REF, "a"), (REF, "b"), ("a", "b")] + + def test_contrast_direction_is_b_over_a(self): + result, _ = run({REF: 1.0, "slow": 1.5}, noise=0.01) + cfg = config() + assert result.contrast(REF, "slow", "wall", cfg).ratio == pytest.approx( + 1.5, rel=0.1 + ) + assert result.contrast("slow", REF, "wall", cfg).ratio == pytest.approx( + 1 / 1.5, rel=0.1 + ) + class TestStopping: def test_stops_early_on_precision_when_quiet(self): @@ -215,13 +276,47 @@ def test_never_stops_before_min_rounds(self): result, _ = run(1.0, cfg=cfg, noise=0.0001) assert result.n_rounds >= 25 + def test_precision_waits_for_the_slowest_contrast_to_converge(self): + # One quiet candidate and one noisy one. Stopping as soon as *some* + # contrast is precise would leave the noisy arm unresolved with budget + # still on the table -- and at K arms the slowest contrast to converge + # is exactly the one someone is waiting on. + cfg = config(max_rounds=40) + quiet, _ = run({REF: 1.0, "cand": 1.0}, cfg=cfg, noise=0.005) + assert quiet.stopped_because == "precision" + + runs = FakeRuns({REF: 1.0, "quiet": 1.0, "noisy": 1.0}, noise=0.005) + real_call = runs.__call__ + + def jittery(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: + sample = real_call(argv, env, **kwargs) + if argv[0] != "noisy": + return sample + spike = math.exp(runs.rng.gauss(0.0, 0.5)) + return Sample( + wall_ns=int(sample.wall_ns * spike), + cpu_s=sample.cpu_s * spike, + max_rss_bytes=int(sample.max_rss_bytes * spike), + exit_code=0, + ) + + mixed = run_cell( + "cell", + [arm(REF), arm("quiet"), arm("noisy")], + cfg, + clock=clock_from(runs), + run=jittery, + ) + assert mixed.stopped_because == "max_rounds", ( + "the loop stopped on the quiet contrast and left the noisy one unresolved" + ) + def test_deadline_stops_the_loop(self): - runs = FakeRuns({"base": 1.0, "cand": 1.0}, noise=0.3) + runs = FakeRuns({REF: 1.0, "cand": 1.0}, noise=0.3) clock = clock_from(runs) result = run_cell( "cell", - arm("baseline", "base"), - arm("candidate", "cand"), + [arm(REF), arm("cand")], config(warmup=0, max_rounds=200), deadline=clock() + 60.0, # each round costs ~2 simulated seconds clock=clock, @@ -230,18 +325,34 @@ def test_deadline_stops_the_loop(self): assert result.stopped_because == "budget" assert result.elapsed_s <= 60.0, "a round we could not finish was started" + def test_a_three_arm_round_costs_three_invocations_of_budget(self): + # Rounds get more expensive as arms are added, which is the whole + # reason the default budget scales with K. + runs = FakeRuns({REF: 1.0, "a": 1.0, "b": 1.0}, noise=0.3) + clock = clock_from(runs) + result = run_cell( + "cell", + [arm(REF), arm("a"), arm("b")], + config(warmup=0, max_rounds=200), + deadline=clock() + 60.0, # each round now costs ~3 simulated seconds + clock=clock, + run=runs, + ) + assert result.stopped_because == "budget" + assert len(runs.calls) == 3 * result.n_rounds + assert result.n_rounds < 30, "three-arm rounds cost more than two-arm ones" + def test_warmup_gives_up_when_the_budget_runs_out(self): # The budget's end is absolute, so warm-ups that run past it are # spending the *following* cells' time -- and then reaching the # measured loop with nothing left, paying the whole cost of the cell # for no data at all. Stop at the deadline and say where it went. - runs = FakeRuns({"base": 1.0, "cand": 1.0}) + runs = FakeRuns({REF: 1.0, "cand": 1.0}) clock = clock_from(runs) with pytest.raises(BudgetExhausted, match="warm-up"): run_cell( "cell", - arm("baseline", "base"), - arm("candidate", "cand"), + [arm(REF), arm("cand")], config(warmup=10), deadline=clock() + 4.0, # each round costs ~2 simulated seconds clock=clock, @@ -250,13 +361,12 @@ def test_warmup_gives_up_when_the_budget_runs_out(self): assert len(runs.calls) < 2 * 10, "warm-up ran past its own deadline" def test_budget_below_min_usable_rounds_refuses_a_verdict(self): - runs = FakeRuns({"base": 1.0, "cand": 1.0}) + runs = FakeRuns({REF: 1.0, "cand": 1.0}) clock = clock_from(runs) with pytest.raises(BudgetExhausted, match="below the"): run_cell( "cell", - arm("baseline", "base"), - arm("candidate", "cand"), + [arm(REF), arm("cand")], config(warmup=0), deadline=clock() + 4.0, clock=clock, @@ -334,8 +444,8 @@ def gate(self, candidate_ratio: float, *, noise: float = 0.05, seed: int = 11): def test_planted_25_percent_slowdown_is_caught(self): gate = self.gate(1.25) assert gate.should_fail - assert "encrypt/wall" in gate.regressions - c = gate.comparisons["encrypt/wall"] + assert key("encrypt", "wall") in gate.regressions + c = gate.comparisons[key("encrypt", "wall")] assert c.verdict is stats.Verdict.REGRESSION assert c.ci_low > 1.15, "the interval must exclude the threshold, not just 1.0" assert c.ratio == pytest.approx(1.25, rel=0.1) @@ -343,7 +453,10 @@ def test_planted_25_percent_slowdown_is_caught(self): def test_planted_3_percent_slowdown_is_ignored(self): gate = self.gate(1.03) assert not gate.should_fail - assert gate.comparisons["encrypt/wall"].verdict is not stats.Verdict.REGRESSION + assert ( + gate.comparisons[key("encrypt", "wall")].verdict + is not stats.Verdict.REGRESSION + ) def test_no_effect_does_not_fire(self): gate = self.gate(1.0) @@ -353,8 +466,10 @@ def test_no_effect_does_not_fire(self): def test_planted_speedup_is_reported_not_failed(self): gate = self.gate(0.7) assert not gate.should_fail - assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.IMPROVED - assert "encrypt/wall" in gate.improvements + assert ( + gate.comparisons[key("encrypt", "wall")].verdict is stats.Verdict.IMPROVED + ) + assert key("encrypt", "wall") in gate.improvements def test_the_control_cell_never_fails_the_build(self): # Both arms of the control are the same build, so any verdict it @@ -369,9 +484,11 @@ def test_a_regression_in_an_ungated_metric_does_not_fail(self): gate = analyze([control, measured], cfg) # CPU time moved with everything else and is reported as such; it # simply is not allowed to turn the build red. - assert gate.comparisons["encrypt/cpu"].verdict is stats.Verdict.REGRESSION - assert "encrypt/cpu" not in gate.regressions - assert "encrypt/wall" in gate.regressions + assert ( + gate.comparisons[key("encrypt", "cpu")].verdict is stats.Verdict.REGRESSION + ) + assert key("encrypt", "cpu") not in gate.regressions + assert key("encrypt", "wall") in gate.regressions def test_rss_pinned_to_the_measurement_floor_cannot_report_pass(self): # A command whose peak sits at the floor is not measured, it is @@ -384,14 +501,14 @@ def test_rss_pinned_to_the_measurement_floor_cannot_report_pass(self): measured, _ = run(1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=floor) gate = analyze([control, measured], cfg) - rss = gate.comparisons["encrypt/rss"] + rss = gate.comparisons[key("encrypt", "rss")] assert rss.ratio == pytest.approx(1.0), "the floor clipped both arms" assert rss.verdict is stats.Verdict.INCONCLUSIVE assert "floor" in rss.note - assert "encrypt/rss" not in gate.regressions - assert "encrypt/rss" not in gate.improvements + assert key("encrypt", "rss") not in gate.regressions + assert key("encrypt", "rss") not in gate.improvements # Wall clock is untouched by a memory floor and still does its job. - assert "encrypt/wall" in gate.regressions + assert key("encrypt", "wall") in gate.regressions def test_rss_above_the_floor_is_still_gated(self): cfg = config(max_rounds=40) @@ -402,7 +519,7 @@ def test_rss_above_the_floor_is_still_gated(self): 1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=BASELINE_RSS // 10 ) gate = analyze([control, measured], cfg) - assert "encrypt/rss" in gate.regressions + assert key("encrypt", "rss") in gate.regressions def test_each_sdk_is_judged_against_its_own_control(self): # One control per SDK: they are different harness paths with different @@ -433,9 +550,10 @@ def test_each_sdk_is_judged_against_its_own_control(self): gate = analyze([go_aa, go_cell, java_aa, java_cell], cfg) assert len(gate.noise_by_control) == 2, "one noise floor per SDK" - assert gate.comparisons["go-encrypt/wall"].verdict is stats.Verdict.PASS + assert gate.comparisons[key("go-encrypt", "wall")].verdict is stats.Verdict.PASS assert ( - gate.comparisons["java-encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + gate.comparisons[key("java-encrypt", "wall")].verdict + is stats.Verdict.INCONCLUSIVE ), "java's own control had no power, whatever go's control managed" def test_a_run_with_no_control_cannot_report_pass(self): @@ -443,5 +561,101 @@ def test_a_run_with_no_control_cannot_report_pass(self): measured, _ = run(1.0, cfg=cfg, cell_id="encrypt") gate = analyze([measured], cfg) assert gate.noise is not None and gate.noise.underpowered - assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + assert ( + gate.comparisons[key("encrypt", "wall")].verdict + is stats.Verdict.INCONCLUSIVE + ) assert not gate.should_fail, "an unassessed run warns; it does not fail" + + +class TestGateAtThreeArms: + """What the extra arms buy, and what they must not be allowed to do. + + Every contrast here is a within-round ratio measured on one runner, which + is the only reason a candidate-versus-candidate question is answerable at + all. But only the vs-reference contrasts may fail a build: a bake-off + ranks implementations, it does not decide whether the branch is shippable. + """ + + def gate(self, costs: Mapping[str, float], *, noise: float = 0.05, seed: int = 11): + cfg = config(max_rounds=40) + control_costs = dict.fromkeys(costs, 1.0) + control, _ = run( + control_costs, cfg=cfg, noise=noise, seed=seed, cell_id="aa", control=True + ) + measured, _ = run(costs, cfg=cfg, noise=noise, seed=seed + 1, cell_id="encrypt") + return analyze([control, measured], cfg) + + def test_both_candidates_are_gated_against_the_reference(self): + gate = self.gate({REF: 1.0, "slow": 1.3, "quick": 1.0}) + assert gate.should_fail + assert key("encrypt", "wall", b="slow") in gate.regressions + assert key("encrypt", "wall", b="quick") not in gate.regressions + + def test_a_head_to_head_gap_never_fails_the_build(self): + # Neither candidate regressed against the reference; one is simply + # slower than the other. That is a ranking, and rankings do not turn + # the build red -- invariant #9. + gate = self.gate({REF: 1.3, "slow": 1.3, "quick": 1.0}) + h2h = key("encrypt", "wall", a="slow", b="quick") + assert gate.comparisons[h2h].verdict is stats.Verdict.FASTER + assert not gate.should_fail + assert h2h not in gate.regressions and h2h not in gate.improvements + + def test_a_head_to_head_is_judged_symmetrically(self): + # The one-sided vocabulary would call this PASS or REGRESSION, both of + # which presume an incumbent. Between two candidates there is none. + gate = self.gate({REF: 1.0, "a": 1.0, "b": 1.3}) + h2h = gate.comparisons[key("encrypt", "wall", a="a", b="b")] + assert h2h.verdict is stats.Verdict.SLOWER + assert h2h.verdict not in {stats.Verdict.PASS, stats.Verdict.REGRESSION} + + def test_indistinguishable_candidates_are_tied_not_passed(self): + # PASS is a one-sided claim -- "not slower". For a bake-off the answer + # worth reporting is that the two are the same, and TIED says so. + gate = self.gate({REF: 1.0, "a": 1.0, "b": 1.0}, noise=0.01) + assert ( + gate.comparisons[key("encrypt", "wall", a="a", b="b")].verdict + is stats.Verdict.TIED + ) + + def test_head_to_head_covers_ungated_metrics_too(self): + # "Which arm is faster" has no privileged direction on cpu either, and + # none of these gate, so they all share the symmetric vocabulary. + gate = self.gate({REF: 1.0, "a": 1.0, "b": 1.3}) + assert gate.comparisons[key("encrypt", "cpu", a="a", b="b")].verdict in { + stats.Verdict.FASTER, + stats.Verdict.SLOWER, + stats.Verdict.TIED, + stats.Verdict.INCONCLUSIVE, + } + + def test_a_decided_head_to_head_is_recorded_for_ranking(self): + gate = self.gate({REF: 1.0, "slow": 1.4, "quick": 1.0}) + # ``ranked`` carries the material a report turns into a winner; it is + # keys only, because at this layer an arm id is opaque. + assert key("encrypt", "wall", a="slow", b="quick") in gate.ranked + assert not any( + k in gate.regressions or k in gate.improvements for k in gate.ranked + ) + + def test_the_control_yields_one_contrast_per_pair(self): + gate = self.gate({REF: 1.0, "a": 1.0, "b": 1.0}) + aa = [ + k for k in gate.comparisons if k.startswith("aa/") and k.endswith("/wall") + ] + assert len(aa) == 3, "C(3,2) pairs, not one -- arm 3 drifts further than arm 2" + + def test_the_noise_floor_is_the_worst_control_pair(self): + # A 2-arm control measures adjacent invocations only, and so understates + # the drift carried by the widest contrast the run actually judges. + gate = self.gate({REF: 1.0, "a": 1.0, "b": 1.0}) + aa = [ + k for k in gate.comparisons if k.startswith("aa/") and k.endswith("/wall") + ] + assert len(gate.noise_by_control) == 1, "exactly one of the pairs is the floor" + assert gate.noise is not None + widest = max( + stats.assess_noise_floor(gate.comparisons[k]).width_ratio for k in aa + ) + assert gate.noise.width_ratio == pytest.approx(widest) diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py index 71a94d9ae..2001a10a2 100644 --- a/xtest/test_bench_stats.py +++ b/xtest/test_bench_stats.py @@ -338,6 +338,51 @@ def test_ungated_metric_is_reported_but_cannot_fail(self): assert g.comparisons["cpu"].verdict is Verdict.REGRESSION assert g.regressions == ["wall"], "cpu is reported but never gates" + def test_the_three_families_are_corrected_separately(self): + # A bake-off adds head-to-head contrasts that cannot fail the build. + # Folding them into the gate's family would raise every gated p-value + # for the sake of tests nobody gates on, which is precisely the trade + # the gated/ungated split already refuses to make. + rng = np.random.default_rng(41) + b, c = synth(rng, 1.4, n=60) + regressed = stats.compare(b, c, seed=41, n_resamples=RESAMPLES) + cells = {"wall": regressed, "control": quiet_control()} + alone = stats.apply_multiplicity_control( + cells, gated={"wall"}, controls=all_under_one_control(cells) + ) + + crowded = dict(cells) + for i in range(10): + r = np.random.default_rng(4100 + i) + x, y = synth(r, 1.0, n=60) + crowded[f"h2h{i}"] = stats.compare(x, y, seed=i, n_resamples=RESAMPLES) + with_h2h = stats.apply_multiplicity_control( + crowded, + gated={"wall"}, + symmetric={f"h2h{i}" for i in range(10)}, + controls=all_under_one_control(crowded), + ) + assert with_h2h.comparisons["wall"].p_adjusted == pytest.approx( + alone.comparisons["wall"].p_adjusted + ), "ten head-to-heads must not dilute the one contrast that can gate" + assert with_h2h.regressions == ["wall"] + + def test_a_symmetric_key_that_is_also_gated_stays_gated(self): + # Only reachable through a caller bug, and the safe resolution is the + # rule that can still fail the build rather than the one that cannot. + rng = np.random.default_rng(42) + b, c = synth(rng, 1.4, n=60) + cells = {"wall": stats.compare(b, c, seed=42, n_resamples=RESAMPLES)} + cells["control"] = quiet_control() + g = stats.apply_multiplicity_control( + cells, + gated={"wall"}, + symmetric={"wall"}, + controls=all_under_one_control(cells), + ) + assert g.comparisons["wall"].verdict is Verdict.REGRESSION + assert g.regressions == ["wall"] + def test_empty_run_reports_no_regressions(self): g = stats.apply_multiplicity_control({}) assert not g.should_fail, "nothing measured is not a regression" @@ -359,3 +404,119 @@ def test_a_run_with_comparisons_measured_something(self): stats.compare(b, c, seed=33, n_resamples=RESAMPLES), control=quiet_control() ) assert not g.nothing_measured + + +class TestSymmetricVerdicts: + """The bake-off rule: which of two candidates is faster, if either. + + Neither arm is an incumbent, so the one-sided vocabulary does not apply. + PASS would let a slower arm read as a clean result, and REGRESSION would + imply the other arm was the thing that changed. + """ + + def h2h( + self, + true_ratio: float, + *, + n: int = 60, + seed: int = 51, + sigma: float = NOISE_SIGMA, + ): + rng = np.random.default_rng(seed) + a, b = synth(rng, true_ratio, n=n, sigma=sigma) + cells = { + "h2h": stats.compare(a, b, seed=seed, n_resamples=RESAMPLES), + "control": quiet_control(), + } + g = stats.apply_multiplicity_control( + cells, + gated=set(), + symmetric={"h2h"}, + controls=all_under_one_control(cells), + ) + return g, g.comparisons["h2h"] + + def test_a_clearly_slower_arm_is_slower(self): + _, c = self.h2h(1.5) + assert c.verdict is Verdict.SLOWER + + def test_a_clearly_faster_arm_is_faster(self): + _, c = self.h2h(1 / 1.5) + assert c.verdict is Verdict.FASTER + + def test_indistinguishable_arms_are_tied(self): + # A positive finding, and the most likely honest answer for two + # implementations of the same idea. Not PASS: that is a one-sided + # claim about an incumbent that does not exist here. + _, c = self.h2h(1.0, n=120, sigma=0.02) + assert c.verdict is Verdict.TIED + assert 1 / 1.15 < c.ci_low and c.ci_high < 1.15 + + def test_an_interval_straddling_the_band_edge_is_inconclusive(self): + # A real but unresolved difference. Calling it TIED would claim an + # equivalence the interval does not support. + _, c = self.h2h(1.15, n=20, sigma=0.15) + assert c.verdict is Verdict.INCONCLUSIVE + assert "cannot be ranked" in c.note + + def test_a_head_to_head_never_gates(self): + g, c = self.h2h(1.5) + assert c.verdict is Verdict.SLOWER + assert not g.should_fail, "a bake-off ranks; it does not fail the build" + assert g.regressions == [] and g.improvements == [] + + def test_a_decided_head_to_head_is_ranked(self): + g, _ = self.h2h(1.5) + assert g.ranked == ["h2h"] + + def test_a_tie_is_not_ranked(self): + # Nothing to rank: naming a winner from a tie is the failure mode this + # verdict exists to prevent. + g, c = self.h2h(1.0, n=120, sigma=0.02) + assert c.verdict is Verdict.TIED + assert g.ranked == [] + + def test_without_a_noise_floor_nothing_is_ranked(self): + # Invariant 4 covers TIED as much as PASS: a tie nobody had the power + # to tell from a difference is not a tie. + rng = np.random.default_rng(52) + a, b = synth(rng, 1.0, n=120, sigma=0.02) + cells = {"h2h": stats.compare(a, b, seed=52, n_resamples=RESAMPLES)} + g = stats.apply_multiplicity_control(cells, gated=set(), symmetric={"h2h"}) + assert g.comparisons["h2h"].verdict is Verdict.INCONCLUSIVE + assert g.ranked == [] + + +class TestWorstControlKey: + """A K-arm control yields C(K,2) A/A contrasts, and they are not equal. + + Arm 3 runs two invocations after arm 1, so it carries more within-round + drift. Taking whichever came first would let dict ordering decide how + noisy the run is allowed to look. + """ + + def controls(self) -> dict[str, stats.PairedComparison]: + rng = np.random.default_rng(61) + tight_b, tight_c = synth(rng, 1.0, n=120, sigma=0.02) + wide_b, wide_c = synth(rng, 1.0, n=25, sigma=0.25) + return { + "tight": stats.compare(tight_b, tight_c, seed=61, n_resamples=RESAMPLES), + "wide": stats.compare(wide_b, wide_c, seed=62, n_resamples=RESAMPLES), + } + + def test_the_widest_contrast_is_the_floor(self): + c = self.controls() + assert stats.worst_control_key(c, ["tight", "wide"]) == "wide" + + def test_the_answer_does_not_depend_on_input_order(self): + c = self.controls() + assert stats.worst_control_key(c, ["wide", "tight"]) == "wide" + + def test_a_missing_contrast_is_worse_than_any_real_one(self): + # An absent control is not a quiet one; `assess_noise_floor(None)` + # calls it underpowered, and that must win over a real interval. + c = self.controls() + assert stats.worst_control_key(c, ["tight", "absent"]) == "absent" + + def test_no_keys_means_no_floor(self): + assert stats.worst_control_key({}, []) is None diff --git a/xtest/test_benchmarks.py b/xtest/test_benchmarks.py index 0e4197455..e7b603389 100644 --- a/xtest/test_benchmarks.py +++ b/xtest/test_benchmarks.py @@ -1,7 +1,8 @@ """SDK performance regression cells. -One test per cell: an operation at a payload size, measuring the newest -installed release against the branch build on the same runner, in the same +One test per cell: an operation at a payload size, measuring every arm of the +run -- by default the newest installed release against the branch build, and +in a bake-off several candidates at once -- on the same runner, in the same round, in a randomized order. **These tests do not assert.** Each one records its raw samples and passes. @@ -66,7 +67,7 @@ def bail(reason: str) -> NoReturn: if bench_cell.operation == "decrypt" else None ) - baseline, candidate = bench.build_arms( + cell_arms = bench.build_arms( bench_cell, arms, pt_file=bench_payloads[bench_cell.payload.label], @@ -78,8 +79,7 @@ def bail(reason: str) -> NoReturn: try: result = runner.run_cell( bench_cell.id, - baseline, - candidate, + cell_arms, bench_config, deadline=bench_budget.next_deadline(), control=bench_cell.control, From 3d0620730ed4e248bd2348be2bdc71a7fee16e5e Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 26 Aug 2026 11:01:14 -0400 Subject: [PATCH 09/11] feat(xtest): make benchmark summaries decision-first --- .github/workflows/xtest.yml | 94 +++++- xtest/conftest.py | 11 +- xtest/fixtures/bench.py | 64 +++- xtest/perf/README.md | 71 ++++- xtest/perf/aggregate.py | 266 ++++++++++++++++ xtest/perf/report.py | 584 +++++++++++++++++++++++++++++++--- xtest/perf/stats.py | 92 ++++-- xtest/test_bench_aggregate.py | 114 +++++++ xtest/test_bench_arms.py | 54 ++++ xtest/test_bench_report.py | 109 +++++++ xtest/test_bench_stats.py | 28 +- 11 files changed, 1389 insertions(+), 98 deletions(-) create mode 100644 xtest/perf/aggregate.py create mode 100644 xtest/test_bench_aggregate.py diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 568ce9f3e..3937dcbcc 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -1206,6 +1206,15 @@ jobs: # Already defaulted (and scaled by arm count) in resolve-versions. BENCH_BUDGET_SECONDS: ${{ needs.resolve-versions.outputs.bench-budget-seconds }} BENCH_MAX_ROUNDS: ${{ inputs.bench-max-rounds || '60' }} + # Structured resolver output: original alias, immutable SHA, PR, + # release tag, and source/head classification. The report turns it + # into links instead of guessing provenance from a dist directory. + BENCH_VERSION_INFO: >- + ${{ steps.bench-arms.outputs.version-info + || needs.resolve-versions.outputs[matrix.sdk] }} + # pytest writes a summary template beside the JSON. Publish it only + # after upload-artifact gives us the direct evidence URL. + BENCH_DEFER_SUMMARY: "1" PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main @@ -1223,15 +1232,43 @@ jobs: # surprising result offline beats re-running a 30-minute job to look at # the same numbers again. - name: Upload benchmark results + id: benchmark-results uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: success() || failure() with: - name: ${{ job.status == 'success' && '✅' || '❌' }} bench-${{ matrix.sdk }} + name: bench-result-${{ matrix.sdk }} path: | otdftests/xtest/test-results/benchmarks/*.json + otdftests/xtest/test-results/benchmarks/*.summary.md otdftests/xtest/test-results/*.html if-no-files-found: warn + - name: Publish benchmark summary + if: always() + shell: bash + env: + ARTIFACT_URL: ${{ steps.benchmark-results.outputs.artifact-url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SUMMARY_DIR: otdftests/xtest/test-results/benchmarks + run: |- + python3 - <<'PY' + import os + from pathlib import Path + + summaries = sorted(Path(os.environ["SUMMARY_DIR"]).glob("*.summary.md")) + target = Path(os.environ["GITHUB_STEP_SUMMARY"]) + if not summaries: + target.write_text( + "## SDK performance benchmark — NO SUMMARY\n\n" + "> The benchmark stopped before session-final reporting. " + f"[Inspect the workflow run]({os.environ['RUN_URL']}).\n" + ) + else: + evidence = os.environ.get("ARTIFACT_URL") or f"{os.environ['RUN_URL']}#artifacts" + text = summaries[0].read_text().replace("@@BENCH_ARTIFACT_URL@@", evidence) + target.write_text(text) + PY + - name: Upload server logs on failure uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: failure() @@ -1240,6 +1277,61 @@ jobs: path: ${{ steps.run-platform.outputs.platform-log-file }} if-no-files-found: ignore + benchmark-summary: + name: Performance benchmark roll-up + runs-on: ubuntu-latest + needs: [resolve-versions, bench] + if: >- + always() && ( + github.event.schedule == '30 6 * * *' || + ((github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') + && inputs.run-benchmarks) + ) + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: opentdf/tests + path: otdftests + persist-credentials: false + + - name: Download SDK benchmark results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + pattern: bench-result-* + path: benchmark-results + merge-multiple: true + + - name: Resolve benchmark artifact links + id: benchmark-artifacts + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #v9.0.0 + with: + script: | + const { owner, repo } = context.repo; + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { owner, repo, run_id: context.runId, per_page: 100 }, + ); + const urls = {}; + for (const artifact of artifacts) { + const match = artifact.name.match(/^bench-result-(.+)$/); + if (match) { + urls[match[1]] = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}/artifacts/${artifact.id}`; + } + } + core.setOutput('urls', JSON.stringify(urls)); + + - name: Publish combined benchmark summary + env: + BENCH_ARTIFACT_URLS: ${{ steps.benchmark-artifacts.outputs.urls }} + BENCH_EXPECTED_SDKS: ${{ needs.resolve-versions.outputs.bench-sdks }} + BENCH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: |- + python3 otdftests/xtest/perf/aggregate.py benchmark-results >> "$GITHUB_STEP_SUMMARY" + publish-results: runs-on: ubuntu-latest needs: xct diff --git a/xtest/conftest.py b/xtest/conftest.py index 4c9896086..ac09598ce 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -443,8 +443,15 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): out_dir / f"{name}.json", recorder, bench_config, gate ) - summary = report.markdown(recorder, bench_config, gate) - report.append_step_summary(summary) + artifact_url = ( + report.ARTIFACT_URL_PLACEHOLDER + if os.environ.get("BENCH_DEFER_SUMMARY", "").lower() in {"1", "true", "yes"} + else "" + ) + summary = report.markdown(recorder, bench_config, gate, artifact_url=artifact_url) + report.write_markdown(out_dir / f"{name}.summary.md", summary) + if not artifact_url: + report.append_step_summary(summary) reporter = config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: reporter.write_sep("=", "benchmark results") diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py index 064b9e419..e3f619506 100644 --- a/xtest/fixtures/bench.py +++ b/xtest/fixtures/bench.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import os import platform import random @@ -599,7 +600,8 @@ def runner_metadata(config: pytest.Config) -> dict[str, object]: here feeds the decision rule. It is recorded so that a human reading an old artifact can tell what they are looking at. """ - return { + metadata: dict[str, object] = { + "sdk": os.environ.get("BENCH_SDK", ""), "python": platform.python_version(), "platform": platform.platform(), "processor": platform.processor() or "unknown", @@ -607,9 +609,69 @@ def runner_metadata(config: pytest.Config) -> dict[str, object]: "runner_os": os.environ.get("RUNNER_OS", ""), "runner_arch": os.environ.get("RUNNER_ARCH", ""), "github_run_id": os.environ.get("GITHUB_RUN_ID", ""), + "github_run_url": _github_run_url(), "platform_version": _platform_version(), "seed": config.getoption("--bench-seed"), } + sources, warning = _arm_sources() + metadata["arm_sources"] = sources + if warning: + metadata["arm_sources_warning"] = warning + return metadata + + +def _github_run_url() -> str: + server = os.environ.get("GITHUB_SERVER_URL", "") + repository = os.environ.get("GITHUB_REPOSITORY", "") + run_id = os.environ.get("GITHUB_RUN_ID", "") + if not (server and repository and run_id): + return "" + return f"{server}/{repository}/actions/runs/{run_id}" + + +def _arm_sources() -> tuple[list[dict[str, object]], str]: + """Resolver metadata for the builds, enriched with their GitHub repository. + + CI already paid to resolve every ref to an immutable SHA before installing + it. Carry that result into the benchmark rather than trying to infer a PR, + release, or branch from the flattened dist-directory name afterward. + """ + raw = os.environ.get("BENCH_VERSION_INFO", "").strip() + if not raw: + return [], "" + try: + parsed = json.loads(raw) + except json.JSONDecodeError as e: + return [], f"BENCH_VERSION_INFO was not valid JSON: {e}" + if not isinstance(parsed, list) or not all(isinstance(v, dict) for v in parsed): + return [], "BENCH_VERSION_INFO must be a JSON array of objects" + + sources: list[dict[str, object]] = [] + for value in parsed: + source = {str(k): v for k, v in value.items()} + source["repo_url"] = _repo_url_for(source) + sources.append(source) + return sources, "" + + +def _repo_url_for(source: dict[str, object]) -> str: + sdk = str(source.get("sdk", "")) + if sdk == "java": + return "https://github.com/opentdf/java-sdk" + if sdk == "js": + return "https://github.com/opentdf/web-sdk" + if sdk != "go": + return "" + + # otdfctl moved into the platform monorepo at v0.31.0. Resolver results + # from there use the namespaced release tag; old standalone releases do + # not. Branch/PR/SHA builds resolve against platform first. + release = str(source.get("release", "")) + if release and not release.startswith("otdfctl/"): + match = re.search(r"v?(\d+)\.(\d+)\.(\d+)", release) + if match and tuple(map(int, match.groups())) < (0, 31, 0): + return "https://github.com/opentdf/otdfctl" + return "https://github.com/opentdf/platform" def _platform_version() -> str: diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 8c2574864..ef217230b 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -40,8 +40,9 @@ measure. | Artifact | Where | Contents | | --- | --- | --- | -| Job summary | The Actions run page | The table below, plus the verdict | -| `bench-` artifact | Run artifacts | `.json` with **every raw per-round sample**, and an HTML report | +| SDK job summary | Each matrix job | TL;DR, linked build provenance, attention rows, Unicode effect views, and run facts | +| Workflow roll-up | `Performance benchmark roll-up` job | One bottom line across Go, Java, and JS, with matrix health and links to each artifact | +| `bench-result-` artifact | Run artifacts | `.json` with **every raw per-round sample**, the rendered summary, and an HTML report | | Terminal | Job log tail | One-line summary and the JSON path | The JSON is the useful one. It holds each cell's full per-round vectors for @@ -50,9 +51,37 @@ re-running a 30-minute job to look at the same numbers again. It is `"schema": 2`: each cell carries `arms`, `reference`, and `contrasts` keyed `"_vs_"`. `baseline` and `candidate` are still there for readers that predate the K-arm schema, but past two arms they name only the reference and the *first* -candidate — use `arms` and `contrasts`. - -### The table +candidate — use `arms` and `contrasts`. Resolver metadata beside it records the +immutable commit and whether each arm came from a PR, branch, or release. + +### Reading the summary + +The summary is ordered for progressive disclosure: + +1. **TL;DR** gives the run status and counts of confirmed, unresolved, and + improved gated comparisons. +2. **Compared builds** links each arm to its PR, release/tag, commit, and GitHub + diff against the reference where those links exist. +3. **What changed** shows only regressions, inconclusive rows, and confirmed + improvements. Clean and ungated rows do not crowd the decision. +4. **Effect at a glance** puts those rows on one fixed log-ratio scale. `┆` + marks the practical threshold, `│` means no change, the bracket is the 95% + interval, and `●` is the point estimate. +5. Expandable blocks hold per-round Braille traces, all measurements and + controls, skipped cells, and the statistical rule. +6. **Run facts** closes the SDK summary with rounds, elapsed time, A/A noise, + platform/runner identity, seed, and a direct evidence link. +7. After every matrix job finishes, the workflow roll-up combines verdicts and + run health across SDKs. It deliberately does not compare absolute timings + from different runners. + +The Unicode plots are plain text, so they remain legible in copied comments, +logs, dark mode, and restricted GitHub Markdown without external image assets. + +### The full measurement table + +The complete table is collapsed by default under **All measurements, controls, +and statistical details**: ``` | cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict | @@ -71,7 +100,9 @@ candidate — use `arms` and `contrasts`. `b` is faster. - **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely this run could measure; a wide interval means a noisy runner, not a big change. -- **p (BH)** — one-sided p-value, Benjamini–Hochberg adjusted across the run. +- **p (BH)** — one-sided p-value in the direction of the observed effect, + Benjamini–Hochberg adjusted across the run. Slower and faster tails are + calculated and adjusted separately; the JSON records both. - **n** — paired rounds actually measured (20–60; the loop stops early once the interval is narrow enough). @@ -79,9 +110,10 @@ candidate — use `arms` and `contrasts`. **REGRESSION** — the CI lower bound exceeds the threshold (default **1.15x**, i.e. 15% slower) *and* the adjusted p < 0.05. Both clauses are required, and -neither is redundant: the threshold alone would fire on a reproducible 0.5% -slowdown nobody cares about, and significance alone would fire on noise often -enough to be ignored within a week. This fails the job. +neither is redundant: the CI establishes that the effect exceeds the practical +threshold but is not multiplicity-adjusted, while significance alone would +flag both reproducible 0.5% slowdowns nobody cares about and pure-noise false +positives. This fails the job. **PASS** — not a regression, *and* the run had enough precision to have found one. "We looked and found nothing" only counts when we could have found @@ -396,7 +428,8 @@ A 3-arm 1 GiB run wants roughly `--bench-payloads 1KiB,1GiB | `_launcher.py` | The separate process that actually forks the measured command | | `runner.py` | The paired round loop, the stopping rule, the budget, `analyze()` | | `stats.py` | Pure functions: log-ratios, bootstrap CI, Wilcoxon, BH, the decision rule | -| `report.py` | Session recorder, JSON artifact, step-summary markdown | +| `report.py` | Session recorder, JSON artifact, decision-first SDK summary markdown | +| `aggregate.py` | Pure-stdlib workflow roll-up over downloaded SDK JSON artifacts | | `../fixtures/bench.py` | The pytest glue: arm selection, payloads, ciphertexts, budget | | `../test_benchmarks.py` | One test per cell. **Records; never asserts** | | `../conftest.py` | `--bench*` options, cell parametrization, the session-finish gate | @@ -406,7 +439,8 @@ Offline tests, no platform and no subprocesses needed: ```bash cd xtest uv run pytest -q test_bench_stats.py test_bench_measure.py \ - test_bench_runner.py test_bench_arms.py test_bench_report.py + test_bench_runner.py test_bench_arms.py test_bench_report.py \ + test_bench_aggregate.py ``` These run on every PR via `check.yml`, so the harness is exercised continuously @@ -470,9 +504,18 @@ because a NaN width must read as "keep going" and `NaN > target` is `False`. #### Both clauses of the decision rule A cell is a regression iff the CI lower bound exceeds `threshold` **and** the -BH-adjusted p is below alpha. Clause 1 alone fires on real-but-trivial effects -measured precisely; clause 2 alone fires on noise roughly alpha of the time per -cell, and a run has enough cells that "roughly alpha" becomes "most nights". +BH-adjusted p is below alpha. Clause 1 establishes practical significance but +does not adjust the many intervals examined in a run. Clause 2 supplies +multiplicity control but, alone, fires on real-but-trivial effects and on +pure-noise false positives. Faster findings use a separately computed and +BH-adjusted lower-tail p-value; an adjusted upper-tail probability cannot be +read backwards as evidence for the opposite direction. + +The signed-rank test does not require normal raw latencies and is resistant to +the magnitude of a stray stalled invocation. Its location-test interpretation +does assume the paired *log differences* are approximately symmetric. The log +transform and the harness's multiplicative-jitter model are intended to make +that reasonable; the raw vectors remain in the artifact for checking it. #### The symmetric rule for head-to-heads diff --git a/xtest/perf/aggregate.py b/xtest/perf/aggregate.py new file mode 100644 index 000000000..dfae82af4 --- /dev/null +++ b/xtest/perf/aggregate.py @@ -0,0 +1,266 @@ +"""Pure-stdlib cross-SDK renderer for benchmark JSON artifacts. + +The workflow runs this after downloading every matrix artifact. Keeping it +free of pytest/numpy/scipy means the roll-up needs no environment setup and can +still explain a partially failed matrix whose artifact set is incomplete. +""" + +from __future__ import annotations + +import json +import math +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True, slots=True) +class Run: + sdk: str + document: dict[str, Any] + + @property + def gated(self) -> list[tuple[str, str, dict[str, Any]]]: + metrics = set(self.document.get("config", {}).get("gated_metrics", [])) + rows: list[tuple[str, str, dict[str, Any]]] = [] + for cell in self.document.get("cells", []): + if cell.get("control"): + continue + reference = cell.get("reference") + for name, by_metric in cell.get("contrasts", {}).items(): + if not name.endswith(f"_vs_{reference}"): + continue + for metric, comparison in by_metric.items(): + if metric in metrics: + rows.append((str(cell.get("id", "")), metric, comparison)) + return rows + + @property + def inconclusive(self) -> int: + return sum(c.get("verdict") == "INCONCLUSIVE" for _, _, c in self.gated) + + @property + def improvements(self) -> int: + return sum(c.get("verdict") == "IMPROVED" for _, _, c in self.gated) + + @property + def regressions(self) -> list[tuple[str, str, dict[str, Any]]]: + return [r for r in self.gated if r[2].get("verdict") == "REGRESSION"] + + @property + def status(self) -> str: + if self.document.get("nothing_measured") or not self.gated: + return "NOTHING MEASURED" + if not self.document.get("trustworthy", True): + return "UNTRUSTWORTHY" + if self.regressions: + return "REGRESSION" + if self.inconclusive: + return "INCONCLUSIVE" + return "PASS" + + +def load_runs(root: Path) -> list[Run]: + runs: list[Run] = [] + for path in root.rglob("*.json"): + try: + document = json.loads(path.read_text()) + except OSError, json.JSONDecodeError: + continue + if not isinstance(document, dict) or "config" not in document: + continue + metadata = document.get("metadata", {}) + sdk = str(metadata.get("sdk", "")) if isinstance(metadata, dict) else "" + if not sdk: + cells = document.get("cells", []) + cell_id = str(cells[0].get("id", "")) if cells else path.stem + sdk = cell_id.split("-", 1)[0] + runs.append(Run(sdk=sdk, document=document)) + order = {"go": 0, "java": 1, "js": 2} + return sorted(runs, key=lambda r: (order.get(r.sdk, 99), r.sdk)) + + +def markdown( + runs: list[Run], + *, + artifact_urls: dict[str, str] | None = None, + run_url: str = "", + expected_sdks: list[str] | None = None, +) -> str: + artifact_urls = artifact_urls or {} + expected = expected_sdks or [run.sdk for run in runs] + missing = [sdk for sdk in expected if sdk not in {run.sdk for run in runs}] + overall = _overall_status(runs, missing) + lines = [ + f"# SDK performance benchmark roll-up — {overall}", + "", + "### TL;DR", + "", + f"> **{_overall_headline(runs, missing)}**", + "", + ] + if run_url: + lines += [f"[Workflow run]({run_url})", ""] + if not runs: + return "\n".join( + lines + + [ + "> [!WARNING]", + "> No benchmark JSON artifacts were available. The matrix may have " + "failed before session-final reporting.", + "", + ] + ) + + lines += [ + "| SDK | outcome | compared builds | regressions | unresolved | improvements | A/A noise | evidence |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |", + ] + for run in runs: + noise = run.document.get("noise_floor", {}) + width = noise.get("width_ratio") if isinstance(noise, dict) else None + noise_text = ( + f"±{(float(width) - 1) * 100:.1f}%" + if isinstance(width, (int, float)) and math.isfinite(width) + else "—" + ) + evidence = ( + f"[artifact]({artifact_urls[run.sdk]})" + if artifact_urls.get(run.sdk) + else "—" + ) + lines.append( + f"| **{run.sdk}** | **{run.status}** | {_arms(run)} " + f"| {len(run.regressions)} | {run.inconclusive} | {run.improvements} " + f"| {noise_text} | {evidence} |" + ) + for sdk in missing: + lines.append(f"| **{sdk}** | **MISSING** | — | — | — | — | — | — |") + + regressions = [(run, *row) for run in runs for row in run.regressions] + if regressions: + lines += [ + "", + "### Confirmed regressions", + "", + "| SDK | measurement | metric | change (95% CI) |", + "| --- | --- | --- | --- |", + ] + for run, cell, metric, comparison in regressions: + lines.append(f"| {run.sdk} | `{cell}` | {metric} | {_change(comparison)} |") + + lines += [ + "", + "### Combined run facts", + "", + "| matrix results | measured cells | skipped cells | total measured time | platform versions |", + "| ---: | ---: | ---: | ---: | --- |", + f"| {len(runs)}/{len(expected)} | {sum(_measured_cells(r) for r in runs)} " + f"| {sum(len(r.document.get('skipped', {})) for r in runs)} " + f"| {sum(_elapsed(r) for r in runs):.0f}s " + f"| {', '.join(_platform_versions(runs)) or 'unknown'} |", + "", + "Each SDK was measured on its own runner. Absolute timings are not " + "compared across SDKs; this block combines verdicts and run health only.", + ] + return "\n".join(lines) + "\n" + + +def _overall_status(runs: list[Run], missing: list[str]) -> str: + for status in ("REGRESSION", "UNTRUSTWORTHY", "NOTHING MEASURED"): + if any(r.status == status for r in runs): + return status + if missing: + return "INCOMPLETE" + if any(r.status == "INCONCLUSIVE" for r in runs): + return "INCONCLUSIVE" + return "PASS" if runs else "NO RESULTS" + + +def _overall_headline(runs: list[Run], missing: list[str]) -> str: + if not runs: + return "No benchmark result artifacts were found." + regressions = sum(len(r.regressions) for r in runs) + unresolved = sum(r.inconclusive for r in runs) + outcomes = ", ".join(f"{r.sdk}: {r.status}" for r in runs) + missing_text = f" Missing result(s): {', '.join(missing)}." if missing else "" + return ( + f"{regressions} confirmed regression(s), {unresolved} unresolved gated " + f"comparison(s) across {len(runs)} available SDK result(s). {outcomes}." + f"{missing_text}" + ) + + +def _arms(run: Run) -> str: + sources = run.document.get("metadata", {}).get("arm_sources", []) + if isinstance(sources, list) and sources: + return " → ".join( + f"`{s.get('tag', '?')}`" for s in sources if isinstance(s, dict) + ) + cells = [c for c in run.document.get("cells", []) if not c.get("control")] + return " → ".join(f"`{a}`" for a in cells[0].get("arms", [])) if cells else "—" + + +def _change(comparison: dict[str, Any]) -> str: + def pct(value: object) -> str: + return ( + f"{(float(value) - 1) * 100:+.1f}%" + if isinstance(value, (int, float)) + else "—" + ) + + return ( + f"{pct(comparison.get('ratio'))} " + f"[{pct(comparison.get('ci_low'))}, {pct(comparison.get('ci_high'))}]" + ) + + +def _measured_cells(run: Run) -> int: + return sum(not c.get("control") for c in run.document.get("cells", [])) + + +def _elapsed(run: Run) -> float: + return sum(float(c.get("elapsed_s", 0)) for c in run.document.get("cells", [])) + + +def _platform_versions(runs: list[Run]) -> list[str]: + return sorted( + { + str(r.document.get("metadata", {}).get("platform_version", "")) + for r in runs + if r.document.get("metadata", {}).get("platform_version") + } + ) + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: aggregate.py RESULTS_DIR", file=sys.stderr) + return 2 + try: + urls = json.loads(os.environ.get("BENCH_ARTIFACT_URLS", "{}")) + except json.JSONDecodeError: + urls = {} + try: + expected = json.loads(os.environ.get("BENCH_EXPECTED_SDKS", "[]")) + except json.JSONDecodeError: + expected = [] + print( + markdown( + load_runs(Path(args[0])), + artifact_urls=urls if isinstance(urls, dict) else {}, + run_url=os.environ.get("BENCH_RUN_URL", ""), + expected_sdks=( + [str(sdk) for sdk in expected] if isinstance(expected, list) else [] + ), + ), + end="", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/xtest/perf/report.py b/xtest/perf/report.py index 1068e589f..aa0212a1b 100644 --- a/xtest/perf/report.py +++ b/xtest/perf/report.py @@ -21,8 +21,11 @@ import json import math import os +import statistics +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path +from urllib.parse import quote import pytest @@ -38,6 +41,10 @@ #: The session's recorder, reachable from both fixtures and session hooks. RECORDER_KEY: pytest.StashKey[BenchmarkRecorder] = pytest.StashKey() +# Replaced by the workflow after upload-artifact returns its authenticated URL. +# Kept conspicuous so a failed substitution cannot look like a real link. +ARTIFACT_URL_PLACEHOLDER = "@@BENCH_ARTIFACT_URL@@" + def recorder_for(config: pytest.Config) -> BenchmarkRecorder: """Return the session's recorder, creating it on first use.""" @@ -92,6 +99,19 @@ class BakeOff: detail: str +@dataclass(frozen=True, slots=True) +class ReportRow: + """One reportable contrast/metric with enough context to render it.""" + + result: CellResult + a: str + b: str + metric: str + comparison: stats.PairedComparison + gated: bool + head_to_head: bool + + def bake_offs( recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult ) -> list[BakeOff]: @@ -214,6 +234,7 @@ def to_dict( "noise_floor_by_control": { k: _noise_dict(n) for k, n in gate.noise_by_control.items() }, + "nothing_measured": gate.nothing_measured, "trustworthy": gate.trustworthy, "regressions": gate.regressions, "improvements": gate.improvements, @@ -273,8 +294,13 @@ def _comparison_dict(c: stats.PairedComparison) -> dict[str, object]: "ratio": _jsonable(c.ratio), "ci_low": _jsonable(c.ci_low), "ci_high": _jsonable(c.ci_high), + # `p_value`/`p_adjusted` retain their historical meaning: the + # one-sided "b is slower than a" tail. The faster tail is separate so + # consumers never have to reverse an adjusted upper-tail probability. "p_value": _jsonable(c.p_value), "p_adjusted": _jsonable(c.p_adjusted), + "p_value_faster": _jsonable(c.p_value_faster), + "p_adjusted_faster": _jsonable(c.p_adjusted_faster), "verdict": str(c.verdict), "note": c.note, } @@ -298,6 +324,13 @@ def write_json( return path +def write_markdown(path: Path, text: str) -> Path: + """Write a summary template for CI to publish after artifact upload.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def underpowered_warning( recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult ) -> str | None: @@ -347,41 +380,88 @@ def underpowered_warning( def markdown( - recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult + recorder: BenchmarkRecorder, + config: BenchConfig, + gate: stats.GateResult, + *, + artifact_url: str = "", ) -> str: - """Render the run as a GitHub step summary.""" - threshold_pct = (config.threshold - 1) * 100 + """Render a decision-first GitHub job summary with progressive disclosure.""" + rows = _report_rows(recorder, config, gate) + gated_rows = [r for r in rows if r.gated] + attention = [ + r + for r in gated_rows + if r.comparison.verdict + in ( + stats.Verdict.REGRESSION, + stats.Verdict.INCONCLUSIVE, + stats.Verdict.IMPROVED, + ) + ] + attention.sort(key=_attention_sort_key) + sdk = next((r.sdk for r in recorder.results if r.sdk), "SDK") + status, headline = _headline(gate, gated_rows) n_arms = max((len(r.arm_ids) for r in recorder.results), default=2) + lines = [ - "## SDK performance regression benchmark", + f"## {sdk.upper()} SDK performance — {status}", "", - f"Paired {n_arms}-arm comparison, all arms in the same rounds on one " - f"runner. A contrast against the reference fails only if the 95% CI " - f"lower bound exceeds **{config.threshold:.2f}x** (+{threshold_pct:.0f}%) " - f"*and* the BH-adjusted p < {stats.DEFAULT_ALPHA}.", + "### TL;DR", "", - f"**{gate.summary}**", + f"> **{headline}**", + ">", + f"> {gate.summary}", "", ] + lines += _quick_links(recorder.metadata, artifact_url) + lines += _provenance(recorder) warning = underpowered_warning(recorder, config, gate) if warning: lines += ["> [!WARNING]", f"> {warning}", ""] - noise = gate.noise if noise is not None and noise.detail: lines += [f"> {noise.detail}", ""] - elif noise is not None and math.isfinite(noise.width_ratio): + + lines += ["### What changed", ""] + if attention: + lines += [ + "Only gated regressions, unresolved measurements, and confirmed " + "improvements are shown here. Clean rows and diagnostic metrics are below.", + "", + "| measurement | contrast | observed cost | change (95% CI) | gate margin | result |", + "| --- | --- | --- | --- | ---: | --- |", + ] + for row in attention: + c = row.comparison + lines.append( + f"| {_cell_label(row.result)} · {METRIC_LABELS[row.metric][0]} " + f"| `{row.b}` vs `{row.a}` | {_cost_change(row.metric, c)} " + f"| {_percent_change(c)} | {_gate_margin(c, config.threshold)} " + f"| {_verdict_cell(c)} |" + ) + lines += _effect_overview(attention, config.threshold) + else: lines += [ - f"A/A noise floor: +/-{(noise.width_ratio - 1) * 100:.1f}% " - f"(the smallest effect this run could resolve).", + "No gated comparison requires attention: every measured wall-clock and " + "RSS contrast passed with enough precision.", "", ] - lines += [ - "| cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict |", - "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", - ] + lines += _bake_off_section(recorder, config, gate) + lines += _diagnostics(recorder, attention, config) + lines += _full_measurements(rows) + lines += _not_measured(recorder) + lines += _method(config, n_arms) + lines += _run_facts(recorder, config, gate, artifact_url) + return "\n".join(lines) + "\n" + + +def _report_rows( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> list[ReportRow]: + rows: list[ReportRow] = [] for result in recorder.results: for a, b in result.contrast_pairs(): head_to_head = result.reference not in (a, b) @@ -389,49 +469,441 @@ def markdown( c = gate.comparisons.get(contrast_key(result.cell_id, a, b, metric)) if c is None: continue - gated = ( - metric in config.gated_metrics - and not result.control - and not head_to_head - ) - label = METRIC_LABELS[metric][0] + ("" if gated else " (ungated)") - lines.append( - f"| {result.cell_id} | `{b}` vs `{a}` | {label} " - f"| {format_metric(metric, c.baseline_median)} " - f"| {format_metric(metric, c.candidate_median)} " - f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " - f"| {_verdict_cell(c)} |" + rows.append( + ReportRow( + result=result, + a=a, + b=b, + metric=metric, + comparison=c, + gated=( + metric in config.gated_metrics + and not result.control + and not head_to_head + ), + head_to_head=head_to_head, + ) ) + return rows + + +def _headline(gate: stats.GateResult, gated_rows: list[ReportRow]) -> tuple[str, str]: + inconclusive = sum( + r.comparison.verdict is stats.Verdict.INCONCLUSIVE for r in gated_rows + ) + improvements = sum( + r.comparison.verdict is stats.Verdict.IMPROVED for r in gated_rows + ) + if gate.nothing_measured: + return "NOTHING MEASURED", "The benchmark produced no comparisons." + if not gate.trustworthy: + return ( + "UNTRUSTWORTHY", + "The A/A control detected bias; results are visible but cannot fail the build.", + ) + if gate.regressions: + return ( + "REGRESSION", + f"{len(gate.regressions)} confirmed regression(s); " + f"{inconclusive} unresolved and {improvements} improved gated comparison(s).", + ) + if inconclusive: + return ( + "INCONCLUSIVE", + f"No confirmed regressions, but {inconclusive} gated comparison(s) " + "could not be resolved.", + ) + return ( + "PASS", + f"No confirmed regressions; {improvements} gated comparison(s) improved.", + ) + + +def _quick_links(metadata: Mapping[str, object], artifact_url: str) -> list[str]: + links: list[str] = [] + if artifact_url: + links.append(f"[Download raw samples and HTML report]({artifact_url})") + run_url = str(metadata.get("github_run_url", "")) + if run_url: + links.append(f"[Workflow run]({run_url})") + return [" · ".join(links), ""] if links else [] + + +def _provenance(recorder: BenchmarkRecorder) -> list[str]: + result = next((r for r in recorder.results if not r.control), None) + if result is None: + result = next(iter(recorder.results), None) + if result is None: + return [] + + raw_sources = recorder.metadata.get("arm_sources", []) + sources = ( + { + str(s.get("tag", "")): s + for s in raw_sources + if isinstance(s, dict) and s.get("tag") + } + if isinstance(raw_sources, list) + else {} + ) + lines = [ + "### Compared builds", + "", + "| arm | role | source | commit | compare to reference |", + "| --- | --- | --- | --- | --- |", + ] + reference_source = sources.get(result.reference) + for arm in result.arm_ids: + source = sources.get(arm) + role = "**reference**" if arm == result.reference else "candidate" + label = result.arm_labels.get(arm, arm) + source_link = _source_link(source, label) + commit_link = _commit_link(source) + compare_link = ( + "—" if arm == result.reference else _compare_link(reference_source, source) + ) + lines.append( + f"| `{_md(arm)}` | {role} | {source_link} | {commit_link} | {compare_link} |" + ) + lines.append("") + warning = recorder.metadata.get("arm_sources_warning") + if warning: + lines += [f"> Provenance unavailable: {_md(str(warning))}", ""] + return lines + + +def _source_link(source: object, fallback: str) -> str: + if not isinstance(source, dict): + return _md(fallback) + repo = str(source.get("repo_url", "")) + tag = str(source.get("tag", fallback)) + alias = str(source.get("alias", tag)) + pr = str(source.get("pr", "")) + release = str(source.get("release", "")) + sha = str(source.get("sha", "")) + if repo and pr: + return f"[PR #{_md(pr)}]({repo}/pull/{quote(pr, safe='')}) · `{_md(tag)}`" + if repo and release: + return f"[{_md(tag)}]({repo}/releases/tag/{quote(release, safe='')}) · release" + if repo and sha: + kind = "branch" if source.get("head") else "commit" + return f"[{_md(alias)}]({repo}/tree/{quote(sha, safe='')}) · {kind}" + return _md(fallback) + + +def _commit_link(source: object) -> str: + if not isinstance(source, dict): + return "—" + repo = str(source.get("repo_url", "")) + sha = str(source.get("sha", "")) + if not (repo and sha): + return "—" + return f"[`{_md(sha[:7])}`]({repo}/commit/{quote(sha, safe='')})" + + +def _compare_link(reference: object, candidate: object) -> str: + if not (isinstance(reference, dict) and isinstance(candidate, dict)): + return "—" + repo = str(reference.get("repo_url", "")) + if not repo or repo != str(candidate.get("repo_url", "")): + return "—" + a, b = str(reference.get("sha", "")), str(candidate.get("sha", "")) + if not (a and b): + return "—" + return f"[diff]({repo}/compare/{quote(a, safe='')}...{quote(b, safe='')})" + + +def _md(value: str) -> str: + return value.replace("|", "\\|").replace("\n", " ") + + +def _attention_sort_key(row: ReportRow) -> tuple[int, float, str, int]: + order = { + stats.Verdict.REGRESSION: 0, + stats.Verdict.INCONCLUSIVE: 1, + stats.Verdict.IMPROVED: 2, + } + ratio = row.comparison.ratio + magnitude = abs(math.log(ratio)) if math.isfinite(ratio) and ratio > 0 else 0 + return ( + order.get(row.comparison.verdict, 3), + -round(magnitude, 3), + row.result.cell_id, + METRICS.index(row.metric), + ) + + +def _cell_label(result: CellResult) -> str: + return result.cell_id.removeprefix(f"{result.sdk}-").replace("-", " / ") + +def _cost_change(metric: str, c: stats.PairedComparison) -> str: + if not (math.isfinite(c.baseline_median) and math.isfinite(c.candidate_median)): + return "—" + delta = c.candidate_median - c.baseline_median + sign = "+" if delta >= 0 else "−" + return ( + f"{format_metric(metric, c.baseline_median)} → " + f"{format_metric(metric, c.candidate_median)} " + f"({sign}{format_metric(metric, abs(delta))})" + ) + + +def _pct(ratio: float) -> str: + if not math.isfinite(ratio): + return "—" + return f"{(ratio - 1) * 100:+.1f}%" + + +def _percent_change(c: stats.PairedComparison) -> str: + point = _pct(c.ratio) + if not (math.isfinite(c.ci_low) and math.isfinite(c.ci_high)): + return point + return f"{point} [{_pct(c.ci_low)}, {_pct(c.ci_high)}]" + + +def _gate_margin(c: stats.PairedComparison, threshold: float) -> str: + if c.verdict is stats.Verdict.REGRESSION and math.isfinite(c.ci_low): + return f"+{(c.ci_low - threshold) * 100:.1f} pp" + if c.verdict is stats.Verdict.IMPROVED and math.isfinite(c.ci_high): + return f"+{(1 / threshold - c.ci_high) * 100:.1f} pp" + return "—" + + +def _effect_overview(rows: list[ReportRow], threshold: float) -> list[str]: + labels = [f"{_cell_label(r.result)} {METRIC_LABELS[r.metric][0]}" for r in rows] + label_width = min(30, max(map(len, labels), default=0)) + width = 57 + axis = [" "] * width + for text, start in ( + ("← faster", 0), + ("no change", (width - len("no change")) // 2), + ("slower →", width - len("slower →")), + ): + axis[start : start + len(text)] = text + lines = [ + "", + "#### Effect at a glance", + "", + "Fixed log-ratio scale; `┆` marks ±the practical threshold and `│` no change.", + "", + "```text", + f"{'measurement':<{label_width}} {''.join(axis)}", + ] + for label, row in zip(labels, rows, strict=True): + lines.append( + f"{label[:label_width]:<{label_width}} " + f"{_effect_strip(row.comparison, threshold, width=width)} " + f"{_pct(row.comparison.ratio):>7} {row.comparison.verdict}" + ) + lines += ["```", ""] + return lines + + +def _effect_strip( + c: stats.PairedComparison, threshold: float, *, width: int = 57 +) -> str: + """A fixed-width CI forest strip on a symmetric log-ratio scale.""" + chars = [" "] * width + # Show three threshold-widths in each direction. That keeps the practical + # boundary near the center while leaving enough room to draw the interval + # of an ordinary 20–40% regression instead of collapsing it to an arrow. + span = 3 * math.log(threshold) + + def pos(ratio: float) -> int: + value = math.log(ratio) if math.isfinite(ratio) and ratio > 0 else 0.0 + unit = max(-1.0, min(1.0, value / span)) + return round((unit + 1) * (width - 1) / 2) + + for ratio, marker in ((1 / threshold, "┆"), (1.0, "│"), (threshold, "┆")): + chars[pos(ratio)] = marker + if math.isfinite(c.ci_low) and math.isfinite(c.ci_high): + lo, hi = sorted((pos(c.ci_low), pos(c.ci_high))) + for i in range(lo, hi + 1): + if chars[i] == " ": + chars[i] = "━" + chars[lo], chars[hi] = "[", "]" + if math.isfinite(c.ratio) and c.ratio > 0: + chars[pos(c.ratio)] = "●" + if math.log(c.ratio) < -span: + chars[0] = "◀" + elif math.log(c.ratio) > span: + chars[-1] = "▶" + return "".join(chars) + + +def _bake_off_section( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> list[str]: rankings = bake_offs(recorder, config, gate) - if rankings: - lines += [ - "", - "### Bake-off", - "", - "Head-to-head between candidates, measured in the same rounds as " - "everything else. Ranking only -- these contrasts never fail the " - "build.", - "", - ] - for bo in rankings: - order = " < ".join(f"`{a}`" for a in bo.order) - lines.append( - f"- **{bo.cell_id}** ({METRIC_LABELS[bo.metric][0]}): " - f"{order} -- {bo.detail}" + if not rankings: + return [] + lines = [ + "### Bake-off", + "", + "Candidate-to-candidate ranking only; these contrasts never fail the build.", + "", + ] + for bo in rankings: + order = " < ".join(f"`{a}`" for a in bo.order) + lines.append( + f"- **{_cell_label(next(r for r in recorder.results if r.cell_id == bo.cell_id))}** " + f"({METRIC_LABELS[bo.metric][0]}): {order} — {bo.detail}" + ) + return lines + [""] + + +def _diagnostics( + recorder: BenchmarkRecorder, + attention: list[ReportRow], + config: BenchConfig, +) -> list[str]: + if not attention: + return [] + lines = [ + "
", + "Round stability for attention rows", + "", + "Each Braille glyph carries two rounds at four vertical levels. The scale " + "is centered on 1.0 and is never tighter than the practical threshold.", + "", + "```text", + ] + for row in attention: + values = _paired_ratios(row) + if not values: + continue + slower = sum(v > 1 for v in values) + label = f"{_cell_label(row.result)} {METRIC_LABELS[row.metric][0]}" + lines.append( + f"{label[:28]:<28} {_braille_sparkline(values, config.threshold):<24} " + f"{slower}/{len(values)} rounds slower; median {_pct(statistics.median(values))}" + ) + lines += ["```", "", "
", ""] + return lines + + +def _paired_ratios(row: ReportRow) -> list[float]: + baseline = row.result.samples.get(row.a, {}).get(row.metric, []) + candidate = row.result.samples.get(row.b, {}).get(row.metric, []) + return [c / b for b, c in zip(baseline, candidate, strict=True) if b > 0 and c > 0] + + +def _braille_sparkline( + values: Iterable[float], threshold: float, *, max_chars: int = 24 +) -> str: + """Render positive ratios as a compact two-samples-per-glyph trace.""" + logs = [math.log(v) for v in values if math.isfinite(v) and v > 0] + if not logs: + return "—" + limit = max_chars * 2 + if len(logs) > limit: + # Median bins retain the robust character of the reported estimator. + logs = [ + statistics.median( + logs[round(i * len(logs) / limit) : round((i + 1) * len(logs) / limit)] ) + for i in range(limit) + ] + span = max(math.log(threshold), max(abs(v) for v in logs), 1e-12) + dot_bits = ((0, 1, 2, 6), (3, 4, 5, 7)) + glyphs: list[str] = [] + for i in range(0, len(logs), 2): + bits = 0 + for column, value in enumerate(logs[i : i + 2]): + # Braille rows run top to bottom; positive/slower belongs at top. + row = round((span - value) / (2 * span) * 3) + row = max(0, min(3, row)) + bits |= 1 << dot_bits[column][row] + glyphs.append(chr(0x2800 + bits)) + return "".join(glyphs) + + +def _full_measurements(rows: list[ReportRow]) -> list[str]: + lines = [ + "
", + "All measurements, controls, and statistical details", + "", + "| cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict |", + "| --- | --- | --- | --- | --- | --- | --- | ---: | --- |", + ] + for row in rows: + c = row.comparison + label = METRIC_LABELS[row.metric][0] + ("" if row.gated else " (ungated)") + lines.append( + f"| {row.result.cell_id} | `{row.b}` vs `{row.a}` | {label} " + f"| {format_metric(row.metric, c.baseline_median)} " + f"| {format_metric(row.metric, c.candidate_median)} " + f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " + f"| {_verdict_cell(c)} |" + ) + return lines + ["", "
", ""] + + +def _not_measured(recorder: BenchmarkRecorder) -> list[str]: + if not recorder.skipped: + return [] + lines = [ + "
", + "Not measured", + "", + ] + lines += [f"- `{cid}`: {why}" for cid, why in sorted(recorder.skipped.items())] + return lines + ["", "
", ""] + + +def _method(config: BenchConfig, n_arms: int) -> list[str]: + threshold_pct = (config.threshold - 1) * 100 + return [ + "
", + "How to read this gate", + "", + f"This is a paired {n_arms}-arm comparison: every arm ran in the same " + "randomized rounds on one runner. A reference contrast regresses only " + f"when its 95% CI is wholly beyond +{threshold_pct:.0f}% " + f"and its directional BH-adjusted p-value is below {stats.DEFAULT_ALPHA}. " + "The loop stops on attained interval width, never significance.", + "", + "PASS means the run was precise enough to have found an effect at the " + "threshold; INCONCLUSIVE does not mean no change.", + "", + "
", + "", + ] - if recorder.skipped: - lines += ["", "### Not measured", ""] - lines += [f"- `{cid}`: {why}" for cid, why in sorted(recorder.skipped.items())] - lines += [ +def _run_facts( + recorder: BenchmarkRecorder, + config: BenchConfig, + gate: stats.GateResult, + artifact_url: str, +) -> list[str]: + rounds = [r.n_rounds for r in recorder.results if not r.control] + elapsed = sum(r.elapsed_s for r in recorder.results) + noise = gate.noise + noise_text = ( + f"±{(noise.width_ratio - 1) * 100:.1f}%" + if noise is not None and math.isfinite(noise.width_ratio) + else "unavailable" + ) + metadata = recorder.metadata + artifact = f"[JSON + HTML]({artifact_url})" if artifact_url else "local output" + round_text = f"{min(rounds)}–{max(rounds)}" if rounds else "0" + return [ + "### Run facts", + "", + "| result cells | rounds/cell | elapsed | A/A noise | platform | runner | evidence |", + "| ---: | ---: | ---: | ---: | --- | --- | --- |", + f"| {sum(not r.control for r in recorder.results)} " + f"| {round_text} | {elapsed:.0f}s | {noise_text} " + f"| {_md(str(metadata.get('platform_version', 'unknown')))} " + f"| {_md(str(metadata.get('runner_os') or metadata.get('platform', 'unknown')))} " + f"| {artifact} |", "", - f"seed {config.seed}; warm-up {config.warmup} rounds; " - f"{config.min_rounds}-{config.max_rounds} measured rounds per cell; " - "stopping on attained CI width, never on significance.", + f"seed {config.seed}; {config.warmup} warm-up rounds; " + f"{config.min_rounds}–{config.max_rounds} measured rounds allowed; " + f"{len(recorder.skipped)} cells skipped.", ] - return "\n".join(lines) + "\n" def _ratio_cell(c: stats.PairedComparison) -> str: @@ -443,7 +915,13 @@ def _ratio_cell(c: stats.PairedComparison) -> str: def _p_cell(c: stats.PairedComparison) -> str: - p = c.p_adjusted if c.p_adjusted is not None else c.p_value + # Show the one-sided tail matching the observed effect. For a ratio below + # one that is the explicit faster-tail test; adjusted upper-tail p-values + # cannot be interpreted backwards after BH correction. + if c.ratio < 1: + p = c.p_adjusted_faster if c.p_adjusted_faster is not None else c.p_value_faster + else: + p = c.p_adjusted if c.p_adjusted is not None else c.p_value if p is None or not math.isfinite(p): return "-" return f"{p:.3f}" if p >= 0.001 else "<0.001" diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py index c70498557..f94e29c82 100644 --- a/xtest/perf/stats.py +++ b/xtest/perf/stats.py @@ -28,13 +28,12 @@ Requiring both is deliberate, and neither clause is redundant: -- Clause 1 alone would fire on a real-but-trivial effect measured precisely - enough -- a reproducible 0.5% slowdown is not worth a red build. - It cannot fire on pure noise, since that would require the interval to - exclude an effect that is not there. -- Clause 2 alone would fire on noise roughly ``alpha`` of the time per cell, - and a run has enough cells that "roughly alpha" becomes "most nights". - BH adjustment across cells controls the false discovery rate. +- Clause 1 establishes that the effect is larger than the practical threshold, + but an unadjusted 95% interval on every cell does not control false + discoveries across the run. +- Clause 2 supplies that multiplicity control, but alone would flag both + real-but-trivial effects and pure-noise false positives. A reproducible 0.5% + slowdown is statistically real and still not worth a red build. Together they answer the only question worth gating on: is the slowdown both real and large enough to care about? @@ -122,9 +121,15 @@ class PairedComparison: ratio: float ci_low: float ci_high: float + #: One-sided p-value for "candidate is slower". Retains the original field + #: name for artifact/API compatibility; the opposite direction is explicit. p_value: float + #: One-sided p-value for "candidate is faster". + p_value_faster: float #: Set by :func:`apply_multiplicity_control` once every cell is known. p_adjusted: float | None = None + #: BH-adjusted form of :attr:`p_value_faster`. + p_adjusted_faster: float | None = None verdict: Verdict = Verdict.INCONCLUSIVE note: str = "" @@ -207,18 +212,22 @@ def _bootstrap_ci( return float(res.confidence_interval.low), float(res.confidence_interval.high) -def _one_sided_p(d: np.ndarray) -> float: - """One-sided Wilcoxon signed-rank p-value for "candidate is slower". +def _one_sided_ps(d: np.ndarray) -> tuple[float, float]: + """Wilcoxon signed-rank p-values for slower and faster, respectively. - Signed-rank rather than a t-test because latency distributions are - skewed and occasionally have a stray outlier round; we do not want a - single stalled invocation to drive the verdict. + Signed-rank does not require normally distributed raw latencies and a + single stalled invocation cannot drive it by magnitude alone. Interpreting + it as a location test does assume the *paired log differences* are roughly + symmetric; the log transform and multiplicative-jitter measurement model + are intended to make that a reasonable assumption. """ if np.all(d == 0): # No difference whatsoever. Wilcoxon rejects an all-zero input. - return 1.0 + return 1.0, 1.0 # scipy's stubs type the result as an opaque tuple-like; index and cast. - return cast(float, _scipy_stats.wilcoxon(d, alternative="greater")[1]) + slower = cast(float, _scipy_stats.wilcoxon(d, alternative="greater")[1]) + faster = cast(float, _scipy_stats.wilcoxon(d, alternative="less")[1]) + return slower, faster def compare( @@ -249,12 +258,14 @@ def compare( ci_low=math.nan, ci_high=math.nan, p_value=math.nan, + p_value_faster=math.nan, note=f"only {n} usable rounds; need at least {MIN_USABLE_ROUNDS}", ) lo_log, hi_log = _bootstrap_ci( d, confidence=confidence, seed=seed, n_resamples=n_resamples ) + p_slower, p_faster = _one_sided_ps(d) return PairedComparison( n_rounds=n, baseline_median=b_med, @@ -262,7 +273,8 @@ def compare( ratio=math.exp(float(np.median(d))), ci_low=math.exp(lo_log), ci_high=math.exp(hi_log), - p_value=_one_sided_p(d), + p_value=p_slower, + p_value_faster=p_faster, ) @@ -530,6 +542,7 @@ def apply_multiplicity_control( symmetric_set = set(symmetric_family) rest = [k for k in adjustable if k not in gated_set and k not in symmetric_set] p_adj: dict[str, float] = {} + p_adj_faster: dict[str, float] = {} for family in (gated_family, symmetric_family, rest): p_adj.update( zip( @@ -538,11 +551,23 @@ def apply_multiplicity_control( strict=True, ) ) + # The opposite direction needs its own lower-tail p-value and its own + # adjustment. Reading an adjusted upper-tail value as `p > 1-alpha` + # is invalid: BH controls small p-values and generally pushes large + # ones toward 1, making that backwards test easier rather than safer. + p_adj_faster.update( + zip( + family, + benjamini_hochberg([comparisons[k].p_value_faster for k in family]), + strict=True, + ) + ) result = GateResult(noise=noise, noise_by_control=noise_by_control) for key in keys: c = comparisons[key] pa = p_adj.get(key) + pa_faster = p_adj_faster.get(key) key_noise = noise_by_control.get(controls.get(key, ""), uncontrolled) if key in censored: verdict, note = Verdict.INCONCLUSIVE, censored[key] @@ -553,6 +578,7 @@ def apply_multiplicity_control( verdict, note = _verdict_for( c, pa, + pa_faster, threshold=threshold, alpha=alpha, noise=key_noise, @@ -560,12 +586,18 @@ def apply_multiplicity_control( ) elif key in symmetric_set: verdict, note = _symmetric_verdict_for( - c, pa, threshold=threshold, alpha=alpha, noise=key_noise + c, + pa, + pa_faster, + threshold=threshold, + alpha=alpha, + noise=key_noise, ) else: verdict, note = _verdict_for( c, pa, + pa_faster, threshold=threshold, alpha=alpha, noise=key_noise, @@ -579,7 +611,9 @@ def apply_multiplicity_control( ci_low=c.ci_low, ci_high=c.ci_high, p_value=c.p_value, + p_value_faster=c.p_value_faster, p_adjusted=pa, + p_adjusted_faster=pa_faster, verdict=verdict, note=note or c.note, ) @@ -600,6 +634,7 @@ def apply_multiplicity_control( def _verdict_for( c: PairedComparison, p_adjusted: float | None, + p_adjusted_faster: float | None, *, threshold: float, alpha: float, @@ -609,13 +644,16 @@ def _verdict_for( if c.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite(c.ci_low): return Verdict.INCONCLUSIVE, c.note or "no usable interval" - p = c.p_value if is_control else p_adjusted - if p is None or not math.isfinite(p): + p_slower = c.p_value if is_control else p_adjusted + p_faster = c.p_value_faster if is_control else p_adjusted_faster + if p_slower is None or p_faster is None: + return Verdict.INCONCLUSIVE, "no p-value" + if not (math.isfinite(p_slower) and math.isfinite(p_faster)): return Verdict.INCONCLUSIVE, "no p-value" - if c.ci_low > threshold and p < alpha: + if c.ci_low > threshold and p_slower < alpha: return Verdict.REGRESSION, "" - if c.ci_high < 1 / threshold and p > 1 - alpha: + if c.ci_high < 1 / threshold and p_faster < alpha: return Verdict.IMPROVED, "" # Not a regression. But "we looked and found nothing" only counts as PASS @@ -635,6 +673,7 @@ def _verdict_for( def _symmetric_verdict_for( c: PairedComparison, p_adjusted: float | None, + p_adjusted_faster: float | None, *, threshold: float, alpha: float, @@ -674,15 +713,16 @@ def _symmetric_verdict_for( if noise.underpowered: return Verdict.INCONCLUSIVE, noise.detail - p = p_adjusted - if p is None or not math.isfinite(p): + if p_adjusted is None or p_adjusted_faster is None: + return Verdict.INCONCLUSIVE, "no p-value" + if not (math.isfinite(p_adjusted) and math.isfinite(p_adjusted_faster)): return Verdict.INCONCLUSIVE, "no p-value" - # Direction clauses mirror REGRESSION and IMPROVED exactly, including the - # upper-tail read of the one-sided p for the faster direction. - if c.ci_low > threshold and p < alpha: + # Direction clauses mirror REGRESSION and IMPROVED exactly. Each direction + # uses its own BH-adjusted one-sided p-value. + if c.ci_low > threshold and p_adjusted < alpha: return Verdict.SLOWER, "" - if c.ci_high < 1 / threshold and p > 1 - alpha: + if c.ci_high < 1 / threshold and p_adjusted_faster < alpha: return Verdict.FASTER, "" if c.ci_low > 1 / threshold and c.ci_high < threshold: return Verdict.TIED, "" diff --git a/xtest/test_bench_aggregate.py b/xtest/test_bench_aggregate.py new file mode 100644 index 000000000..ddb62241a --- /dev/null +++ b/xtest/test_bench_aggregate.py @@ -0,0 +1,114 @@ +"""Tests for the final workflow-level benchmark summary.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from perf import aggregate + + +def document( + sdk: str, + verdict: str, + *, + elapsed: float = 30, + skipped: int = 0, +) -> dict[str, object]: + ratio = { + "REGRESSION": 1.3, + "IMPROVED": 0.7, + "INCONCLUSIVE": 1.1, + "PASS": 1.0, + }[verdict] + return { + "schema": 2, + "metadata": { + "sdk": sdk, + "platform_version": "v0.4.50", + "arm_sources": [{"tag": "v1"}, {"tag": "main"}], + }, + "config": {"gated_metrics": ["wall", "rss"]}, + "noise_floor": {"assessed": True, "width_ratio": 1.04}, + "trustworthy": True, + "skipped": {f"skipped-{i}": "unsupported" for i in range(skipped)}, + "cells": [ + { + "id": f"{sdk}-encrypt-1MiB", + "control": False, + "reference": "v1", + "arms": ["v1", "main"], + "elapsed_s": elapsed, + "contrasts": { + "main_vs_v1": { + "wall": { + "verdict": verdict, + "ratio": ratio, + "ci_low": ratio - 0.02, + "ci_high": ratio + 0.02, + } + } + }, + } + ], + } + + +def test_load_runs_ignores_unrelated_json_and_uses_stable_sdk_order(tmp_path: Path): + (tmp_path / "java.json").write_text(json.dumps(document("java", "PASS"))) + (tmp_path / "go.json").write_text(json.dumps(document("go", "PASS"))) + (tmp_path / "unrelated.json").write_text('{"hello": "world"}') + + assert [run.sdk for run in aggregate.load_runs(tmp_path)] == ["go", "java"] + + +def test_rollup_puts_the_cross_sdk_bottom_line_first_and_combines_run_facts(): + runs = [ + aggregate.Run("go", document("go", "REGRESSION", elapsed=20)), + aggregate.Run("java", document("java", "PASS", elapsed=30, skipped=1)), + aggregate.Run("js", document("js", "INCONCLUSIVE", elapsed=40)), + ] + md = aggregate.markdown( + runs, + artifact_urls={ + "go": "https://example.test/go", + "java": "https://example.test/java", + }, + run_url="https://example.test/run", + expected_sdks=["go", "java", "js"], + ) + + assert md.startswith("# SDK performance benchmark roll-up — REGRESSION\n") + assert "go: REGRESSION, java: PASS, js: INCONCLUSIVE" in md + assert md.index("### TL;DR") < md.index("| SDK | outcome") + assert md.index("| SDK | outcome") < md.index("### Confirmed regressions") + assert "| go | `go-encrypt-1MiB` | wall | +30.0% [+28.0%, +32.0%] |" in md + assert "| 3/3 | 3 | 1 | 90s | v0.4.50 |" in md + assert "[artifact](https://example.test/go)" in md + assert "[Workflow run](https://example.test/run)" in md + assert "Absolute timings are not compared across SDKs" in md + + +def test_rollup_explains_when_no_matrix_artifact_survived(): + md = aggregate.markdown([], run_url="https://example.test/run") + + assert "NO RESULTS" in md + assert "No benchmark JSON artifacts were available" in md + + +def test_rollup_makes_a_missing_matrix_result_visible(): + md = aggregate.markdown( + [aggregate.Run("go", document("go", "PASS"))], + expected_sdks=["go", "java"], + ) + + assert "roll-up — INCOMPLETE" in md + assert "Missing result(s): java" in md + assert "| **java** | **MISSING** |" in md + + +def test_a_control_without_a_result_cell_is_nothing_measured(): + doc = document("go", "PASS") + doc["cells"] = [{"id": "go-control", "control": True, "contrasts": {}}] + + assert aggregate.Run("go", doc).status == "NOTHING MEASURED" diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index a35229126..d570ff2bf 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -10,6 +10,7 @@ ``tmp_path``. """ +import json import shutil from collections.abc import Sequence from pathlib import Path @@ -211,6 +212,59 @@ def test_the_arm_count_follows_the_refs(self): assert bench.arm_count(options(bench_refs="go@a,go@b,go@c")) == 3 +class TestRunnerMetadata: + def test_resolver_provenance_is_preserved_and_enriched( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv( + "BENCH_VERSION_INFO", + json.dumps( + [ + { + "sdk": "go", + "tag": "v0.30.0", + "release": "v0.30.0", + "sha": "a" * 40, + }, + { + "sdk": "go", + "tag": "feature", + "pr": 42, + "head": True, + "sha": "b" * 40, + }, + ] + ), + ) + + sources, warning = bench._arm_sources() + + assert warning == "" + assert sources[0]["repo_url"] == "https://github.com/opentdf/otdfctl" + assert sources[1]["repo_url"] == "https://github.com/opentdf/platform" + + def test_bad_resolver_metadata_degrades_to_an_explanation( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv("BENCH_VERSION_INFO", "not-json") + + sources, warning = bench._arm_sources() + + assert sources == [] + assert "not valid JSON" in warning + + def test_workflow_url_needs_all_three_parts(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com") + monkeypatch.setenv("GITHUB_REPOSITORY", "opentdf/tests") + monkeypatch.setenv("GITHUB_RUN_ID", "123") + assert bench._github_run_url() == ( + "https://github.com/opentdf/tests/actions/runs/123" + ) + + monkeypatch.delenv("GITHUB_RUN_ID") + assert bench._github_run_url() == "" + + class TestDefaultBudget: def test_two_arms_keep_the_number_the_default_was_chosen_for(self): assert bench.default_budget_seconds(2) == BenchConfig().budget_seconds diff --git a/xtest/test_bench_report.py b/xtest/test_bench_report.py index d80b7cb70..076db53a7 100644 --- a/xtest/test_bench_report.py +++ b/xtest/test_bench_report.py @@ -163,6 +163,15 @@ def test_the_bake_off_is_in_the_artifact(self, tmp_path: Path): wall = next(b for b in doc["bake_off"] if b["metric"] == "wall") assert wall["winner"] == "quick" + def test_both_directional_p_values_are_recorded(self, tmp_path: Path): + doc = self.artifact(tmp_path, {REF: 1.0, "cand": 0.7}) + cell = next(c for c in doc["cells"] if c["id"] == "encrypt") + wall = cell["contrasts"][f"cand_vs_{REF}"]["wall"] + assert wall["p_value"] is not None + assert wall["p_adjusted"] is not None + assert wall["p_value_faster"] is not None + assert wall["p_adjusted_faster"] is not None + def test_the_file_is_valid_json_despite_nan(self, tmp_path: Path): # A cell with no usable interval produces NaN, which `json.dumps` # would happily write as a bare `NaN` that no strict parser accepts. @@ -193,3 +202,103 @@ def test_a_bake_off_section_appears_only_with_candidates_to_rank(self): assert "### Bake-off" not in report.markdown(two, cfg, two.gate(cfg)) three = recorder({REF: 1.0, "a": 1.0, "b": 1.3}, cfg=cfg, noise=0.02) assert "### Bake-off" in report.markdown(three, cfg, three.gate(cfg)) + + def test_the_bottom_line_precedes_supporting_detail(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "cand": 1.35}, cfg=cfg, noise=0.01) + md = report.markdown(rec, cfg, rec.gate(cfg)) + + assert md.index("### TL;DR") < md.index("### Compared builds") + assert md.index("### Compared builds") < md.index("### What changed") + assert md.index("### What changed") < md.index("All measurements") + assert md.index("All measurements") < md.index("### Run facts") + + def test_provenance_links_releases_prs_commits_and_the_diff(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "cand": 1.35}, cfg=cfg, noise=0.01) + repo = "https://github.com/opentdf/platform" + rec.metadata = { + "github_run_url": "https://github.com/opentdf/tests/actions/runs/42", + "arm_sources": [ + { + "tag": REF, + "alias": "latest", + "release": "otdfctl/v0.40.0", + "sha": "a" * 40, + "repo_url": repo, + }, + { + "tag": "cand", + "alias": "feature", + "pr": 123, + "head": True, + "sha": "b" * 40, + "repo_url": repo, + }, + ], + } + md = report.markdown( + rec, + cfg, + rec.gate(cfg), + artifact_url="https://github.com/opentdf/tests/actions/runs/42/artifacts/7", + ) + + assert f"{repo}/releases/tag/otdfctl%2Fv0.40.0" in md + assert f"{repo}/pull/123" in md + assert f"{repo}/commit/{'b' * 40}" in md + assert f"{repo}/compare/{'a' * 40}...{'b' * 40}" in md + assert "actions/runs/42/artifacts/7" in md + assert "actions/runs/42" in md + + def test_the_primary_table_omits_clean_rows_but_the_full_table_keeps_them(self): + cfg = config(min_rounds=10, max_rounds=60) + rec = recorder({REF: 1.0, "clean": 1.0, "slow": 1.4}, cfg=cfg, noise=0.005) + md = report.markdown(rec, cfg, rec.gate(cfg)) + primary = md.split("### What changed", 1)[1].split("### Bake-off", 1)[0] + + assert "`slow` vs `base`" in primary + assert "`clean` vs `base`" not in primary + assert "`clean` vs `base`" in md + + def test_attention_rows_get_fixed_scale_unicode_views(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "cand": 1.35}, cfg=cfg, noise=0.01) + md = report.markdown(rec, cfg, rec.gate(cfg)) + + assert "#### Effect at a glance" in md + assert "┆" in md and "│" in md and "●" in md + assert "Round stability for attention rows" in md + assert any("\u2800" <= char <= "\u28ff" for char in md) + + def test_run_facts_end_the_summary_with_reproducibility_context(self): + cfg = config(max_rounds=40, seed=91) + rec = recorder({REF: 1.0, "cand": 1.0}, cfg=cfg, noise=0.01) + rec.metadata = { + "platform_version": "v0.4.50", + "runner_os": "Linux", + } + md = report.markdown(rec, cfg, rec.gate(cfg)) + + assert "### Run facts" in md + assert "v0.4.50" in md and "Linux" in md + assert "seed 91" in md + assert md.rstrip().endswith("cells skipped.") + + def test_braille_trace_is_compact_and_deterministic(self): + values = [0.9, 1.0, 1.1, 1.2] * 20 + trace = report._braille_sparkline(values, 1.15) + + assert trace == report._braille_sparkline(values, 1.15) + assert len(trace) == 24 + assert all("\u2800" <= char <= "\u28ff" for char in trace) + + +class TestMarkdownTemplate: + def test_it_can_be_published_after_the_artifact_url_is_known(self, tmp_path: Path): + path = report.write_markdown( + tmp_path / "go.summary.md", + f"[evidence]({report.ARTIFACT_URL_PLACEHOLDER})\n", + ) + + assert report.ARTIFACT_URL_PLACEHOLDER in path.read_text() diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py index 2001a10a2..b76e9caef 100644 --- a/xtest/test_bench_stats.py +++ b/xtest/test_bench_stats.py @@ -130,6 +130,7 @@ def test_identical_inputs_give_unit_ratio_and_no_significance(self): r = stats.compare(v, v, seed=0, n_resamples=RESAMPLES) assert r.ratio == pytest.approx(1.0) assert r.p_value == 1.0 + assert r.p_value_faster == 1.0 def test_constant_offset_has_degenerate_interval(self): # Every round shows exactly a 2x slowdown: there is no sampling @@ -185,7 +186,10 @@ def test_planted_speedup_is_reported_but_never_fails(self): g = gate_one( stats.compare(b, c, seed=12, n_resamples=RESAMPLES), control=quiet_control() ) - assert g.comparisons["cell"].verdict is Verdict.IMPROVED + result = g.comparisons["cell"] + assert result.verdict is Verdict.IMPROVED + assert result.p_adjusted_faster is not None + assert result.p_adjusted_faster < stats.DEFAULT_ALPHA assert not g.should_fail def test_borderline_effect_without_power_is_inconclusive_not_pass(self): @@ -301,6 +305,26 @@ def test_bh_passes_nan_through(self): assert math.isnan(adj[1]) assert all(math.isfinite(a) for a in (adj[0], adj[2])) + def test_faster_tail_is_adjusted_directly(self): + cells = {} + for i, ratio in enumerate((0.70, 0.75, 0.80)): + rng = np.random.default_rng(2900 + i) + b, c = synth(rng, ratio, n=60) + cells[f"cell{i}"] = stats.compare(b, c, seed=i, n_resamples=RESAMPLES) + cells["control"] = quiet_control() + + g = stats.apply_multiplicity_control( + cells, controls=all_under_one_control(cells) + ) + expected = stats.benjamini_hochberg( + [cells[f"cell{i}"].p_value_faster for i in range(3)] + ) + actual = [g.comparisons[f"cell{i}"].p_adjusted_faster for i in range(3)] + assert actual == pytest.approx(expected) + assert all( + g.comparisons[f"cell{i}"].verdict is Verdict.IMPROVED for i in range(3) + ) + def test_correction_suppresses_lone_lucky_cell(self): # 20 pure-noise cells: without BH one of them firing is expected. cells = {} @@ -443,6 +467,8 @@ def test_a_clearly_slower_arm_is_slower(self): def test_a_clearly_faster_arm_is_faster(self): _, c = self.h2h(1 / 1.5) assert c.verdict is Verdict.FASTER + assert c.p_adjusted_faster is not None + assert c.p_adjusted_faster < stats.DEFAULT_ALPHA def test_indistinguishable_arms_are_tied(self): # A positive finding, and the most likely honest answer for two From d776bc1210c7863b2ff1220fbc7fd9902261cfdf Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 26 Aug 2026 12:46:09 -0400 Subject: [PATCH 10/11] fix(xtest): keep extreme benchmark effects legible --- .github/workflows/xtest.yml | 14 +++++- xtest/perf/README.md | 9 ++-- xtest/perf/aggregate.py | 8 ++-- xtest/perf/report.py | 96 +++++++++++++++++++++++++++++-------- xtest/test_bench_report.py | 14 +++++- 5 files changed, 114 insertions(+), 27 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 3937dcbcc..ef38cf3ef 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -1297,6 +1297,16 @@ jobs: path: otdftests persist-credentials: false + # aggregate.py follows xtest's Python target. The hosted runner's system + # Python can lag it (Ubuntu 24.04 currently supplies 3.12), so run through + # the same pinned uv/Python toolchain as the benchmark rather than hoping + # stdlib-only also means syntax-compatible. + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.14" + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - name: Download SDK benchmark results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true @@ -1330,7 +1340,9 @@ jobs: BENCH_EXPECTED_SDKS: ${{ needs.resolve-versions.outputs.bench-sdks }} BENCH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: |- - python3 otdftests/xtest/perf/aggregate.py benchmark-results >> "$GITHUB_STEP_SUMMARY" + uv run --project otdftests/xtest --frozen --no-sync \ + python otdftests/xtest/perf/aggregate.py benchmark-results \ + >> "$GITHUB_STEP_SUMMARY" publish-results: runs-on: ubuntu-latest diff --git a/xtest/perf/README.md b/xtest/perf/README.md index ef217230b..6505b8a29 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -64,9 +64,12 @@ The summary is ordered for progressive disclosure: diff against the reference where those links exist. 3. **What changed** shows only regressions, inconclusive rows, and confirmed improvements. Clean and ungated rows do not crowd the decision. -4. **Effect at a glance** puts those rows on one fixed log-ratio scale. `┆` - marks the practical threshold, `│` means no change, the bracket is the 95% - interval, and `●` is the point estimate. +4. **Effect at a glance** puts those rows on one shared, tail-compressed + log-ratio scale. This preserves space around the practical gate while still + fitting unusually large effects. `┆` marks the threshold, `│` means no + change, the bracket is the 95% interval, `●` is the point estimate, and `◆` + is an interval narrower than one character. Stable candidate letters map + duplicate measurement names back to the **Compared builds** table. 5. Expandable blocks hold per-round Braille traces, all measurements and controls, skipped cells, and the statistical rule. 6. **Run facts** closes the SDK summary with rounds, elapsed time, A/A noise, diff --git a/xtest/perf/aggregate.py b/xtest/perf/aggregate.py index dfae82af4..fa973e7fa 100644 --- a/xtest/perf/aggregate.py +++ b/xtest/perf/aggregate.py @@ -1,8 +1,10 @@ """Pure-stdlib cross-SDK renderer for benchmark JSON artifacts. -The workflow runs this after downloading every matrix artifact. Keeping it -free of pytest/numpy/scipy means the roll-up needs no environment setup and can -still explain a partially failed matrix whose artifact set is incomplete. +The workflow runs this after downloading every matrix artifact. It has no +third-party imports, so the roll-up needs no dependency sync and can still +explain a partially failed matrix whose artifact set is incomplete. It does +run through xtest's pinned interpreter: stdlib-only does not imply that a +hosted runner's older Python understands the repository's target syntax. """ from __future__ import annotations diff --git a/xtest/perf/report.py b/xtest/perf/report.py index aa0212a1b..bbb8270a4 100644 --- a/xtest/perf/report.py +++ b/xtest/perf/report.py @@ -555,7 +555,8 @@ def _provenance(recorder: BenchmarkRecorder) -> list[str]: reference_source = sources.get(result.reference) for arm in result.arm_ids: source = sources.get(arm) - role = "**reference**" if arm == result.reference else "candidate" + role_name = _arm_role(result, arm) + role = f"**{role_name}**" if arm == result.reference else role_name label = result.arm_labels.get(arm, arm) source_link = _source_link(source, label) commit_link = _commit_link(source) @@ -637,6 +638,19 @@ def _cell_label(result: CellResult) -> str: return result.cell_id.removeprefix(f"{result.sdk}-").replace("-", " / ") +def _arm_role(result: CellResult, arm: str) -> str: + """A short, stable role that maps plot rows back to the build table.""" + if arm == result.reference: + return "reference" + candidates = [ + candidate for candidate in result.arm_ids if candidate != result.reference + ] + try: + return f"candidate {chr(ord('A') + candidates.index(arm))}" + except ValueError: + return arm + + def _cost_change(metric: str, c: stats.PairedComparison) -> str: if not (math.isfinite(c.baseline_median) and math.isfinite(c.candidate_median)): return "—" @@ -672,8 +686,26 @@ def _gate_margin(c: stats.PairedComparison, threshold: float) -> str: def _effect_overview(rows: list[ReportRow], threshold: float) -> list[str]: labels = [f"{_cell_label(r.result)} {METRIC_LABELS[r.metric][0]}" for r in rows] + roles = [_arm_role(row.result, row.b) for row in rows] label_width = min(30, max(map(len, labels), default=0)) + role_width = max(map(len, roles), default=0) width = 57 + threshold_log = math.log(threshold) + observed_logs = [ + abs(math.log(value)) + for row in rows + for value in ( + row.comparison.ratio, + row.comparison.ci_low, + row.comparison.ci_high, + ) + if math.isfinite(value) and value > 0 + ] + # Preserve room around the practical band, but expand far enough that the + # largest interval gets brackets rather than an off-scale arrow. The tail + # transform in `_effect_strip` prevents an epic result from squeezing the + # gate markers and every merely-large result into the center character. + span = max(3 * threshold_log, max(observed_logs, default=0) * 1.05) axis = [" "] * width for text, start in ( ("← faster", 0), @@ -685,15 +717,18 @@ def _effect_overview(rows: list[ReportRow], threshold: float) -> list[str]: "", "#### Effect at a glance", "", - "Fixed log-ratio scale; `┆` marks ±the practical threshold and `│` no change.", + "Shared tail-compressed log-ratio scale; `┆` marks ±the practical " + "threshold and `│` no change. Candidate letters match the build table; " + "exact changes are printed at right, and `◆` means the CI is narrower " + "than one character.", "", "```text", - f"{'measurement':<{label_width}} {''.join(axis)}", + f"{'arm':<{role_width}} {'measurement':<{label_width}} {''.join(axis)}", ] - for label, row in zip(labels, rows, strict=True): + for role, label, row in zip(roles, labels, rows, strict=True): lines.append( - f"{label[:label_width]:<{label_width}} " - f"{_effect_strip(row.comparison, threshold, width=width)} " + f"{role:<{role_width}} {label[:label_width]:<{label_width}} " + f"{_effect_strip(row.comparison, threshold, width=width, span=span)} " f"{_pct(row.comparison.ratio):>7} {row.comparison.verdict}" ) lines += ["```", ""] @@ -701,30 +736,53 @@ def _effect_overview(rows: list[ReportRow], threshold: float) -> list[str]: def _effect_strip( - c: stats.PairedComparison, threshold: float, *, width: int = 57 + c: stats.PairedComparison, + threshold: float, + *, + width: int = 57, + span: float | None = None, ) -> str: - """A fixed-width CI forest strip on a symmetric log-ratio scale.""" + """A fixed-width CI forest strip on a symmetric compressed-log scale.""" chars = [" "] * width - # Show three threshold-widths in each direction. That keeps the practical - # boundary near the center while leaving enough room to draw the interval - # of an ordinary 20–40% regression instead of collapsing it to an arrow. - span = 3 * math.log(threshold) + threshold_log = math.log(threshold) + span = span or 3 * threshold_log + + def unit(value: float) -> float: + """Keep the gate at 1/3 width and compress the dynamic tails.""" + magnitude = abs(value) + if magnitude <= threshold_log: + scaled = magnitude / threshold_log / 3 + else: + tail = max(span - threshold_log, 1e-12) + # Keep ten percent of either edge as breathing room: the largest + # observed effect should look extreme without masquerading as a + # clipped value. + scaled = 1 / 3 + (0.9 - 1 / 3) * ( + math.log1p((magnitude - threshold_log) / threshold_log) + / math.log1p(tail / threshold_log) + ) + return math.copysign(max(-1.0, min(1.0, scaled)), value) def pos(ratio: float) -> int: value = math.log(ratio) if math.isfinite(ratio) and ratio > 0 else 0.0 - unit = max(-1.0, min(1.0, value / span)) - return round((unit + 1) * (width - 1) / 2) + return round((unit(value) + 1) * (width - 1) / 2) for ratio, marker in ((1 / threshold, "┆"), (1.0, "│"), (threshold, "┆")): chars[pos(ratio)] = marker + collapsed_ci_at: int | None = None if math.isfinite(c.ci_low) and math.isfinite(c.ci_high): lo, hi = sorted((pos(c.ci_low), pos(c.ci_high))) - for i in range(lo, hi + 1): - if chars[i] == " ": - chars[i] = "━" - chars[lo], chars[hi] = "[", "]" + if lo == hi: + collapsed_ci_at = lo + chars[lo] = "◆" + else: + for i in range(lo, hi + 1): + if chars[i] == " ": + chars[i] = "━" + chars[lo], chars[hi] = "[", "]" if math.isfinite(c.ratio) and c.ratio > 0: - chars[pos(c.ratio)] = "●" + point = pos(c.ratio) + chars[point] = "◆" if point == collapsed_ci_at else "●" if math.log(c.ratio) < -span: chars[0] = "◀" elif math.log(c.ratio) > span: diff --git a/xtest/test_bench_report.py b/xtest/test_bench_report.py index 076db53a7..63f5f20b4 100644 --- a/xtest/test_bench_report.py +++ b/xtest/test_bench_report.py @@ -250,6 +250,7 @@ def test_provenance_links_releases_prs_commits_and_the_diff(self): assert f"{repo}/compare/{'a' * 40}...{'b' * 40}" in md assert "actions/runs/42/artifacts/7" in md assert "actions/runs/42" in md + assert "candidate A" in md def test_the_primary_table_omits_clean_rows_but_the_full_table_keeps_them(self): cfg = config(min_rounds=10, max_rounds=60) @@ -261,7 +262,7 @@ def test_the_primary_table_omits_clean_rows_but_the_full_table_keeps_them(self): assert "`clean` vs `base`" not in primary assert "`clean` vs `base`" in md - def test_attention_rows_get_fixed_scale_unicode_views(self): + def test_attention_rows_get_shared_scale_unicode_views(self): cfg = config(max_rounds=40) rec = recorder({REF: 1.0, "cand": 1.35}, cfg=cfg, noise=0.01) md = report.markdown(rec, cfg, rec.gate(cfg)) @@ -271,6 +272,17 @@ def test_attention_rows_get_fixed_scale_unicode_views(self): assert "Round stability for attention rows" in md assert any("\u2800" <= char <= "\u28ff" for char in md) + def test_extreme_effects_fit_and_name_their_candidate(self): + cfg = config(min_rounds=12, max_rounds=40) + rec = recorder({REF: 1.0, "epic": 0.02, "fast": 0.5}, cfg=cfg, noise=0.005) + md = report.markdown(rec, cfg, rec.gate(cfg)) + effect = md.split("#### Effect at a glance", 1)[1].split("### Bake-off", 1)[0] + + assert "candidate A" in effect and "candidate B" in effect + assert "-98.0%" in effect and "-49.7%" in effect + assert "◀" not in effect + assert "◆" in effect + def test_run_facts_end_the_summary_with_reproducibility_context(self): cfg = config(max_rounds=40, seed=91) rec = recorder({REF: 1.0, "cand": 1.0}, cfg=cfg, noise=0.01) From 0313e374826410c6c8bf73b0c49098e81d1c6a0b Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 26 Aug 2026 13:24:10 -0400 Subject: [PATCH 11/11] fix(xtest): clarify equivalent benchmark arms and controls --- .github/workflows/xtest.yml | 9 ++ xtest/conftest.py | 14 ++- xtest/fixtures/bench.py | 16 +++ xtest/perf/README.md | 15 ++- xtest/perf/aggregate.py | 61 +++++++++++- xtest/perf/report.py | 177 ++++++++++++++++++++++++++++------ xtest/test_bench_aggregate.py | 50 ++++++++++ xtest/test_bench_arms.py | 23 +++++ xtest/test_bench_report.py | 64 ++++++++++++ 9 files changed, 388 insertions(+), 41 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index ef38cf3ef..ed36334c7 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -1212,6 +1212,15 @@ jobs: BENCH_VERSION_INFO: >- ${{ steps.bench-arms.outputs.version-info || needs.resolve-versions.outputs[matrix.sdk] }} + # Keep the names the caller asked for even when the resolver folds + # aliases at one SHA into a single install. That is how reporting can + # distinguish "main == latest" from a benchmark that broke before it + # found its second arm. + BENCH_REQUESTED_REFS: >- + ${{ needs.resolve-versions.outputs.bench-refs + || (matrix.sdk == 'go' && (inputs.otdfctl-ref || 'main latest')) + || (matrix.sdk == 'java' && (inputs.java-ref || 'main latest')) + || (inputs.js-ref || 'main latest') }} # pytest writes a summary template beside the JSON. Publish it only # after upload-artifact gives us the direct evidence URL. BENCH_DEFER_SUMMARY: "1" diff --git a/xtest/conftest.py b/xtest/conftest.py index ac09598ce..aef9ca10f 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -455,7 +455,11 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): reporter = config.pluginmanager.get_plugin("terminalreporter") if reporter is not None: reporter.write_sep("=", "benchmark results") - reporter.write_line(gate.summary) + reporter.write_line( + str(recorder.metadata.get("comparison_note", gate.summary)) + if report.same_commit(recorder.metadata) + else gate.summary + ) # Repeated on the terminal as well as in the step summary: "the run # was too short for the number of arms you asked for" is the one # finding a reader is most likely to mistake for a real result. @@ -474,8 +478,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): # regression. --bench is an explicit request for a measurement; answering # it with a green tick and an empty table is the one outcome nobody # inspects, so a benchmark that has quietly stopped measuring can survive - # indefinitely. Every reason a cell skips is already in the report. - if gate.should_fail or gate.nothing_measured: + # indefinitely. The exception is two requested names resolving to one SHA: + # that is a complete, neutral answer (there is no code difference to test), + # not a harness that failed to measure an existing difference. + if gate.should_fail or ( + gate.nothing_measured and not report.same_commit(recorder.metadata) + ): session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py index e3f619506..d3a12627f 100644 --- a/xtest/fixtures/bench.py +++ b/xtest/fixtures/bench.py @@ -615,6 +615,16 @@ def runner_metadata(config: pytest.Config) -> dict[str, object]: } sources, warning = _arm_sources() metadata["arm_sources"] = sources + requested = _requested_refs() + metadata["requested_refs"] = requested + if len(requested) >= 2 and len(sources) == 1 and sources[0].get("sha"): + sha = str(sources[0].get("sha", "")) + names = ", ".join(requested) + metadata["comparison_status"] = "same_commit" + metadata["comparison_note"] = ( + f"{names} resolve to the same commit" + f"{f' {sha[:7]}' if sha else ''}; there is no code difference to benchmark." + ) if warning: metadata["arm_sources_warning"] = warning return metadata @@ -629,6 +639,12 @@ def _github_run_url() -> str: return f"{server}/{repository}/actions/runs/{run_id}" +def _requested_refs() -> list[str]: + """The caller's names, retained even when resolution deduplicates a SHA.""" + value = os.environ.get("BENCH_REQUESTED_REFS", "") + return [part for part in re.split(r"[,\s]+", value.strip()) if part] + + def _arm_sources() -> tuple[list[dict[str, object]], str]: """Resolver metadata for the builds, enriched with their GitHub repository. diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 6505b8a29..19deab9cb 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -151,6 +151,11 @@ reported as TIED rather than PASS because PASS is a one-sided claim. **inconclusive** — the CI straddles a band edge, so the run cannot say which of the three it is. +**SAME COMMIT** — the caller requested two names (typically `main latest`) but +both resolved to one immutable SHA. There is no code difference that could +produce a performance difference, so no cell runs and the job remains neutral. +This is distinct from an empty or broken benchmark. + **NOTHING MEASURED** — no cell produced a comparison at all, usually because only one build was installed so there was no baseline to compare against. This **fails the job**. An empty run and a clean run have the same empty list of @@ -160,10 +165,9 @@ lists the reason for each cell. > One cause looks like a bug and is not. If an SDK's newest release tags the same > commit as `main` — java sat at `v0.18.0 == main == dev == 57d070b0` through -> August 2026 — then `main latest` resolves both arms to one SHA, `otdf-sdk-mgr` -> installs a single build, and every cell skips with *"no final release to compare -> against; installed: main"*. The message is true from where the harness stands, but -> the release it is looking for does exist; the two arms are just the same code. +> August 2026 — then `main latest` resolves both arms to one SHA and +> `otdf-sdk-mgr` installs a single build. The report calls this **SAME COMMIT**, +> not NOTHING MEASURED: the release exists, but the two arms are the same code. > Check with `otdf-sdk-mgr versions resolve main latest` — one entry back > instead of two means there is nothing to measure until `main` moves. @@ -717,7 +721,8 @@ two builds doing different amounts of work) is invisible in the output. 5. Never let the arms differ in anything but the build. 6. Never run the measured command from a process holding memory. 7. Never run the benchmark in parallel with anything, including itself. -8. Never let a run that measured nothing report success. +8. Never let a run that measured nothing report success, unless every requested + arm resolved to the same immutable commit and there is no difference to test. 9. Never gate a contrast that does not involve the reference. A bake-off ranks; it does not fail the build. diff --git a/xtest/perf/aggregate.py b/xtest/perf/aggregate.py index fa973e7fa..a69129599 100644 --- a/xtest/perf/aggregate.py +++ b/xtest/perf/aggregate.py @@ -16,6 +16,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any +from urllib.parse import quote @dataclass(frozen=True, slots=True) @@ -51,8 +52,16 @@ def improvements(self) -> int: def regressions(self) -> list[tuple[str, str, dict[str, Any]]]: return [r for r in self.gated if r[2].get("verdict") == "REGRESSION"] + @property + def same_commit(self) -> bool: + return ( + self.document.get("metadata", {}).get("comparison_status") == "same_commit" + ) + @property def status(self) -> str: + if self.same_commit: + return "SAME COMMIT" if self.document.get("nothing_measured") or not self.gated: return "NOTHING MEASURED" if not self.document.get("trustworthy", True): @@ -178,6 +187,8 @@ def _overall_status(runs: list[Run], missing: list[str]) -> str: return "INCOMPLETE" if any(r.status == "INCONCLUSIVE" for r in runs): return "INCONCLUSIVE" + if runs and all(r.same_commit for r in runs): + return "SAME COMMIT" return "PASS" if runs else "NO RESULTS" @@ -186,23 +197,63 @@ def _overall_headline(runs: list[Run], missing: list[str]) -> str: return "No benchmark result artifacts were found." regressions = sum(len(r.regressions) for r in runs) unresolved = sum(r.inconclusive for r in runs) + same = sum(r.same_commit for r in runs) outcomes = ", ".join(f"{r.sdk}: {r.status}" for r in runs) missing_text = f" Missing result(s): {', '.join(missing)}." if missing else "" return ( f"{regressions} confirmed regression(s), {unresolved} unresolved gated " - f"comparison(s) across {len(runs)} available SDK result(s). {outcomes}." + f"comparison(s), and {same} same-commit result(s) across {len(runs)} " + f"available SDK result(s). {outcomes}." f"{missing_text}" ) def _arms(run: Run) -> str: sources = run.document.get("metadata", {}).get("arm_sources", []) - if isinstance(sources, list) and sources: + source_list = [source for source in sources if isinstance(source, dict)] + by_tag = {str(source.get("tag", "")): source for source in source_list} + if run.same_commit: + requested = run.document.get("metadata", {}).get("requested_refs", []) + source = source_list[0] if source_list else None + if isinstance(requested, list): + return ( + " = ".join(_aggregate_arm(str(arm), source) for arm in requested) + + " (same commit)" + ) + cells = [c for c in run.document.get("cells", []) if not c.get("control")] + if cells: return " → ".join( - f"`{s.get('tag', '?')}`" for s in sources if isinstance(s, dict) + _aggregate_arm(str(arm), by_tag.get(str(arm))) + for arm in cells[0].get("arms", []) ) - cells = [c for c in run.document.get("cells", []) if not c.get("control")] - return " → ".join(f"`{a}`" for a in cells[0].get("arms", [])) if cells else "—" + if source_list: + return " → ".join( + _aggregate_arm(str(source.get("tag", "?")), source) + for source in source_list + ) + return "—" + + +def _aggregate_arm(arm: str, source: object) -> str: + url = _aggregate_source_url(source) + code = f"`{arm}`" + return f"[{code}]({url})" if url else code + + +def _aggregate_source_url(source: object) -> str: + if not isinstance(source, dict): + return "" + repo = str(source.get("repo_url", "")) + pr = str(source.get("pr", "")) + release = str(source.get("release", "")) + sha = str(source.get("sha", "")) + if repo and pr: + return f"{repo}/pull/{quote(pr, safe='')}" + if repo and release: + return f"{repo}/releases/tag/{quote(release, safe='')}" + if repo and sha: + return f"{repo}/tree/{quote(sha, safe='')}" + return "" def _change(comparison: dict[str, Any]) -> str: diff --git a/xtest/perf/report.py b/xtest/perf/report.py index bbb8270a4..6021bc414 100644 --- a/xtest/perf/report.py +++ b/xtest/perf/report.py @@ -400,9 +400,18 @@ def markdown( ) ] attention.sort(key=_attention_sort_key) - sdk = next((r.sdk for r in recorder.results if r.sdk), "SDK") - status, headline = _headline(gate, gated_rows) + sdk = next( + (r.sdk for r in recorder.results if r.sdk), + str(recorder.metadata.get("sdk") or "SDK"), + ) + equivalent = same_commit(recorder.metadata) + status, headline = _headline(gate, gated_rows, recorder.metadata) n_arms = max((len(r.arm_ids) for r in recorder.results), default=2) + summary = ( + str(recorder.metadata.get("comparison_note", gate.summary)) + if equivalent + else gate.summary + ) lines = [ f"## {sdk.upper()} SDK performance — {status}", @@ -411,7 +420,7 @@ def markdown( "", f"> **{headline}**", ">", - f"> {gate.summary}", + f"> {summary}", "", ] lines += _quick_links(recorder.metadata, artifact_url) @@ -421,11 +430,17 @@ def markdown( if warning: lines += ["> [!WARNING]", f"> {warning}", ""] noise = gate.noise - if noise is not None and noise.detail: + if noise is not None and noise.detail and not equivalent: lines += [f"> {noise.detail}", ""] lines += ["### What changed", ""] - if attention: + if equivalent: + lines += [ + "No performance comparison was necessary: the requested refs identify " + "the same source commit.", + "", + ] + elif attention: lines += [ "Only gated regressions, unresolved measurements, and confirmed " "improvements are shown here. Clean rows and diagnostic metrics are below.", @@ -442,6 +457,12 @@ def markdown( f"| {_verdict_cell(c)} |" ) lines += _effect_overview(attention, config.threshold) + elif gate.nothing_measured: + lines += [ + "No comparison ran. Expand **Not measured** below for the cell-level " + "reasons; this is not a passing performance result.", + "", + ] else: lines += [ "No gated comparison requires attention: every measured wall-clock and " @@ -451,9 +472,10 @@ def markdown( lines += _bake_off_section(recorder, config, gate) lines += _diagnostics(recorder, attention, config) - lines += _full_measurements(rows) + lines += _full_measurements(rows, recorder.metadata) lines += _not_measured(recorder) - lines += _method(config, n_arms) + if not equivalent: + lines += _method(config, n_arms) lines += _run_facts(recorder, config, gate, artifact_url) return "\n".join(lines) + "\n" @@ -487,13 +509,24 @@ def _report_rows( return rows -def _headline(gate: stats.GateResult, gated_rows: list[ReportRow]) -> tuple[str, str]: +def same_commit(metadata: Mapping[str, object]) -> bool: + """Whether multiple requested names resolved to one immutable commit.""" + return metadata.get("comparison_status") == "same_commit" + + +def _headline( + gate: stats.GateResult, + gated_rows: list[ReportRow], + metadata: Mapping[str, object], +) -> tuple[str, str]: inconclusive = sum( r.comparison.verdict is stats.Verdict.INCONCLUSIVE for r in gated_rows ) improvements = sum( r.comparison.verdict is stats.Verdict.IMPROVED for r in gated_rows ) + if same_commit(metadata): + return "SAME COMMIT", "No code difference to benchmark." if gate.nothing_measured: return "NOTHING MEASURED", "The benchmark produced no comparisons." if not gate.trustworthy: @@ -533,25 +566,29 @@ def _provenance(recorder: BenchmarkRecorder) -> list[str]: result = next((r for r in recorder.results if not r.control), None) if result is None: result = next(iter(recorder.results), None) - if result is None: - return [] - - raw_sources = recorder.metadata.get("arm_sources", []) - sources = ( - { - str(s.get("tag", "")): s - for s in raw_sources - if isinstance(s, dict) and s.get("tag") - } - if isinstance(raw_sources, list) - else {} - ) + sources = _sources_by_tag(recorder.metadata) lines = [ "### Compared builds", "", "| arm | role | source | commit | compare to reference |", "| --- | --- | --- | --- | --- |", ] + if result is None: + if not (same_commit(recorder.metadata) and sources): + return [] + source = next(iter(sources.values())) + requested = recorder.metadata.get("requested_refs", []) + arms = [str(arm) for arm in requested] if isinstance(requested, list) else [] + for index, arm in enumerate(arms): + resolution = ( + _source_link(source, arm) if index == 0 else "same resolved source" + ) + lines.append( + f"| {_linked_arm(arm, sources, fallback_source=source)} " + f"| **same commit** | {resolution} | {_commit_link(source)} | — |" + ) + return lines + [""] + reference_source = sources.get(result.reference) for arm in result.arm_ids: source = sources.get(arm) @@ -564,7 +601,8 @@ def _provenance(recorder: BenchmarkRecorder) -> list[str]: "—" if arm == result.reference else _compare_link(reference_source, source) ) lines.append( - f"| `{_md(arm)}` | {role} | {source_link} | {commit_link} | {compare_link} |" + f"| {_linked_arm(arm, sources)} | {role} | {source_link} " + f"| {commit_link} | {compare_link} |" ) lines.append("") warning = recorder.metadata.get("arm_sources_warning") @@ -573,6 +611,49 @@ def _provenance(recorder: BenchmarkRecorder) -> list[str]: return lines +def _sources_by_tag(metadata: Mapping[str, object]) -> dict[str, dict[str, object]]: + raw_sources = metadata.get("arm_sources", []) + if not isinstance(raw_sources, list): + return {} + sources: dict[str, dict[str, object]] = {} + for source in raw_sources: + if not isinstance(source, dict) or not source.get("tag"): + continue + normalized = {str(key): value for key, value in source.items()} + sources[str(normalized["tag"])] = normalized + return sources + + +def _source_url(source: object) -> str: + if not isinstance(source, dict): + return "" + repo = str(source.get("repo_url", "")) + pr = str(source.get("pr", "")) + release = str(source.get("release", "")) + sha = str(source.get("sha", "")) + if repo and pr: + return f"{repo}/pull/{quote(pr, safe='')}" + if repo and release: + return f"{repo}/releases/tag/{quote(release, safe='')}" + if repo and sha: + return f"{repo}/tree/{quote(sha, safe='')}" + return "" + + +def _linked_arm( + arm: str, + sources: Mapping[str, object], + *, + fallback_source: object = None, +) -> str: + base, separator, copy = arm.partition("#") + label = f"{base} copy {copy}" if separator else arm + source = sources.get(base, fallback_source) + url = _source_url(source) + code = f"`{_md(label)}`" + return f"[{code}]({url})" if url else code + + def _source_link(source: object, fallback: str) -> str: if not isinstance(source, dict): return _md(fallback) @@ -877,19 +958,44 @@ def _braille_sparkline( return "".join(glyphs) -def _full_measurements(rows: list[ReportRow]) -> list[str]: +def _full_measurements( + rows: list[ReportRow], metadata: Mapping[str, object] +) -> list[str]: + if not rows: + return [] + sources = _sources_by_tag(metadata) lines = [ "
", "All measurements, controls, and statistical details", "", - "| cell | contrast | metric | a | b | ratio (95% CI) | p (BH) | n | verdict |", + ] + if any(row.result.control for row in rows): + lines += [ + "**A/A control** rows intentionally run multiple copies of the reference " + "build against itself on a fixed 1 MiB encrypt operation. They measure " + "runner noise; they are not candidate comparisons.", + "", + ] + lines += [ + "| cell | contrast | metric | a median | b median | ratio b/a (95% CI) | p (BH) | n | verdict |", "| --- | --- | --- | --- | --- | --- | --- | ---: | --- |", ] for row in rows: c = row.comparison - label = METRIC_LABELS[row.metric][0] + ("" if row.gated else " (ungated)") + qualifier = ( + "A/A control" + if row.result.control + else "head-to-head" + if row.head_to_head + else "" + if row.gated + else "ungated" + ) + label = METRIC_LABELS[row.metric][0] + (f" ({qualifier})" if qualifier else "") lines.append( - f"| {row.result.cell_id} | `{row.b}` vs `{row.a}` | {label} " + f"| {_detail_cell_label(row.result)} " + f"| {_linked_arm(row.b, sources)} vs {_linked_arm(row.a, sources)} " + f"| {label} " f"| {format_metric(row.metric, c.baseline_median)} " f"| {format_metric(row.metric, c.candidate_median)} " f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " @@ -898,12 +1004,26 @@ def _full_measurements(rows: list[ReportRow]) -> list[str]: return lines + ["", "
", ""] +def _detail_cell_label(result: CellResult) -> str: + label = _cell_label(result) + if result.control: + operation = label.removesuffix(" / control").replace("1MiB", "1 MiB") + return f"**A/A control** · {operation}" + return label + + def _not_measured(recorder: BenchmarkRecorder) -> list[str]: if not recorder.skipped: return [] lines = [ "
", - "Not measured", + "" + + ( + "Cells not run (same commit)" + if same_commit(recorder.metadata) + else "Not measured" + ) + + "", "", ] lines += [f"- `{cid}`: {why}" for cid, why in sorted(recorder.skipped.items())] @@ -960,7 +1080,8 @@ def _run_facts( "", f"seed {config.seed}; {config.warmup} warm-up rounds; " f"{config.min_rounds}–{config.max_rounds} measured rounds allowed; " - f"{len(recorder.skipped)} cells skipped.", + f"{len(recorder.skipped)} " + f"{'cell' if len(recorder.skipped) == 1 else 'cells'} skipped.", ] diff --git a/xtest/test_bench_aggregate.py b/xtest/test_bench_aggregate.py index ddb62241a..043468124 100644 --- a/xtest/test_bench_aggregate.py +++ b/xtest/test_bench_aggregate.py @@ -112,3 +112,53 @@ def test_a_control_without_a_result_cell_is_nothing_measured(): doc["cells"] = [{"id": "go-control", "control": True, "contrasts": {}}] assert aggregate.Run("go", doc).status == "NOTHING MEASURED" + + +def test_same_commit_is_neutral_and_names_both_requested_refs(): + doc = document("java", "PASS") + doc["nothing_measured"] = True + doc["cells"] = [] + metadata = doc["metadata"] + assert isinstance(metadata, dict) + metadata.update( + { + "comparison_status": "same_commit", + "requested_refs": ["main", "latest"], + "arm_sources": [ + { + "tag": "main", + "sha": "a" * 40, + "repo_url": "https://github.com/opentdf/java-sdk", + } + ], + } + ) + run = aggregate.Run("java", doc) + md = aggregate.markdown([run], expected_sdks=["java"]) + + assert run.status == "SAME COMMIT" + assert "roll-up — SAME COMMIT" in md + assert "`main`" in md and "`latest`" in md and "same commit" in md + + +def test_rollup_uses_measured_reference_order_and_links_sources(): + doc = document("go", "PASS") + metadata = doc["metadata"] + assert isinstance(metadata, dict) + metadata["arm_sources"] = [ + { + "tag": "main", + "sha": "b" * 40, + "repo_url": "https://github.com/opentdf/platform", + }, + { + "tag": "v1", + "release": "otdfctl/v1.0.0", + "sha": "a" * 40, + "repo_url": "https://github.com/opentdf/platform", + }, + ] + md = aggregate.markdown([aggregate.Run("go", doc)]) + + assert "releases/tag/otdfctl%2Fv1.0.0" in md + assert md.index("`v1`") < md.index("`main`") diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index d570ff2bf..32763f32f 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -264,6 +264,29 @@ def test_workflow_url_needs_all_three_parts(self, monkeypatch: pytest.MonkeyPatc monkeypatch.delenv("GITHUB_RUN_ID") assert bench._github_run_url() == "" + def test_requested_names_survive_resolver_deduplication( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv("BENCH_REQUESTED_REFS", "main latest") + + assert bench._requested_refs() == ["main", "latest"] + + def test_one_resolved_sha_from_two_names_is_marked_same_commit( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv("BENCH_REQUESTED_REFS", "main latest") + monkeypatch.setattr( + bench, + "_arm_sources", + lambda: ([{"tag": "main", "sha": "a" * 40}], ""), + ) + monkeypatch.setattr(bench, "_platform_version", lambda: "test") + + metadata = bench.runner_metadata(options(bench_seed="0")) + + assert metadata["comparison_status"] == "same_commit" + assert "main, latest" in str(metadata["comparison_note"]) + class TestDefaultBudget: def test_two_arms_keep_the_number_the_default_was_chosen_for(self): diff --git a/xtest/test_bench_report.py b/xtest/test_bench_report.py index 63f5f20b4..662024981 100644 --- a/xtest/test_bench_report.py +++ b/xtest/test_bench_report.py @@ -252,6 +252,70 @@ def test_provenance_links_releases_prs_commits_and_the_diff(self): assert "actions/runs/42" in md assert "candidate A" in md + def test_short_arm_names_are_links_and_controls_explain_the_same_build(self): + cfg = config(max_rounds=40) + rec = recorder({REF: 1.0, "cand": 1.0}, cfg=cfg, noise=0.01) + repo = "https://github.com/opentdf/platform" + rec.metadata = { + "arm_sources": [ + { + "tag": REF, + "release": "otdfctl/v0.37.0", + "sha": "a" * 40, + "repo_url": repo, + }, + { + "tag": "cand", + "pr": 321, + "sha": "b" * 40, + "repo_url": repo, + }, + ] + } + md = report.markdown(rec, cfg, rec.gate(cfg)) + + assert f"[`{REF}`]({repo}/releases/tag/otdfctl%2Fv0.37.0)" in md + assert f"[`cand`]({repo}/pull/321)" in md + assert "**A/A control** rows intentionally run multiple copies" in md + sources = report._sources_by_tag(rec.metadata) + assert report._linked_arm(f"{REF}#2", sources) == ( + f"[`{REF} copy 2`]({repo}/releases/tag/otdfctl%2Fv0.37.0)" + ) + assert "ratio b/a (95% CI)" in md + + def test_same_commit_is_neutral_and_explains_why_nothing_ran(self): + cfg = config(max_rounds=25) + rec = report.BenchmarkRecorder( + skipped={"java-encrypt-1KiB": "only one build installed"}, + metadata={ + "sdk": "java", + "comparison_status": "same_commit", + "comparison_note": ( + "main, latest resolve to the same commit 57d070b; " + "there is no code difference to benchmark." + ), + "requested_refs": ["main", "latest"], + "arm_sources": [ + { + "tag": "main", + "head": True, + "sha": "57d070b075a9134a11f8926b8898ac19b8cb6718", + "repo_url": "https://github.com/opentdf/java-sdk", + } + ], + }, + ) + gate = rec.gate(cfg) + md = report.markdown(rec, cfg, gate) + + assert gate.nothing_measured + assert report.same_commit(rec.metadata) + assert md.startswith("## JAVA SDK performance — SAME COMMIT") + assert "No performance comparison was necessary" in md + assert "every measured wall-clock" not in md + assert "### Compared builds" in md + assert "[`latest`](https://github.com/opentdf/java-sdk/tree/57d070" in md + def test_the_primary_table_omits_clean_rows_but_the_full_table_keeps_them(self): cfg = config(min_rounds=10, max_rounds=60) rec = recorder({REF: 1.0, "clean": 1.0, "slow": 1.4}, cfg=cfg, noise=0.005)