ci: make the CodSpeed memory benchmarks deterministic

.github/workflows/benchmarks.yml

@@ -16,9 +16,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+# The runner image is pinned: CodSpeed compares a pull request against its
+# base commit's run, and `ubuntu-latest` moving to a new image between the two
+# changes the numbers of every benchmark at once. Update it in a pull request of
+# its own, so the shift is attributed to it.
+
 jobs:
   benchmark:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       id-token: write # Required for OIDC authentication with CodSpeed
@@ -43,7 +48,7 @@ jobs:
           mode: "simulation"
 
   memory-benchmark:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       id-token: write # Required for OIDC authentication with CodSpeed

benchmark/README.md

@@ -30,8 +30,12 @@ the wrapper detects the CodSpeed runner environment and switches to
 instruction-counting mode automatically.
 
 The V8 flags in `package.json` (`--no-opt --predictable --hash-seed=1` etc.)
-are required by CodSpeed's instrumentation mode for deterministic results —
-do not drop them.
+are required by CodSpeed's instrumentation and memory modes for deterministic
+results — do not drop them. `benchmark:memory` needs them as much as
+`benchmark`: without them, V8 optimizes functions on background threads, and
+the compiler's own allocations land in whichever benchmark happens to be
+measured at the time, so the same commit reports different memory numbers
+from run to run. CodSpeed warns `missing required flags` when any is absent.
 
 ### Optional: running real instruction counts locally
 

benchmark/with-codspeed.mjs

@@ -15,6 +15,7 @@
  * Modes (via getCodspeedRunnerMode() from @codspeed/core):
  *   "disabled"   — returns the bench untouched (local runs)
  *   "simulation" — overrides bench.run/runSync for CodSpeed instrumentation
+ *   "memory"     — same as "simulation"; CodSpeed tracks allocations instead
  *   "walltime"   — left untouched; tinybench's built-in timing is used
  */
 
@@ -93,7 +94,7 @@ export function withCodSpeed(bench) {
 	const mode = getCodspeedRunnerMode();
 	if (mode === "disabled" || mode === "walltime") return bench;
 
-	// --- simulation mode ---
+	// --- simulation and memory modes ---
 
 	const meta = getOrCreateMeta(bench);
 	const rawAdd = bench.add.bind(bench);
@@ -107,7 +108,7 @@ export function withCodSpeed(bench) {
 
 	const setup = () => {
 		setupCore();
-		console.log("[CodSpeed] running in simulation mode");
+		console.log(`[CodSpeed] running in ${mode} mode`);
 	};
 
 	const teardown = () => {

package.json

@@ -34,7 +34,7 @@
     "test:legacy": "jest",
     "test:legacy:coverage": "jest --coverage",
     "benchmark": "node --max-old-space-size=4096 --hash-seed=1 --random-seed=1 --no-opt --predictable --predictable-gc-schedule --interpreted-frames-native-stack --allow-natives-syntax --expose-gc --no-concurrent-sweeping ./benchmark/run.mjs cases",
-    "benchmark:memory": "node --expose-gc --max-old-space-size=4096 ./benchmark/run.mjs memory",
+    "benchmark:memory": "node --max-old-space-size=4096 --hash-seed=1 --random-seed=1 --no-opt --predictable --predictable-gc-schedule --interpreted-frames-native-stack --allow-natives-syntax --expose-gc --no-concurrent-sweeping ./benchmark/run.mjs memory",
     "version": "changeset version && npm run fmt",
     "release": "changeset publish"
   },