From ad8d3b04ea6bf94520308d1be2167849e1b88c4e Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Fri, 17 Jul 2026 22:04:59 +0000 Subject: [PATCH 01/10] fix: standardize dax benchmark resources to 8 vCPU / 16 GiB Each provider gets 8 vCPUs and 16 GiB RAM for fair comparison: - E2B: cpuCount=8, memoryMB=16384 - Modal: cpu=4 (4 physical = 8 vCPUs), memoryMiB=16384 - Tensorlake: cpus=8, memoryMb=16384 - isorun: vcpus=8, memMiB=16384 - runloop: customCpuCores=8, customMemoryGb=16 - daytona: resources.cpu=8, resources.memory=16 (GiB) - upstash: size=large (8 cores, 16 GB) - vercel: resources.vcpus=8 (no memory control) - blaxel: memory=16384 (CPU derived: 16384/2048=8 cores) Providers without CPU/memory control (archil, cloudflare, northflank, beam, declaw, hopx, codesandbox) use their defaults. --- src/sandbox/dax.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index e1670ae4..f05b0fda 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -6,6 +6,28 @@ import { withTimeout } from '../util/timeout.js'; const BENCH_SCRIPT_URL = 'https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh'; +// Standardized resource sizing for fair comparison across providers. +// Target: 8 vCPU, 16 GiB RAM. +// Each provider uses different parameter names and units, so we map per-provider. +// Providers not listed here don't support CPU/memory configuration at sandbox creation time. +const DAX_RESOURCE_OPTIONS: Record> = { + e2b: { cpuCount: 8, memoryMB: 16384 }, + modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs + tensorlake: { cpus: 8, memoryMb: 16384 }, + isorun: { vcpus: 8, memMiB: 16384 }, + runloop: { customCpuCores: 8, customMemoryGb: 16 }, + daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB + upstash: { size: 'large' }, // large = 8 cores, 16 GB + vercel: { resources: { vcpus: 8 } }, // no memory control + blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 +}; + +function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { + const resourceOpts = DAX_RESOURCE_OPTIONS[providerName]; + if (!resourceOpts) return baseOptions ?? {}; + return { ...baseOptions, ...resourceOpts }; +} + export interface DaxTimingResult { totalMs: number; phasesCompleted?: number; @@ -74,7 +96,7 @@ export async function runDaxBenchmark(config: ProviderConfig): Promise Date: Fri, 17 Jul 2026 22:11:56 +0000 Subject: [PATCH 02/10] fix: remove E2B from resource options - CPU/mem set at template build time E2B's Sandbox.create() accepts SandboxOpts which does not include cpuCount/memoryMB. Those are template build options only. --- src/sandbox/dax.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index f05b0fda..edd6adc0 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -10,8 +10,8 @@ const BENCH_SCRIPT_URL = 'https://raw.githubusercontent.com/anomalyco/opencode/p // Target: 8 vCPU, 16 GiB RAM. // Each provider uses different parameter names and units, so we map per-provider. // Providers not listed here don't support CPU/memory configuration at sandbox creation time. +// Note: E2B sets CPU/memory at template build time, not at sandbox creation. const DAX_RESOURCE_OPTIONS: Record> = { - e2b: { cpuCount: 8, memoryMB: 16384 }, modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs tensorlake: { cpus: 8, memoryMb: 16384 }, isorun: { vcpus: 8, memMiB: 16384 }, From bca8ec723a47dacd7e309d70c26f6402d63f181a Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 01:56:19 +0000 Subject: [PATCH 03/10] fix: standardize dax resource specs for all providers to 8 vCPU / 16 GiB - Fix runloop: nest resource params in launch_parameters with snake_case - Add beam: cpu=8, memory=16384 MiB - Add codesandbox: vmTier=VMTier.Small (8 CPU / 16 GiB) - Add northflank: deploymentPlan resolved via API at runtime - Add E2B template build prereq (base-8cpu-16gb with 8 vCPU / 16 GiB) - Add Northflank plan discovery prereq script --- .github/workflows/sandbox-dax-benchmarks.yml | 10 +++ scripts/build-e2b-template.ts | 36 +++++++++ scripts/find-northflank-plan.ts | 81 ++++++++++++++++++++ src/sandbox/dax.ts | 6 +- src/sandbox/providers.ts | 1 + 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 scripts/build-e2b-template.ts create mode 100644 scripts/find-northflank-plan.ts diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index 476e8c21..10869eca 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -76,6 +76,16 @@ jobs: else npm ci fi + - name: Build E2B template with 8 vCPU / 16 GiB + if: matrix.provider == 'e2b' + env: + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + run: npx tsx scripts/build-e2b-template.ts + - name: Find Northflank plan for 8 vCPU / 16 GiB + if: matrix.provider == 'northflank' + env: + NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} + run: npx tsx scripts/find-northflank-plan.ts - name: Clear stale results from checkout run: rm -rf results/ - name: Run dax benchmark diff --git a/scripts/build-e2b-template.ts b/scripts/build-e2b-template.ts new file mode 100644 index 00000000..79b26be7 --- /dev/null +++ b/scripts/build-e2b-template.ts @@ -0,0 +1,36 @@ +import 'dotenv/config'; +import { Template, defaultBuildLogger } from 'e2b'; + +async function main() { + const apiKey = process.env.E2B_API_KEY; + if (!apiKey) { + throw new Error('E2B_API_KEY environment variable is not set'); + } + + const template = Template().fromTemplate('base'); + + try { + await Template.build(template, 'base-8cpu-16gb', { + cpuCount: 8, + memoryMB: 16384, + onBuildLogs: defaultBuildLogger(), + }); + console.log('E2B template base-8cpu-16gb built successfully'); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if ( + message.toLowerCase().includes('already exists') || + message.toLowerCase().includes('already taken') || + message.toLowerCase().includes('duplicate') + ) { + console.log('Template already exists, skipping'); + return; + } + throw error; + } +} + +main().catch((error) => { + console.error('Failed to build E2B template:', error); + process.exit(1); +}); diff --git a/scripts/find-northflank-plan.ts b/scripts/find-northflank-plan.ts new file mode 100644 index 00000000..c218c6ec --- /dev/null +++ b/scripts/find-northflank-plan.ts @@ -0,0 +1,81 @@ +import 'dotenv/config'; +import fs from 'fs'; + +interface Plan { + id: string; + cpuResource?: number; + ramResource?: number; + [key: string]: any; +} + +interface PlansResponse { + plans?: Plan[]; +} + +async function main() { + const token = process.env.NORTHFLANK_TOKEN; + if (!token) { + throw new Error('NORTHFLANK_TOKEN environment variable is not set'); + } + + const res = await fetch('https://api.northflank.com/v1/plans', { + headers: { Authorization: `Bearer ${token}` }, + }); + + if (!res.ok) { + throw new Error(`Failed to list Northflank plans: ${res.status} ${res.statusText}`); + } + + const body = (await res.json()) as PlansResponse | Plan[]; + const plans = Array.isArray(body) ? body : body.plans ?? []; + + if (plans.length === 0) { + throw new Error('No plans returned from Northflank API'); + } + + const targetCpu = 8; + const targetRam = 16384; // 16 GiB in MB + + const exactMatch = plans.find( + (p) => p.cpuResource === targetCpu && p.ramResource === targetRam, + ); + + let selectedPlan: Plan; + if (exactMatch) { + selectedPlan = exactMatch; + console.log(`Found Northflank plan with exact match: ${selectedPlan.id} (${targetCpu} vCPU, ${targetRam} MB)`); + } else { + const suitable = plans.filter( + (p) => (p.cpuResource ?? 0) >= targetCpu && (p.ramResource ?? 0) >= targetRam, + ); + if (suitable.length === 0) { + throw new Error( + `No Northflank plan meets the target resources of ${targetCpu} vCPU / ${targetRam} MB`, + ); + } + // Pick the closest plan by minimising normalised excess (1 vCPU == 2048 MB). + suitable.sort((a, b) => { + const scoreA = (a.cpuResource ?? 0) + (a.ramResource ?? 0) / 2048; + const scoreB = (b.cpuResource ?? 0) + (b.ramResource ?? 0) / 2048; + return scoreA - scoreB; + }); + selectedPlan = suitable[0]!; + console.warn( + `Warning: no exact match for ${targetCpu} vCPU / ${targetRam} MB. ` + + `Using closest suitable plan: ${selectedPlan.id} ` + + `(${selectedPlan.cpuResource ?? '?'} vCPU, ${selectedPlan.ramResource ?? '?'} MB)`, + ); + } + + console.log(`NORTHFLANK_DEPLOYMENT_PLAN=${selectedPlan.id}`); + + const githubEnv = process.env.GITHUB_ENV; + if (githubEnv) { + fs.appendFileSync(githubEnv, `NORTHFLANK_DEPLOYMENT_PLAN=${selectedPlan.id}\n`); + } +} + +main().catch((error) => { + console.error('Failed to find Northflank plan:', error); + process.exit(1); +}); diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index edd6adc0..2fbdeb66 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -3,6 +3,7 @@ import os from 'os'; import type { ProviderConfig, Stats } from './types.js'; import { computeStats } from '../util/stats.js'; import { withTimeout } from '../util/timeout.js'; +import { VMTier } from '@codesandbox/sdk'; const BENCH_SCRIPT_URL = 'https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh'; @@ -15,11 +16,14 @@ const DAX_RESOURCE_OPTIONS: Record> = { modal: { cpu: 4, cpuLimit: 4, memoryMiB: 16384 }, // Modal: 1 core = 2 vCPUs, so 4 cores = 8 vCPUs tensorlake: { cpus: 8, memoryMb: 16384 }, isorun: { vcpus: 8, memMiB: 16384 }, - runloop: { customCpuCores: 8, customMemoryGb: 16 }, + runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB upstash: { size: 'large' }, // large = 8 cores, 16 GB vercel: { resources: { vcpus: 8 } }, // no memory control blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 + beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB + codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB + northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50' }, // resolved by scripts/find-northflank-plan.ts }; function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { diff --git a/src/sandbox/providers.ts b/src/sandbox/providers.ts index 03d096ff..ec5ad89e 100644 --- a/src/sandbox/providers.ts +++ b/src/sandbox/providers.ts @@ -94,6 +94,7 @@ export const providers: ProviderConfig[] = [ name: 'e2b', requiredEnvVars: ['E2B_API_KEY'], createCompute: () => e2b({ apiKey: process.env.E2B_API_KEY! }), + sandboxOptions: { template: 'base-8cpu-16gb' }, }, { name: 'hopx', From 19740f5676507df65e766db69d94ca3c280400d2 Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 02:02:13 +0000 Subject: [PATCH 04/10] feat: local benchmark script with apt/dnf and arch detection - Add scripts/provider-benchmark.sh: vendored from upstream with dnf support (RHEL/Fedora) alongside apt-get (Debian/Ubuntu) and aarch64 architecture detection for Node.js and Bun downloads - Update dax.ts to load script from local filesystem instead of curl, eliminating curl dependency for providers that don't ship it --- scripts/provider-benchmark.sh | 254 ++++++++++++++++++++++++++++++++++ src/sandbox/dax.ts | 33 +++-- 2 files changed, 279 insertions(+), 8 deletions(-) create mode 100755 scripts/provider-benchmark.sh diff --git a/scripts/provider-benchmark.sh b/scripts/provider-benchmark.sh new file mode 100755 index 00000000..8403bdf6 --- /dev/null +++ b/scripts/provider-benchmark.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# Cold OpenCode clone/install/typecheck benchmark for a fresh VM (Debian/Ubuntu x86_64 or aarch64). + +set -euo pipefail + +REPO_URL="${BENCH_REPO_URL:-https://github.com/anomalyco/opencode.git}" +COMMIT="${BENCH_COMMIT:-08fb47373509ba64b13441061314eeacf4264f51}" +BUN_VERSION="${BENCH_BUN_VERSION:-1.3.14}" +NODE_VERSION="${BENCH_NODE_VERSION:-24.14.1}" +ROOT="${BENCH_ROOT:-/tmp/opencode-provider-benchmark}" +KEEP_ROOT="${BENCH_KEEP_ROOT:-false}" +PROVIDER="${BENCH_PROVIDER:-unknown}" +REGION="${BENCH_REGION:-unknown}" + +# Detect architecture for Node.js and Bun downloads. +# x86_64 -> Node: linux-x64, Bun: bun-linux-x64-baseline.zip (path: bun-linux-x64-baseline/bun) +# aarch64 -> Node: linux-arm64, Bun: bun-linux-aarch64.zip (path: bun-linux-aarch64/bun) +ARCH="$(uname -m)" +case "$ARCH" in + x86_64) + NODE_ARCH="linux-x64" + BUN_ARCH="x64" + BUN_BASELINE_SUFFIX="-baseline" + ;; + aarch64|arm64) + NODE_ARCH="linux-arm64" + BUN_ARCH="aarch64" + BUN_BASELINE_SUFFIX="" + ;; + *) + printf 'BENCH_ERROR\tprepare\tunsupported_arch_%s\n' "$ARCH" >&2 + exit 1 + ;; +esac +BUN_FILENAME="bun-linux-${BUN_ARCH}${BUN_BASELINE_SUFFIX}.zip" +BUN_INTERNAL_PATH="${BUN_FILENAME}/bun" + +# Detect the available package manager (apt-get on Debian/Ubuntu, dnf on RHEL/Fedora). +PKG_MANAGER="" +if command -v apt-get >/dev/null 2>&1; then + PKG_MANAGER="apt" +elif command -v dnf >/dev/null 2>&1; then + PKG_MANAGER="dnf" +else + printf 'BENCH_ERROR\tprepare\tno_package_manager_found\n' >&2 + exit 1 +fi + +declare -A PHASE_MS=() + +timestamp() { + date +%s%N +} + +phase() { + local name="$1" + shift + local start end + start="$(timestamp)" + set +e + "$@" + local status=$? + set -e + end="$(timestamp)" + PHASE_MS["$name"]="$(( (end - start) / 1000000 ))" + printf 'BENCH_PHASE\t%s\t%s\n' "$name" "${PHASE_MS[$name]}" + return "$status" +} + +seconds() { + awk -v milliseconds="${1:-0}" 'BEGIN { printf "%.3fs", milliseconds / 1000 }' +} + +render_table() { + local result="$1" + local typecheck="—" + local workload="—" + if [[ -n "${PHASE_MS[typecheck]:-}" ]]; then + typecheck="$(seconds "${PHASE_MS[typecheck]}")" + fi + if [[ "$result" == "✅" ]]; then + workload="$(seconds "$(( ${PHASE_MS[clone]} + ${PHASE_MS[install]} + ${PHASE_MS[typecheck]} ))")" + elif [[ -n "${PHASE_MS[typecheck]:-}" ]]; then + typecheck="${typecheck} (failed)" + fi + local memory + memory="$(awk '/MemTotal/{printf "%.2f GiB", $2 / 1048576}' /proc/meminfo)" + local cpu_model + cpu_model="$(awk -F: '/model name/{gsub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo)" + printf '\n| Provider | CPU / RAM | Region / CPU | Clone | Install | Typecheck | Workload total | Result |\n' + printf '|---|---|---|---:|---:|---:|---:|---|\n' + printf '| **%s** | %s CPU / %s | %s, %s | %s | %s | %s | %s | %s |\n' \ + "$PROVIDER" \ + "$(getconf _NPROCESSORS_ONLN)" \ + "$memory" \ + "$REGION" \ + "$cpu_model" \ + "$(seconds "${PHASE_MS[clone]:-0}")" \ + "$(seconds "${PHASE_MS[install]:-0}")" \ + "$typecheck" \ + "$workload" \ + "$result" +} + +if [[ "$(id -u)" -eq 0 ]]; then + SUDO=() +elif command -v sudo >/dev/null; then + SUDO=(sudo) +else + printf 'BENCH_ERROR\tprepare\troot_or_sudo_required\n' >&2 + exit 1 +fi + +prepare() { + case "$PKG_MANAGER" in + apt) + "${SUDO[@]}" apt-get update -qq + "${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + bash build-essential ca-certificates curl git python3 python3-setuptools unzip xz-utils + ;; + dnf) + "${SUDO[@]}" dnf makecache --quiet + "${SUDO[@]}" dnf install -y \ + bash gcc gcc-c++ make ca-certificates curl git python3 python3-setuptools unzip xz-utils + ;; + esac + + python3 -c 'import setuptools' + + if [[ "$(node --version 2>/dev/null || true)" != "v${NODE_VERSION}" ]]; then + local archive="node-v${NODE_VERSION}-${NODE_ARCH}.tar.xz" + local prefix="/opt/node-v${NODE_VERSION}-${NODE_ARCH}" + curl -fsSL "https://nodejs.org/download/release/v${NODE_VERSION}/${archive}" -o "/tmp/${archive}" + "${SUDO[@]}" rm -rf "$prefix" + "${SUDO[@]}" mkdir -p "$prefix" + "${SUDO[@]}" tar -xJf "/tmp/${archive}" --strip-components=1 -C "$prefix" + for executable in node npm npx corepack; do + "${SUDO[@]}" ln -sfn "$prefix/bin/$executable" "/usr/local/bin/$executable" + done + fi + test "$(node --version)" = "v${NODE_VERSION}" +} + +download_bun() { + curl -fsSL \ + "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/${BUN_FILENAME}" \ + -o "$ROOT/bun.zip" +} + +unpack_bun() { + unzip -q -j "$ROOT/bun.zip" "$BUN_INTERNAL_PATH" -d "$BUN_INSTALL/bin" + chmod +x "$BUN_INSTALL/bin/bun" +} + +clone_repo() { + mkdir "$ROOT/repo" + cd "$ROOT/repo" + git init -q + git remote add origin "$REPO_URL" + git fetch -q --depth=1 origin "$COMMIT" + git checkout -q --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$COMMIT" +} + +install_dependencies() { + cd "$ROOT/repo" + bun install + git diff --exit-code -- bun.lock package.json +} + +typecheck() { + cd "$ROOT/repo" + bun typecheck +} + +disk() { + du -sx --block-size=1 "$ROOT" | awk '{print $1}' +} + +clear_caches() { + rm -rf "$ROOT" + "${SUDO[@]}" sync + if "${SUDO[@]}" sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null; then + printf 'BENCH_CACHE\tguest_page_cache\tdropped\n' + else + printf 'BENCH_CACHE\tguest_page_cache\tunavailable\n' + fi + printf 'BENCH_CACHE\tworkspace\tfresh\n' + printf 'BENCH_CACHE\tbun\tempty\n' + printf 'BENCH_CACHE\tturbo\tempty\n' +} + +cleanup() { + local status=$? + if [[ "$KEEP_ROOT" != "true" ]]; then + rm -rf "$ROOT" + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +total_start="$(timestamp)" +if ! phase prepare prepare; then + render_table "❌ prepare" + exit 1 +fi + +phase cache_clear clear_caches +mkdir -p "$ROOT/bun/bin" "$ROOT/home" "$ROOT/bun-cache" +export HOME="$ROOT/home" +export BUN_INSTALL="$ROOT/bun" +export BUN_INSTALL_CACHE_DIR="$ROOT/bun-cache" +export PATH="$BUN_INSTALL/bin:$PATH" +export CI=true +export OPENCODE_DISABLE_SHARE=true +export TURBO_TELEMETRY_DISABLED=1 + +printf 'BENCH_META\tcommit\t%s\n' "$COMMIT" +printf 'BENCH_META\tarchitecture\t%s\n' "$(uname -m)" +printf 'BENCH_META\tkernel\t%s\n' "$(uname -sr)" +printf 'BENCH_META\tlogical_cpus\t%s\n' "$(getconf _NPROCESSORS_ONLN)" +printf 'BENCH_META\tcpu_model\t%s\n' "$(awk -F: '/model name/{gsub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo)" +printf 'BENCH_META\tmemory_kib\t%s\n' "$(awk '/MemTotal/{print $2}' /proc/meminfo)" + +if ! phase bun_download download_bun; then + render_table "❌ Bun download" + exit 1 +fi +if ! phase bun_unpack unpack_bun; then + render_table "❌ Bun unpack" + exit 1 +fi +printf 'BENCH_META\tbun_version\t%s\n' "$(bun --version)" +printf 'BENCH_META\tnode_version\t%s\n' "$(node --version)" + +if ! phase clone clone_repo; then + render_table "❌ clone" + exit 1 +fi +printf 'BENCH_DISK\tafter_clone\t%s\n' "$(disk)" +if ! phase install install_dependencies; then + render_table "❌ install" + exit 1 +fi +printf 'BENCH_DISK\tafter_install\t%s\n' "$(disk)" +if ! phase typecheck typecheck; then + render_table "❌ typecheck" + exit 1 +fi +printf 'BENCH_DISK\tafter_typecheck\t%s\n' "$(disk)" +printf 'BENCH_DONE\t%s\n' "$(git -C "$ROOT/repo" rev-parse HEAD)" +total_end="$(timestamp)" +printf 'BENCH_PHASE\ttotal\t%s\n' "$(( (total_end - total_start) / 1000000 ))" +render_table "✅" diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index 2fbdeb66..f1a0c6d1 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -1,11 +1,16 @@ import fs from 'fs'; import os from 'os'; +import path from 'path'; import type { ProviderConfig, Stats } from './types.js'; import { computeStats } from '../util/stats.js'; import { withTimeout } from '../util/timeout.js'; import { VMTier } from '@codesandbox/sdk'; -const BENCH_SCRIPT_URL = 'https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh'; +// The benchmark script is now loaded from the local filesystem (scripts/provider-benchmark.sh) +// rather than fetched over HTTP from upstream. This avoids a curl dependency inside the +// sandbox (some providers don't ship curl). The previous upstream URL was: +// https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh +const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../../scripts/provider-benchmark.sh'); // Standardized resource sizing for fair comparison across providers. // Target: 8 vCPU, 16 GiB RAM. @@ -154,20 +159,32 @@ export async function runDaxBenchmark(config: ProviderConfig): Promise { + // Load the benchmark script from the local filesystem rather than fetching + // it over HTTP inside the sandbox. This eliminates a curl dependency + // (several providers don't ship curl in their sandboxes). + const benchScript = fs.readFileSync(BENCH_SCRIPT_PATH, 'utf8'); const script = String.raw` const { spawnSync } = require('child_process'); const { performance } = require('perf_hooks'); -const scriptUrl = ${JSON.stringify(BENCH_SCRIPT_URL)}; +const benchScript = ${JSON.stringify(benchScript)}; const provider = ${JSON.stringify(providerName)}; const start = performance.now(); -// Download and run the dax benchmark script via curl|bash. -// The script emits structured BENCH_PHASE / BENCH_META / BENCH_DISK / BENCH_DONE -// lines on stdout and BENCH_ERROR on stderr that we parse below. -// If curl is not available the script will fail (which is expected and tracked). -const result = spawnSync('bash', ['-c', 'curl -fsSL ' + scriptUrl + ' | BENCH_PROVIDER=' + provider + ' BENCH_REGION=unknown bash'], { +// Write the benchmark script to /tmp inside the sandbox via a single-quoted +// heredoc (so $ and backticks in the script are not expanded) and execute it. +// A random marker avoids collisions with anything appearing on its own line +// inside the script. The script emits the same BENCH_PHASE / BENCH_META / +// BENCH_DISK / BENCH_DONE / BENCH_ERROR structure consumed by the parser below. +const marker = '__DAX_BENCH_HEREDOC_' + Math.random().toString(36).slice(2) + '__'; +const shellCmd = + "cat > /tmp/provider-benchmark.sh <<'" + marker + "'\n" + + benchScript + "\n" + + marker + "\n" + + "BENCH_PROVIDER=" + provider + " BENCH_REGION=unknown bash /tmp/provider-benchmark.sh"; + +const result = spawnSync('bash', ['-c', shellCmd], { encoding: 'utf8', timeout: 540000, stdio: ['ignore', 'pipe', 'pipe'], @@ -339,7 +356,7 @@ export async function writeDaxResultsJson(results: DaxBenchmarkResult[], outPath version: '1.0', timestamp: new Date().toISOString(), environment: { node: process.version, platform: os.platform(), arch: os.arch() }, - config: { mode: 'sandbox-dax', timeoutMs: 600000, scriptUrl: BENCH_SCRIPT_URL }, + config: { mode: 'sandbox-dax', timeoutMs: 600000, scriptSource: 'local', scriptPath: BENCH_SCRIPT_PATH }, results: cleanResults, }; From 5f71355be68d48023c20569ea238d1655983b92f Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 02:18:02 +0000 Subject: [PATCH 05/10] rename: provider-benchmark.sh to dax-benchmark.sh --- scripts/{provider-benchmark.sh => dax-benchmark.sh} | 0 src/sandbox/dax.ts | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename scripts/{provider-benchmark.sh => dax-benchmark.sh} (100%) diff --git a/scripts/provider-benchmark.sh b/scripts/dax-benchmark.sh similarity index 100% rename from scripts/provider-benchmark.sh rename to scripts/dax-benchmark.sh diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index f1a0c6d1..829c41da 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -6,11 +6,11 @@ import { computeStats } from '../util/stats.js'; import { withTimeout } from '../util/timeout.js'; import { VMTier } from '@codesandbox/sdk'; -// The benchmark script is now loaded from the local filesystem (scripts/provider-benchmark.sh) +// The benchmark script is now loaded from the local filesystem (scripts/dax-benchmark.sh) // rather than fetched over HTTP from upstream. This avoids a curl dependency inside the // sandbox (some providers don't ship curl). The previous upstream URL was: // https://raw.githubusercontent.com/anomalyco/opencode/provider-benchmark/script/provider-benchmark.sh -const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../../scripts/provider-benchmark.sh'); +const BENCH_SCRIPT_PATH = path.resolve(import.meta.dirname, '../../scripts/dax-benchmark.sh'); // Standardized resource sizing for fair comparison across providers. // Target: 8 vCPU, 16 GiB RAM. @@ -179,10 +179,10 @@ const start = performance.now(); // BENCH_DISK / BENCH_DONE / BENCH_ERROR structure consumed by the parser below. const marker = '__DAX_BENCH_HEREDOC_' + Math.random().toString(36).slice(2) + '__'; const shellCmd = - "cat > /tmp/provider-benchmark.sh <<'" + marker + "'\n" + + "cat > /tmp/dax-benchmark.sh <<'" + marker + "'\n" + benchScript + "\n" + marker + "\n" + - "BENCH_PROVIDER=" + provider + " BENCH_REGION=unknown bash /tmp/provider-benchmark.sh"; + "BENCH_PROVIDER=" + provider + " BENCH_REGION=unknown bash /tmp/dax-benchmark.sh"; const result = spawnSync('bash', ['-c', shellCmd], { encoding: 'utf8', From d6497ade8eb65272445f24042102479248b77485 Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 02:30:31 +0000 Subject: [PATCH 06/10] feat: add namespace provider to dax benchmark (16 vCPU / 32 GiB) --- .github/workflows/sandbox-dax-benchmarks.yml | 2 ++ src/sandbox/providers.ts | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index 10869eca..2ef575c8 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -57,6 +57,7 @@ jobs: - isorun - lightning - modal + - namespace - northflank - runloop - superserve @@ -113,6 +114,7 @@ jobs: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} + NSC_TOKEN: ${{ secrets.NSC_TOKEN }} NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }} RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }} diff --git a/src/sandbox/providers.ts b/src/sandbox/providers.ts index ec5ad89e..e24ca1f6 100644 --- a/src/sandbox/providers.ts +++ b/src/sandbox/providers.ts @@ -14,7 +14,7 @@ import { isorun } from '@computesdk/isorun'; // import { lelantos } from '@computesdk/lelantos'; import { lightning } from '@computesdk/lightning'; import { modal } from '@computesdk/modal'; -// import { namespace } from '@computesdk/namespace'; +import { namespace } from '@computesdk/namespace'; import { northflank } from '@computesdk/northflank'; // import { quilt } from '@computesdk/quilt'; // import { railway } from '@computesdk/railway'; @@ -122,12 +122,16 @@ export const providers: ProviderConfig[] = [ requiredEnvVars: ['MODAL_TOKEN_ID', 'MODAL_TOKEN_SECRET'], createCompute: () => modal({ tokenId: process.env.MODAL_TOKEN_ID!, tokenSecret: process.env.MODAL_TOKEN_SECRET!, scalableSandboxes: true }), }, - // { - // name: 'namespace', - // requiredEnvVars: ['NSC_TOKEN'], - // createCompute: () => namespace({ token: process.env.NSC_TOKEN! }), - // sandboxOptions: { image: 'node:22' }, - // }, + { + name: 'namespace', + requiredEnvVars: ['NSC_TOKEN'], + createCompute: () => + namespace({ + token: process.env.NSC_TOKEN!, + virtualCpu: 16, + memoryMegabytes: 32768, + }), + }, { name: 'northflank', requiredEnvVars: ['NORTHFLANK_TOKEN', 'NORTHFLANK_PROJECT_ID'], From f0efe46e221316fc666f3859e10318d598daa872 Mon Sep 17 00:00:00 2001 From: Shivam Nayak Date: Sat, 18 Jul 2026 08:02:50 +0530 Subject: [PATCH 07/10] dax: standardize declaw resources via node-large template (#197) Adds declaw to DAX_RESOURCE_OPTIONS so it targets the same 8 vCPU / 16 GiB profile as the other providers in the map. The node-large template provisions 8 vCPU / 16 GiB RAM with an 8 GiB disk overlay. Co-authored-by: shivam-declaw <272314190+shivam-declaw@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) --- src/sandbox/dax.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index 829c41da..45abf734 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -29,6 +29,7 @@ const DAX_RESOURCE_OPTIONS: Record> = { beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50' }, // resolved by scripts/find-northflank-plan.ts + declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk }; function getSandboxOptionsWithResources(providerName: string, baseOptions?: Record): Record { From c7f4c47be999195a5434bbd5a12b7344ad9c8d2a Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 02:40:20 +0000 Subject: [PATCH 08/10] fix: bun zip path, northflank API fallback, daytona snapshot, namespace node image - Fix BUN_INTERNAL_PATH: remove .zip extension from zip-internal path (was bun-linux-x64-baseline.zip/bun, should be bun-linux-x64-baseline/bun) - Northflank: robust API response parsing, graceful fallback to nf-compute-50, sanitize plan ID before writing to GITHUB_ENV - Daytona: remove from DAX_RESOURCE_OPTIONS (resources not supported with snapshot-based sandbox creation) - Namespace: add sandboxOptions image node:22 (dax benchmark requires node) --- scripts/dax-benchmark.sh | 2 +- scripts/find-northflank-plan.ts | 36 +++++++++++++++++++++++++++++++-- src/sandbox/dax.ts | 2 +- src/sandbox/providers.ts | 1 + 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/scripts/dax-benchmark.sh b/scripts/dax-benchmark.sh index 8403bdf6..dac1172e 100755 --- a/scripts/dax-benchmark.sh +++ b/scripts/dax-benchmark.sh @@ -33,7 +33,7 @@ case "$ARCH" in ;; esac BUN_FILENAME="bun-linux-${BUN_ARCH}${BUN_BASELINE_SUFFIX}.zip" -BUN_INTERNAL_PATH="${BUN_FILENAME}/bun" +BUN_INTERNAL_PATH="bun-linux-${BUN_ARCH}${BUN_BASELINE_SUFFIX}/bun" # Detect the available package manager (apt-get on Debian/Ubuntu, dnf on RHEL/Fedora). PKG_MANAGER="" diff --git a/scripts/find-northflank-plan.ts b/scripts/find-northflank-plan.ts index c218c6ec..0bbcdc29 100644 --- a/scripts/find-northflank-plan.ts +++ b/scripts/find-northflank-plan.ts @@ -10,6 +10,7 @@ interface Plan { interface PlansResponse { plans?: Plan[]; + data?: { plans?: Plan[] }; } async function main() { @@ -27,10 +28,32 @@ async function main() { } const body = (await res.json()) as PlansResponse | Plan[]; - const plans = Array.isArray(body) ? body : body.plans ?? []; + // Log the raw response structure (keys only, not the full body) for debugging. + if (body && typeof body === 'object' && !Array.isArray(body)) { + console.log('Northflank plans response keys:', Object.keys(body).join(', ')); + if ('data' in body && body.data && typeof body.data === 'object') { + console.log('Northflank plans response.data keys:', Object.keys(body.data as object).join(', ')); + } + } else if (Array.isArray(body)) { + console.log('Northflank plans response is an array of length', body.length); + } else { + console.log('Northflank plans response is of type', typeof body); + } + + // The API may return plans at body.plans, body.data.plans, or as a bare array. + const plans: Plan[] = Array.isArray(body) + ? body + : body.plans ?? body.data?.plans ?? []; if (plans.length === 0) { - throw new Error('No plans returned from Northflank API'); + console.warn('Warning: No plans returned from Northflank API, falling back to nf-compute-50'); + const fallback = 'nf-compute-50'; + console.log(`NORTHFLANK_DEPLOYMENT_PLAN=${fallback}`); + const githubEnv = process.env.GITHUB_ENV; + if (githubEnv) { + fs.appendFileSync(githubEnv, `NORTHFLANK_DEPLOYMENT_PLAN=${fallback}\n`); + } + return; } const targetCpu = 8; @@ -67,6 +90,15 @@ async function main() { ); } + // Sanitise the plan ID before writing to GITHUB_ENV to avoid injection of + // newlines or other shell/environment-file control characters. + if (!/^[a-zA-Z0-9_-]+$/.test(selectedPlan.id)) { + throw new Error( + `Invalid Northflank plan ID: ${JSON.stringify(selectedPlan.id)} ` + + `(must match /^[a-zA-Z0-9_-]+$/)`, + ); + } + console.log(`NORTHFLANK_DEPLOYMENT_PLAN=${selectedPlan.id}`); const githubEnv = process.env.GITHUB_ENV; diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index 45abf734..c95f1066 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -22,12 +22,12 @@ const DAX_RESOURCE_OPTIONS: Record> = { tensorlake: { cpus: 8, memoryMb: 16384 }, isorun: { vcpus: 8, memMiB: 16384 }, runloop: { launch_parameters: { resource_size_request: 'CUSTOM_SIZE', custom_cpu_cores: 8, custom_gb_memory: 16 } }, - daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB upstash: { size: 'large' }, // large = 8 cores, 16 GB vercel: { resources: { vcpus: 8 } }, // no memory control blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB + // daytona: omitted - resources cannot be specified when creating from a snapshot (uses defaults). northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50' }, // resolved by scripts/find-northflank-plan.ts declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk }; diff --git a/src/sandbox/providers.ts b/src/sandbox/providers.ts index e24ca1f6..47409d6f 100644 --- a/src/sandbox/providers.ts +++ b/src/sandbox/providers.ts @@ -131,6 +131,7 @@ export const providers: ProviderConfig[] = [ virtualCpu: 16, memoryMegabytes: 32768, }), + sandboxOptions: { image: 'node:22' }, }, { name: 'northflank', From 58b9d5e6b420d4eacfd511bb368e49cc50fd7afa Mon Sep 17 00:00:00 2001 From: Garrison Snelling Date: Sat, 18 Jul 2026 03:33:02 +0000 Subject: [PATCH 09/10] fix: northflank smallest plan selection + E2B command timeout workaround - Northflank: pick smallest suitable plan instead of closest to reduce chance of exceeding project resource allowance - E2B: workaround for 60s command timeout by setting defaultProcessConnectionTimeout=0 via getInstance() while waiting for upstream wrapper fix to forward timeoutMs --- scripts/find-northflank-plan.ts | 12 +++++++----- src/sandbox/dax.ts | 8 ++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/find-northflank-plan.ts b/scripts/find-northflank-plan.ts index 0bbcdc29..403402c1 100644 --- a/scripts/find-northflank-plan.ts +++ b/scripts/find-northflank-plan.ts @@ -76,16 +76,18 @@ async function main() { `No Northflank plan meets the target resources of ${targetCpu} vCPU / ${targetRam} MB`, ); } - // Pick the closest plan by minimising normalised excess (1 vCPU == 2048 MB). + // Pick the SMALLEST suitable plan by total resource footprint to minimise + // the chance of exceeding Northflank project allowance. (1 vCPU ~= 2048 MB + // for normalisation against RAM in MB.) suitable.sort((a, b) => { - const scoreA = (a.cpuResource ?? 0) + (a.ramResource ?? 0) / 2048; - const scoreB = (b.cpuResource ?? 0) + (b.ramResource ?? 0) / 2048; - return scoreA - scoreB; + const sizeA = (a.cpuResource ?? 0) * 2048 + (a.ramResource ?? 0); + const sizeB = (b.cpuResource ?? 0) * 2048 + (b.ramResource ?? 0); + return sizeA - sizeB; }); selectedPlan = suitable[0]!; console.warn( `Warning: no exact match for ${targetCpu} vCPU / ${targetRam} MB. ` + - `Using closest suitable plan: ${selectedPlan.id} ` + + `Using smallest suitable plan: ${selectedPlan.id} ` + `(${selectedPlan.cpuResource ?? '?'} vCPU, ${selectedPlan.ramResource ?? '?'} MB)`, ); } diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index c95f1066..4706d632 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -107,6 +107,14 @@ export async function runDaxBenchmark(config: ProviderConfig): Promise Date: Sat, 18 Jul 2026 03:55:37 +0000 Subject: [PATCH 10/10] fix: daytona use image-based creation to enable resource configuration Add image: 'node:22' to daytona sandboxOptions to switch from snapshot-based to image-based creation, which allows specifying resources. Re-add daytona to DAX_RESOURCE_OPTIONS with 8 vCPU / 16 GiB. --- src/sandbox/dax.ts | 2 +- src/sandbox/providers.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sandbox/dax.ts b/src/sandbox/dax.ts index 4706d632..b40f5d30 100644 --- a/src/sandbox/dax.ts +++ b/src/sandbox/dax.ts @@ -27,7 +27,7 @@ const DAX_RESOURCE_OPTIONS: Record> = { blaxel: { memory: 16384 }, // CPU derived: cores = memory_MB / 2048 = 8 beam: { cpu: 8, memory: 16384 }, // cpu = cores, memory = MiB codesandbox: { vmTier: VMTier.Small }, // Small = 8 CPU, 16 GiB - // daytona: omitted - resources cannot be specified when creating from a snapshot (uses defaults). + daytona: { resources: { cpu: 8, memory: 16 } }, // memory in GiB; requires image-based creation (see providers.ts) northflank: { deploymentPlan: process.env.NORTHFLANK_DEPLOYMENT_PLAN || 'nf-compute-50' }, // resolved by scripts/find-northflank-plan.ts declaw: { templateId: 'node-large' }, // node-large template: 8 vCPU / 16 GiB RAM / 8 GiB disk }; diff --git a/src/sandbox/providers.ts b/src/sandbox/providers.ts index 47409d6f..84a1df34 100644 --- a/src/sandbox/providers.ts +++ b/src/sandbox/providers.ts @@ -83,7 +83,7 @@ export const providers: ProviderConfig[] = [ name: 'daytona', requiredEnvVars: ['DAYTONA_API_KEY'], createCompute: () => daytona({ apiKey: process.env.DAYTONA_API_KEY! }), - sandboxOptions: { autoStopInterval: 15, autoDeleteInterval: 0 }, + sandboxOptions: { autoStopInterval: 15, autoDeleteInterval: 0, image: 'node:22' }, }, { name: 'declaw',