Skip to content

fix: standardize dax benchmark resources to 8 vCPU / 16 GiB #196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HeyGarrison wants to merge 10 commits into
base: master
Choose a base branch
from
Open

fix: standardize dax benchmark resources to 8 vCPU / 16 GiB #196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ad8d3b0
fix: standardize dax benchmark resources to 8 vCPU / 16 GiB
HeyGarrison Jul 17, 2026
27f9ee4
fix: remove E2B from resource options - CPU/mem set at template build…
HeyGarrison Jul 17, 2026
bca8ec7
fix: standardize dax resource specs for all providers to 8 vCPU / 16 GiB
HeyGarrison Jul 18, 2026
19740f5
feat: local benchmark script with apt/dnf and arch detection
HeyGarrison Jul 18, 2026
5f71355
rename: provider-benchmark.sh to dax-benchmark.sh
HeyGarrison Jul 18, 2026
d6497ad
feat: add namespace provider to dax benchmark (16 vCPU / 32 GiB)
HeyGarrison Jul 18, 2026
f0efe46
dax: standardize declaw resources via node-large template (#197)
shivam-declaw Jul 18, 2026
c7f4c47
fix: bun zip path, northflank API fallback, daytona snapshot, namespa…
HeyGarrison Jul 18, 2026
58b9d5e
fix: northflank smallest plan selection + E2B command timeout workaround
HeyGarrison Jul 18, 2026
67813b6
fix: daytona use image-based creation to enable resource configuration
HeyGarrison Jul 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/workflows/sandbox-dax-benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ jobs:
- isorun
- lightning
- modal
- namespace
- northflank
- runloop
- superserve
Expand All @@ -76,6 +77,16 @@ jobs:
else
npm ci
fi
- name: Build E2B template with 8 vCPU / 16 GiB
if: matrix.provider == 'e2b'
env:
E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
run: npx tsx scripts/build-e2b-template.ts
- name: Find Northflank plan for 8 vCPU / 16 GiB
if: matrix.provider == 'northflank'
env:
NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }}
run: npx tsx scripts/find-northflank-plan.ts
- name: Clear stale results from checkout
run: rm -rf results/
- name: Run dax benchmark
Expand Down Expand Up @@ -103,6 +114,7 @@ jobs:
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }}
NSC_TOKEN: ${{ secrets.NSC_TOKEN }}
NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }}
RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }}
SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }}
Expand Down
36 changes: 36 additions & 0 deletions scripts/build-e2b-template.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import 'dotenv/config';
import { Template, defaultBuildLogger } from 'e2b';

async function main() {
const apiKey = process.env.E2B_API_KEY;
if (!apiKey) {
throw new Error('E2B_API_KEY environment variable is not set');
}

const template = Template().fromTemplate('base');

try {
await Template.build(template, 'base-8cpu-16gb', {
cpuCount: 8,
memoryMB: 16384,
onBuildLogs: defaultBuildLogger(),
});
console.log('E2B template base-8cpu-16gb built successfully');
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (
message.toLowerCase().includes('already exists') ||
message.toLowerCase().includes('already taken') ||
message.toLowerCase().includes('duplicate')
) {
console.log('Template already exists, skipping');
return;
}
throw error;
}
}

main().catch((error) => {
console.error('Failed to build E2B template:', error);
process.exit(1);
});
254 changes: 254 additions & 0 deletions scripts/dax-benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,254 @@
#!/usr/bin/env bash
# Cold OpenCode clone/install/typecheck benchmark for a fresh VM (Debian/Ubuntu x86_64 or aarch64).

set -euo pipefail

REPO_URL="${BENCH_REPO_URL:-https://github.com/anomalyco/opencode.git}"
COMMIT="${BENCH_COMMIT:-08fb47373509ba64b13441061314eeacf4264f51}"
BUN_VERSION="${BENCH_BUN_VERSION:-1.3.14}"
NODE_VERSION="${BENCH_NODE_VERSION:-24.14.1}"
ROOT="${BENCH_ROOT:-/tmp/opencode-provider-benchmark}"
KEEP_ROOT="${BENCH_KEEP_ROOT:-false}"
PROVIDER="${BENCH_PROVIDER:-unknown}"
REGION="${BENCH_REGION:-unknown}"

# Detect architecture for Node.js and Bun downloads.
# x86_64 -> Node: linux-x64, Bun: bun-linux-x64-baseline.zip (path: bun-linux-x64-baseline/bun)
# aarch64 -> Node: linux-arm64, Bun: bun-linux-aarch64.zip (path: bun-linux-aarch64/bun)
ARCH="$(uname -m)"
case "$ARCH" in
x86_64)
NODE_ARCH="linux-x64"
BUN_ARCH="x64"
BUN_BASELINE_SUFFIX="-baseline"
;;
aarch64|arm64)
NODE_ARCH="linux-arm64"
BUN_ARCH="aarch64"
BUN_BASELINE_SUFFIX=""
;;
*)
printf 'BENCH_ERROR\tprepare\tunsupported_arch_%s\n' "$ARCH" >&2
exit 1
;;
esac
BUN_FILENAME="bun-linux-${BUN_ARCH}${BUN_BASELINE_SUFFIX}.zip"
BUN_INTERNAL_PATH="bun-linux-${BUN_ARCH}${BUN_BASELINE_SUFFIX}/bun"

# Detect the available package manager (apt-get on Debian/Ubuntu, dnf on RHEL/Fedora).
PKG_MANAGER=""
if command -v apt-get >/dev/null 2>&1; then
PKG_MANAGER="apt"
elif command -v dnf >/dev/null 2>&1; then
PKG_MANAGER="dnf"
else
printf 'BENCH_ERROR\tprepare\tno_package_manager_found\n' >&2
exit 1
fi

declare -A PHASE_MS=()

timestamp() {
date +%s%N
}

phase() {
local name="$1"
shift
local start end
start="$(timestamp)"
set +e
"$@"
local status=$?
set -e
end="$(timestamp)"
PHASE_MS["$name"]="$(( (end - start) / 1000000 ))"
printf 'BENCH_PHASE\t%s\t%s\n' "$name" "${PHASE_MS[$name]}"
return "$status"
}

seconds() {
awk -v milliseconds="${1:-0}" 'BEGIN { printf "%.3fs", milliseconds / 1000 }'
}

render_table() {
local result="$1"
local typecheck="—"
local workload="—"
if [[ -n "${PHASE_MS[typecheck]:-}" ]]; then
typecheck="$(seconds "${PHASE_MS[typecheck]}")"
fi
if [[ "$result" == "✅" ]]; then
workload="$(seconds "$(( ${PHASE_MS[clone]} + ${PHASE_MS[install]} + ${PHASE_MS[typecheck]} ))")"
elif [[ -n "${PHASE_MS[typecheck]:-}" ]]; then
typecheck="${typecheck} (failed)"
fi
local memory
memory="$(awk '/MemTotal/{printf "%.2f GiB", $2 / 1048576}' /proc/meminfo)"
local cpu_model
cpu_model="$(awk -F: '/model name/{gsub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo)"
printf '\n| Provider | CPU / RAM | Region / CPU | Clone | Install | Typecheck | Workload total | Result |\n'
printf '|---|---|---|---:|---:|---:|---:|---|\n'
printf '| **%s** | %s CPU / %s | %s, %s | %s | %s | %s | %s | %s |\n' \
"$PROVIDER" \
"$(getconf _NPROCESSORS_ONLN)" \
"$memory" \
"$REGION" \
"$cpu_model" \
"$(seconds "${PHASE_MS[clone]:-0}")" \
"$(seconds "${PHASE_MS[install]:-0}")" \
"$typecheck" \
"$workload" \
"$result"
}

if [[ "$(id -u)" -eq 0 ]]; then
SUDO=()
elif command -v sudo >/dev/null; then
SUDO=(sudo)
else
printf 'BENCH_ERROR\tprepare\troot_or_sudo_required\n' >&2
exit 1
fi

prepare() {
case "$PKG_MANAGER" in
apt)
"${SUDO[@]}" apt-get update -qq
"${SUDO[@]}" env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
bash build-essential ca-certificates curl git python3 python3-setuptools unzip xz-utils
;;
dnf)
"${SUDO[@]}" dnf makecache --quiet
"${SUDO[@]}" dnf install -y \
bash gcc gcc-c++ make ca-certificates curl git python3 python3-setuptools unzip xz-utils
;;
esac

python3 -c 'import setuptools'

if [[ "$(node --version 2>/dev/null || true)" != "v${NODE_VERSION}" ]]; then
local archive="node-v${NODE_VERSION}-${NODE_ARCH}.tar.xz"
local prefix="/opt/node-v${NODE_VERSION}-${NODE_ARCH}"
curl -fsSL "https://nodejs.org/download/release/v${NODE_VERSION}/${archive}" -o "/tmp/${archive}"

@superagent-security superagent-security Bot Jul 18, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Benchmark script downloads and executes binaries without checksum verification

Downloads Node.js and Bun archives without verifying checksums or signatures before execution.

Verify SHA256 checksums of downloaded archives against official manifests before extraction.

AI prompt 
Check if this security scanner issue is valid. If so, understand the root cause and fix it. If appropriate, update or add tests. Keep the change focused and preserve intended behavior.

<file name="scripts/dax-benchmark.sh">
<violation number="1" location="scripts/dax-benchmark.sh:133">
<priority>P2</priority>
<title>Benchmark script downloads and executes binaries without checksum verification</title>
<evidence>The script uses curl to download Node.js and Bun release archives and then directly extracts and executes them without verifying SHA256 checksums, signatures, or release manifests.</evidence>
<recommendation>Add checksum verification for downloaded Node.js and Bun archives before extraction. For Node.js, verify against the official SHASUMS256.txt file; for Bun, use the published checksum files or hardcode known-good checksums for each supported version.</recommendation>
</violation>
</file>

"${SUDO[@]}" rm -rf "$prefix"
"${SUDO[@]}" mkdir -p "$prefix"
"${SUDO[@]}" tar -xJf "/tmp/${archive}" --strip-components=1 -C "$prefix"
for executable in node npm npx corepack; do
"${SUDO[@]}" ln -sfn "$prefix/bin/$executable" "/usr/local/bin/$executable"
done
fi
test "$(node --version)" = "v${NODE_VERSION}"
}

download_bun() {
curl -fsSL \
"https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/${BUN_FILENAME}" \
-o "$ROOT/bun.zip"
}

unpack_bun() {
unzip -q -j "$ROOT/bun.zip" "$BUN_INTERNAL_PATH" -d "$BUN_INSTALL/bin"
chmod +x "$BUN_INSTALL/bin/bun"
}

clone_repo() {
mkdir "$ROOT/repo"
cd "$ROOT/repo"
git init -q
git remote add origin "$REPO_URL"
git fetch -q --depth=1 origin "$COMMIT"
git checkout -q --detach FETCH_HEAD
test "$(git rev-parse HEAD)" = "$COMMIT"
}

install_dependencies() {
cd "$ROOT/repo"
bun install
git diff --exit-code -- bun.lock package.json
}

typecheck() {
cd "$ROOT/repo"
bun typecheck
}

disk() {
du -sx --block-size=1 "$ROOT" | awk '{print $1}'
}

clear_caches() {
rm -rf "$ROOT"
"${SUDO[@]}" sync
if "${SUDO[@]}" sh -c 'echo 3 > /proc/sys/vm/drop_caches' 2>/dev/null; then
printf 'BENCH_CACHE\tguest_page_cache\tdropped\n'
else
printf 'BENCH_CACHE\tguest_page_cache\tunavailable\n'
fi
printf 'BENCH_CACHE\tworkspace\tfresh\n'
printf 'BENCH_CACHE\tbun\tempty\n'
printf 'BENCH_CACHE\tturbo\tempty\n'
}

cleanup() {
local status=$?
if [[ "$KEEP_ROOT" != "true" ]]; then
rm -rf "$ROOT"
fi
exit "$status"
}
trap cleanup EXIT INT TERM

total_start="$(timestamp)"
if ! phase prepare prepare; then
render_table "❌ prepare"
exit 1
fi

phase cache_clear clear_caches
mkdir -p "$ROOT/bun/bin" "$ROOT/home" "$ROOT/bun-cache"
export HOME="$ROOT/home"
export BUN_INSTALL="$ROOT/bun"
export BUN_INSTALL_CACHE_DIR="$ROOT/bun-cache"
export PATH="$BUN_INSTALL/bin:$PATH"
export CI=true
export OPENCODE_DISABLE_SHARE=true
export TURBO_TELEMETRY_DISABLED=1

printf 'BENCH_META\tcommit\t%s\n' "$COMMIT"
printf 'BENCH_META\tarchitecture\t%s\n' "$(uname -m)"
printf 'BENCH_META\tkernel\t%s\n' "$(uname -sr)"
printf 'BENCH_META\tlogical_cpus\t%s\n' "$(getconf _NPROCESSORS_ONLN)"
printf 'BENCH_META\tcpu_model\t%s\n' "$(awk -F: '/model name/{gsub(/^[ \t]+/, "", $2); print $2; exit}' /proc/cpuinfo)"
printf 'BENCH_META\tmemory_kib\t%s\n' "$(awk '/MemTotal/{print $2}' /proc/meminfo)"

if ! phase bun_download download_bun; then
render_table "❌ Bun download"
exit 1
fi
if ! phase bun_unpack unpack_bun; then
render_table "❌ Bun unpack"
exit 1
fi
printf 'BENCH_META\tbun_version\t%s\n' "$(bun --version)"
printf 'BENCH_META\tnode_version\t%s\n' "$(node --version)"

if ! phase clone clone_repo; then
render_table "❌ clone"
exit 1
fi
printf 'BENCH_DISK\tafter_clone\t%s\n' "$(disk)"
if ! phase install install_dependencies; then
render_table "❌ install"
exit 1
fi
printf 'BENCH_DISK\tafter_install\t%s\n' "$(disk)"
if ! phase typecheck typecheck; then
render_table "❌ typecheck"
exit 1
fi
printf 'BENCH_DISK\tafter_typecheck\t%s\n' "$(disk)"
printf 'BENCH_DONE\t%s\n' "$(git -C "$ROOT/repo" rev-parse HEAD)"
total_end="$(timestamp)"
printf 'BENCH_PHASE\ttotal\t%s\n' "$(( (total_end - total_start) / 1000000 ))"
render_table "✅"
Loading
Loading