diff --git a/.github/workflows/browser-throughput-benchmarks.yml b/.github/workflows/browser-throughput-benchmarks.yml index 5e5d3d36..fa4f85c9 100644 --- a/.github/workflows/browser-throughput-benchmarks.yml +++ b/.github/workflows/browser-throughput-benchmarks.yml @@ -60,8 +60,9 @@ jobs: # Body links appear as protocol-relative (//en.wikipedia.org/wiki/) # or absolute (https://en.wikipedia.org/wiki/) URLs. html=$(curl -sL -A 'ComputeSDK-Benchmark' "$url") - if echo "$html" | grep -oP 'href="[^"]*"' \ - | grep -qP '/wiki/[^:]*$'; then + if echo "$html" | grep -qP 'href="(//en\.wikipedia\.org)?/wiki/[^"]*" *' \ + && echo "$html" | grep -oP 'href="[^"]*"' \ + | grep -qP '/wiki/[^:]*$'; then break fi done diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml new file mode 100644 index 00000000..476e8c21 --- /dev/null +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -0,0 +1,248 @@ +name: Sandbox Dax Benchmark + +on: + pull_request: + paths: + - 'src/sandbox/dax.ts' + - 'src/run.ts' + - 'src/merge-results.ts' + - 'package.json' + - '.github/workflows/sandbox-dax-benchmarks.yml' + schedule: + - cron: '0 0 * * 5' # Weekly on Friday at midnight UTC + workflow_dispatch: + inputs: + iterations: + description: 'Iterations per provider' + required: false + default: '1' + provider: + description: 'Provider to run (leave empty for all)' + required: false + default: '' + dry_run: + description: 'Run without ingesting or committing results' + required: false + default: false + type: boolean + +concurrency: + group: dax-benchmarks + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +jobs: + dax: + name: Dax ${{ matrix.provider }} + runs-on: namespace-profile-default + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + provider: + - archil + - beam + - blaxel + - cloud-run + - cloudflare + - codesandbox + - createos + - daytona + - declaw + - e2b + - hopx + - isorun + - lightning + - modal + - northflank + - runloop + - superserve + - tensorlake + - upstash + - vercel + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'npm' + - name: Install dependencies + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + npm update + else + npm ci + fi + - name: Clear stale results from checkout + run: rm -rf results/ + - name: Run dax benchmark + env: + COMPUTESDK_API_KEY: ${{ secrets.COMPUTESDK_API_KEY }} + ARCHIL_API_KEY: ${{ secrets.ARCHIL_API_KEY }} + ARCHIL_REGION: ${{ secrets.ARCHIL_REGION }} + ARCHIL_DISK_ID: ${{ secrets.ARCHIL_DISK_ID }} + BEAM_TOKEN: ${{ secrets.BEAM_TOKEN }} + BEAM_WORKSPACE_ID: ${{ secrets.BEAM_WORKSPACE_ID }} + CLOUD_RUN_SANDBOX_URL: ${{ secrets.CLOUD_RUN_SANDBOX_URL }} + CLOUD_RUN_SANDBOX_SECRET: ${{ secrets.CLOUD_RUN_SANDBOX_SECRET }} + BL_API_KEY: ${{ secrets.BL_API_KEY }} + BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} + CLOUDFLARE_SANDBOX_URL: ${{ secrets.CLOUDFLARE_SANDBOX_URL }} + CLOUDFLARE_SANDBOX_SECRET: ${{ secrets.CLOUDFLARE_SANDBOX_SECRET }} + CSB_API_KEY: ${{ secrets.CSB_API_KEY }} + CREATEOS_SANDBOX_API_KEY: ${{ secrets.CREATEOS_SANDBOX_API_KEY }} + DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} + DECLAW_API_KEY: ${{ secrets.DECLAW_API_KEY }} + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + HOPX_API_KEY: ${{ secrets.HOPX_API_KEY }} + ISORUN_API_KEY: ${{ secrets.ISORUN_API_KEY }} + LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} + NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }} + RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} + SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }} + TENSORLAKE_API_KEY: ${{ secrets.TENSORLAKE_API_KEY }} + UPSTASH_BOX_API_KEY: ${{ secrets.UPSTASH_BOX_API_KEY }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + run: | + npm run bench -- \ + --provider ${{ matrix.provider }} \ + --mode sandbox-dax \ + --iterations ${{ github.event.inputs.iterations || '1' }} + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: results-${{ matrix.provider }}-sandbox-dax + path: results/ + if-no-files-found: ignore + retention-days: 7 + + collect: + name: Collect Dax Results + runs-on: namespace-profile-default + needs: dax + if: always() + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: 'npm' + - name: Install dependencies + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + npm update + else + npm ci + fi + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + pattern: results-* + - name: Merge results + run: npx tsx src/merge-results.ts --input artifacts + - name: Ingest results to platform + if: github.event_name != 'pull_request' && github.event.inputs.dry_run != 'true' + continue-on-error: true + env: + INGEST_URL: ${{ secrets.INGEST_URL }} + INGEST_SECRET: ${{ secrets.INGEST_SECRET }} + run: npx tsx src/ingest.ts --type sandbox + - name: Post results to PR + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const latestPath = path.join('results', 'sandbox-dax', 'latest.json'); + let body = '## Sandbox Dax Benchmark Results\n\n'; + let hasResults = false; + + if (fs.existsSync(latestPath)) { + const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); + const results = data.results.filter(r => !r.skipped); + if (results.length > 0) { + hasResults = true; + body += '| Provider | Phases | Total | Prepare | Clone | Install | Typecheck | Status |\n'; + body += '|----------|--------|-------|---------|-------|---------|-----------|--------|\n'; + for (const r of results) { + const latest = [...r.iterations].reverse().find(i => i.phasesCompleted != null); + const score = latest ? `${latest.phasesCompleted}/${latest.phasesTotal}` : '--'; + const ok = r.iterations.filter(it => !it.error).length; + const total = r.iterations.length; + const total_s = (r.summary.totalMs.median / 1000).toFixed(2) + 's'; + const prepare_s = (r.summary.prepareMs.median / 1000).toFixed(2) + 's'; + const clone_s = (r.summary.cloneMs.median / 1000).toFixed(2) + 's'; + const install_s = (r.summary.installMs.median / 1000).toFixed(2) + 's'; + const typecheck_s = (r.summary.typecheckMs.median / 1000).toFixed(2) + 's'; + body += `| ${r.provider} | ${score} | ${total_s} | ${prepare_s} | ${clone_s} | ${install_s} | ${typecheck_s} | ${ok}/${total} OK |\n`; + } + } + } + + if (!hasResults) { + body += '> No dax benchmark results were generated.\n\n'; + } + + body += `---\n*[View full run](${runUrl})*`; + + const marker = '## Sandbox Dax Benchmark Results'; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + + const existing = comments.find(c => c.body.startsWith(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + - name: Commit and push + if: github.event_name != 'pull_request' && github.event.inputs.dry_run != 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add results/sandbox-dax/ + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "chore: update dax benchmark results [skip ci]" + + branch="${GITHUB_REF#refs/heads/}" + for attempt in 1 2 3 4 5; do + git fetch origin "${branch}" + git rebase "origin/${branch}" || { git rebase --abort; exit 1; } + if git push origin "HEAD:${branch}"; then + echo "Pushed on attempt ${attempt}" + exit 0 + fi + echo "Push rejected (attempt ${attempt}); will rebase and retry" + done + echo "Failed to push after multiple attempts" >&2 + exit 1 diff --git a/.github/workflows/sandbox-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml similarity index 69% rename from .github/workflows/sandbox-benchmarks.yml rename to .github/workflows/sandbox-tti-benchmarks.yml index 28e0847f..f5d10323 100644 --- a/.github/workflows/sandbox-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -1,4 +1,4 @@ -name: Sandbox Benchmark +name: Sandbox TTI Benchmark on: pull_request: @@ -13,9 +13,9 @@ on: workflow_dispatch: inputs: iterations: - description: 'Iterations per provider (TTI scheduled runs use 100)' + description: 'Iterations per provider' required: false - default: '3' + default: '100' concurrency: description: 'Concurrent sandboxes (sequential/staggered/burst only)' required: false @@ -39,7 +39,6 @@ on: - sequential - staggered - burst - - sandbox-dax concurrency: group: benchmarks @@ -52,7 +51,6 @@ permissions: jobs: bench: name: Bench ${{ matrix.provider }} - if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.mode != 'sandbox-dax' }} runs-on: namespace-profile-default timeout-minutes: 60 strategy: @@ -163,99 +161,10 @@ jobs: if-no-files-found: ignore retention-days: 7 - dax: - name: Dax ${{ matrix.provider }} - if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.mode == '' || github.event.inputs.mode == 'sandbox-dax' }} - runs-on: namespace-profile-default - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - provider: - - archil - - beam - - blaxel - - cloud-run - - cloudflare - - codesandbox - - createos - - daytona - - declaw - - e2b - - hopx - - isorun - - modal - - northflank - - runloop - - superserve - - tensorlake - - upstash - - vercel - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: 'npm' - - name: Install dependencies - run: | - if [ "${{ github.event_name }}" = "schedule" ]; then - npm update - else - npm ci - fi - - name: Clear stale results from checkout - run: rm -rf results/ - - name: Run dax benchmark - env: - COMPUTESDK_API_KEY: ${{ secrets.COMPUTESDK_API_KEY }} - ARCHIL_API_KEY: ${{ secrets.ARCHIL_API_KEY }} - ARCHIL_REGION: ${{ secrets.ARCHIL_REGION }} - ARCHIL_DISK_ID: ${{ secrets.ARCHIL_DISK_ID }} - BEAM_TOKEN: ${{ secrets.BEAM_TOKEN }} - BEAM_WORKSPACE_ID: ${{ secrets.BEAM_WORKSPACE_ID }} - CLOUD_RUN_SANDBOX_URL: ${{ secrets.CLOUD_RUN_SANDBOX_URL }} - CLOUD_RUN_SANDBOX_SECRET: ${{ secrets.CLOUD_RUN_SANDBOX_SECRET }} - BL_API_KEY: ${{ secrets.BL_API_KEY }} - BL_WORKSPACE: ${{ secrets.BL_WORKSPACE }} - CLOUDFLARE_SANDBOX_URL: ${{ secrets.CLOUDFLARE_SANDBOX_URL }} - CLOUDFLARE_SANDBOX_SECRET: ${{ secrets.CLOUDFLARE_SANDBOX_SECRET }} - CSB_API_KEY: ${{ secrets.CSB_API_KEY }} - CREATEOS_SANDBOX_API_KEY: ${{ secrets.CREATEOS_SANDBOX_API_KEY }} - DAYTONA_API_KEY: ${{ secrets.DAYTONA_API_KEY }} - DECLAW_API_KEY: ${{ secrets.DECLAW_API_KEY }} - E2B_API_KEY: ${{ secrets.E2B_API_KEY }} - HOPX_API_KEY: ${{ secrets.HOPX_API_KEY }} - ISORUN_API_KEY: ${{ secrets.ISORUN_API_KEY }} - MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} - MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} - NORTHFLANK_TOKEN: ${{ secrets.NORTHFLANK_TOKEN }} - NORTHFLANK_PROJECT_ID: ${{ secrets.NORTHFLANK_PROJECT_ID }} - RUNLOOP_API_KEY: ${{ secrets.RUNLOOP_API_KEY }} - SUPERSERVE_API_KEY: ${{ secrets.SUPERSERVE_API_KEY }} - TENSORLAKE_API_KEY: ${{ secrets.TENSORLAKE_API_KEY }} - UPSTASH_BOX_API_KEY: ${{ secrets.UPSTASH_BOX_API_KEY }} - VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} - VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} - VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} - run: | - npm run bench -- \ - --provider ${{ matrix.provider }} \ - --mode sandbox-dax \ - --iterations ${{ github.event.inputs.iterations || '1' }} - - name: Upload results - if: always() - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.provider }}-sandbox-dax - path: results/ - if-no-files-found: ignore - retention-days: 7 - collect: name: Collect Results runs-on: namespace-profile-default - needs: [bench, dax] + needs: bench if: always() steps: - uses: actions/checkout@v4 @@ -349,33 +258,6 @@ jobs: body += '\n'; } - const workloadModes = [ - { key: 'sandbox-dax', label: 'Sandbox Dax', metric: r => { const l = [...r.iterations].reverse().find(i => i.phasesCompleted != null); const s = l ? `${l.phasesCompleted}/${l.phasesTotal}` : '--'; return `${s} phases · total ${(r.summary.totalMs.median / 1000).toFixed(2)}s · prepare ${(r.summary.prepareMs.median / 1000).toFixed(2)}s · clone ${(r.summary.cloneMs.median / 1000).toFixed(2)}s · install ${(r.summary.installMs.median / 1000).toFixed(2)}s · typecheck ${(r.summary.typecheckMs.median / 1000).toFixed(2)}s`; } }, - ]; - for (const mode of workloadModes) { - const latestPath = path.join('results', mode.key, 'latest.json'); - if (!fs.existsSync(latestPath)) continue; - - const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8')); - const results = data.results.filter(r => !r.skipped); - if (results.length === 0) continue; - hasResults = true; - - body += `### ${mode.label}\n\n`; - body += '| Provider | Metric | Status |\n'; - body += '|----------|--------|--------|\n'; - - results.forEach(r => { - const ok = r.iterations.filter(it => !it.error).length; - const total = r.iterations.length; - const latest = [...r.iterations].reverse().find(i => i.phasesCompleted != null); - const status = latest ? `${latest.phasesCompleted}/${latest.phasesTotal} phases` : `${ok}/${total} OK`; - body += `| ${r.provider} | ${mode.metric(r)} | ${status} |\n`; - }); - - body += '\n'; - } - if (!hasResults) { body += '> No sandbox benchmark results were generated.\n\n'; }