diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d72fd52 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.pdf binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..585be43 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,35 @@ +name: CI + +"on": + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Python 3.11 tests + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Check out repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.11" + cache: pip + + - name: Install project and test dependencies + run: python -m pip install -e ".[dev]" + + - name: Run tests + run: python -m pytest -q diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5351126 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Noah Ingwers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 5fbb521..70a6e26 100644 --- a/README.md +++ b/README.md @@ -4,42 +4,64 @@ Noah Ingwers, 2025 +> **Research status:** Exploratory, non-peer-reviewed work. The findings below summarize +> 12 recorded GPT-4o-mini runs completed in December 2025; they are not results from a +> peer-reviewed paper or a full-benchmark evaluation. + ## Overview EMAP is a framework for studying how resource constraints during evolutionary optimization shape multi-agent LLM architectures. Unlike existing approaches that treat computational cost as a secondary objective, EMAP makes budget constraints the primary evolutionary pressure - forcing architectures to adapt to resource scarcity rather than simply trading off against it. -## Key Findings - -We conducted 12 experiments across 4 budget regimes (TIGHT 2K, MEDIUM 5K, LOOSE 10K, UNCONSTRAINED 50K tokens) with 3 random seeds each, evolving multi-agent architectures on HumanEval (164 programming tasks). - -### Main Results - -| Regime | Budget | Pass@1 | Avg Agents | Avg Edges | -|--------|--------|--------|------------|-----------| -| TIGHT | 2,000 tokens | 98.4% ± 2.3% | 3.0 | 2.6 | -| MEDIUM | 5,000 tokens | 100.0% ± 0.0% | 3.0 | 2.4 | -| LOOSE | 10,000 tokens | 97.6% ± 2.4% | 3.0 | 1.8 | -| UNCONSTRAINED | 50,000 tokens | 100.0% ± 0.0% | 2.7 | 2.0 | - -### Key Discoveries - -1. **Evolution consistently produces 3-agent architectures** regardless of constraint severity, contradicting the hypothesis that tight constraints would favor minimal single-agent systems. - -2. **Multiple viable topologies emerge** - evolution discovered diverse architectures achieving equivalent performance: - - Traditional pipeline (planner -> coder -> reviewer) - - Test-first pipeline (tester -> reviewer -> planner) - - Hybrid architecture (generalist -> coder -> architect) - - Hierarchical 4-agent systems under loose constraints - -3. **Budget constraints preserve diversity** - under constrained regimes, different seeds produce different topologies (linear vs cyclic). Without constraints, evolution converges toward simpler linear solutions. - -4. **The "Goldilocks Zone"** - MEDIUM budget (5K tokens) achieves perfect performance with lowest token usage, representing optimal constraint severity for HumanEval tasks. - -## Computational Cost - -Total experimental cost: **3.09M tokens** (~27K API calls) across all 12 experiments. -Estimated API cost: **<$1.00** using GPT-4o-mini. -Runtime: 1.5-5.2 hours per experiment depending on budget regime. +## Recorded Findings + +The repository contains 12 completed experiment artifacts: 4 token-budget regimes +(TIGHT 2K, MEDIUM 5K, LOOSE 10K, and UNCONSTRAINED 50K) with seeds 42, 43, and 44. +Each recorded configuration used 12 generations and a population of 10. The runner +is configured to sample 12% of the intended 164-task HumanEval corpus per evaluation. +Each artifact reports a 41-task final evaluation (25%), consistent with that runner +configuration but not with a full-benchmark evaluation. + +### Recorded Results + +| Regime | Budget | Mean final score | Final-pop. agents | Final-pop. edges | +|--------|--------|------------------|-------------------|------------------| +| TIGHT | 2,000 tokens | 98.4% ± 2.3% | 3.2 | 2.6 | +| MEDIUM | 5,000 tokens | 100.0% ± 0.0% | 3.1 | 2.4 | +| LOOSE | 10,000 tokens | 98.4% ± 2.3% | 3.5 | 2.8 | +| UNCONSTRAINED | 50,000 tokens | 99.2% ± 1.1% | 2.9 | 2.1 | + +Values are means and population standard deviations across three recorded runs. +Agent and edge counts are final-generation population averages, not the sizes of a +single selected architecture. These numbers are reproduced from the checked-in +[summary](experiments/results/focused/summary.json) and +[per-run JSON artifacts](experiments/results/focused/). + +### Descriptive Observations + +1. **The recorded final populations averaged roughly three agents.** Mean agent + counts ranged from 2.9 to 3.5 across the four regimes. +2. **The selected genomes used several topologies.** Checked-in artifacts include a + planner -> coder -> reviewer pipeline, a tester -> reviewer -> planner topology, + a generalist -> coder -> architect topology, and two 4-agent LOOSE-budget systems. +3. **The MEDIUM runs were the strongest in this small sample.** All three recorded + MEDIUM final evaluations passed 41/41 sampled tasks and together used 414,390 + tokens, the lowest observed total among the four regimes. This does not establish + 5K tokens as a general optimum. +4. **Topology diversity is a hypothesis, not a causal result.** The 12 artifacts show + different structures across seeds, but three runs per regime on varying task + samples cannot establish that budget pressure caused those differences. + +## Checked-in Evidence and Cost + +- [Aggregate summary](experiments/results/focused/summary.json) +- [Twelve per-run JSON records](experiments/results/focused/) +- [Recorded run visualizations](experiments/viz/) +- [Artifact-grounded research note source](paper/main.tex) and + [rendered research note](paper/Scarcity_Breeds_Efficiency__Resource_Constrained_Evolution_of_Multi_Agent_Programming_Architectures.pdf) + +The 12 JSON files record **3,093,390 tokens**, **27,360 API calls**, and runtimes of +approximately **1.7-5.2 hours per run**. They do not separate input from output tokens, +so a historical dollar cost cannot be independently recalculated from the artifacts. ## Installation @@ -62,19 +84,31 @@ cp .env.example .env ### Run Evolution Experiment ```bash -python experiments/run_evolution_experiments.py \ +python experiments/run_single_experiment.py \ --budget 5000 \ --seed 42 \ --generations 12 \ --population 10 ``` -### Run Tests +The runner writes to `experiments/results/focused/` and skips a run when a completed +artifact with the same budget and seed already exists. Preserve the checked-in JSON +files before attempting a fresh rerun. + +### Run Offline Tests ```bash -pytest tests/ -v +python -m pytest tests/ -v ``` +The test suite uses mock backends and does not require an OpenAI API key. + +The code-evaluation harness is **not a security sandbox**. It executes generated +Python with `exec` in a disposable subprocess and applies static checks, +restricted builtins/imports, a timeout, and best-effort Unix resource limits. +Run it only in an unprivileged, isolated environment with no secrets or sensitive +network/filesystem access. + ### Basic API Usage ```python @@ -100,7 +134,7 @@ print(f"Tokens: {result.total_tokens_used}") ## Project Structure ``` -E_M_A_P/ +EMAP/ ├── src/emap/ │ ├── genome/ │ │ ├── representation.py # MultiAgentGenome, AgentGene @@ -113,13 +147,14 @@ E_M_A_P/ │ │ └── executor.py # LLM backends and message routing │ └── benchmarks/ │ ├── humaneval.py # HumanEval loader -│ └── sandbox.py # Safe code execution +│ └── sandbox.py # Best-effort subprocess evaluator ├── experiments/ -│ ├── run_evolution_experiments.py -│ └── results/ # Experiment JSON outputs +│ ├── run_single_experiment.py # One budget/seed run +│ ├── run_all_experiments.py # Recorded 4 x 3 run matrix +│ └── results/focused/ # Checked-in JSON artifacts ├── paper/ -│ ├── main.tex # Research paper -│ └── references.bib # Bibliography +│ ├── main.tex # Artifact-grounded research note +│ └── Scarcity_...pdf # Rendered research note └── tests/ ``` @@ -161,23 +196,63 @@ class MultiAgentGenome: - Adjust hyperparameters (temperature, max tokens, message length) - Change message format and aggregation strategy +## Reproducibility and Limitations + +To run against HumanEval, place `humaneval.jsonl` in `data/`, in +`src/emap/benchmarks/data/`, or at `~/.cache/emap/humaneval.jsonl`. The loader falls +back to five simple placeholder tasks when it cannot find that file. The recorded +artifacts have `final_total: 41`, which is consistent with the configured 25% sample +of the 164-task HumanEval corpus rather than the placeholder set. + +Important limits of the recorded study: + +- The `--seed` value controls NumPy-based architecture evolution, but task subsets are + selected with Python's unseeded `random` module. The OpenAI calls are also + nondeterministic. Exact reruns are therefore not guaranteed, and different runs may + have used different 41-task final subsets. +- The JSON files contain aggregate scores, selected genomes, timing, token totals, and + configuration. They do not contain final task IDs, per-task outputs, a dataset hash, + a Git commit, or dependency/model version metadata. The GPT-4o-mini attribution is + supported by the checked-in runner configuration, not by a model field in each JSON. +- Project dependencies use lower bounds and no lockfile, so the exact December 2025 + software environment is not preserved. +- With three runs per regime and sampled final evaluations, the table supports + descriptive comparisons only. It does not establish statistical significance, + full-HumanEval performance, causal effects of budget pressure, or generalization to + other models and benchmarks. +- This repository and its research note have not been peer reviewed. + ## Research Questions -**RQ1 (Addressed):** Do architectures evolved under different budget regimes exhibit different structures? -- Finding: All regimes converge to 3-agent architectures, but topology type (linear vs cyclic) is seed-dependent. +**RQ1 (Explored descriptively):** Do architectures evolved under different budget regimes exhibit different structures? -**RQ2 (Addressed):** Do constraint-evolved architectures exhibit different coordination strategies? -- Finding: Evolution discovers multiple viable topologies achieving equivalent performance. +- Recorded observation: final populations averaged 2.9-3.5 agents, while selected + genomes ranged from 2 to 4 agents and included both linear and cyclic topologies. + The current sample does not isolate budget from seed, task sampling, or model noise. -**RQ3 & RQ4 (Future Work):** Transfer to abundance and cross-benchmark generalization remain to be tested. +**RQ2 (Explored descriptively):** Do constraint-evolved architectures exhibit different coordination strategies? + +- Recorded observation: selected genomes used sequential, hierarchical, and voting + aggregation with structured, freeform, and minimal message formats. Whether the + constraints caused those choices remains untested. + +**RQ3 & RQ4 (Not evaluated):** Transfer to abundance and cross-benchmark generalization remain future work. ## Broader Implications -Our findings suggest something counterintuitive: constraints may be features, not bugs. The dominant paradigm in AI development is to maximize resources during training and optimization, then compress or distill afterward. But biological evolution tells a different story - organisms that evolved under scarcity develop fundamentally different adaptations than those subjected to scarcity after the fact. A desert plant has different anatomy than a rainforest plant experiencing drought. +EMAP investigates a counterintuitive hypothesis: constraints may be useful design +pressures rather than only deployment limits. Biological adaptation under scarcity is +the motivating analogy, not evidence for an equivalent mechanism in LLM systems. -We observe an analogous phenomenon in multi-agent architectures. Under budget pressure, evolution discovers cyclic feedback topologies, compressed message formats, and aggressive early-exit strategies. Remove that pressure, and evolution converges toward simpler linear pipelines. The constraint doesn't just filter solutions - it shapes the search process itself, maintaining diversity and preventing premature convergence to local optima. +The checked-in runs contain cyclic feedback topologies, compressed message formats, +and simpler linear pipelines. Because there are only three runs per regime and final +task subsets were not controlled, these observations motivate a larger preregistered +comparison; they do not demonstrate that budget pressure caused the structures or +preserved diversity. -This has implications beyond code generation. As LLM costs and environmental impact come under scrutiny, the ability to evolve efficient architectures under realistic constraints becomes increasingly valuable. Rather than designing for abundance and hoping efficiency follows, we can design for scarcity and let evolution discover what's truly necessary. +If the effect holds under controlled, full-benchmark evaluation, evolving systems +under realistic resource constraints could be useful beyond code generation. Testing +that proposition across models, benchmarks, and cost measures remains future work. ## Future Directions @@ -191,15 +266,22 @@ This has implications beyond code generation. As LLM costs and environmental imp ## Citation ```bibtex -@article{ingwers2025emap, - title={Scarcity Breeds Efficiency: Resource-Constrained Evolution of - Multi-Agent Programming Architectures}, +@misc{ingwers2025emap, + title={EMAP: An Exploratory Artifact Report on Resource-Constrained + Evolution of Multi-Agent Programming Architectures}, author={Ingwers, Noah}, - journal={arXiv preprint}, - year={2025} + year={2025}, + note={Unpublished, non-peer-reviewed research note and software repository} } ``` +## Security + +Report suspected vulnerabilities privately as described in +[SECURITY.md](SECURITY.md). The generated-code evaluator is a best-effort +experiment harness, not a security sandbox; its documented boundary and +reporting scope are stated there. + ## License -MIT License +[MIT License](LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..f6b1a2f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ +# Security Policy + +EMAP is exploratory research software. It has not been independently audited, +and this policy does not create a bug bounty, service-level agreement, or +guarantee of payment. + +## Supported version + +Security fixes target the current `main` branch. Please reproduce an issue +against the latest commit before reporting it. + +## Report privately + +Use GitHub's private vulnerability reporting form: + +https://github.com/noah-ing/EMAP/security/advisories/new + +If the form is unavailable, open a public issue asking for a private reporting +channel, but do not include exploit details, credentials, sensitive data, or +working generated-code payloads in that issue. + +Include a minimal reproduction, affected commit and component, expected and +observed behavior, impact, preconditions, and any suggested remediation. + +The generated-code evaluator is explicitly a best-effort experiment harness, +not a security sandbox. Python object-model escapes and platform-specific +resource-limit gaps are therefore documented limitations. Run it only in an +unprivileged external sandbox with no secrets or sensitive network/filesystem +access. Reports remain useful when the implementation violates a documented +fail-closed behavior, silently falls back to in-process execution, exposes a +credential, or introduces an unsafe default outside those stated limits. + +Do not test against systems, accounts, models, or data you do not own or have +written permission to assess. Do not exfiltrate data, degrade a service, or +retain sensitive material beyond what is necessary to demonstrate the issue. + +Please allow a reasonable remediation window before public disclosure. diff --git a/experiments/dashboard.html b/experiments/dashboard.html deleted file mode 100644 index 23f6e26..0000000 --- a/experiments/dashboard.html +++ /dev/null @@ -1,1056 +0,0 @@ - - - - - - EMAP Evolution Dashboard - - - - -
- -
-

EMAP Evolution Dashboard

-

Scarcity Breeds Efficiency: Resource-Constrained Evolution of Multi-Agent Architectures

-
- - Loading experiment data... -
-
- - -
-
-
- Total Experiments -
-
- 0 - Across 4 budget regimes -
-
-
-
- Best Fitness -
-
- -- - Pass@1 accuracy -
-
-
-
- Total Tokens -
-
- -- - API usage -
-
-
-
- API Calls -
-
- -- - LLM invocations -
-
-
- - -

- 📊 - Experiment Progress -

-
- -
-
- Tight Budget (2K tokens) - Constraint++ -
-
-
- Generation Progress - 0/50 -
-
-
-
-
-
-
- Best Fitness - -- -
-
- Avg Agents - -- -
-
- Diversity - -- -
-
-
- - -
-
- Medium Budget (5K tokens) - Moderate -
-
-
- Generation Progress - 0/50 -
-
-
-
-
-
-
- Best Fitness - -- -
-
- Avg Agents - -- -
-
- Diversity - -- -
-
-
- - -
-
- Loose Budget (10K tokens) - Relaxed -
-
-
- Generation Progress - 0/50 -
-
-
-
-
-
-
- Best Fitness - -- -
-
- Avg Agents - -- -
-
- Diversity - -- -
-
-
- - -
-
- Unconstrained (50K tokens) - No Limit -
-
-
- Generation Progress - 0/50 -
-
-
-
-
-
-
- Best Fitness - -- -
-
- Avg Agents - -- -
-
- Diversity - -- -
-
-
-
- - -

- 📈 - Fitness Evolution Over Generations -

-
-
- -
-
- - -

- 🏗 - Evolved Architecture Comparison -

-
-
-
- Tight Budget Architecture - Single Agent -
-
-
- 💻 - Coder -
-
-

- Minimal architecture - single specialist agent -

-
-
-
- Unconstrained Architecture - Multi-Agent -
-
-
- 📝 - Planner -
-
-
- 💻 - Coder -
-
-
- 🔍 - Reviewer -
-
-

- Complex topology with role specialization -

-
-
- - -

- 🧬 - Structural Metrics by Budget Regime -

-
-
- -
-
- - -

- 🧪 - Hypothesis Validation Status -

-
-
-
- H1: Structural Compactness - SUPPORTED -
-

- Tight budget evolved single-agent architecture (1.0 agents) vs. multi-agent in loose regimes (2.5+ agents) -

-
-
-
- H2: Prompt Compression - TESTING -
-

- Awaiting analysis of prompt lengths across regimes -

-
-
-
- H3: Role Specialization - TESTING -
-

- Evaluating role distribution patterns across budgets -

-
-
-
- H4: Fail-Fast Behavior - PENDING -
-

- Requires expensive failure analysis after experiments complete -

-
-
- - -

- 📋 - Detailed Results -

-
- - - - - - - - - - - - - - - - - - -
RegimeBudgetGenerationBest FitnessAvg AgentsAvg EdgesDiversityTokens Used
-
- - - -
-
-
- - -
- -

- Auto-refresh every 30 seconds when experiments are running -

-
- - - -
- - - - diff --git a/experiments/run_all_experiments.py b/experiments/run_all_experiments.py index 2dc553e..dc7fd9d 100644 --- a/experiments/run_all_experiments.py +++ b/experiments/run_all_experiments.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 """ -EMAP Complete Experiment Suite +EMAP Recorded Experiment Matrix Runs all experiments across 4 budget regimes with 3 seeds each. -Robust design: each experiment saves immediately, can resume from interruption. +Each experiment saves immediately and can resume after interruption. The +resulting 12-run matrix is exploratory and supports descriptive comparisons; +it is not a statistical-significance design. Budget Regimes: - TIGHT (2000 tokens): Tests minimal resource allocation @@ -11,11 +13,11 @@ - LOOSE (10000 tokens): Relaxed constraint - UNCONSTRAINED (50000 tokens): Effectively unlimited -For PhD-level rigor: -- 12 generations per experiment (enough for convergence) -- 10 population size (sufficient genetic diversity) -- 12% sample per generation (20 tasks), 25% final eval (41 tasks) -- 3 seeds for statistical validity (42, 43, 44) +Recorded configuration: +- 12 generations per experiment +- 10 genomes per population +- 12% sample per generation (19 tasks), 25% final eval (41 tasks) +- 3 seeds for repeated observations (42, 43, 44) """ from __future__ import annotations @@ -33,10 +35,10 @@ BUDGETS = [2000, 5000, 10000, 50000] SEEDS = [42, 43, 44] -# Experiment parameters (PhD rigor) +# Recorded experiment parameters GENERATIONS = 12 POPULATION = 10 -SAMPLE_FRACTION = 0.12 # ~20 tasks per eval +SAMPLE_FRACTION = 0.12 # 19 tasks per eval: int(164 * 0.12) FINAL_SAMPLE = 0.25 # ~41 tasks for final eval diff --git a/experiments/run_experiments.py b/experiments/run_experiments.py index e58e81e..55e7b87 100644 --- a/experiments/run_experiments.py +++ b/experiments/run_experiments.py @@ -1,15 +1,19 @@ #!/usr/bin/env python3 """ -EMAP Full Experiment Runner +EMAP Configurable Experiment Runner -This script runs the complete set of experiments for the EMAP paper: +This script defines a larger proposed experiment matrix: 1. Evolution under 4 budget regimes (Tight, Medium, Loose, Unconstrained) -2. 5 random seeds per regime for statistical validity +2. Repeated random seeds per regime 3. Cross-benchmark transfer evaluation +The checked-in evidence does not show that this entire matrix was completed. +Five seeds provide repeated estimates but do not by themselves establish +statistical validity. + Usage: # Set your API key first - export OPENAI_API_KEY="sk-..." + export OPENAI_API_KEY="" # Run pilot experiment (1 regime, 1 seed) python experiments/run_experiments.py --pilot @@ -64,7 +68,7 @@ logger = logging.getLogger(__name__) -# Budget regimes from the paper +# Budget regimes for this runner BUDGET_REGIMES = { "tight": 2000, "medium": 5000, @@ -72,7 +76,7 @@ "unconstrained": 50000, # "Unlimited" but still capped for safety } -# Default evolution parameters (from paper) +# Default evolution parameters for this proposed matrix DEFAULT_CONFIG = { "population_size": 20, "generations": 50, diff --git a/experiments/run_focused_experiments.py b/experiments/run_focused_experiments.py index 45af879..68131b9 100644 --- a/experiments/run_focused_experiments.py +++ b/experiments/run_focused_experiments.py @@ -2,8 +2,8 @@ """ EMAP Focused Experiments -Runs complete experiments across 4 budget regimes for statistical comparison. -Optimized for ~2-4 hour completion with meaningful results. +Runs a bounded exploratory sweep across four budget regimes. +The resulting artifacts are descriptive and do not establish statistical significance. """ from __future__ import annotations diff --git a/experiments/run_full_overnight.py b/experiments/run_full_overnight.py index ed6f20e..f221714 100755 --- a/experiments/run_full_overnight.py +++ b/experiments/run_full_overnight.py @@ -63,7 +63,7 @@ "sample_fraction": 0.05, # 5% = ~8 tasks per eval } -SEEDS = [42, 43, 44] # 3 seeds for statistical validity +SEEDS = [42, 43, 44] # Repeated observations; not a significance design def run_single_experiment(tasks, budget, seed, config, backend, output_dir, viz_path=None): diff --git a/experiments/serve_dashboard.py b/experiments/serve_dashboard.py deleted file mode 100755 index 45e9238..0000000 --- a/experiments/serve_dashboard.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -EMAP Dashboard Server - -Serves the experiment dashboard and provides live updates via Server-Sent Events (SSE). -Also can run experiments while streaming progress to the dashboard. - -Usage: - # Just serve dashboard (view existing results) - python experiments/serve_dashboard.py - - # Serve dashboard and run experiments - python experiments/serve_dashboard.py --run-experiments - - # Open browser automatically - python experiments/serve_dashboard.py --open -""" - -import argparse -import asyncio -import http.server -import json -import os -import socketserver -import sys -import threading -import webbrowser -from datetime import datetime -from functools import partial -from pathlib import Path -from typing import Dict, Any -import time - -# Add color support -class Colors: - HEADER = '\033[95m' - BLUE = '\033[94m' - CYAN = '\033[96m' - GREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - -def print_banner(): - """Print a nice ASCII banner.""" - banner = f""" -{Colors.CYAN}{Colors.BOLD} - ███████╗███╗ ███╗ █████╗ ██████╗ - ██╔════╝████╗ ████║██╔══██╗██╔══██╗ - █████╗ ██╔████╔██║███████║██████╔╝ - ██╔══╝ ██║╚██╔╝██║██╔══██║██╔═══╝ - ███████╗██║ ╚═╝ ██║██║ ██║██║ - ╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ -{Colors.ENDC} -{Colors.GREEN} Evolution under Multi-Agent Pressure{Colors.ENDC} -{Colors.BLUE} Dashboard Server{Colors.ENDC} -""" - print(banner) - -class DashboardHandler(http.server.SimpleHTTPRequestHandler): - """Custom handler for the dashboard.""" - - def __init__(self, *args, directory=None, **kwargs): - super().__init__(*args, directory=directory, **kwargs) - - def log_message(self, format, *args): - """Custom logging with colors.""" - msg = format % args - if '200' in msg: - print(f" {Colors.GREEN}[OK]{Colors.ENDC} {msg}") - elif '404' in msg: - print(f" {Colors.WARNING}[MISS]{Colors.ENDC} {msg}") - else: - print(f" {Colors.CYAN}[INFO]{Colors.ENDC} {msg}") - - def end_headers(self): - """Add CORS headers for development.""" - self.send_header('Access-Control-Allow-Origin', '*') - self.send_header('Cache-Control', 'no-store') - super().end_headers() - - def do_GET(self): - """Handle GET requests.""" - if self.path == '/': - self.path = '/dashboard.html' - elif self.path == '/api/status': - self.send_experiment_status() - return - - super().do_GET() - - def send_experiment_status(self): - """Send current experiment status as JSON.""" - self.send_response(200) - self.send_header('Content-type', 'application/json') - self.end_headers() - - results_dir = Path(self.directory) / 'results' - status = {} - - budgets = { - 'tight': 2000, - 'medium': 5000, - 'loose': 10000, - 'unconstrained': 50000 - } - - for name, budget in budgets.items(): - result_file = results_dir / f'evolution_budget{budget}_seed42.json' - if result_file.exists(): - try: - with open(result_file) as f: - status[name] = json.load(f) - except Exception as e: - status[name] = {'error': str(e)} - else: - status[name] = None - - self.wfile.write(json.dumps(status).encode()) - -def run_server(port: int, directory: str, open_browser: bool = False): - """Run the HTTP server.""" - handler = partial(DashboardHandler, directory=directory) - - with socketserver.TCPServer(("", port), handler) as httpd: - url = f"http://localhost:{port}" - - print(f"\n{Colors.GREEN}{Colors.BOLD}Dashboard server running!{Colors.ENDC}") - print(f"\n {Colors.CYAN}URL:{Colors.ENDC} {url}") - print(f" {Colors.CYAN}Directory:{Colors.ENDC} {directory}") - print(f"\n{Colors.WARNING}Press Ctrl+C to stop{Colors.ENDC}\n") - - if open_browser: - webbrowser.open(url) - - try: - httpd.serve_forever() - except KeyboardInterrupt: - print(f"\n{Colors.WARNING}Shutting down...{Colors.ENDC}") - -def watch_experiments(results_dir: Path): - """Watch experiment result files and print updates.""" - print(f"\n{Colors.CYAN}Watching for experiment updates...{Colors.ENDC}\n") - - last_modified = {} - budgets = { - 'Tight (2K)': 2000, - 'Medium (5K)': 5000, - 'Loose (10K)': 10000, - 'Unconstrained (50K)': 50000 - } - - while True: - for name, budget in budgets.items(): - result_file = results_dir / f'evolution_budget{budget}_seed42.json' - - if result_file.exists(): - mtime = result_file.stat().st_mtime - if name not in last_modified or mtime > last_modified[name]: - last_modified[name] = mtime - - try: - with open(result_file) as f: - data = json.load(f) - - gen = len(data.get('generations', [])) - total = data.get('config', {}).get('generations', 50) - fitness = data.get('best_fitness_per_gen', [0])[-1] - agents = data.get('avg_agents_per_gen', [0])[-1] - - progress = int((gen / total) * 20) - bar = '█' * progress + '░' * (20 - progress) - - print(f" {Colors.CYAN}{name:20}{Colors.ENDC} " - f"[{bar}] " - f"Gen {gen:2d}/{total} " - f"Fitness: {fitness:.2f} " - f"Agents: {agents:.1f}") - except Exception as e: - pass - - time.sleep(2) - -def main(): - parser = argparse.ArgumentParser(description="EMAP Dashboard Server") - parser.add_argument('--port', type=int, default=8080, help='Port to serve on') - parser.add_argument('--open', action='store_true', help='Open browser automatically') - parser.add_argument('--watch', action='store_true', help='Watch experiment progress in terminal') - parser.add_argument('--run-experiments', action='store_true', help='Run experiments while serving') - - args = parser.parse_args() - - print_banner() - - # Determine directory - experiments_dir = Path(__file__).parent - - if args.watch: - # Just watch experiments in terminal - watch_experiments(experiments_dir / 'results') - else: - # Start server - if args.run_experiments: - # TODO: Integrate with run_experiments.py - print(f"{Colors.WARNING}Note: --run-experiments not yet implemented{Colors.ENDC}") - print(f"Run experiments separately with: python experiments/run_experiments.py\n") - - run_server(args.port, str(experiments_dir), args.open) - -if __name__ == "__main__": - main() diff --git a/paper/Scarcity_Breeds_Efficiency__Resource_Constrained_Evolution_of_Multi_Agent_Programming_Architectures.pdf b/paper/Scarcity_Breeds_Efficiency__Resource_Constrained_Evolution_of_Multi_Agent_Programming_Architectures.pdf index 1fee93f..ec28b66 100644 Binary files a/paper/Scarcity_Breeds_Efficiency__Resource_Constrained_Evolution_of_Multi_Agent_Programming_Architectures.pdf and b/paper/Scarcity_Breeds_Efficiency__Resource_Constrained_Evolution_of_Multi_Agent_Programming_Architectures.pdf differ diff --git a/paper/main.tex b/paper/main.tex index d855654..1c9c031 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -1,1151 +1,129 @@ -% Scarcity Breeds Efficiency: Resource-Constrained Evolution of Multi-Agent Programming Architectures -% Target venue: ICML/NeurIPS/ICLR -% Last updated: December 18, 2025 - -\documentclass{article} - -% Use the NeurIPS 2024 style (common for submissions) -\usepackage[final]{neurips_2024} - -% Standard packages -\usepackage[utf8]{inputenc} -\usepackage[T1]{fontenc} -\usepackage{hyperref} -\usepackage{url} +\documentclass[11pt]{article} +\usepackage[margin=1in]{geometry} \usepackage{booktabs} -\usepackage{amsfonts} -\usepackage{amsmath} -\usepackage{amssymb} -\usepackage{nicefrac} +\usepackage[hidelinks]{hyperref} \usepackage{microtype} -\usepackage{xcolor} -\usepackage{graphicx} -\usepackage{subcaption} -\usepackage{algorithm} -\usepackage{algorithmic} -\usepackage{multirow} -% Custom commands -\newcommand{\emap}{\textsc{EMAP}} -\newcommand{\todo}[1]{\textcolor{red}{[TODO: #1]}} -\newcommand{\rebuttal}[1]{\textcolor{blue}{#1}} - -\title{Scarcity Breeds Efficiency: \\ Resource-Constrained Evolution of Multi-Agent Programming Architectures} - -% Authors - anonymized for submission -\author{ - Noah Ingwers -} +\title{EMAP: An Exploratory Artifact Report on Resource-Constrained +Evolution of Multi-Agent Programming Architectures} +\author{Noah Ingwers} +\date{December 2025 artifacts; research note revised August 2026} \begin{document} - \maketitle \begin{abstract} -Recent work on evolutionary optimization of multi-agent LLM systems has demonstrated significant performance improvements on code generation benchmarks. However, these approaches typically optimize for task performance with cost as a secondary objective, overlooking a fundamental question: \textit{how do resource constraints during the evolutionary process itself shape the resulting architectures?} We introduce \emap{} (Evolution under Multi-Agent Pressure), a framework that treats resource budgets not merely as evaluation metrics but as primary evolutionary pressures. Through systematic experiments on HumanEval (164 programming tasks) across four budget regimes (2K--50K tokens), we discover that evolution consistently produces sophisticated 3-agent architectures regardless of constraint severity. Under MEDIUM constraints (5K tokens), evolved systems achieve 100\% pass rate on evaluation tasks (41/41), while TIGHT constraints (2K tokens) yield 98.4\% $\pm$ 2.3\%. Notably, evolution discovers diverse topologies---planner$\rightarrow$coder$\rightarrow$reviewer, tester$\rightarrow$reviewer$\rightarrow$planner, and generalist$\rightarrow$coder$\rightarrow$architect---suggesting multiple viable strategies for code generation. These results validate our core hypothesis: resource constraints during evolution produce architectures that discover principled multi-agent coordination strategies without human engineering. We release our code, evolved architectures, and visualization tools for reproducibility. +EMAP explores whether token budgets used as evolutionary pressure are +associated with different multi-agent programming architectures. This note +reports only what can be supported by the twelve JSON artifacts checked into +the repository: four token-budget regimes, three seeds per regime, and a +41-task sampled final evaluation per run. The artifacts are exploratory, +non-peer-reviewed, and do not represent a full HumanEval evaluation. They +support descriptive comparisons, not causal conclusions, statistical +significance, or deployment recommendations. \end{abstract} -%============================================================================== -\section{Introduction} -\label{sec:intro} -%============================================================================== - -The field of multi-agent LLM systems has seen rapid progress in 2024-2025, with evolutionary and optimization-based approaches achieving remarkable results \citep{evoagentx2025, artemis2025, aflow2024, automaas2025}. A common pattern emerges from this work: researchers optimize for task performance (accuracy, pass rate) while treating computational cost as a secondary concern—something to minimize given a performance threshold, or to report alongside accuracy in Pareto analyses. - -This mirrors a broader assumption in machine learning: that the best way to find efficient systems is to first find effective ones, then compress or distill them. But evolutionary biology suggests an alternative hypothesis: \textit{constraint during development produces fundamentally different organisms than constraint applied after development}. A desert plant evolved under water scarcity has different anatomy than a rainforest plant subjected to drought—and when both are given abundant water, the desert plant may exhibit surprising advantages in water utilization efficiency that persist despite no longer being necessary. - -We term this the \textit{evolutionary pressure hypothesis}: resource constraints experienced during the optimization process itself—not merely during evaluation—shape the resulting systems in qualitatively different ways. Just as organisms exhibit island dwarfism, desert metabolic efficiency, and communication compression under evolutionary pressure, we hypothesize that multi-agent LLM architectures evolved under token budget constraints will exhibit analogous adaptations: fewer agents, compressed prompts, specialized roles, and fail-fast strategies that avoid ``expensive failures.'' - -\paragraph{Research Questions.} We ask: -\begin{enumerate} - \item[\textbf{RQ1}] Do multi-agent architectures evolved under strict resource constraints differ \textit{structurally} from those evolved without constraints? - \item[\textbf{RQ2}] Do constraint-evolved architectures exhibit qualitatively different coordination strategies compared to unconstrained evolution? - \item[\textbf{RQ3}] When given abundant resources, do constraint-evolved architectures outperform architectures that never experienced scarcity? (The ``transfer to abundance'' question.) \textit{[Deferred to future work]} - \item[\textbf{RQ4}] Do architectures evolved on one benchmark transfer to others, and does constraint during evolution affect transferability? \textit{[Deferred to future work]} -\end{enumerate} - -\paragraph{Contributions.} We make the following contributions: -\begin{itemize} - \item We introduce \emap{}, a framework for evolving multi-agent programming architectures under explicit resource constraints, treating constraint as evolutionary pressure rather than evaluation metric. - \item We formalize the distinction between \textit{hard constraint evolution} (zero fitness for budget violations) and \textit{soft Pareto evolution} (cost as one of multiple objectives), providing theoretical grounding for why these produce different selective pressures. - \item We provide the first systematic study of how varying resource budgets during evolution affects the resulting architectures (structure, communication patterns, role differentiation). - \item We discover that evolution consistently produces 3-agent architectures across all budget regimes, with diverse topologies achieving equivalent performance---suggesting a multi-modal fitness landscape. - \item We release our code, evolved architectures, and analysis tools for reproducibility. -\end{itemize} - -%============================================================================== -\section{Related Work} -\label{sec:related} -%============================================================================== - -\paragraph{Evolutionary Optimization of LLM Agents.} -The past year has seen an explosion of work applying evolutionary and search-based methods to LLM agent optimization. \citet{evoagentx2025} introduce EvoAgentX, which automates generation, execution, and evolutionary optimization of multi-agent workflows, demonstrating state-of-the-art results on code generation benchmarks. \citet{artemis2025} propose ARTEMIS, using semantically-aware genetic operators (paraphrase, simplify, elaborate) for prompt optimization, though focused on single-agent systems. AFlow \citep{aflow2024} reformulates workflow optimization as code-represented search via Monte Carlo Tree Search, enabling complex workflow discovery. AutoMaAS \citep{automaas2025} applies neural architecture search principles with dynamic cost-aware optimization. AgentNet \citep{agentnet2025} evolves multi-agent topologies using graph-based genetic operators. - -A key observation across this literature: while all these systems can incorporate cost into fitness (typically as a secondary Pareto objective), \textbf{none study how constraints during evolution shape architecture}. Cost is something to minimize or trade off—not an environmental pressure that shapes adaptation. - -\paragraph{Resource-Efficient LLM Agents.} -\citet{sweeffi2025} introduce metrics for evaluating AI agent ``effectiveness'' (accuracy/cost tradeoff) and document two critical failure patterns: the ``token snowball'' (harder tasks consume disproportionately more tokens) and ``expensive failures'' (agents waste resources on ultimately unsolvable problems). These patterns motivate our hypothesis: evolution under budget constraints might naturally select \textit{against} architectures prone to these failure modes. - -\citet{corl2025} use reinforcement learning to train budget-aware multi-agent coordination policies. \citet{curriculum2025} apply curriculum learning for constraint-aware training, progressively tightening resource limits. Both learn \textit{behaviors} for budget awareness; we evolve \textit{architectures} under constraint—a fundamentally different approach. - -\paragraph{Multi-Objective vs. Constraint-Based Optimization.} -MALBO \citep{malbo2025} applies Bayesian optimization to find Pareto-optimal LLM team compositions trading accuracy for cost. This exemplifies the dominant paradigm: cost is one of multiple objectives to balance. Our formulation differs fundamentally: - -\begin{equation} -\text{Pareto:} \quad \max_A \left[ \text{Accuracy}(A),---\text{Cost}(A) \right] \quad \text{(multi-objective)} -\end{equation} -\begin{equation} -\text{Hard constraint:} \quad \max_A \text{Accuracy}(A) \quad \text{s.t.} \quad \text{Cost}(A) \leq B \quad \text{(feasibility)} -\end{equation} - -In Pareto optimization, high-cost architectures can survive if sufficiently accurate. In hard constraint optimization, architectures that cannot survive within budget are eliminated regardless of accuracy—creating genuine selective pressure for adaptation. - -\paragraph{Biological Precedent.} -The evolutionary pressure hypothesis draws from biological observations. The theory of island biogeography \citep{macarthur1967} established that isolated populations evolve under distinct selective pressures. Foster's rule \citep{foster1964} documents systematic body size changes in insular populations, later generalized by \citet{lomolino2005} as the ``island rule'': vertebrates on resource-limited islands tend toward dwarfism while small species may gigantism. Desert organisms exhibit convergent evolution toward metabolic efficiency. Crucially, these adaptations often prove advantageous when constraints are relaxed—desert plants show superior water utilization even with abundant water. We test whether analogous ``learned frugality'' emerges in evolved multi-agent systems. - -%============================================================================== -\section{Method} -\label{sec:method} -%============================================================================== - -\subsection{Problem Formulation} - -Let $\mathcal{A}$ denote the space of multi-agent architectures, where each architecture $A \in \mathcal{A}$ specifies: -\begin{itemize} - \item A set of agents $\{a_1, \ldots, a_n\}$ with associated prompts and role specifications - \item A communication topology $G = (V, E)$ defining message flow between agents - \item Aggregation and decision mechanisms for combining agent outputs - \item Per-agent token limits and message compression settings -\end{itemize} - -Given a benchmark $\mathcal{B}$ with tasks $\{t_1, \ldots, t_m\}$ and a resource budget $B \in \mathbb{R}^+$ (measured in tokens), we define the \textbf{hard constraint fitness function}: - -\begin{equation} -\text{Fitness}_B(A) = -\begin{cases} -\frac{1}{m}\sum_{i=1}^{m} \mathbf{1}[\text{passes}(A, t_i)] & \text{if } \forall i: \text{Cost}(A, t_i) \leq B \\ -0 & \text{otherwise} -\end{cases} -\label{eq:fitness} -\end{equation} - -This formulation differs crucially from soft Pareto approaches in two ways: - -\textbf{(1) Zero fitness for any violation.} An architecture that exceeds budget on even one task receives zero fitness, regardless of performance on other tasks. This creates strong selective pressure against ``risky'' architectures that might occasionally exceed limits. - -\textbf{(2) No accuracy-cost tradeoff.} Within budget, fitness depends only on accuracy. An architecture using 1,900 tokens has the same fitness as one using 200 tokens (if both are within a 2,000 token budget). This allows evolution to discover that sometimes ``using your full budget wisely'' is better than minimizing cost. - -\paragraph{Theoretical Justification.} -The hard constraint formulation induces a different fitness landscape than Pareto optimization. In Pareto, the gradient always points toward reducing cost (among other directions). In hard constraint, the gradient is zero with respect to cost until the constraint boundary is approached—then it becomes infinite. This means: - -\begin{itemize} - \item Architectures learn to \textit{use} their budget effectively, not minimize it - \item Selection pressure focuses on robustness (never exceeding) rather than efficiency (minimizing average) - \item Architectures that occasionally fail to fit within budget are heavily penalized, selecting for conservative strategies -\end{itemize} - -\subsection{Evolutionary Framework} - -\emap{} implements evolutionary search with the following components: - -\paragraph{Genome Representation.} -We represent architectures as typed graphs where nodes are agents (with associated prompts) and edges are communication channels. The genome encodes: -\begin{itemize} - \item Number of agents $n \in \{1, \ldots, N_{\max}\}$ - \item Agent types (from a discrete set: planner, coder, reviewer, debugger, etc.) - \item Prompt templates (parameterized by slots for task-specific content) - \item Topology structure (adjacency specification) - \item Message compression level (bits allocated per inter-agent message) -\end{itemize} - -\paragraph{Genetic Operators.} - -\textit{Mutation} operators include: -\begin{itemize} - \item Agent addition/removal (structural mutation) - \item Prompt token reduction/expansion (efficiency mutation) - \item Communication edge rewiring (topology mutation) - \item Role type change (functional mutation) -\end{itemize} - -\textit{Crossover} operates on compatible topologies: -\begin{itemize} - \item Subgraph exchange between parent architectures - \item Prompt recombination within agent types -\end{itemize} - -\paragraph{Selection.} -We use tournament selection with elitism, maintaining the top $k$ architectures across generations. - -\paragraph{Algorithm.} -Algorithm~\ref{alg:emap} presents the complete evolutionary procedure. - -\begin{algorithm}[t] -\caption{\emap{}: Resource-Constrained Evolution}\label{alg:emap} -\begin{algorithmic}[1] -\REQUIRE Benchmark $\mathcal{B}$, budget $B$, generations $G$, population size $N$ -\ENSURE Best evolved architecture $A^*$ -\STATE $\mathcal{P}_0 \leftarrow \textsc{InitializePopulation}(N)$ \COMMENT{Diverse initial architectures} -\FOR{$g = 1$ to $G$} - \FOR{$A \in \mathcal{P}_{g-1}$} - \STATE $\text{Fitness}(A) \leftarrow \textsc{Evaluate}(A, \mathcal{B}, B)$ \COMMENT{Eq.~\ref{eq:fitness}} - \ENDFOR - \STATE $\mathcal{E} \leftarrow \textsc{ElitistSelection}(\mathcal{P}_{g-1}, k)$ - \STATE $\mathcal{P}_g \leftarrow \mathcal{E}$ - \WHILE{$|\mathcal{P}_g| < N$} - \STATE $p_1 \leftarrow \textsc{TournamentSelect}(\mathcal{P}_{g-1})$ - \STATE $p_2 \leftarrow \textsc{TournamentSelect}(\mathcal{P}_{g-1})$ - \STATE $c \leftarrow \textsc{Crossover}(p_1, p_2)$ with probability $p_c$ - \STATE $c \leftarrow \textsc{Mutate}(c)$ with probability $p_m$ - \STATE $\mathcal{P}_g \leftarrow \mathcal{P}_g \cup \{c\}$ - \ENDWHILE -\ENDFOR -\STATE $A^* \leftarrow \arg\max_{A \in \mathcal{P}_G} \text{Fitness}(A)$ -\RETURN $A^*$ -\end{algorithmic} -\end{algorithm} - -\subsection{Constraint Regimes} - -We study evolution under four budget regimes: -\begin{itemize} - \item \textbf{Tight} ($B_T = 2\text{K tokens}$): Severe constraint forcing minimal communication - \item \textbf{Medium} ($B_M = 5\text{K tokens}$): Moderate constraint allowing structured collaboration - \item \textbf{Loose} ($B_L = 10\text{K tokens}$): Mild constraint; most reasonable architectures fit - \item \textbf{Unconstrained} ($B_\infty$): No limit; baseline for comparison -\end{itemize} - -\subsection{Hypothesized Emergent Behaviors} - -Drawing from evolutionary biology and the failure patterns documented by \citet{sweeffi2025}, we hypothesize that tight budget evolution will produce architectures exhibiting: - -\paragraph{H1: Structural Compactness.} -Fewer agents, simpler topologies, and shorter prompts. Under severe constraints, the overhead of multi-agent coordination becomes prohibitive; evolution should favor lean architectures that accomplish tasks with minimal communication. - -\paragraph{H2: Fail-Fast Strategies.} -Architectures that quickly recognize difficult or unsolvable problems and abandon them before consuming significant resources. This directly counters the ``expensive failure'' pattern where agents persist on hopeless tasks. We expect constraint-evolved architectures to develop implicit difficulty estimation. - -\paragraph{H3: Communication Compression.} -When inter-agent messages consume limited budget, evolution should favor compressed, structured communication over verbose natural language. We expect to see emergence of task-specific message formats and abbreviations. - -\paragraph{H4: Role Specialization.} -Under constraint, generalist agents that attempt everything may be outcompeted by specialists that excel at narrow subtasks. Evolution should produce clearer division of labor as budget tightens. - -\paragraph{H5: Transferable Efficiency.} -The efficiency strategies learned under constraint should persist—and prove advantageous—when constraints are relaxed. An architecture that learned not to waste tokens on expensive failures will continue this behavior even with unlimited budget. - -\subsection{Benchmarks} - -We evaluate on: -\begin{itemize} - \item \textbf{HumanEval} \citep{humaneval}: 164 Python programming problems - \item \textbf{MBPP} \citep{mbpp}: 974 Python programming problems - \item \textbf{SWE-bench-lite}: Subset of real GitHub issues (for transfer analysis) -\end{itemize} - -%============================================================================== -\section{Experiments} -\label{sec:experiments} -%============================================================================== - -\subsection{Research Questions} - -\begin{enumerate} - \item[\textbf{RQ1}:] \textit{Structural differences.} Do architectures evolved under different budget regimes exhibit different structures (number of agents, topology, role distribution)? - - \item[\textbf{RQ2}:] \textit{Failure avoidance.} Do constraint-evolved architectures exhibit fewer ``expensive failures'' compared to unconstrained-evolved architectures? - - \item[\textbf{RQ3}:] \textit{Transfer to abundance.} When given unlimited resources, how do constraint-evolved architectures perform relative to unconstrained-evolved architectures? - - \item[\textbf{RQ4}:] \textit{Cross-benchmark transfer.} Do architectures evolved on HumanEval transfer to MBPP, and vice versa? -\end{enumerate} - -\subsection{Experimental Setup} - -\paragraph{Evolution parameters.} -Population size: 10. Generations: 12. Tournament size: 2. Elitism: top 2. Mutation rate: 0.4. Crossover rate: 0.5. Sample fraction: 12\% ($\sim$20 tasks per generation evaluation). Final evaluation: 25\% ($\sim$41 tasks). - -\paragraph{Base LLM.} -We use GPT-4o-mini as the underlying model for all agents, controlling for model capability while maintaining cost efficiency. - -\paragraph{Evaluation protocol.} -During evolution, each architecture is evaluated on a 12\% sample ($\sim$20 tasks) for efficiency. Final best architectures are evaluated on 25\% of HumanEval ($\sim$41 tasks) for statistical validity. - -\paragraph{Runs.} -3 independent runs per constraint regime (seeds 42, 43, 44), totaling 12 experiments across 4 budget regimes. - -\paragraph{Computational Cost.} -Total experimental cost: \textbf{3.09M tokens} ($\sim$27K API calls) across all 12 experiments. Using GPT-4o-mini pricing ($\sim$\$0.15/1M input, \$0.60/1M output), estimated total cost: \textbf{$<$\$1.00}. Individual experiment runtimes ranged from 1.5--5.2 hours depending on budget regime, with UNCONSTRAINED experiments taking longest due to larger per-evaluation token allowances. This demonstrates that evolutionary architecture search is cost-effective even for academic budgets. - -\subsection{Metrics} - -We report the following metrics, following best practices from \citet{sweeffi2025}: - -\begin{itemize} - \item \textbf{Pass@1}: Fraction of tasks where the first generated solution passes all tests - \item \textbf{Avg. Tokens}: Mean tokens consumed per task (regardless of success) - \item \textbf{Efficiency} ($\eta$): Pass@1 divided by normalized token usage, i.e., $\eta = \frac{\text{Pass@1}}{\text{Tokens}/B_\infty}$ - \item \textbf{Expensive Failure Rate} (EFR): Fraction of failed tasks where $>50\%$ of budget was consumed before failure - \item \textbf{Structural Compactness}: Average number of agents in evolved architectures -\end{itemize} - -\subsection{Results} - -All results are averaged over 3 random seeds (42, 43, 44) per budget regime, with standard deviation reported. We ran a total of 12 experiments across 4 budget regimes, each evolving for 12 generations with population size 10. - -\subsubsection{RQ1: Structural Analysis} - -Table~\ref{tab:structural} presents the structural properties of best-performing architectures evolved under each constraint regime. - -\begin{table}[h] -\centering -\caption{Performance across budget regimes (HumanEval benchmark, 3 seeds per regime, 12 generations, population 10). All regimes converge to 3-agent architectures. TIGHT constraint introduces variance while MEDIUM achieves perfect fitness.} -\label{tab:structural} -\begin{tabular}{lcccc} -\toprule -Regime & Budget (tokens) & Pass@1 & Avg. Agents & Avg. Edges \\ -\midrule -TIGHT & 2,000 & 98.4\% $\pm$ 2.3\% & 3.0 & 2.6 \\ -MEDIUM & 5,000 & 100.0\% $\pm$ 0.0\% & 3.0 & 2.4 \\ -LOOSE & 10,000 & 97.6\% $\pm$ 2.4\% & 3.0 & 1.8 \\ -UNCONSTRAINED & 50,000 & 100.0\% $\pm$ 0.0\% & 2.7 & 2.0 \\ -\bottomrule -\end{tabular} -\vspace{0.5em} - -\textit{Key finding: All regimes consistently achieve near-perfect performance (97.6--100\%). UNCONSTRAINED produces slightly simpler architectures (2.7 agents vs. 3.0) but maintains topological diversity, including both linear and cyclic configurations.} -\end{table} - -\subsubsection{Per-Seed Detailed Results} - -Table~\ref{tab:per-seed} provides a comprehensive breakdown of results for each individual experiment, revealing the diversity of evolved architectures. - -\begin{table}[h] -\centering -\caption{Detailed per-seed results across all budget regimes. Each experiment ran for 12 generations with population size 10. Final evaluation on 41 HumanEval tasks (25\% sample).} -\label{tab:per-seed} -\small -\begin{tabular}{llccccc} -\toprule -Regime & Seed & Pass@1 & Agents & Edges & Topology Type & Message Format \\ -\midrule -\multirow{3}{*}{TIGHT (2K)} & 42 & 95.1\% & 3 & 2 & Linear pipeline & structured \\ - & 43 & 100.0\% & 3 & 3 & Cyclic feedback & freeform \\ - & 44 & 100.0\% & 3 & 3 & Cyclic feedback & minimal \\ -\midrule -\multirow{3}{*}{MEDIUM (5K)} & 42 & 100.0\% & 3 & 2 & Linear pipeline & structured \\ - & 43 & 100.0\% & 3 & 3 & Cyclic feedback & freeform \\ - & 44 & 100.0\% & 3 & 3 & Cyclic feedback & minimal \\ -\midrule -\multirow{3}{*}{LOOSE (10K)} & 42 & 95.1\% & 4 & 3 & Complex voting & structured \\ - & 43 & 100.0\% & 3 & 2 & Linear pipeline & structured \\ - & 44 & 100.0\% & 4 & 4 & Hierarchical 4-agent & structured \\ -\midrule -\multirow{3}{*}{UNCON. (50K)} & 42 & 100.0\% & 2 & 1 & Minimal pair & structured \\ - & 43 & 100.0\% & 3 & 2 & Linear pipeline & structured \\ - & 44 & 100.0\% & 3 & 3 & Cyclic feedback & structured \\ -\bottomrule -\end{tabular} -\end{table} - -\paragraph{Topology Diversity.} A striking finding is the \textbf{diversity of successful topologies} that evolution discovers. Across our experiments, we identified five distinct architectural patterns, each achieving near-optimal performance: - -\begin{enumerate} - \item \textbf{Traditional Pipeline} (planner$\rightarrow$coder$\rightarrow$reviewer): The classic software engineering workflow---plan first, code second, review third. This emerged in TIGHT seed 42 and MEDIUM seed 42, achieving 95.1\% and 100\% respectively. - - \item \textbf{Test-First Pipeline} (tester$\rightarrow$reviewer$\rightarrow$planner): An unconventional topology where testing initiates the workflow, followed by review and planning. This achieved 100\% in both TIGHT seed 43 and MEDIUM seed 43. Notably, this topology includes a \textit{cyclic} edge from planner back to reviewer, enabling iterative refinement. - - \item \textbf{Hybrid Architecture} (generalist$\rightarrow$coder$\rightarrow$architect): Emerged in TIGHT seed 44, featuring a generalist agent that provides initial guidance, a dedicated coder, and an architect for structural oversight. Uses \textit{minimal} message format, suggesting compression as a response to tight constraints. - - \item \textbf{Complex Voting} (4-agent with planner, generalist, architect, coder): Appeared in LOOSE seed 42, where the larger budget allows for more sophisticated coordination through voting-based aggregation. - - \item \textbf{Hierarchical 4-Agent} (planner, architect, coder, tester): Emerged in LOOSE seed 44, with architect$\rightarrow$planner, coder$\rightarrow$planner, and tester$\rightarrow$coder edges. This architecture achieved 100\% and represents the most complex topology discovered, with avg agents evolving from 2.4 to 4.3 over 12 generations. - - \item \textbf{Minimal Pair} (debugger$\rightarrow$planner or coder$\rightarrow$reviewer): Unique to UNCONSTRAINED regime. With no budget pressure, evolution converges to the simplest effective solution: just 2 agents with a single edge. This demonstrates that multi-agent complexity in constrained regimes is a \textit{necessary response} to constraints, not inherently optimal. -\end{enumerate} - -\subsubsection{Evolutionary Dynamics} - -Figure~\ref{fig:evolution-trajectories} shows how fitness and population diversity evolve over generations across different budget regimes. - -\paragraph{Convergence Patterns.} We observe three distinct convergence patterns: - -\begin{enumerate} - \item \textbf{Immediate Convergence}: In some seeds (TIGHT 42, 43; LOOSE 43), the best genome achieves 100\% fitness from generation 0 or 1, maintaining this throughout evolution. This suggests that effective architectures exist in the initial random population. - - \item \textbf{Gradual Improvement}: TIGHT seed 44 shows gradual improvement from 10\% average fitness in generation 0 to 100\% by generation 6, demonstrating evolutionary search discovering viable configurations over time. - - \item \textbf{Fitness Valley}: MEDIUM seed 42 shows a striking pattern: 0\% best fitness for generations 0-5, then sudden jump to 100\% at generation 6. This ``fitness valley'' occurs because early populations contain architectures that exceed the 5K budget (receiving zero fitness), and evolution must discover budget-compliant designs through mutation. -\end{enumerate} - -\paragraph{Diversity Dynamics.} Population diversity (measured as fraction of unique genomes) consistently declines from 1.0 to 0.3-0.5 over 12 generations, indicating healthy convergence while maintaining exploration. The cyclic topology seeds (TIGHT 43, MEDIUM 43) show faster diversity collapse (reaching 0.3 by generation 5), suggesting these represent strong attractors in the fitness landscape. - -\subsubsection{Structural Complexity Evolution} - -Table~\ref{tab:complexity-evolution} shows how architectural complexity changes during evolution. - -\begin{table}[h] -\centering -\caption{Evolution of structural complexity (average agents and edges per population) from initial to final generation.} -\label{tab:complexity-evolution} -\begin{tabular}{lcccc} -\toprule -Regime & Initial Agents & Final Agents & Initial Edges & Final Edges \\ -\midrule -TIGHT (2K) & 2.2 $\pm$ 0.2 & 3.2 $\pm$ 0.3 & 1.3 $\pm$ 0.2 & 2.6 $\pm$ 0.5 \\ -MEDIUM (5K) & 2.1 $\pm$ 0.0 & 3.2 $\pm$ 0.2 & 1.3 $\pm$ 0.1 & 2.1 $\pm$ 0.3 \\ -LOOSE (10K) & 2.1 $\pm$ 0.0 & 3.2 $\pm$ 0.2 & 1.2 $\pm$ 0.1 & 2.3 $\pm$ 0.2 \\ -\bottomrule -\end{tabular} -\vspace{0.5em} - -\textit{All regimes show consistent growth from $\sim$2 to $\sim$3 agents, suggesting 3-agent architectures are optimal for HumanEval tasks regardless of budget.} -\end{table} - -\paragraph{Key Observation: Complexity Growth.} Across all budget regimes, populations evolve from simpler configurations (avg 2.1 agents, 1.2 edges) toward more complex ones (avg 3.2 agents, 2.3 edges). This contradicts our initial hypothesis (H1) that tight constraints would favor minimal architectures. Instead, the fitness benefit of multi-agent coordination outweighs token overhead costs even under severe 2K constraints. - -\textbf{Hypothesis H1 (Structural Compactness)}: We expected tight-budget evolution to produce architectures with fewer agents and simpler topologies. - -\paragraph{Finding: Consistent Multi-Agent Emergence.} Contrary to our hypothesis, evolution on the full HumanEval benchmark produced 3-agent architectures \textit{across all budget regimes}. This suggests that the complexity of real coding tasks creates selection pressure for multi-agent coordination that outweighs the token overhead cost, even under severe constraints. - -Remarkably, evolution discovered \textit{diverse} topologies that achieve equivalent performance: -\begin{itemize} - \item \textbf{Planner $\rightarrow$ Coder $\rightarrow$ Reviewer}: Traditional software engineering pipeline - \item \textbf{Tester $\rightarrow$ Reviewer $\rightarrow$ Planner}: Test-first methodology with validation - \item \textbf{Generalist $\rightarrow$ Coder $\rightarrow$ Architect}: Hybrid approach with architectural oversight -\end{itemize} - -This diversity suggests evolution navigates a multi-modal fitness landscape where multiple architectural strategies achieve similar performance. The MEDIUM budget (5K tokens) appears optimal: tight enough to prevent wasteful patterns but permissive enough for effective multi-agent coordination, achieving 100\% pass rate across all seeds. - -\subsubsection{RQ2: Evolved Architecture Case Studies} - -We present detailed case studies of three architectures that emerged from evolution, each representing a distinct coordination strategy. - -\paragraph{Case Study 1: Traditional Pipeline (TIGHT seed 42).} -\begin{quote} -\texttt{planner $\rightarrow$ coder $\rightarrow$ reviewer} -\end{quote} - -This architecture mirrors established software engineering practices: -\begin{itemize} - \item \textbf{Planner Agent}: ``Break down programming tasks into clear, actionable steps. Output a numbered plan.'' - \item \textbf{Coder Agent}: ``Write clean, correct Python code that solves the given task. Include necessary imports.'' - \item \textbf{Reviewer Agent}: ``Analyze code for bugs, edge cases, and improvements. Be specific about issues found.'' -\end{itemize} - -Configuration: \texttt{max\_rounds=3}, \texttt{early\_exit\_confidence=0.9}, \texttt{aggregation=best\_of\_n}. This architecture achieved 95.1\% (39/41) on final evaluation, using 386K tokens over 12 generations. The sequential flow ensures each agent builds on the previous one's output. - -\paragraph{Case Study 2: Test-First Pipeline (TIGHT seed 43).} -\begin{quote} -\texttt{tester $\rightarrow$ reviewer $\rightarrow$ planner} (with cyclic edge: planner $\rightarrow$ reviewer) -\end{quote} - -This unconventional topology inverts the traditional order: -\begin{itemize} - \item \textbf{Tester Agent}: ``Write comprehensive test cases that cover edge cases and validate correctness.'' - \item \textbf{Reviewer Agent}: ``Analyze code for bugs, edge cases, and improvements. Be specific about issues found.'' - \item \textbf{Planner Agent}: ``Break down programming tasks into clear, actionable steps. Output a numbered plan.'' -\end{itemize} - -Configuration: \texttt{max\_rounds=2}, \texttt{message\_format=freeform}, \texttt{max\_message\_length=134}, \texttt{aggregation=hierarchical}. This architecture achieved \textbf{100\%} (41/41) on final evaluation using only 286K tokens---26\% fewer than Case Study 1. The cyclic edge from planner back to reviewer enables iterative refinement within the tight 2K budget. - -Key insight: The freeform message format and reduced max\_message\_length (134 vs 200) represent evolved adaptations to tight constraints, compressing inter-agent communication. - -\paragraph{Case Study 3: Hybrid Architecture (TIGHT seed 44).} -\begin{quote} -\texttt{generalist $\rightarrow$ coder $\rightarrow$ architect} (with cyclic edges: generalist $\leftrightarrow$ coder) -\end{quote} - -A novel configuration discovered by evolution: -\begin{itemize} - \item \textbf{Generalist Agent}: ``Solve the given task by writing correct, efficient Python code.'' - \item \textbf{Coder Agent}: ``Write clean, correct Python code that solves the given task. Include necessary imports.'' - \item \textbf{Architect Agent}: ``Design the overall structure and approach before implementation begins.'' -\end{itemize} - -Configuration: \texttt{message\_format=minimal}, \texttt{max\_message\_length=228}, \texttt{early\_exit\_confidence=0.74}. This architecture achieved \textbf{100\%} (41/41) using 183K tokens---the most efficient of all TIGHT experiments. The \textit{minimal} message format only appeared in this seed, representing maximum communication compression. - -\subsubsection{RQ2: Baseline Comparison} - -Table~\ref{tab:baseline} compares evolved architectures against hand-designed baselines to understand the value evolution provides. - -\begin{table}[h] -\centering -\caption{Baseline performance comparison on full HumanEval (164 tasks). Planning provides the key benefit; review has diminishing returns.} -\label{tab:baseline} -\begin{tabular}{lcccc} -\toprule -Architecture & Agents & Pass@1 & Tokens & Efficiency ($\eta$) \\ -\midrule -Planner $\rightarrow$ Coder & 2 & \textbf{97.6\%} & 26,637 & 3.67 \\ -Planner $\rightarrow$ Coder $\rightarrow$ Reviewer & 3 & 96.3\% & 43,946 & 2.19 \\ -Single Coder & 1 & 95.7\% & 15,078 & \textbf{6.35} \\ -Coder $\rightarrow$ Reviewer & 2 & 95.7\% & 31,463 & 3.04 \\ -\midrule -\textit{Evolved (TIGHT seed 44)} & 3 & 100.0\% & 183K\textsuperscript{*} &---- \\ -\textit{Evolved (MEDIUM seed 43)} & 3 & 100.0\% & 97K\textsuperscript{*} &---- \\ -\bottomrule -\end{tabular} -\vspace{0.3em} - -\textsuperscript{*}Token counts for evolved architectures are total evolution cost, not per-task cost. - -\vspace{0.5em} -\textit{Key finding: The planner $\rightarrow$ coder pipeline achieves the highest accuracy (97.6\%), beating single-agent by 2\%. Adding a reviewer \textbf{hurts} performance (96.3\% < 97.6\%), suggesting coordination overhead exceeds benefit for this benchmark. Evolved architectures achieve 100\% on sampled tasks.} -\end{table} - -\textbf{Analysis:} These baselines reveal a nuanced picture: -\begin{itemize} - \item \textbf{Planning adds value}: The planner stage enables better problem decomposition, improving pass rate from 95.7\% to 97.6\%. - \item \textbf{Review has diminishing returns}: The reviewer stage adds 17K tokens but \textit{decreases} accuracy, suggesting the overhead of multi-round communication exceeds the benefit of code review for relatively simple tasks. - \item \textbf{Evolution discovered novel configurations}: The test-first and hybrid architectures represent designs that would be unlikely to emerge from human engineering, yet achieve equal or better performance. - \item \textbf{Evolution optimizes holistically}: Unlike hand-designed architectures, evolved systems optimize the \textit{combination} of topology, message format, aggregation strategy, and hyperparameters simultaneously. -\end{itemize} - -\subsubsection{RQ3 \& RQ4: Transfer Experiments (Future Work)} - -We originally posed research questions about transfer to abundance (RQ3) and cross-benchmark generalization (RQ4). Due to computational constraints, we defer these experiments to future work. The current study focuses on RQ1 (structural analysis) and RQ2 (evolved architecture characterization), which provide sufficient novelty for initial publication. We hypothesize that constraint-evolved architectures will exhibit transferable efficiency, but this remains to be empirically validated. - -\textbf{Planned experiments}: (1) Evaluate evolved architectures with unlimited token budgets to test transfer to abundance; (2) Cross-benchmark evaluation on MBPP to test generalization beyond HumanEval. - -%============================================================================== -\section{Analysis} -\label{sec:analysis} -%============================================================================== - -\subsection{Emergence of Multi-Agent Coordination} - -Our most striking finding is that \textbf{evolution consistently produced 3-agent architectures across all budget regimes} (Table~\ref{tab:structural}). This contradicts our initial hypothesis (H1) that tight constraints would favor minimal single-agent systems. Instead, the complexity of HumanEval's 164 programming problems creates strong selection pressure for multi-agent coordination that outweighs token overhead costs. - -This result has significant implications: - -\begin{enumerate} - \item \textbf{Task Complexity Dominates Constraint Effects}: Real coding tasks require sufficient complexity that multi-agent coordination provides benefit even under severe (2K token) constraints. The fitness improvement from coordination outweighs the token cost. - - \item \textbf{Diverse Optima in Fitness Landscape}: Evolution discovered multiple distinct topologies---planner$\rightarrow$coder$\rightarrow$reviewer, tester$\rightarrow$reviewer$\rightarrow$planner, generalist$\rightarrow$coder$\rightarrow$architect---all achieving near-optimal performance (98.4--100\%). This suggests a multi-modal fitness landscape. - - \item \textbf{Constraint Severity Affects Variance, Not Structure}: TIGHT constraints (2K tokens) introduce performance variance ($\pm$2.3\%) while MEDIUM constraints (5K tokens) achieve consistent 100\% across seeds. The 5K budget appears optimal for this benchmark. -\end{enumerate} - -\subsection{The Multi-Modal Fitness Landscape} - -A key finding is that evolution discovers \textbf{multiple equally-viable architectural strategies}. This suggests the fitness landscape for multi-agent code generation is multi-modal---containing multiple peaks of similar height rather than a single global optimum. - -\paragraph{Evidence for Multi-Modality.} Three distinct topologies achieve 100\% pass rate: -\begin{itemize} - \item \textbf{Traditional} (planner$\rightarrow$coder$\rightarrow$reviewer): Sequential refinement - \item \textbf{Test-First} (tester$\rightarrow$reviewer$\rightarrow$planner): Validation-driven development - \item \textbf{Hybrid} (generalist$\rightarrow$coder$\rightarrow$architect): Parallel expertise -\end{itemize} - -These represent fundamentally different problem-solving strategies, yet evolution independently discovers each under identical conditions (same budget, different random seed). This has practical implications: practitioners can choose among multiple validated architectures based on secondary criteria (interpretability, latency, cost preference) without sacrificing performance. - -\paragraph{Cyclic vs. Linear Topologies.} Analysis of topology patterns across budget regimes reveals a nuanced relationship between constraints, seeds, and topology. Under constrained budgets (TIGHT, MEDIUM, LOOSE), seed 42 consistently evolved linear pipelines while seeds 43 and 44 evolved cyclic feedback topologies---suggesting that initial population composition establishes a trajectory that persists throughout evolution. However, the UNCONSTRAINED regime (50K tokens) shows different behavior: seeds 42 and 43 both produce linear topologies (2-agent minimal pair and 3-agent pipeline respectively), while only seed 44 maintains a cyclic architecture. This suggests that \textit{budget constraints may help preserve topological diversity}---without resource pressure, evolution more readily converges to simpler linear solutions. - -\subsection{Hyperparameter Evolution} - -Beyond topology, evolution also optimizes continuous hyperparameters. Table~\ref{tab:hyperparam-evolution} summarizes the evolved values across experiments. +\section{Artifact scope} -\begin{table}[h] -\centering -\caption{Distribution of evolved hyperparameters across all experiments. Evolution discovers distinct configurations for different constraint regimes.} -\label{tab:hyperparam-evolution} -\begin{tabular}{lcccc} -\toprule -Hyperparameter & TIGHT Range & MEDIUM Range & LOOSE Range & Default \\ -\midrule -\texttt{max\_rounds} & 2--3 & 2--3 & 2--3 & 3 \\ -\texttt{early\_exit\_confidence} & 0.74--0.92 & 0.85--0.92 & 0.85--0.88 & 0.9 \\ -\texttt{max\_message\_length} & 134--228 & 154--195 & 188--255 & 200 \\ -\texttt{temperature} & 0.7--0.88 & 0.7--0.88 & 0.7 & 0.7 \\ -\bottomrule -\end{tabular} -\end{table} +The repository contains twelve completed runs in +\texttt{experiments/results/focused/}: TIGHT (2,000 tokens), MEDIUM (5,000), +LOOSE (10,000), and UNCONSTRAINED (50,000), each with seeds 42, 43, and 44. +The checked-in summary records 12 generations, a population of 10, a +12-percent task sample during evolution, and a 25-percent final sample. Each +per-run artifact reports \texttt{final\_total: 41}. -\paragraph{Key Observations:} -\begin{enumerate} - \item \textbf{Message Length Adapts to Constraints}: TIGHT regimes evolve shorter \texttt{max\_message\_length} (134--228) compared to LOOSE (188--255), directly compressing communication to fit within budget. +Across the twelve artifacts, the recorded totals are 3,093,390 tokens and +27,360 API calls. Recorded runtimes range from approximately 1.7 to 5.2 hours +per run. The artifacts do not separate input and output tokens, so a historical +dollar cost cannot be reconstructed from them. - \item \textbf{Early Exit Confidence Varies}: TIGHT seed 44 evolved notably lower \texttt{early\_exit\_confidence} (0.74) than other seeds ($\sim$0.9), suggesting a strategy of terminating earlier on difficult problems rather than exhausting the budget. +\section{Method represented by the artifacts} - \item \textbf{Temperature Stability}: Unlike other hyperparameters, temperature remained relatively stable near the default (0.7), suggesting this value is already near-optimal for code generation. -\end{enumerate} +The runner was intended to use the 164-task HumanEval corpus, sampling tasks +during evolution and for final evaluation. A 41-task final total is consistent +with the configured 25-percent sample, but the artifacts do not preserve task +identifiers, per-task outputs, or a dataset hash. Consequently, this note does +not treat the recorded scores as full-HumanEval results and cannot establish +that different seeds were evaluated on identical task subsets. -\subsection{Token Efficiency Analysis} +The seed controls NumPy-based architecture evolution. Task selection uses +Python's \texttt{random} module without a recorded seed, and model calls are +nondeterministic. The model name is also absent from the JSON artifacts; +GPT-4o-mini attribution comes from the checked-in runner configuration rather +than from a model field in each record. -Table~\ref{tab:token-efficiency} compares token usage across regimes, revealing how constraint shapes resource utilization. +\section{Recorded results} \begin{table}[h] \centering -\caption{Token usage across budget regimes. Total tokens consumed during complete 12-generation evolutionary runs (population=10, 3 seeds each).} -\label{tab:token-efficiency} -\begin{tabular}{lccccc} +\begin{tabular}{lrrrr} \toprule -Regime & Seed 42 & Seed 43 & Seed 44 & Mean & Per-Gen \\ -\midrule -TIGHT (2K) & 387K & 286K & 183K & 285K & 24K \\ -MEDIUM (5K) & 135K & 97K & 183K & 138K & 12K \\ -LOOSE (10K) & 302K & 97K & 482K & 294K & 24K \\ -UNCONSTRAINED (50K) & 206K & 296K & 441K & 314K & 26K \\ +Regime & Budget & Mean final score & Mean agents & Mean edges \\ \midrule -\textbf{Total} & \multicolumn{5}{c}{3.09M tokens across 12 experiments} \\ +TIGHT & 2,000 & 98.4\% $\pm$ 2.3\% & 3.2 & 2.6 \\ +MEDIUM & 5,000 & 100.0\% $\pm$ 0.0\% & 3.1 & 2.4 \\ +LOOSE & 10,000 & 98.4\% $\pm$ 2.3\% & 3.5 & 2.8 \\ +UNCONSTRAINED & 50,000 & 99.2\% $\pm$ 1.1\% & 2.9 & 2.1 \\ \bottomrule \end{tabular} +\caption{Means and population standard deviations across three recorded runs +per regime. Agent and edge counts are final-generation population averages, +not the size of one selected architecture.} \end{table} -\paragraph{Paradoxical Efficiency.} TIGHT constraints produce higher total token usage than MEDIUM constraints (285K vs 116K average). This occurs because: -\begin{enumerate} - \item \textbf{More Exploration Required}: Tight constraints create a smaller feasible region in architecture space, requiring more generations to find viable configurations. - \item \textbf{Higher Evaluation Cost}: Each evaluation under tight constraints still requires multiple agent invocations, and the hard constraint fitness function forces re-evaluation of borderline architectures. - \item \textbf{The ``Goldilocks Zone''}: MEDIUM (5K) budget appears optimal---sufficient headroom to avoid expensive exploration while still constraining wasteful architectures. -\end{enumerate} - -\subsection{Evolutionary Dynamics} - -We analyze evolutionary trajectories across all 12 experiments to understand how architectures emerge and stabilize. - -\paragraph{Convergence Analysis.} Table~\ref{tab:convergence} shows the generation at which each experiment first achieved 100\% fitness on the evaluation sample. - -\begin{table}[h] -\centering -\caption{Convergence speed across experiments. ``First 100\%'' indicates the first generation achieving perfect fitness on evaluation sample. All experiments completed 12 generations.} -\label{tab:convergence} -\begin{tabular}{lcccc} -\toprule -Regime & Seed 42 & Seed 43 & Seed 44 & Mean Gen. \\ -\midrule -TIGHT (2K) & Gen 1 & Gen 1 & Gen 1 & 1.0 \\ -MEDIUM (5K) & Gen 7 & Gen 1 & Gen 1 & 3.0 \\ -LOOSE (10K) & Gen 1 & Gen 1 & Gen 1 & 1.0 \\ -UNCONSTRAINED (50K) & Gen 1 & Gen 1 & Gen 1 & 1.0 \\ -\bottomrule -\end{tabular} -\end{table} - -\paragraph{Rapid Convergence with Continued Exploration.} -A striking finding is that 11 of 12 experiments achieved 100\% fitness by the very first generation, indicating that randomly initialized populations often contain viable multi-agent architectures. This suggests that the search space, while large, contains many high-fitness regions accessible from random starting points. - -The exception---MEDIUM seed 42---required 7 generations to reach 100\%, representing genuine evolutionary search through the architecture space. This experiment's initial population happened to lack immediately viable solutions, forcing evolution to discover them through mutation and crossover. Notably, this longer search path did not produce a qualitatively different final architecture (still a 3-agent linear pipeline), suggesting multiple paths lead to similar optima. - -\paragraph{Structural Evolution Patterns.} -Despite rapid fitness convergence, architectural structure evolved substantially across generations. We observed three distinct patterns: - -\textit{Complexity Growth}: Several experiments (TIGHT seed 43, MEDIUM seed 42, LOOSE seeds 42 and 44, UNCONSTRAINED seeds 42 and 43) began with single-agent or minimal architectures (1 agent, 0 edges) and evolved toward multi-agent systems (2--4 agents, 1--4 edges). This demonstrates that evolution can bootstrap complex coordination from simple starting points. - -\textit{Structural Stability}: Other experiments (TIGHT seeds 42 and 44, MEDIUM seeds 43 and 44, LOOSE seed 43, UNCONSTRAINED seed 44) maintained consistent structure throughout---starting and ending with 3 agents. These populations initialized with viable multi-agent architectures and refined them without major structural changes. - -\textit{UNCONSTRAINED Simplification}: Uniquely, UNCONSTRAINED seed 42 evolved \textit{toward} simplicity, maintaining a 2-agent minimal architecture despite having budget for larger systems. This confirms that multi-agent complexity in constrained regimes reflects genuine necessity rather than evolutionary drift. - -\paragraph{Implications for Architecture Search.} -These dynamics suggest that evolutionary multi-agent architecture search is efficient: viable architectures emerge quickly, and continued evolution serves primarily to explore alternative topologies rather than to improve fitness. For practitioners, this implies that short evolutionary runs (5--10 generations) may suffice for finding effective architectures, with longer runs providing diversity in solutions rather than quality improvements. - -\subsection{Implications for Multi-Agent System Design} - -These results validate the ``complexity threshold'' hypothesis: multi-agent architectures provide benefit when task complexity exceeds coordination overhead. The emergence of the reviewer-coder-planner pipeline suggests that evolution can discover principled multi-agent designs without human engineering. - -Key design insight: \textbf{The evolved architecture mirrors best practices in software engineering}---code review before implementation, planning for coordination. Evolution independently discovered these patterns, suggesting they represent genuinely optimal strategies for code generation tasks. - -\subsection{Emergent Communication Strategies} - -Contrary to our hypothesis (H3), tight budget constraints did \textit{not} eliminate inter-agent communication. Instead, evolution found ways to maintain multi-agent coordination within the constraint. All regimes converged to architectures with 2--3 communication edges on average (range: 1--4), suggesting that some level of multi-agent communication is essential for code generation tasks on HumanEval. - -Examination of edge counts reveals that topology complexity is influenced by both seed and budget constraint. Within constrained regimes (TIGHT, MEDIUM, LOOSE), the same seeds produce consistent edge counts: seed 42 evolves 2-edge linear topologies, while seeds 43 and 44 evolve 3--4 edge cyclic topologies. The UNCONSTRAINED regime (50K tokens) shows partial convergence: seeds 42 and 43 produce simpler linear topologies (1--2 edges), while seed 44 maintains a 3-edge cyclic architecture. This suggests that budget pressure \textit{maintains diversity}---constrained evolution preserves multiple viable topologies, while unconstrained evolution allows greater convergence toward simpler solutions. - -\subsection{Practical Recommendations} - -Based on our results, for code generation tasks of HumanEval's complexity, practitioners should: -\begin{enumerate} - \item \textbf{Default to 3-agent architectures}: Evolution consistently discovers this as optimal, regardless of constraint level - \item \textbf{Use 5K token budgets}: This appears to be the ``sweet spot'' achieving perfect performance with minimal variance - \item \textbf{Experiment with diverse topologies}: Multiple architectures achieve equivalent performance, so practitioners can choose based on secondary criteria (interpretability, latency, etc.) -\end{enumerate} - -%============================================================================== -\section{Discussion} -\label{sec:discussion} -%============================================================================== - -\subsection{The Evolutionary Pressure Principle} - -Our central thesis---that constraints experienced \textit{during optimization} produce fundamentally different systems than constraints applied \textit{after optimization}---finds partial support in our experiments. While we did not observe the structural minimization we initially hypothesized (H1), we did observe qualitative differences in how architectures adapt to constraints: - -\begin{enumerate} - \item \textbf{Topology Adaptation}: Tight constraints favor cyclic topologies that maximize information reuse; loose constraints allow simpler linear pipelines. - \item \textbf{Communication Compression}: Message formats evolve from structured to freeform to minimal as constraints tighten, directly compressing inter-agent communication. - \item \textbf{Hyperparameter Tuning}: Early exit confidence, message length, and round limits all show constraint-dependent adaptation. -\end{enumerate} - -This suggests a refined principle: \textit{constraint during evolution produces architectures that adapt their coordination strategies, not their fundamental complexity, to resource limits.} The 3-agent structure appears to be a fundamental requirement for HumanEval-level tasks---what varies is \textit{how} those agents coordinate. - -\subsection{Comparison to Related Work} - -\paragraph{Relation to EvoAgentX.} \citet{evoagentx2025} evolve multi-agent workflows but optimize for task performance with cost as secondary objective. Our hard constraint formulation produces qualitatively different selection pressure: architectures must \textit{survive} within budget, not merely minimize cost. Our results suggest this produces more efficient coordination strategies (cyclic topologies, compressed messages) that soft Pareto optimization might not discover. +All three MEDIUM artifacts report 41 of 41 sampled final tasks passed. The +three MEDIUM runs also record 414,390 total tokens, the lowest aggregate among +the four regimes. In this small, uncontrolled sample, those values make MEDIUM +the strongest observed regime. They do not establish 5,000 tokens as an +optimum or justify a general Goldilocks-zone claim. + +The final populations averaged roughly three agents across regimes. Selected +genomes in the records range from two to four agents and include linear and +cyclic structures with different roles, aggregation modes, and message +formats. These observations document diversity in the recorded outcomes; they +do not show that budget pressure caused that diversity or that three-agent +systems are generally optimal. + +\section{Reproducibility limits} + +\begin{itemize} + \item There are three runs per regime, with no preregistered analysis or + statistical significance test. + \item Final task identifiers, per-task outputs, prompt transcripts, dataset + hashes, Git commits, dependency versions, and model versions are not + preserved in the artifacts. + \item Task sampling and remote model responses are not fully seeded, so exact + reruns are not guaranteed. + \item The project specifies dependency lower bounds and has no lockfile for + the December 2025 environment. + \item The study does not evaluate a full benchmark, cross-benchmark transfer, + other models, or deployment performance. +\end{itemize} + +\section{Execution-harness safety} + +The benchmark harness evaluates generated Python with \texttt{exec} inside a +disposable subprocess. It applies static denylist checks, restricted builtins +and imports, a parent-process timeout, and best-effort Unix limits for CPU, +address space, file size, and file descriptors. These controls reduce +accidental risk but are not a secure sandbox. Python object-model escapes and +platform differences remain possible. The evaluator should run only on an +unprivileged, isolated host without secrets or sensitive network/filesystem +access, and it fails closed if subprocess isolation cannot be established. -\paragraph{Relation to AFlow.} \citet{aflow2024} use Monte Carlo Tree Search to discover workflows represented as code. While powerful, MCTS requires many samples to explore the space. Our evolutionary approach achieves comparable results (100\% on sampled tasks) with only 12 generations and population 10---suggesting evolutionary search may be more sample-efficient for constrained optimization. - -\paragraph{Relation to MALBO.} \citet{malbo2025} apply Bayesian optimization for Pareto-optimal team composition. Our approach differs in two ways: (1) we use hard constraints rather than Pareto objectives, and (2) we evolve topology in addition to composition. The diversity of topologies we discovered (linear, cyclic, hierarchical) suggests topology is a critical dimension that composition-only approaches miss. - -\subsection{Theoretical Implications} - -\paragraph{The Complexity Threshold Hypothesis.} Our finding that 3-agent architectures emerge regardless of constraint level suggests a \textit{complexity threshold}: below a certain task complexity, single agents suffice; above it, multi-agent coordination provides sufficient fitness benefit to justify token overhead. HumanEval's 164 problems apparently exceed this threshold. - -This has implications for architecture selection: practitioners should expect multi-agent systems to emerge when task complexity exceeds coordination overhead, \textit{even under severe constraints}. The question becomes not ``how many agents?'' but ``how should agents coordinate?'' - -\paragraph{The Multi-Modal Landscape Hypothesis.} The diversity of successful topologies suggests the fitness landscape for multi-agent code generation is multi-modal. This has implications for optimization: (1) random restarts (different seeds) can discover qualitatively different solutions; (2) ensemble methods combining multiple topologies might further improve performance; (3) human designers should not assume a single ``best'' architecture exists. - -\subsection{Practical Guidelines} - -Based on our findings, we offer the following guidelines for practitioners: - -\begin{enumerate} - \item \textbf{Default to 3-agent architectures} for code generation tasks of HumanEval complexity or greater. Our results suggest this is optimal regardless of budget constraints. - - \item \textbf{Use 5K token budgets} when possible. This ``Goldilocks zone'' achieves perfect performance with minimal variance, avoiding both the expensive exploration of tight constraints and the wastefulness of loose ones. - - \item \textbf{Consider cyclic topologies} for very tight constraints. Feedback loops appear to maximize information density when linear communication is too expensive. - - \item \textbf{Compress message formats} under constraint. Evolution consistently discovers that freeform and minimal formats outperform structured messages in tight budgets. - - \item \textbf{Run multiple seeds}. The multi-modal fitness landscape means different random initializations discover different (but equally valid) architectures. This diversity provides options for secondary optimization criteria. -\end{enumerate} - -\subsection{Limitations} - -Several limitations bound our conclusions: - -\begin{itemize} - \item \textbf{Single LLM backbone}: All experiments use GPT-4o-mini. Different base models (e.g., Claude, Gemini, open-source models) may produce different evolutionary dynamics. Larger models might reduce the benefit of multi-agent coordination; smaller models might increase it. - - \item \textbf{Code generation focus}: We study programming benchmarks; other domains (mathematical reasoning, retrieval-augmented generation, creative writing) may exhibit different patterns. Multi-agent coordination might be less beneficial for domains with clearer single-agent solutions. - - \item \textbf{Token-based constraints}: We focus on token budgets; other resource constraints (wall-clock latency, API cost, memory) may induce different adaptations. Latency constraints, for example, might favor parallel topologies over sequential ones. - - \item \textbf{Evolutionary hyperparameters}: Our results depend on specific choices (population=10, generations=12, mutation=0.4). Different evolutionary algorithms (genetic programming, neuroevolution, quality-diversity) may find different optima. - - \item \textbf{Sample-based evaluation}: We evaluate on 12--25\% of HumanEval during evolution. While computationally necessary, this introduces noise that might affect generalization. - - \item \textbf{Fixed agent roles}: We use a predefined set of roles (planner, coder, reviewer, etc.). Allowing evolution to discover novel roles might produce different architectures. -\end{itemize} - -\subsection{Future Work} - -Several directions merit investigation: - -\begin{enumerate} - \item \textbf{Progressive constraint schedules}: Rather than fixed budgets, evolve under gradually tightening constraints (curriculum learning for constraints). This might produce architectures that gracefully degrade under varying resource availability. - - \item \textbf{Multi-constraint evolution}: Simultaneously constrain tokens, latency, and memory. Different constraint combinations might produce different architectural adaptations. - - \item \textbf{Cross-benchmark transfer}: We observed that evolved architectures achieve near-perfect performance on HumanEval samples. Do they transfer to MBPP, SWE-bench, or other code generation benchmarks? - - \item \textbf{Prompt evolution}: Our current approach uses fixed prompts per role. Allowing prompt mutation might discover more efficient communication strategies. - - \item \textbf{Meta-evolution}: Can we evolve the evolutionary process itself---discovering optimal mutation operators, selection pressures, or constraint schedules? - - \item \textbf{Interpretability}: What makes constraint-evolved architectures efficient? Can we extract human-understandable design principles from evolved topologies? -\end{enumerate} - -\subsection{Broader Impact} - -\paragraph{Environmental Benefits.} Efficient multi-agent systems reduce computational cost and environmental impact of AI deployments. If constrained evolution produces 2--5$\times$ more efficient architectures (as suggested by our MEDIUM vs TIGHT token usage), widespread adoption could significantly reduce the carbon footprint of LLM-based applications. - -\paragraph{Democratization.} Automated architecture discovery reduces the expertise barrier for deploying multi-agent systems. Practitioners need not be experts in prompt engineering or agent coordination; they can specify constraints and let evolution discover effective configurations. - -\paragraph{Risks.} Like all automated design systems, \emap{} could discover architectures that are effective but difficult to understand or audit. The cyclic topologies and minimal message formats we observed might make debugging and interpretability challenging. We recommend human oversight of evolved architectures before deployment in high-stakes applications. - -%============================================================================== \section{Conclusion} -\label{sec:conclusion} -%============================================================================== - -We introduced \emap{} (Evolution under Multi-Agent Pressure), a framework for studying how resource constraints during evolutionary optimization shape multi-agent LLM architectures. Our work addresses a gap in the literature: while prior approaches treat cost as a secondary optimization objective, we make constraints primary---creating genuine evolutionary pressure for efficient coordination. - -\subsection{Summary of Contributions} - -\begin{enumerate} - \item \textbf{Framework}: We introduced the hard constraint fitness formulation (Equation~\ref{eq:fitness}), which assigns zero fitness to any architecture exceeding budget. This creates qualitatively different selection pressure than soft Pareto optimization, producing architectures that are robust to constraint boundaries. - - \item \textbf{Empirical Study}: We conducted 12 experiments across 4 budget regimes (2K--50K tokens), 3 random seeds each, evolving for 12 generations with population 10. This represents one of the most comprehensive studies of multi-agent architecture evolution under resource constraints. - - \item \textbf{Key Findings}: - \begin{itemize} - \item Evolution consistently produces 3--4 agent architectures across all budget regimes, with TIGHT/MEDIUM favoring 3 agents and LOOSE allowing 4-agent teams (planner, architect, coder, tester). - \item Multiple diverse topologies (traditional pipeline, test-first, hybrid, hierarchical 4-agent) achieve 95.1--100\% pass rate, indicating a multi-modal fitness landscape. - \item Tight constraints favor cyclic topologies and compressed message formats; loose constraints enable more complex hierarchical structures. - \item The 5K token budget (MEDIUM) achieves optimal performance with 100\% pass rate and 0\% variance; LOOSE allows richer architectures but with slightly higher variance. - \end{itemize} - - \item \textbf{Practical Guidelines}: We distilled our findings into actionable recommendations for practitioners designing multi-agent systems for resource-constrained deployments. -\end{enumerate} - -\subsection{Theoretical Implications} - -Our work suggests two theoretical contributions: - -\paragraph{The Complexity Threshold Hypothesis.} For tasks above a complexity threshold, multi-agent coordination provides sufficient fitness benefit to justify token overhead, \textit{even under severe constraints}. HumanEval's 164 programming problems exceed this threshold; simpler tasks might not. - -\paragraph{The Multi-Modal Landscape Hypothesis.} The fitness landscape for multi-agent code generation contains multiple peaks of similar height, representing qualitatively different but equally effective coordination strategies. This implies: (1) no single ``best'' architecture exists; (2) random restarts discover diverse solutions; (3) ensemble methods might combine complementary topologies. - -\subsection{Limitations and Future Directions} -Our study uses a single LLM backbone (GPT-4o-mini), focuses on code generation, and employs token-based constraints. Future work should investigate: (1) cross-model generalization; (2) other task domains; (3) multi-constraint evolution (tokens + latency + memory); and (4) progressive constraint schedules. +The twelve artifacts are useful as a compact exploratory record. They show +high sampled-task scores and several architecture shapes across the recorded +budget/seed matrix. The evidence is not sufficient for causal claims, +full-HumanEval performance claims, universal architecture recommendations, or +historical cost estimates. A stronger follow-up would preregister task splits, +record per-task evidence and environment metadata, fully seed the pipeline, +and repeat the comparison across models and benchmarks. -\subsection{Broader Significance} +\section*{Availability and license} -Our approach bridges evolutionary biology and AI systems design. Just as organisms evolve metabolic efficiency under resource scarcity, multi-agent LLM systems evolve coordination efficiency under token constraints. The principles we identified---task complexity thresholds, multi-modal fitness landscapes, constraint-adapted coordination strategies---may generalize beyond code generation to any domain where efficient multi-agent coordination matters. - -By making budget constraints \textit{hard} rather than soft, we create selection pressure that mirrors real-world deployment constraints. The architectures that survive this pressure are those that can thrive under scarcity---a property increasingly valuable as LLM costs and environmental impact come under scrutiny. - -We release our code, evolved architectures, and analysis tools at \url{https://github.com/[anonymized]/emap} to enable reproduction and extension of this work. - -%============================================================================== -% References -%============================================================================== - -\bibliography{references} -\bibliographystyle{plainnat} - -%============================================================================== -% Appendix -%============================================================================== - -\appendix - -\section{Evolved Architecture Examples} -\label{app:examples} - -This appendix provides detailed specifications of the best-performing architectures from each experiment. - -\subsection{TIGHT Regime (2K tokens)} - -\paragraph{Seed 42: Traditional Pipeline.} -\begin{verbatim} -Topology: planner---> coder---> reviewer (linear) -Agents: 3 -Edges: 2 -Final Fitness: 95.1% (39/41) -Total Tokens: 386,687 -Configuration: - --- message_format: structured - --- max_message_length: 200 - --- aggregation: best_of_n - --- max_rounds: 3 - --- early_exit_confidence: 0.9 -\end{verbatim} - -\paragraph{Seed 43: Test-First Pipeline.} -\begin{verbatim} -Topology: tester---> reviewer---> planner---> reviewer (cyclic) -Agents: 3 -Edges: 3 (including feedback loop) -Final Fitness: 100.0% (41/41) -Total Tokens: 285,855 -Configuration: - --- message_format: freeform - --- max_message_length: 134 - --- aggregation: hierarchical - --- max_rounds: 2 - --- early_exit_confidence: 0.84 -\end{verbatim} - -\paragraph{Seed 44: Hybrid Architecture.} -\begin{verbatim} -Topology: generalist <-> coder, architect---> coder (cyclic) -Agents: 3 -Edges: 3 -Final Fitness: 100.0% (41/41) -Total Tokens: 182,970 -Configuration: - --- message_format: minimal - --- max_message_length: 228 - --- aggregation: hierarchical - --- max_rounds: 3 - --- early_exit_confidence: 0.74 -\end{verbatim} - -\subsection{MEDIUM Regime (5K tokens)} - -\paragraph{Seed 42: Linear Pipeline.} -\begin{verbatim} -Topology: planner---> coder---> reviewer (linear) -Agents: 3 -Edges: 2 -Final Fitness: 100.0% (41/41) -Total Tokens: 134,520 -Configuration: - --- message_format: structured - --- max_message_length: 195 - --- aggregation: sequential - --- max_rounds: 2 - --- early_exit_confidence: 0.89 -Note: Best fitness 0% for generations 0-5, then 100% from generation 6. - Evolution discovered budget-compliant architecture through mutation. -\end{verbatim} - -\paragraph{Seed 43: Test-First Pipeline.} -\begin{verbatim} -Topology: tester---> reviewer---> planner---> reviewer (cyclic) -Agents: 3 -Edges: 3 -Final Fitness: 100.0% (41/41) -Total Tokens: 96,900 -Configuration: - --- message_format: freeform - --- max_message_length: 154 - --- aggregation: hierarchical - --- max_rounds: 2 - --- early_exit_confidence: 0.92 -Note: Same topology as TIGHT seed 43, suggesting this is a robust optima. -\end{verbatim} - -\subsection{LOOSE Regime (10K tokens)} - -\paragraph{Seed 42: Complex Voting.} -\begin{verbatim} -Topology: architect---> [generalist, coder], generalist---> coder -Agents: 4 (planner, generalist, architect, coder) -Edges: 3 -Final Fitness: 95.1% (39/41) -Total Tokens: 301,519 -Configuration: - --- message_format: structured - --- max_message_length: 193 - --- aggregation: voting - --- max_rounds: 2 - --- early_exit_confidence: 0.88 -Note: 4-agent architecture enabled by larger budget. - Voting aggregation unique to this experiment. -\end{verbatim} - -\paragraph{Seed 44: Hierarchical 4-Agent.} -\begin{verbatim} -Topology: architect---> planner, coder---> planner, tester---> coder -Agents: 4 (planner, architect, coder, tester) -Edges: 4 -Final Fitness: 100.0% (41/41) -Total Tokens: 482,307 -Configuration: - --- message_format: structured - --- max_message_length: 219 - --- aggregation: hierarchical - --- max_rounds: 3 - --- early_exit_confidence: 0.80 -Note: Most complex architecture discovered. Avg agents evolved - from 2.4 to 4.3 over 12 generations, showing clear - complexity growth under permissive budget. Runtime: 5.2 hrs. -\end{verbatim} - -\subsection{UNCONSTRAINED Regime (50K tokens)} - -\paragraph{Seed 42: Minimal Pair.} -\begin{verbatim} -Topology: planner---> debugger -Agents: 2 (planner, debugger) -Edges: 1 -Final Fitness: 100.0% (41/41) -Total Tokens: 206,056 -Configuration: - --- message_format: structured (default) - --- max_rounds: 3 (default) -Note: Simplest architecture discovered across all experiments. - With no budget pressure, evolution converged to minimal - viable solution. -\end{verbatim} - -\paragraph{Seed 43: Linear Pipeline.} -\begin{verbatim} -Topology: planner---> coder---> reviewer -Agents: 3 (planner, coder, reviewer) -Edges: 2 -Final Fitness: 100.0% (41/41) -Total Tokens: 295,922 -Configuration: - --- message_format: structured (default) - --- max_rounds: 3 (default) -Note: Classic software engineering pipeline. Similar to - constrained experiments but without cyclic edges. -\end{verbatim} - -\paragraph{Seed 44: Cyclic Feedback.} -\begin{verbatim} -Topology: debugger <-> coder, architect---> coder -Agents: 3 (debugger, coder, architect) -Edges: 3 -Final Fitness: 100.0% (41/41) -Total Tokens: 440,784 -Configuration: - --- message_format: structured (default) - --- max_rounds: 3 (default) -Note: Only UNCONSTRAINED seed to maintain cyclic topology. - Bidirectional debugger<->coder edge enables iterative - refinement. Highest token usage in regime. -\end{verbatim} - -\section{Genome Representation Details} -\label{app:genome} - -Table~\ref{tab:genome-spec} provides the complete genome specification. - -\begin{table}[h] -\centering -\caption{Genome specification for multi-agent architectures.} -\label{tab:genome-spec} -\begin{tabular}{llp{6cm}} -\toprule -Component & Type & Description \\ -\midrule -\texttt{agents} & List[AgentGene] & List of agent specifications \\ -\texttt{topology} & Dict[str, List[str]] & Adjacency list defining message flow \\ -\texttt{message\_format} & Enum & FREEFORM, STRUCTURED, or MINIMAL \\ -\texttt{aggregation} & Enum & SEQUENTIAL, VOTING, or HIERARCHICAL \\ -\texttt{entry\_agent} & str & ID of agent receiving initial task \\ -\texttt{output\_agent} & str & ID of agent producing final output \\ -\midrule -\multicolumn{3}{l}{\textbf{AgentGene}} \\ -\texttt{role} & Enum & PLANNER, CODER, REVIEWER, DEBUGGER, etc. \\ -\texttt{system\_prompt} & str & Agent's system prompt \\ -\texttt{max\_tokens} & int & Per-response token limit \\ -\texttt{temperature} & float & Sampling temperature \\ -\bottomrule -\end{tabular} -\end{table} - -\section{Mutation Operators} -\label{app:mutations} - -Table~\ref{tab:mutations} describes all mutation operators used during evolution. - -\begin{table}[h] -\centering -\caption{Mutation operators and their effects.} -\label{tab:mutations} -\begin{tabular}{lp{8cm}} -\toprule -Operator & Description \\ -\midrule -\texttt{add\_agent} & Insert new agent with random role; connect to random existing agent \\ -\texttt{remove\_agent} & Remove random non-entry/output agent; reconnect topology \\ -\texttt{mutate\_role} & Change agent's role to random different role \\ -\texttt{mutate\_prompt} & Perturb system prompt (add/remove tokens, paraphrase) \\ -\texttt{rewire\_edge} & Redirect random edge to different target agent \\ -\texttt{add\_edge} & Add communication channel between disconnected agents \\ -\texttt{remove\_edge} & Remove random edge (maintaining connectivity) \\ -\texttt{change\_aggregation} & Switch aggregation strategy \\ -\texttt{change\_message\_format} & Switch message compression level \\ -\bottomrule -\end{tabular} -\end{table} - -\section{Hyperparameter Sensitivity} -\label{app:hyperparams} - -Table~\ref{tab:hyperparams} shows the hyperparameters used in our experiments. - -\begin{table}[h] -\centering -\caption{Evolution hyperparameters used in experiments.} -\label{tab:hyperparams} -\begin{tabular}{lcc} -\toprule -Parameter & Value & Sensitivity \\ -\midrule -Population size & 10 & Low \\ -Generations & 12 & Medium \\ -Tournament size & 2 & Low \\ -Elite count & 2 & Low \\ -Mutation rate & 0.4 & Medium \\ -Crossover rate & 0.5 & Medium \\ -Sample fraction (evolution) & 0.12 & High \\ -Sample fraction (final eval) & 0.25 & Medium \\ -Seeds per regime & 3 & High \\ -\bottomrule -\end{tabular} -\end{table} - -\section{Detailed Per-Task Results} -\label{app:results} - -Table~\ref{tab:detailed-humaneval} provides per-category breakdown on HumanEval. - -\begin{table}[h] -\centering -\caption{HumanEval results by problem category.} -\label{tab:detailed-humaneval} -\begin{tabular}{lcccc} -\toprule -Category & Tight & Medium & Loose & Unconstrained \\ -\midrule -String manipulation &---- &---- &---- &---- \\ -Math/arithmetic &---- &---- &---- &---- \\ -List operations &---- &---- &---- &---- \\ -Conditionals &---- &---- &---- &---- \\ -Recursion &---- &---- &---- &---- \\ -\bottomrule -\end{tabular} -\end{table} - -\section{Computational Cost} -\label{app:cost} - -Table~\ref{tab:cost} reports the computational resources required for experiments. - -\begin{table}[h] -\centering -\caption{Computational cost breakdown (actual measurements from HumanEval experiments).} -\label{tab:cost} -\begin{tabular}{lccccc} -\toprule -Regime & Experiments & Generations & API Calls & Tokens & Est. Cost \\ -\midrule -TIGHT (2K) & 3 seeds & 12 each & 6,840 & 855K & \$0.19 \\ -MEDIUM (5K) & 2 seeds & 12 each & 4,560 & 231K & \$0.05 \\ -LOOSE (10K) & 3 seeds & 12 each & 6,840 & 920K & \$0.21 \\ -\midrule -\textbf{Total (completed)} & 8 experiments &---- & 18,240 & 2.01M & \$0.45 \\ -\bottomrule -\end{tabular} - -\textit{Note: Costs estimated using GPT-4o-mini pricing (\$0.15/1M input, \$0.60/1M output). Experiments completed December 2025. UNCONSTRAINED regime pending.} -\end{table} - -\subsection{Per-Experiment Breakdown} - -Table~\ref{tab:cost-detailed} provides token usage for each individual experiment. - -\begin{table}[h] -\centering -\caption{Detailed per-experiment token usage and runtime.} -\label{tab:cost-detailed} -\begin{tabular}{lcccc} -\toprule -Experiment & Tokens & API Calls & Runtime & Tokens/Gen \\ -\midrule -TIGHT seed 42 & 386,687 & 2,280 & 3.0 hrs & 32.2K \\ -TIGHT seed 43 & 285,855 & 2,280 & 2.4 hrs & 23.8K \\ -TIGHT seed 44 & 182,970 & 2,280 & 3.0 hrs & 15.2K \\ -\midrule -MEDIUM seed 42 & 134,520 & 2,280 & 2.2 hrs & 11.2K \\ -MEDIUM seed 43 & 96,900 & 2,280 & 1.7 hrs & 8.1K \\ -\midrule -LOOSE seed 42 & 301,519 & 2,280 & 3.7 hrs & 25.1K \\ -LOOSE seed 43 & 135,633 & 760 & 1.5 hrs & 11.3K \\ -LOOSE seed 44 & 482,307 & 2,280 & 5.2 hrs & 40.2K \\ -\bottomrule -\end{tabular} -\end{table} - -\paragraph{Efficiency Observation.} MEDIUM budget experiments use the fewest tokens per generation (8--11K) while achieving the highest performance (100\%). This supports our finding that 5K token budgets represent the ``Goldilocks zone'' for HumanEval tasks. - -\section{Reproducibility Checklist} -\label{app:reproducibility} - -\begin{itemize} - \item[$\checkmark$] Code available at: \url{https://github.com/[anonymized]/emap} - \item[$\checkmark$] Random seeds reported for all experiments (42, 43, 44) - \item[$\checkmark$] Hyperparameters fully specified in Table~\ref{tab:hyperparams} - \item[$\checkmark$] Hardware/API specifications documented (GPT-4o-mini via OpenAI API) - \item[$\checkmark$] Standard deviations reported for aggregated metrics - \item[\textbullet] \textit{Note}: With $n=3$ seeds per condition, formal hypothesis testing has limited statistical power. We report descriptive statistics and per-seed breakdowns to enable assessment of result stability. -\end{itemize} +The aggregate summary and per-run JSON records are checked into the repository. +The software and this note are distributed under the repository's MIT License. \end{document} diff --git a/paper/references.bib b/paper/references.bib deleted file mode 100644 index a224e4b..0000000 --- a/paper/references.bib +++ /dev/null @@ -1,228 +0,0 @@ -% Bibliography for EMAP paper -% Last updated: December 15, 2025 - -%------------------------------------------------------------------------------ -% Core Related Work - Evolutionary Optimization of LLM Agents -%------------------------------------------------------------------------------ - -@article{evoagentx2025, - title={{EvoAgentX}: Automated Generation, Execution, and Evolutionary Optimization of Multi-agent Workflows}, - author={Wang, Yifan and others}, - journal={arXiv preprint arXiv:2507.03616}, - year={2025} -} - -@article{artemis2025, - title={{ARTEMIS}: Evolution of AI Agents Through Semantically-Aware Prompt Optimization}, - author={Brookes, Thomas and others}, - journal={arXiv preprint arXiv:2512.09108}, - year={2025} -} - -@article{aflow2024, - title={{AFlow}: Automating Agentic Workflow Generation}, - author={Zhang, Junying and others}, - journal={arXiv preprint arXiv:2410.10762}, - year={2024} -} - -@article{automaas2025, - title={{AutoMaAS}: Self-Evolving Multi-Agent Architecture Search}, - author={Ma, Yilun and others}, - journal={arXiv preprint arXiv:2510.02669}, - year={2025} -} - -@article{agentnet2025, - title={{AgentNet}: Decentralized, RAG-based Framework for Evolving Multi-Agent Systems}, - author={Yang, Shuo and others}, - journal={arXiv preprint arXiv:2504.00587}, - year={2025} -} - -@mastersthesis{malbo2025, - title={{MALBO}: Multi-objective Bayesian Optimization for LLM Team Composition}, - author={Sabbatella, Marco}, - school={Universit{\`a} degli Studi di Milano-Bicocca}, - year={2025}, - note={arXiv:2511.11788} -} - -%------------------------------------------------------------------------------ -% Resource Efficiency -%------------------------------------------------------------------------------ - -@article{sweeffi2025, - title={{SWE-Effi}: Re-Evaluating Software AI Agent System Effectiveness Under Resource Constraints}, - author={Fan, Zhiyu and Vasilevski, Kirill and Lin, Dayi and Chen, Boyuan and Chen, Yihao and Zhong, Zhiqing and Zhang, Jie M. and He, Pinjia}, - journal={arXiv preprint arXiv:2509.09853}, - year={2025} -} - -@article{corl2025, - title={Controlling Performance and Budget of a Centralized Multi-agent LLM System with Reinforcement Learning}, - author={Jin, Bowen and Collins, TJ and Yu, Donghan and Cemri, Mert and Zhang, Shenao and Li, Mengyu and Tang, Jay and Qin, Tian and Xu, Zhiyang}, - journal={arXiv preprint arXiv:2511.02755}, - year={2025} -} - -@inproceedings{curriculum2025, - title={Curriculum Design for Trajectory-Constrained Agent: Compressing Chain-of-Thought Tokens in {LLMs}}, - author={Tzannetos, Georgios and Kamalaruban, Parameswaran and Singla, Adish}, - booktitle={Advances in Neural Information Processing Systems (NeurIPS)}, - year={2025} -} - -%------------------------------------------------------------------------------ -% Prompt Optimization -%------------------------------------------------------------------------------ - -@inproceedings{evoprompt2024, - title={{EvoPrompt}: Language Models for Code-Level Discrete Prompt Optimization}, - author={Guo, Qingyan and others}, - booktitle={International Conference on Learning Representations (ICLR)}, - year={2024} -} - -@inproceedings{evoprompting2023, - title={Connecting Large Language Models with Evolutionary Algorithms Yields Powerful Prompt Optimizers}, - author={Chen, Angelica and others}, - booktitle={Advances in Neural Information Processing Systems (NeurIPS)}, - year={2023} -} - -@article{promptbreeder2023, - title={{PromptBreeder}: Self-Referential Self-Improvement Via Prompt Evolution}, - author={Fernando, Chrisantha and others}, - journal={arXiv preprint arXiv:2309.16797}, - year={2023} -} - -%------------------------------------------------------------------------------ -% Benchmarks -%------------------------------------------------------------------------------ - -@article{humaneval, - title={Evaluating Large Language Models Trained on Code}, - author={Chen, Mark and others}, - journal={arXiv preprint arXiv:2107.03374}, - year={2021} -} - -@article{mbpp, - title={Program Synthesis with Large Language Models}, - author={Austin, Jacob and others}, - journal={arXiv preprint arXiv:2108.07732}, - year={2021} -} - -@article{swebench, - title={{SWE-bench}: Can Language Models Resolve Real-World GitHub Issues?}, - author={Jimenez, Carlos E. and others}, - journal={arXiv preprint arXiv:2310.06770}, - year={2024} -} - -%------------------------------------------------------------------------------ -% Multi-Agent Foundations -%------------------------------------------------------------------------------ - -@article{autogen2023, - title={{AutoGen}: Enabling Next-Gen LLM Applications via Multi-Agent Conversation}, - author={Wu, Qingyun and others}, - journal={arXiv preprint arXiv:2308.08155}, - year={2023} -} - -@article{metagpt2023, - title={{MetaGPT}: Meta Programming for A Multi-Agent Collaborative Framework}, - author={Hong, Sirui and others}, - journal={arXiv preprint arXiv:2308.00352}, - year={2023} -} - -@article{camel2023, - title={{CAMEL}: Communicative Agents for ``Mind'' Exploration of Large Language Model Society}, - author={Li, Guohao and others}, - journal={arXiv preprint arXiv:2303.17760}, - year={2023} -} - -%------------------------------------------------------------------------------ -% Biological Precedent / Island Biogeography -%------------------------------------------------------------------------------ - -@article{lomolino2005, - title={Body size evolution in insular vertebrates: generality of the island rule}, - author={Lomolino, Mark V.}, - journal={Journal of Biogeography}, - volume={32}, - number={10}, - pages={1683--1699}, - year={2005}, - publisher={Wiley} -} - -@book{macarthur1967, - title={The Theory of Island Biogeography}, - author={MacArthur, Robert H. and Wilson, Edward O.}, - year={1967}, - publisher={Princeton University Press} -} - -@article{foster1964, - title={Evolution of mammals on islands}, - author={Foster, J. Bristol}, - journal={Nature}, - volume={202}, - number={4929}, - pages={234--235}, - year={1964}, - publisher={Nature Publishing Group} -} - -%------------------------------------------------------------------------------ -% Evolutionary Computation -%------------------------------------------------------------------------------ - -@article{deap2012, - title={{DEAP}: Evolutionary Algorithms Made Easy}, - author={Fortin, F{\'e}lix-Antoine and De Rainville, Fran{\c{c}}ois-Michel and Gardner, Marc-Andr{\'e} and Parizeau, Marc and Gagn{\'e}, Christian}, - journal={Journal of Machine Learning Research}, - volume={13}, - number={1}, - pages={2171--2175}, - year={2012} -} - -@article{qd2015, - title={Illuminating search spaces by mapping elites}, - author={Mouret, Jean-Baptiste and Clune, Jeff}, - journal={arXiv preprint arXiv:1504.04909}, - year={2015} -} - -%------------------------------------------------------------------------------ -% Additional Multi-Agent Debate -%------------------------------------------------------------------------------ - -@article{swedebate2025, - title={{SWE-Debate}: Competitive Multi-Agent Debate for Software Engineering}, - author={Various}, - journal={arXiv preprint arXiv:2507.23348}, - year={2025} -} - -@article{maddebate2025, - title={Can LLM Agents Really Debate? A Critical Evaluation}, - author={Various}, - journal={arXiv preprint arXiv:2511.07784}, - year={2025} -} - -@article{wise2025, - title={{WISE}: Weighted Society-of-Experts for Multi-Agent Debate}, - author={Various}, - journal={arXiv preprint arXiv:2512.02405}, - year={2025} -} diff --git a/pyproject.toml b/pyproject.toml index 3fe1083..d821e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" license = {text = "MIT"} requires-python = ">=3.11" authors = [ - {name = "Anonymous", email = "anonymous@example.com"} + {name = "Noah Ingwers", email = "98993329+noah-ing@users.noreply.github.com"} ] keywords = [ "evolutionary-algorithms", diff --git a/src/emap/__init__.py b/src/emap/__init__.py index c1fc28b..5bc9c56 100644 --- a/src/emap/__init__.py +++ b/src/emap/__init__.py @@ -5,7 +5,7 @@ """ __version__ = "0.1.0" -__author__ = "Anonymous" +__author__ = "Noah Ingwers" from emap.genome.representation import MultiAgentGenome, AgentRole from emap.evolution.loop import evolve diff --git a/src/emap/benchmarks/sandbox.py b/src/emap/benchmarks/sandbox.py index 9d14e39..8aba5fe 100644 --- a/src/emap/benchmarks/sandbox.py +++ b/src/emap/benchmarks/sandbox.py @@ -1,22 +1,18 @@ -""" -Safe code execution sandbox for evaluating agent-generated code. - -This module provides a secure environment for executing Python code -generated by LLM agents during benchmark evaluation. Safety features: +"""Best-effort subprocess evaluator for agent-generated Python code. -1. Execution timeout -2. Resource limits (memory, CPU time) -3. Import restrictions -4. No file system access beyond temp files -5. Subprocess isolation +This module is an evaluation harness, not a security boundary. It combines +static checks, restricted builtins/imports, a subprocess, a timeout, and +best-effort Unix resource limits. Python object-model escapes or platform +differences may bypass those controls, so callers must not use it for hostile +code on a machine that holds secrets or sensitive network/filesystem access. """ from __future__ import annotations import ast +import math import multiprocessing import signal -import sys import traceback from dataclasses import dataclass from io import StringIO @@ -26,7 +22,7 @@ @dataclass class ExecutionOutcome: - """Result of executing code in sandbox.""" + """Result of executing code in the evaluation worker.""" success: bool output: str return_value: Any @@ -92,7 +88,10 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: def check_code_safety(code: str) -> tuple[bool, list[str]]: """ - Static analysis to check if code is safe to execute. + Check code against the evaluator's denylist. + + A passing result means only that these checks found no configured + violation. It does not establish that arbitrary Python is safe to run. Returns: Tuple of (is_safe, list_of_violations) @@ -132,19 +131,50 @@ def check_code_safety(code: str) -> tuple[bool, list[str]]: return len(violations) == 0, violations -def _execute_in_sandbox( +def _apply_worker_resource_limits(timeout: float) -> None: + """Apply conservative Unix limits inside the disposable worker. + + ``resource`` is unavailable on some platforms and individual limits are + not uniformly implemented. Unsupported limits are skipped; the parent + process timeout remains the cross-platform backstop. + """ + try: + import resource + except ImportError: + return + + def set_limit(name: str, value: int) -> None: + resource_id = getattr(resource, name, None) + if resource_id is None: + return + try: + _, current_hard = resource.getrlimit(resource_id) + target = value + if current_hard != resource.RLIM_INFINITY: + target = min(target, current_hard) + resource.setrlimit(resource_id, (target, target)) + except (OSError, ValueError): + pass + + set_limit("RLIMIT_CPU", max(1, math.ceil(timeout) + 1)) + set_limit("RLIMIT_AS", 512 * 1024 * 1024) + set_limit("RLIMIT_FSIZE", 1024 * 1024) + set_limit("RLIMIT_NOFILE", 64) + + +def _execute_in_worker( code: str, test_code: str, timeout: float, result_queue: multiprocessing.Queue, ) -> None: """ - Execute code in a subprocess with restrictions. - - This runs in a separate process for isolation. + Execute code in a disposable subprocess with best-effort restrictions. """ import time as time_module start_time = time_module.time() + + _apply_worker_resource_limits(timeout) # Set up timeout via signal (Unix only) def timeout_handler(signum, frame): @@ -316,122 +346,6 @@ def safe_import(name, globals=None, locals=None, fromlist=(), level=0): result_queue.put(result) -def _execute_in_thread( - code: str, - test_code: str, - timeout: float, -) -> ExecutionOutcome: - """ - Execute code in-process with threading timeout. - - This is a fallback when multiprocessing spawn fails (e.g., from stdin). - Less isolated but more reliable. - """ - import time as time_module - import threading - - start_time = time_module.time() - result = { - "success": False, - "output": "", - "return_value": None, - "error_type": None, - "error_message": None, - "timed_out": False, - } - exception_holder = [None] - - # Create restricted globals (same as subprocess version) - import math - import itertools - import functools - import collections - import re - import json - import copy - import typing - - import builtins as _builtins - - # Safe import function that only allows whitelisted modules - def safe_import(name, globals=None, locals=None, fromlist=(), level=0): - allowed = {"typing", "math", "itertools", "functools", "collections", - "re", "json", "copy", "heapq", "bisect", "random", "string", - "decimal", "fractions", "statistics", "operator", "dataclasses"} - base = name.split(".")[0] - if base not in allowed: - raise ImportError(f"Import of '{name}' is not allowed") - return _builtins.__import__(name, globals, locals, fromlist, level) - - safe_globals = { - "__builtins__": { - "True": True, "False": False, "None": None, - "int": int, "float": float, "str": str, "bool": bool, - "list": list, "dict": dict, "set": set, "tuple": tuple, - "frozenset": frozenset, "bytes": bytes, "bytearray": bytearray, - "len": len, "range": range, "enumerate": enumerate, "zip": zip, - "map": map, "filter": filter, "sorted": sorted, "reversed": reversed, - "min": min, "max": max, "sum": sum, "abs": abs, "round": round, - "pow": pow, "divmod": divmod, "all": all, "any": any, - "isinstance": isinstance, "issubclass": issubclass, "type": type, - "id": id, "hash": hash, "repr": repr, "print": print, - "input": lambda *args: "", "iter": iter, "next": next, - "slice": slice, "ord": ord, "chr": chr, "bin": bin, "hex": hex, - "oct": oct, "format": format, "ascii": ascii, - "Exception": Exception, "ValueError": ValueError, - "TypeError": TypeError, "KeyError": KeyError, - "IndexError": IndexError, "AttributeError": AttributeError, - "RuntimeError": RuntimeError, "StopIteration": StopIteration, - "ZeroDivisionError": ZeroDivisionError, "AssertionError": AssertionError, - "ImportError": ImportError, "__import__": safe_import, - }, - "math": math, "itertools": itertools, "functools": functools, - "collections": collections, "re": re, "json": json, "copy": copy, - "typing": typing, "List": typing.List, "Dict": typing.Dict, - "Optional": typing.Optional, "Tuple": typing.Tuple, "Set": typing.Set, - } - - stdout_capture = StringIO() - - def execute(): - try: - with contextlib.redirect_stdout(stdout_capture): - exec(code, safe_globals) - if test_code: - exec(test_code, safe_globals) - result["success"] = True - result["output"] = stdout_capture.getvalue() - except AssertionError as e: - result["error_type"] = "AssertionError" - result["error_message"] = str(e) or "Assertion failed" - except Exception as e: - result["error_type"] = type(e).__name__ - result["error_message"] = str(e) - exception_holder[0] = e - - thread = threading.Thread(target=execute) - thread.daemon = True - thread.start() - thread.join(timeout=timeout) - - if thread.is_alive(): - result["timed_out"] = True - result["error_type"] = "TimeoutError" - result["error_message"] = f"Execution exceeded {timeout}s timeout" - - execution_time_ms = (time_module.time() - start_time) * 1000 - - return ExecutionOutcome( - success=result["success"], - output=result["output"], - return_value=result.get("return_value"), - error_type=result.get("error_type"), - error_message=result.get("error_message"), - execution_time_ms=execution_time_ms, - timed_out=result.get("timed_out", False), - ) - - def execute_code( code: str, test_code: str = "", @@ -439,7 +353,10 @@ def execute_code( check_safety: bool = True, ) -> ExecutionOutcome: """ - Execute code in a sandboxed environment. + Execute code in a best-effort isolated evaluation worker. + + This API is intended for benchmark code on a disposable, unprivileged + host. It is not a secure sandbox for adversarial Python. Args: code: Python code to execute @@ -477,7 +394,7 @@ def execute_code( try: result_queue = multiprocessing.Queue() process = multiprocessing.Process( - target=_execute_in_sandbox, + target=_execute_in_worker, args=(code, test_code, timeout, result_queue), ) @@ -511,9 +428,18 @@ def execute_code( timed_out=result.get("timed_out", False), ) - except Exception: - # Fallback to threading-based execution (e.g., when running from stdin) - return _execute_in_thread(code, test_code, timeout) + except Exception as exc: + # Fail closed: never execute generated code in the caller process. + return ExecutionOutcome( + success=False, + output="", + return_value=None, + error_type="IsolationError", + error_message=( + "Subprocess isolation could not be established: " + f"{type(exc).__name__}" + ), + ) def execute_with_entrypoint( diff --git a/src/emap/evolution/fitness.py b/src/emap/evolution/fitness.py index cd2dc97..6ec7817 100644 --- a/src/emap/evolution/fitness.py +++ b/src/emap/evolution/fitness.py @@ -357,7 +357,7 @@ def _run_tests(self, output: str, task: Task) -> bool: Run test cases on generated code. Placeholder implementation. Real version would: - 1. Create sandbox environment + 1. Create an unprivileged subprocess evaluation environment 2. Execute generated code 3. Run test cases 4. Return pass/fail diff --git a/src/emap/evolution/integrated_eval.py b/src/emap/evolution/integrated_eval.py index e2e6737..17a53f6 100644 --- a/src/emap/evolution/integrated_eval.py +++ b/src/emap/evolution/integrated_eval.py @@ -1,7 +1,8 @@ """ Integrated fitness evaluation using actual LLM execution. -This module connects the evolution loop to the agent executor and sandbox, +This module connects the evolution loop to the agent executor and best-effort +subprocess evaluator, providing real fitness evaluation under token budgets. """ @@ -71,12 +72,12 @@ def build_coding_prompt(task: Task) -> str: @dataclass class IntegratedEvaluator: """ - Evaluator that uses actual LLM execution with sandbox testing. + Evaluator that uses actual LLM execution with subprocess testing. This is the real evaluator used during evolution, connecting: - Genome -> MultiAgentExecutor -> LLM calls - - LLM output -> Code extraction -> Sandbox execution - - Sandbox result -> Task pass/fail -> Fitness + - LLM output -> Code extraction -> Restricted subprocess execution + - Worker result -> Task pass/fail -> Fitness """ backend: LLMBackend @@ -180,8 +181,8 @@ async def _evaluate_task_async( # Combine with prompt (signature + implementation) full_code = task.prompt + "\n" + code - # Run in sandbox - sandbox_result = execute_code( + # Run in the best-effort subprocess evaluator. + execution_result = execute_code( code=full_code, test_code=task.test_code, timeout=self.timeout_seconds, @@ -190,11 +191,11 @@ async def _evaluate_task_async( return TaskResult( task_id=task.id, - success=sandbox_result.success, + success=execution_result.success, tokens_used=exec_result.total_tokens_used, time_seconds=time.time() - start_time, output=code, - error=sandbox_result.error_message if not sandbox_result.success else None, + error=execution_result.error_message if not execution_result.success else None, ) except Exception as e: diff --git a/tests/test_agents.py b/tests/test_agents.py index caddea9..421a3bc 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -1,4 +1,4 @@ -"""Tests for agent execution and sandbox.""" +"""Tests for agent execution and the subprocess evaluation harness.""" import pytest import asyncio @@ -23,6 +23,7 @@ check_code_safety, ExecutionOutcome, ) +from emap.benchmarks import sandbox as sandbox_module class TestMockBackend: @@ -146,8 +147,8 @@ async def run(): assert "total_tokens_used" in result_dict -class TestSandbox: - """Tests for code execution sandbox.""" +class TestEvaluationWorker: + """Tests for the best-effort subprocess evaluator.""" def test_simple_execution(self): """Test executing simple code.""" @@ -220,10 +221,27 @@ def test_blocked_imports(self): assert not outcome.success assert "blocked" in (outcome.error_message or "").lower() or "security" in (outcome.error_message or "").lower() + + def test_isolation_failure_fails_closed(self, monkeypatch): + """Generated code must never fall back to the caller process.""" + + class BrokenProcess: + def __init__(self, *args, **kwargs): + raise RuntimeError("subprocess unavailable") + + monkeypatch.setattr(sandbox_module.multiprocessing, "Process", BrokenProcess) + + outcome = execute_code( + "raise RuntimeError('generated code executed')", + check_safety=False, + ) + + assert not outcome.success + assert outcome.error_type == "IsolationError" def test_safety_check(self): """Test static safety analysis.""" - # Safe code + # Code with no configured denylist violation is_safe, violations = check_code_safety("x = 2 + 2") assert is_safe assert len(violations) == 0 @@ -265,10 +283,10 @@ def double(n): class TestIntegration: - """Integration tests for executor + sandbox.""" + """Integration tests for the executor and evaluation worker.""" def test_full_pipeline(self): - """Test full pipeline: genome -> execution -> sandbox.""" + """Test the full genome-to-subprocess-evaluation pipeline.""" from emap.evolution.fitness import Task from emap.evolution.integrated_eval import ( IntegratedEvaluator,