diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..c8cfe39 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/eval/agent_common.py b/eval/agent_common.py new file mode 100644 index 0000000..5f85f1f --- /dev/null +++ b/eval/agent_common.py @@ -0,0 +1,304 @@ +"""Shared, backend-agnostic primitives for the BEAVER agent harnesses +(eval/myagent = Codex CLI, eval/claudeagent = `claude -p`). + +Only the per-backend CLI invocation (`_codex_call` / `_claude_call`) and the +per-backend mode orchestration live in each dir's `agent.py`; everything in +this module is backend- and config-free (timeouts/row-caps are passed in), so +both harnesses share ONE copy of: + + * response parsing : clean_sql + * prompt flattening : render_prompt + * subprocess hardening : cli_env (strips DB creds from the CLI env) + * read-only SQL guard : is_read_only (+ helpers) + * mediated DB access : connect / execute_sql / query_preview / timed + * shared prompt fragments : fix_prompt, EXPLORE_PROTOCOL, RUN_SQL_RE + +All DB access is mediated by the harness process: the model emits queries +through a text protocol, we execute them read-only and feed back the rows. The +model is never told where the gold files live, and DB credentials are stripped +from the CLI subprocess environment (cli_env), so it cannot reach the database +off the mediated read-only path. This is defense-in-depth for a cooperative +model, not a hard boundary — the sandbox still allows filesystem reads. +""" +import os +import re +import threading + +READ_STMTS = ("select", "with", "show", "describe", "desc", "explain") +MAX_FEEDBACK_CHARS = 4000 +MAX_CELL_CHARS = 80 + + +_HTML_ENTITY = re.compile(r"&(?:[a-zA-Z]+|#\d+|#x[0-9a-fA-F]+);") + + +def _unescape_entities(sql: str) -> str: + """Undo HTML entity escaping (< > & ...) some models apply inside + spans (first seen with gpt-5.6) — it turns comparison operators into + 1064 syntax errors. Only touches strings that actually contain entities.""" + if _HTML_ENTITY.search(sql): + import html + return html.unescape(sql) + return sql + + +def clean_sql(text: str) -> str: + """Extract the SQL from a model response. + + Precedence: an ... span, else the first fenced code block + (```sql ... ``` or a plain ``` ... ```), else the raw text with a leading + 'SQL:' label removed. A plain fence must not drop the SQL (an earlier + version returned the prose *before* an un-tagged ``` fence).""" + if not text: + return "" + text = text.strip() + if "" not in text and "<ans>" in text: + text = _unescape_entities(text) + if "" in text: + after = text.split("", 1)[1] + return _unescape_entities(after.split("", 1)[0].strip()) + if "```" in text: + after = text.split("```", 1)[1] + # Drop a leading language tag line (```sql / ```mysql / bare ```). + if "\n" in after: + first_line, rest = after.split("\n", 1) + if first_line.strip().lower() in ("sql", "mysql", ""): + after = rest + return _unescape_entities(after.split("```", 1)[0].strip()) + if text.lower().startswith("sql:"): + text = text[len("sql:"):].strip() + return _unescape_entities(text.strip()) + + +def render_prompt(instance: dict) -> str: + """Flatten the chat-style prompt into the single string the CLIs expect.""" + blocks = [] + user_seen = 0 + for msg in instance["prompt"]: + role, content = msg["role"], msg["content"] + if role == "system": + blocks.append(content) + elif role == "assistant": + blocks.append(f"### Example answer\n{content}") + elif role == "user": + user_seen += 1 + header = "### Example input" if user_seen == 1 else "### Now answer this" + blocks.append(f"{header}\n{content}") + return "\n\n".join(blocks) + + +def cli_env(): + """Environment for the model-CLI subprocess: strip DB credentials so the + model cannot reach the database off the mediated read-only path.""" + return {k: v for k, v in os.environ.items() if not k.upper().startswith("MYSQL")} + + +# ----------------------- read-only SQL guard ----------------------- + +_INTO_FILE = re.compile(r"\binto\s+(outfile|dumpfile)\b", re.IGNORECASE) +_WRITE_IN_CTE = re.compile(r"\b(insert|update|delete|replace)\b", re.IGNORECASE) + + +def strip_string_literals(sql: str) -> str: + """Blank out quoted string contents so keyword scans don't match literals.""" + return re.sub(r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\"", "''", sql) + + +def strip_leading_noise(sql: str) -> str: + """Drop leading SQL comments and opening parens so the first real keyword can + be identified — a valid read may start with `-- note`, `/* */`, or `(SELECT`.""" + s = sql.strip() + while s: + if s.startswith("--"): + nl = s.find("\n") + s = "" if nl == -1 else s[nl + 1:].lstrip() + elif s.startswith("/*"): + end = s.find("*/") + s = "" if end == -1 else s[end + 2:].lstrip() + elif s.startswith("("): + s = s[1:].lstrip() + else: + break + return s + + +def first_keyword(sql: str) -> str: + s = strip_leading_noise(sql or "") + return s.split(None, 1)[0].lower() if s else "" + + +def is_read_only(sql: str) -> bool: + """True only for genuinely read-only statements. Handles leading + comments/parens (so valid reads aren't rejected) and blocks write-capable + constructs whose first keyword looks like a read: `SELECT ... INTO + OUTFILE/DUMPFILE` and data-modifying CTEs (`WITH ... UPDATE/DELETE`).""" + if not sql or not sql.strip(): + return False + first = first_keyword(sql) + if first not in READ_STMTS: + return False + scrubbed = strip_string_literals(sql) + if _INTO_FILE.search(scrubbed): + return False + if first == "with" and _WRITE_IN_CTE.search(scrubbed): + return False + return True + + +# ----------------------- mediated DB access (gold-blind) ----------------------- + +class DBUnavailable(RuntimeError): + """The database could not be reached (bad creds / server down) — distinct + from a SQL error in the model's query, so callers must not treat it as one.""" + + +def load_db_creds(): + if not os.getenv("MYSQL_HOST"): + try: + from dotenv import load_dotenv, find_dotenv + load_dotenv(find_dotenv(usecwd=True)) + except Exception: + pass + return os.getenv("MYSQL_HOST", "localhost"), os.getenv("MYSQL_USER", "root"), os.getenv("MYSQL_PASSWORD", "") + + +def connect(db): + import mysql.connector + h, u, p = load_db_creds() + try: + return mysql.connector.connect(host=h, user=u, password=p, database=db, connection_timeout=10) + except Exception as e: + raise DBUnavailable(str(e)) + + +def timed(fn, timeout_s): + """Run fn() in a daemon thread with a wall-clock cap; raise TimeoutError if + it doesn't finish. The SET SESSION cap is server-side and silently absent on + some servers (MariaDB, old MySQL), so this is the actual backstop against a + runaway query hanging a worker forever.""" + box = {} + + def _run(): + try: + box["ok"] = fn() + except Exception as e: # noqa: BLE001 - re-raised on the caller thread + box["err"] = e + + t = threading.Thread(target=_run, daemon=True) + t.start() + t.join(timeout_s) + if t.is_alive(): + raise TimeoutError(f"query exceeded {timeout_s:.0f}s client-side timeout") + if "err" in box: + raise box["err"] + return box.get("ok") + + +def execute_sql(sql: str, db: str, timeout_ms: int): + """Run read-only under a wall-clock cap that mirrors the scorer (execute + + fetchall). Returns (ok, error_str). Raises DBUnavailable if the DB can't be + reached. Never inspects rows.""" + if not is_read_only(sql): + return False, f"refusing to execute non-read statement (starts with {first_keyword(sql)!r})" + conn = connect(db) + timeout_s = timeout_ms / 1000.0 + try: + cur = conn.cursor() + try: + cur.execute(f"SET SESSION max_execution_time={timeout_ms}") + except Exception: + pass + + def _do(): + cur.execute(sql) + cur.fetchall() + + timed(_do, timeout_s) + return True, "" + except Exception as e: + return False, str(e) + finally: + try: + conn.close() + except Exception: + pass + + +def query_preview(sql: str, db: str, max_rows: int, timeout_ms: int): + """Run read-only and return a compact text preview of up to max_rows rows. + Returns feedback string for the model (cols + rows, or the error).""" + if not is_read_only(sql): + return "ERROR: only read-only queries (SELECT/WITH/SHOW/DESCRIBE/EXPLAIN) are allowed here." + try: + conn = connect(db) + except DBUnavailable as e: + return f"ERROR: could not connect to the `{db}` database (not a problem with your SQL): {e}" + timeout_s = timeout_ms / 1000.0 + try: + cur = conn.cursor() + try: + cur.execute(f"SET SESSION max_execution_time={timeout_ms}") + except Exception: + pass + + def _do(): + cur.execute(sql) + fetched = cur.fetchmany(max_rows + 1) # +1 to detect real truncation + cols = [d[0] for d in cur.description] if cur.description else [] + return fetched, cols + + fetched, cols = timed(_do, timeout_s) + truncated = len(fetched) > max_rows + rows = fetched[:max_rows] + extra = f"\n... (truncated at {max_rows} rows)" if truncated else "" + + def fmt(v): + s = "NULL" if v is None else str(v) + return s if len(s) <= MAX_CELL_CHARS else s[:MAX_CELL_CHARS] + "…" + + lines = [" | ".join(cols)] + [" | ".join(fmt(v) for v in r) for r in rows] + body = "\n".join(lines) + extra + if len(body) > MAX_FEEDBACK_CHARS: + body = body[:MAX_FEEDBACK_CHARS] + "\n… (output truncated)" + return f"{len(rows)} row(s) returned:\n{body}" + except Exception as e: + return f"ERROR: {e}" + finally: + try: + conn.close() + except Exception: + pass + + +# ----------------------- shared prompt fragments ----------------------- + +def fix_prompt(base, db, bad_sql, error): + return ( + base + f"\n\n### Execution feedback\nThe query below was run against the `{db}` MySQL " + f"database and FAILED TO EXECUTE. Fix it so it runs without error, keeping the intended " + f"logic.\n\nFailed SQL:\n{bad_sql}\n\nDatabase error:\n{error}\n\n" + f"Return only the corrected MySQL query wrapped in ." + ) + + +EXPLORE_PROTOCOL = """\ + +### Database access (read-only) — verification is REQUIRED +You have live read-only access to the `{db}` MySQL database. You will be shown the rows YOUR queries return — you will NOT be shown the expected/correct answer. + +You MUST run at least one query to CHECK your candidate answer before finalizing: inspect the real tables (sample rows, distinct values, counts, ranges) to confirm your assumptions, then run your candidate query and verify the returned rows make sense for the question (right columns, plausible row count, filters/joins working, not empty when it shouldn't be). Revise if the results look wrong. + +Respond with EXACTLY ONE of the following each turn (nothing else): + +1) To run a read-only query (SELECT/WITH/SHOW/DESCRIBE/EXPLAIN), output: +RUN_SQL: + + +2) Only after you have verified, output your final answer as: +YOUR FINAL MYSQL QUERY + +You have at most {steps} queries. Do not give until you have run at least one verification query.\ +""" + + +RUN_SQL_RE = re.compile(r"^\s*RUN_SQL:\s*", re.IGNORECASE | re.MULTILINE) diff --git a/eval/claudeagent/.gitignore b/eval/claudeagent/.gitignore new file mode 100644 index 0000000..1f9d07d --- /dev/null +++ b/eval/claudeagent/.gitignore @@ -0,0 +1,4 @@ +# Raw per-run generation artifacts (regenerable; can be large). The meaningful +# results are captured in RESULTS.md; full SQL outputs live under +# eval/unified-output/ (gitignored: contains gold SQL from the gated dataset). +output/ diff --git a/eval/claudeagent/README.md b/eval/claudeagent/README.md new file mode 100644 index 0000000..a60ed60 --- /dev/null +++ b/eval/claudeagent/README.md @@ -0,0 +1,57 @@ +# claudeagent — Claude Code (`claude -p`) text-to-SQL method + +A drop-in BEAVER method folder (mirrors `myagent/`) whose agent delegates SQL +generation to **Claude Code in headless print mode** (`claude -p`). + +## Backend +`agent.py::run_agent` builds one prompt from the question + schema (+ hints for +the active `--setting`), pipes it to `claude -p --output-format text` over stdin +(BEAVER prompts are ~30 KB), and extracts the SQL (`` / ```sql wrappers are +stripped by `clean_sql`). + +### Config (env vars, optional) +| var | default | meaning | +|-----|---------|---------| +| `CLAUDE_MODEL` | claude's default | value for `--model` | +| `CLAUDE_TIMEOUT` | `300` | per claude-call seconds | +| `CLAUDE_BIN` | `claude` | path to the claude binary | + +### Execution-guided modes (optional, gold-blind) +Same capabilities as `myagent`, ported to the Claude backend. All DB access is +mediated read-only by the agent process; the model never sees gold rows. + +- **self-fix** (`CLAUDE_SQL_FIX=1`) — run own SQL; on an execution error, feed + back *only the DB error* and ask for a corrected query (up to `CLAUDE_FIX_ATTEMPTS`). +- **explore/verify** (`CLAUDE_SQL_EXPLORE=1`) — run read-only queries against the + real tables, inspect the rows *its own* queries return, self-check, then finalize + (`CLAUDE_EXPLORE_STEPS`, `CLAUDE_EXPLORE_ROWS`). Combine with `CLAUDE_SQL_FIX=1` + for a final error-repair pass. + +```bash +CLAUDE_SQL_EXPLORE=1 CLAUDE_SQL_FIX=1 ./run.sh --dataset dw --setting 2 +``` +Needs the dataset's MySQL DB loaded + `MYSQL_*` creds (env or nearest `.env`). + +## Files +| file | role | edit? | +|------|------|-------| +| `agent.py` | Claude-backed `run_agent` (the seam) | swap backend here | +| `prompt.py` | per-question context + hints per `--setting` (shared) | rarely | +| `execute.py` | runs the agent in parallel, resume support (shared) | no | +| `unify.py` | reshapes into `unified-output/claudeagent//{generated,gold}` | no | +| `run.sh` | CLI wrapper: execute → unify | no | + +## Run +```bash +cd eval/claudeagent +./run.sh --dataset dw --setting 1 --q_fn dev_one # 1-question smoke test +./run.sh --dataset dw --setting 1 # full dev_sampled (100) +# CLAUDE_MODEL=claude-sonnet-4-6 ./run.sh --dataset dw --setting 1 +``` + +## Score (from `eval/`) +```bash +RUN= +uv run python evaluate_ex_acc.py --dataset dw --input_dir unified-output/claudeagent/$RUN +``` +`evaluate_ex_acc.py` needs the dataset's MySQL DB loaded + creds in `.env`. diff --git a/eval/claudeagent/RESULTS.md b/eval/claudeagent/RESULTS.md new file mode 100644 index 0000000..30bbeae --- /dev/null +++ b/eval/claudeagent/RESULTS.md @@ -0,0 +1,55 @@ +# claudeagent — Claude vs Codex on BEAVER `dw`: results + +Evaluation of the `claude -p` agent (`agent.py`), and a head-to-head against the +Codex agent (`../myagent`). **Execution accuracy** on the 100-question +`dev_sampled` set, scored against the live MySQL `dw` database. Same pipeline, +prompts, settings, and gold-blind enhancements as `myagent` — only the backend +differs. See `../myagent/RESULTS.md` for the per-lever Codex study. + +## Head-to-head — best config (setting 2 + explore/verify + fix, effort high) + +| Backend | exec acc | correct | SQL errors | empty | mismatch | +|---------|:--------:|:-------:|:----------:|:-----:|:--------:| +| Codex (`gpt-5.5`) — `myagent` | 34% | 34 | 0 | 2 | 64 | +| Claude (`opus`) — `claudeagent` | 32% | 32 | 0 | 2 | 66 | + +The 2-point gap is within run-to-run noise; both drive SQL errors to 0 and share +a near-identical (mismatch-dominated) failure profile. + +## Key finding: the backends are complementary + +| | count | +|--|:-----:| +| both correct | 23 | +| only Codex | 11 | +| only Claude | 9 | +| **union (either)** | **43** | + +Only 23 of their ~33 correct answers overlap — **~20 questions are solved by +exactly one model**. The union (**43%**) is far above either alone (+9 / +11). +Backend diversity is a larger lever than any single-backend knob measured +(hints +7, explore +3–4, effort/self-fix ~0), which points to **cross-model +ensembling** (generate with both, majority-vote on the execution result — still +gold-blind) as the path to ~40 %+. + +## Backend +`agent.py` runs `claude -p --output-format text` (prompt over stdin; ~30 KB). +Supports `--model` (`CLAUDE_MODEL`, e.g. `opus`/`sonnet`) and `--effort` +(`CLAUDE_EFFORT`, e.g. `high`), plus the same gold-blind execution-guided modes +as `myagent`: +- **self-fix** (`CLAUDE_SQL_FIX=1`) — repair execution errors from DB error feedback. +- **explore/verify** (`CLAUDE_SQL_EXPLORE=1`) — run read-only queries against the + real tables, inspect the rows its own queries return, self-check, finalize. + +## Reproduce +```bash +cd eval/claudeagent +CLAUDE_MODEL=opus CLAUDE_EFFORT=high CLAUDE_SQL_EXPLORE=1 CLAUDE_SQL_FIX=1 \ + ./run.sh --dataset dw --setting 2 + +cd .. && uv run python evaluate_ex_acc.py --dataset dw \ + --input_dir unified-output/claudeagent/ +``` +Requires the `dw` MySQL DB loaded + `MYSQL_*` creds in `.env`. Per-run SQL +outputs live under `eval/unified-output/` (gitignored — contains gold SQL from +the gated dataset). diff --git a/eval/claudeagent/agent.py b/eval/claudeagent/agent.py new file mode 100644 index 0000000..31ef43a --- /dev/null +++ b/eval/claudeagent/agent.py @@ -0,0 +1,215 @@ +""" +============================================================================ + Claude Code (`claude -p`) backed agent for BEAVER text-to-SQL. + Three generation modes (pick via env), all gold-blind: + * plain : one `claude -p` call -> SQL (default) + * self-fix : CLAUDE_SQL_FIX=1 -> run own SQL, fix execution errors + * explore+verify : CLAUDE_SQL_EXPLORE=1 -> run read-only queries against the + real tables, see the rows returned, self-check, finalize +============================================================================ + +The backend-agnostic machinery (response parsing, the read-only SQL guard, +mediated DB access, credential stripping, the explore protocol) lives in +eval/agent_common.py and is shared with eval/myagent. This file holds only the +`claude -p` invocation and the Claude-specific mode orchestration. + +NOTE on isolation: `claude -p` runs with its default (read) tool access and no +filesystem jail, so a determined model could still read files by absolute path. +This is defense-in-depth for a cooperative model, not a hard security boundary — +true isolation would require running the CLI in a container without the data/ +tree mounted (and/or restricting its tools). + +The prompt is passed to `claude -p` over stdin (BEAVER prompts are ~30 KB). + + instance fields (built in prompt.py::build_instances): + id, question, db, tables, prompt (chat messages), hints, record + -> NOTE: do not read record['sql']; that is the gold answer. + +Config (env vars, all optional): + CLAUDE_BIN path to the claude binary (default: claude) + CLAUDE_MODEL value for `--model` (default: unset -> claude default) + CLAUDE_TIMEOUT per claude-call seconds (default: 300) + CLAUDE_MAX_RETRIES retries for a failed/timed-out claude call (default: 2) + + CLAUDE_SQL_FIX 1 -> fix execution errors via error feedback (default 0) + CLAUDE_FIX_ATTEMPTS max fix rounds (default 2) + + CLAUDE_SQL_EXPLORE 1 -> explore/verify loop (overrides SQL_FIX alone) (default 0) + CLAUDE_EXPLORE_STEPS max exploratory query rounds (default 4) + CLAUDE_EXPLORE_ROWS max rows returned per exploratory query (default 20) + CLAUDE_FIX_TIMEOUT_MS SELECT execution cap, ms (default 10000; keep <= the + scorer's QUERY_TIMEOUT so a query the agent verifies + as OK also passes scoring) + MYSQL_HOST/USER/PASSWORD DB creds (env or nearest .env) +""" +import os +import sys +import time +import subprocess + +# Shared, backend-agnostic primitives (eval/agent_common.py). eval/ is this +# file's grandparent; add it to the path so the import resolves regardless of +# the working directory execute.py is launched from. +_EVAL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _EVAL_DIR not in sys.path: + sys.path.append(_EVAL_DIR) + +from agent_common import ( # noqa: E402 + clean_sql, + render_prompt, + cli_env as _cli_env, + is_read_only as _is_read_only, + timed as _timed, + DBUnavailable as _DBUnavailable, + execute_sql as _execute_sql_raw, + query_preview as _query_preview_raw, + fix_prompt as _fix_prompt, + EXPLORE_PROTOCOL as _EXPLORE_PROTOCOL, + RUN_SQL_RE as _RUN_SQL_RE, +) + +CLAUDE_BIN = os.getenv("CLAUDE_BIN", "claude") +CLAUDE_MODEL = os.getenv("CLAUDE_MODEL") # None -> claude's default model; e.g. "opus", "sonnet" +CLAUDE_EFFORT = os.getenv("CLAUDE_EFFORT") # None -> default; e.g. "high" (--effort) +CLAUDE_TIMEOUT = int(os.getenv("CLAUDE_TIMEOUT", "300")) +CLAUDE_MAX_RETRIES = int(os.getenv("CLAUDE_MAX_RETRIES", "2")) + +CLAUDE_SQL_FIX = os.getenv("CLAUDE_SQL_FIX", "0") not in ("0", "", "false", "False") +CLAUDE_FIX_ATTEMPTS = int(os.getenv("CLAUDE_FIX_ATTEMPTS", "2")) + +CLAUDE_SQL_EXPLORE = os.getenv("CLAUDE_SQL_EXPLORE", "0") not in ("0", "", "false", "False") +CLAUDE_EXPLORE_STEPS = int(os.getenv("CLAUDE_EXPLORE_STEPS", "4")) +CLAUDE_EXPLORE_ROWS = int(os.getenv("CLAUDE_EXPLORE_ROWS", "20")) +CLAUDE_FIX_TIMEOUT_MS = int(os.getenv("CLAUDE_FIX_TIMEOUT_MS", "10000")) + + +# ----------------------- claude -p invocation ----------------------- + +def _claude_call(prompt: str) -> str: + """One headless `claude -p` call -> raw stdout text. Retries a failed/timed-out + call up to CLAUDE_MAX_RETRIES times so a transient CLI failure does not + silently become an empty prediction.""" + cmd = [CLAUDE_BIN, "-p", "--output-format", "text"] + if CLAUDE_MODEL: + cmd += ["--model", CLAUDE_MODEL] + if CLAUDE_EFFORT: + cmd += ["--effort", CLAUDE_EFFORT] + last_err = None + for attempt in range(CLAUDE_MAX_RETRIES + 1): + try: + proc = subprocess.run( + cmd, input=prompt, capture_output=True, text=True, + timeout=CLAUDE_TIMEOUT, env=_cli_env(), + ) + except subprocess.TimeoutExpired: + last_err = f"claude -p timed out after {CLAUDE_TIMEOUT}s" + else: + out = proc.stdout or "" + if out.strip() or proc.returncode == 0: + return out + last_err = f"claude -p failed (rc={proc.returncode}): {(proc.stderr or '')[-800:].strip()}" + if attempt < CLAUDE_MAX_RETRIES: + time.sleep(2 * (attempt + 1)) + raise RuntimeError(last_err or "claude -p failed") + + +def _claude_generate(prompt: str) -> str: + return clean_sql(_claude_call(prompt)) + + +# DB access with this backend's execution timeout bound in (the primitives take +# the cap as a parameter; these thin wrappers keep the internal call sites tidy). +def _execute_sql(sql: str, db: str): + return _execute_sql_raw(sql, db, CLAUDE_FIX_TIMEOUT_MS) + + +def _query_preview(sql: str, db: str, max_rows: int): + return _query_preview_raw(sql, db, max_rows, CLAUDE_FIX_TIMEOUT_MS) + + +# ------------------------------- modes ------------------------------- + +def _fix_loop(base, db, sql): + """Given a candidate SQL, repair execution errors (error feedback only).""" + if not sql: + return sql + for _ in range(CLAUDE_FIX_ATTEMPTS): + try: + ok, err = _execute_sql(sql, db) + except _DBUnavailable as e: + # Can't verify (DB down / bad creds): do NOT rewrite a possibly-correct query. + print(f"DB unavailable; skipping fix loop, keeping candidate SQL: {e}") + break + if ok: + break + fixed = _claude_generate(_fix_prompt(base, db, sql, err)) + if not fixed or fixed == sql: + break + sql = fixed + return sql + + +def _parse_action(text: str): + """Return ('answer', sql) or ('run', query) from a model turn. + + A RUN_SQL request takes precedence over a bare '' *mention* (e.g. the + model restating the protocol); only a complete ... span is + treated as a final answer that overrides an accompanying RUN_SQL.""" + if not text: + return "answer", "" + m = _RUN_SQL_RE.search(text) + has_complete_ans = "" in text and "" in text + if m and not has_complete_ans: + q = text[m.end():].split("", 1)[0] + q = clean_sql(q) if "```" in q else q.strip() + return "run", q.strip() + return "answer", clean_sql(text) + + +def _run_explore(base, db): + transcript = base + _EXPLORE_PROTOCOL.format(db=db, steps=CLAUDE_EXPLORE_STEPS) + queries_run = 0 + nudged = False + for step in range(CLAUDE_EXPLORE_STEPS): + resp = _claude_call(transcript) + action, payload = _parse_action(resp) + if action == "answer" and payload: + # Enforce the protocol's "verify first" rule with a single nudge. + if queries_run == 0 and not nudged and step < CLAUDE_EXPLORE_STEPS - 1: + nudged = True + transcript += ( + f"\n\n### Proposed answer (NOT yet verified)\n{payload}\n\n" + "You have not run any verification query. As required, run at least one " + "read-only RUN_SQL query to check this answer before giving ." + ) + continue + return payload + if action == "run" and payload: + queries_run += 1 + result = _query_preview(payload, db, CLAUDE_EXPLORE_ROWS) + transcript += ( + f"\n\n### Your query (step {step + 1})\n{payload}\n\n### Result\n{result}\n\n" + f"Run another RUN_SQL query, or output your final ...." + ) + else: + break + # Out of steps (or no parseable action): force a final answer. Do NOT fall + # back to the last exploratory probe — a DESCRIBE/COUNT probe is not an answer. + final = _claude_call( + transcript + "\n\nYou must now output ONLY your final answer as YOUR MYSQL QUERY.", + ) + return clean_sql(final) + + +def run_agent(instance: dict, model: str = None) -> str: + # `model` is a run label only (used for the output-dir name); the backend + # model is selected by the CLAUDE_MODEL env var, not this argument. + base = render_prompt(instance) + db = instance.get("db") or "dw" + if CLAUDE_SQL_EXPLORE: + sql = _run_explore(base, db) + else: + sql = _claude_generate(base) + if CLAUDE_SQL_FIX and sql: # final error-repair pass + sql = _fix_loop(base, db, sql) + return sql diff --git a/eval/claudeagent/execute.py b/eval/claudeagent/execute.py new file mode 100644 index 0000000..8846229 --- /dev/null +++ b/eval/claudeagent/execute.py @@ -0,0 +1,98 @@ +"""Generation driver for a custom agent. + +Builds one instance per question, runs your agent (agent.run_agent) over them in +parallel with resume support, and writes each prediction to + //predicted_0.sql (+ generation.log) +which unify.py then reshapes into the unified generated/ + gold/ layout. + +You normally do NOT edit this file — put your agent in agent.py. +""" +import os +import argparse +import traceback +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from prompt import build_instances +from utils import EvalConfig +from agent import run_agent + + +def _output_paths(output_dir, instance_id): + instance_dir = os.path.join(output_dir, instance_id) + return instance_dir, os.path.join(instance_dir, "predicted_0.sql"), os.path.join(instance_dir, "generation.log") + + +def _process(instance, model, output_dir): + instance_id = instance["id"] + instance_dir, sql_file, log_file = _output_paths(output_dir, instance_id) + try: + sql = run_agent(instance, model) or "" + log = f"Question:\n{instance['question']}\n\nGenerated SQL:\n{sql}" + except Exception as e: + sql = "" + log = f"Error: {e}\n{traceback.format_exc()}" + print(f"Error on {instance_id}: {e}") + + # Writes are inside their own guard so a single failed write (bad locale, + # disk full, permissions) fails just this instance instead of aborting the + # whole batch via fut.result(). + try: + os.makedirs(instance_dir, exist_ok=True) + with open(sql_file, "w", encoding="utf-8") as f: + f.write(sql) + with open(log_file, "w", encoding="utf-8") as f: + f.write(log) + except Exception as e: + print(f"Failed to write output for {instance_id}: {e}") + return instance_id, False + return instance_id, bool(sql) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, default="codex") + parser.add_argument("--dataset", type=str, default="dw") + parser.add_argument("--q_fn", type=str, default="dev_sampled") + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--data_dir", type=str, default="../../data") + parser.add_argument("--num_workers", type=int, default=4) + # --setting flags (set by run.sh) + parser.add_argument("--gold_tables", action="store_true") + parser.add_argument("--join_keys", action="store_true") + parser.add_argument("--mapping", action="store_true") + parser.add_argument("--knowledge", action="store_true") + parser.add_argument("--decomp", action="store_true") + args = parser.parse_args() + + eval_config = EvalConfig( + gold_tables=args.gold_tables, join_keys=args.join_keys, + mapping=args.mapping, knowledge=args.knowledge, decomp=args.decomp, + ) + print(eval_config) + + instances = build_instances(args.dataset, args.q_fn, eval_config, args.data_dir) + os.makedirs(args.output_dir, exist_ok=True) + + # Resume: skip instances that already have a non-empty prediction. + todo = [] + for inst in instances: + _, sql_file, _ = _output_paths(args.output_dir, inst["id"]) + if not os.path.exists(sql_file) or os.path.getsize(sql_file) == 0: + todo.append(inst) + + print(f"Output directory: {args.output_dir}") + if not todo: + print("All instances already processed.") + return + print(f"Running agent on {len(todo)} / {len(instances)} instances (model={args.model})...") + + with ThreadPoolExecutor(max_workers=args.num_workers) as ex: + futures = [ex.submit(_process, inst, args.model, args.output_dir) for inst in todo] + for fut in tqdm(futures, total=len(futures)): + fut.result() + + +if __name__ == "__main__": + main() diff --git a/eval/claudeagent/prompt.py b/eval/claudeagent/prompt.py new file mode 100644 index 0000000..c930239 --- /dev/null +++ b/eval/claudeagent/prompt.py @@ -0,0 +1,180 @@ +"""Build per-question instances for the agent. + +This mirrors the context/hint assembly used by the `fewshot` baseline +(`eval/fewshot/prompt.py`) so the --setting flag behaves identically, but it +returns a list of *aligned* instance dicts instead of two parallel lists. Each +instance carries both: + * a ready-to-send OpenAI-style chat `prompt` (system + few-shot example + user) + for agents that just want to call a chat model, and + * the structured fields (question, db, tables, and the active hint strings) for + agents that do their own multi-step reasoning / tool use. +""" +from pathlib import Path + +from utils import ( + read_json, format_tables, EvalConfig, format_join, system, user, assistant, +) + + +def get_mapping(mapping): + desc = [] + for sq in mapping: + cols_desc = [] + for col in mapping[sq]: + table_name, col = col.split(".") + cols_desc.append(f"column {col} in table {table_name.lower()}") + desc.append(f'"{sq}" in the user question refers to {", ".join(cols_desc)}') + return "\n".join(desc) + + +def get_join_keys(join_keys): + return "\n".join(format_join(jk) for jk in join_keys) + + +def get_knowledge(domain_knowledge): + return "\n".join(domain_knowledge) + + +def get_decomp(decomps): + return "\n".join(f"Subquery {i + 1}: {sq}" for i, sq in enumerate(decomps)) + + +def get_user_prompt(q, tables, corpus_tables, eval_config: EvalConfig): + desc = [ + format_tables(tables, corpus_tables, eval_config.instances), + f"User question: {q['question']}", + ] + if eval_config.mapping: + desc.append(f"Mapping:\n{get_mapping(q['column_mapping'])}") + if eval_config.join_keys: + desc.append(f"Join keys:\n{get_join_keys(q['join_keys'])}") + if eval_config.knowledge and get_knowledge(q['domain_knowledge']) != '': + desc.append(f"Domain knowledge:\n{get_knowledge(q['domain_knowledge'])}") + if eval_config.decomp and get_decomp(q["sub_questions"]) != "": + desc.append(f"Query decomposition:\n{get_decomp(q['sub_questions'])}") + return user("\n\n".join(desc)) + + +def get_retrieved_tables(dataset: str, data_dir="../../data"): + retrieved_fn = Path(f"{data_dir}/{dataset}/retrieval/retrieved_tables.json") + reranked_fn = Path(f"{data_dir}/{dataset}/retrieval/reranked_tables.json") + if not retrieved_fn.exists(): + raise FileNotFoundError( + f"No retrieved tables at {retrieved_fn}. At --setting 0 the agent gets the " + f"retrieved candidate tables; run `python retrieve/retrieve.py --dataset {dataset} ...` " + f"first, or use --setting 1/2 which inject gold tables and need no retrieval." + ) + if reranked_fn.exists(): + print(f"Loading reranked tables from {reranked_fn}") + return read_json(reranked_fn) + print(f"Loading retrieved tables from {retrieved_fn}") + return read_json(retrieved_fn) + + +def _build_instruction(eval_config, q_knowledge, q_decomp, q, structures): + db_type = "MySQL" + instruction = ["You are given a list of tables", "a user question"] + if eval_config.join_keys: + instruction.append("join keys among the provided tables") + if eval_config.mapping: + instruction.append("a mapping from information mentioned in the user question to columns in the provided tables") + if q_knowledge: + instruction.append("domain knowledge") + if q_decomp: + instruction.append("decomposition of the user question") + instruction[-1] = f"and {instruction[-1]}" + instruction = ", ".join(instruction) + ", " + + instruction += ( + f"your task is output a {db_type} SQL statement that can be used to answer the user " + f"question based on the provided information. You need to ensure that syntax and functions " + f"used in your SQL statement are appropriate for {db_type} database. If you are unable to " + f"determine the SQL statement, output None. " + ) + if eval_config.mapping: + instruction += "You should use the provided mapping to determine which columns and tables should be used in the SQL statement. " + if eval_config.join_keys: + instruction += "You should use the provided join keys to determine how to connect the tables in the SQL statement. " + if q_knowledge: + instruction += "You should use the provided domain knowledge to determine which tables, columns, and literals should be used in the SQL statement. " + if q_decomp: + instruction += ( + "You must answer each subquery individually and then combine them to form the complete " + "SQL statement. Each subquery you generate must be explicitly used in the final SQL " + "statement, without being simplified. " + ) + instruction += ( + "Below is the structure of the SQL statement with subqueries denoted. Each provided " + "subquery is used in the final SQL statement in such a structure." + ) + structure_name = q.get("detailed_category") + if structure_name and structure_name != 'real' and structure_name in structures: + structure = structures[structure_name] + instruction += f"\n\n{structure['structure']}" + instruction += f"\n\n{structure['subquery_decomposition']} " + + instruction += "The SQL statement need to be wrapped in tags." + return instruction + + +def build_instances(dataset: str, q_fn: str, eval_config: EvalConfig, data_dir: str = "../../data"): + """Return a list of aligned instance dicts, one per question that has table context. + + Each instance: + id : question id + question : natural-language question + db : target database name + tables : list of candidate table names provided to the agent + prompt : OpenAI-style chat messages (system + few-shot example + user) + hints : {mapping, join_keys, domain_knowledge, decomposition} active strings + record : full question/gold dict from .json (avoid peeking at gold `sql`) + """ + structures = read_json(f"{data_dir}/template_structure.json") + qs = read_json(f"{data_dir}/{dataset}/{q_fn}.json") + example = read_json(f"{data_dir}/{dataset}/example.json") + dev_tables = read_json(f"{data_dir}/{dataset}/dev_tables.json") + + retrieved_tables = None if eval_config.gold_tables else get_retrieved_tables(dataset, data_dir) + + example_prompt = [ + get_user_prompt(example, example["tables"], dev_tables, eval_config), + assistant(f"SQL: {example['sql']}"), + ] + + instances = [] + skipped = 0 + for q in qs: + q_id = q['id'] + q_knowledge = eval_config.knowledge and get_knowledge(q['domain_knowledge']) != '' + q_decomp = eval_config.decomp and get_decomp(q["sub_questions"]) != '' + + if eval_config.gold_tables: + tables = q["tables"] + else: + if q_id not in retrieved_tables: + skipped += 1 + continue + tables = retrieved_tables[q_id] + + instruction = _build_instruction(eval_config, q_knowledge, q_decomp, q, structures) + prompt = [system(instruction)] + example_prompt + [get_user_prompt(q, tables, dev_tables, eval_config)] + + instances.append({ + "id": q_id, + "question": q["question"], + "db": q.get("db", dataset), + "tables": tables, + "prompt": prompt, + "hints": { + "mapping": get_mapping(q['column_mapping']) if eval_config.mapping else None, + "join_keys": get_join_keys(q['join_keys']) if eval_config.join_keys else None, + "domain_knowledge": get_knowledge(q['domain_knowledge']) if q_knowledge else None, + "decomposition": get_decomp(q['sub_questions']) if q_decomp else None, + }, + "record": q, + }) + + if skipped: + print(f"Skipped {skipped} question(s) with no retrieved tables.") + print(f"#instances: {len(instances)}") + return instances diff --git a/eval/claudeagent/run.sh b/eval/claudeagent/run.sh new file mode 100755 index 0000000..e029f93 --- /dev/null +++ b/eval/claudeagent/run.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -e + +# ============================================================================ +# Claude Code (`claude -p`) generation pipeline for BEAVER +# Usage: ./run.sh --dataset dw --setting 1 +# +# Step 1 (execute.py): run `claude -p` over every question (agent.py) +# Step 2 (unify.py): reshape outputs into unified-output/claudeagent// +# +# Score from eval/ with evaluate_ex_acc.py / evaluate_subtasks.py. +# ============================================================================ + +MODEL="claude" +DATASET="dw" +SETTING=0 +NUM_WORKERS=4 +Q_FN="dev_sampled" +RESUME_DIR="" + +while [[ $# -gt 0 ]]; do + case $1 in + --model) MODEL="$2"; shift 2 ;; + --dataset) DATASET="$2"; shift 2 ;; + --setting) SETTING="$2"; shift 2 ;; + --num_workers) NUM_WORKERS="$2"; shift 2 ;; + --q_fn) Q_FN="$2"; shift 2 ;; + --resume) RESUME_DIR="$2"; shift 2 ;; + --help|-h) + echo "Usage: ./run.sh --dataset --setting <0|1|2> [--num_workers N] [--q_fn FILE] [--resume DIR]" + echo " --model run label for the output dir (default: claude); does NOT select the" + echo " model — set CLAUDE_MODEL to pick the actual model." + echo " --dataset dw | dw_real | neutron | nova (default: dw)" + echo " --setting 0=no hints, 1=schema hints, 2=all hints (default: 0)" + echo " --num_workers parallel claude -p calls (default: 4)" + echo " --q_fn question file stem (default: dev_sampled; use dev_one for a 1-eval)" + echo " --resume reuse an existing output dir; execute.py then skips questions that" + echo " already have a prediction." + echo "" + echo " Env (optional): CLAUDE_MODEL, CLAUDE_TIMEOUT (default 300)." + exit 0 + ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +case "$SETTING" in + 0|1|2) ;; + *) echo "Invalid --setting: '$SETTING' (expected 0, 1, or 2)"; exit 1 ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +DATA_DIR="$(cd "${SCRIPT_DIR}/../../data" && pwd)" + +if [ -f "${EVAL_DIR}/../.env" ]; then + set -a; source "${EVAL_DIR}/../.env"; set +a +fi + +HINTS="" +if [ "$SETTING" -eq 1 ]; then + HINTS="--gold_tables --mapping --join_keys" +elif [ "$SETTING" -eq 2 ]; then + HINTS="--gold_tables --mapping --join_keys --knowledge --decomp" +fi + +# Resume into an existing dir if asked, else mint a fresh timestamped one. +if [ -n "$RESUME_DIR" ]; then + OUTPUT_DIR="$RESUME_DIR" +else + TIMESTAMP=$(date +"%Y%m%d-%H%M%S") + OUTPUT_DIR="${SCRIPT_DIR}/output/${MODEL}-beaver-${DATASET}-setting${SETTING}-log-${TIMESTAMP}" +fi +mkdir -p "$OUTPUT_DIR" + +echo "================================" +echo "ClaudeAgent (claude -p) on BEAVER ${DATASET} (setting ${SETTING}, q_fn ${Q_FN})" +echo "Output Path: $OUTPUT_DIR" +echo "================================" + +echo "" +echo "Step 1: Generating Predictions" +echo "==============================" +cd "$SCRIPT_DIR" +uv run python execute.py \ + --model "$MODEL" \ + --dataset "$DATASET" \ + --data_dir "$DATA_DIR" \ + --output_dir "$OUTPUT_DIR" \ + --q_fn "$Q_FN" \ + --num_workers "$NUM_WORKERS" \ + $HINTS + +echo "" +echo "Step 2: Unify Predictions" +echo "==============================" +uv run python unify.py \ + --input_dir "$OUTPUT_DIR" \ + --gold_file "${DATA_DIR}/${DATASET}/${Q_FN}.json" \ + --dataset "$DATASET" + +echo "========================================" +echo "ClaudeAgent generation complete." +echo "Run name: $(basename "$OUTPUT_DIR")" +echo "Now score it from eval/ with evaluate_ex_acc.py / evaluate_subtasks.py." diff --git a/eval/claudeagent/unify.py b/eval/claudeagent/unify.py new file mode 100644 index 0000000..79397fb --- /dev/null +++ b/eval/claudeagent/unify.py @@ -0,0 +1,84 @@ +"""Reshape raw per-instance outputs into the unified layout the scorers read: + + eval/unified-output/// + ├── generated/.sql (your agent's prediction) + └── gold/.sql (gold SQL from the question file) + + is this folder's name. For each id-subdir under --input_dir it grabs +result.sql, or the first *.sql it finds (here: predicted_0.sql). Generic — you +should not need to edit this. +""" +import os +import sys +import json +import argparse +import glob +import re + +from tqdm import tqdm + +eval_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +sys.path.append(eval_dir) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", type=str, required=True, help="Directory containing generated outputs") + parser.add_argument("--gold_file", type=str, required=True, help="Path to dev.json or dev_sampled.json") + parser.add_argument("--dataset", type=str, required=True, help="Dataset name, e.g. dw") + args = parser.parse_args() + + args.input_dir = args.input_dir.rstrip('/') + run_name = os.path.basename(args.input_dir) + + baseline_name = os.path.basename(os.path.dirname(os.path.abspath(__file__))) + unified_dir = os.path.join(eval_dir, "unified-output", baseline_name, run_name) + generated_dir = os.path.join(unified_dir, "generated") + gold_dir = os.path.join(unified_dir, "gold") + os.makedirs(generated_dir, exist_ok=True) + os.makedirs(gold_dir, exist_ok=True) + + with open(args.gold_file, 'r', encoding="utf-8") as f: + gold_data = json.load(f) + id_to_entry = {entry['id']: entry for entry in gold_data} + + subdirs = sorted(d for d in glob.glob(os.path.join(args.input_dir, "*")) if os.path.isdir(d)) + print(f"Processing {len(subdirs)} subdirectories for {baseline_name}...") + for subdir_path in tqdm(subdirs): + subdir_name = os.path.basename(subdir_path) + + if subdir_name in id_to_entry: + gold_entry = id_to_entry[subdir_name] + else: + # Fallback for index-based dir names (e.g. ..._001) + match = re.search(r'_(\d+)$', subdir_name) + if match and int(match.group(1)) < len(gold_data): + gold_entry = gold_data[int(match.group(1))] + else: + continue + + real_id = gold_entry['id'] + gold_sql = gold_entry.get("sql", gold_entry.get("oracle_sql", gold_entry.get("gold_sql", ""))) + + with open(os.path.join(gold_dir, f"{real_id}.sql"), "w", encoding="utf-8") as f: + f.write(gold_sql) + + result_sql_path = os.path.join(subdir_path, "result.sql") + if not os.path.exists(result_sql_path): + sql_files = glob.glob(os.path.join(subdir_path, "*.sql")) + if sql_files: + result_sql_path = sql_files[0] + + pred_sql = "" + if os.path.exists(result_sql_path): + with open(result_sql_path, "r", encoding="utf-8") as f: + pred_sql = f.read().strip() + + with open(os.path.join(generated_dir, f"{real_id}.sql"), "w", encoding="utf-8") as f: + f.write(pred_sql) + + print(f"Saved SQL files to {unified_dir}") + + +if __name__ == "__main__": + main() diff --git a/eval/claudeagent/utils.py b/eval/claudeagent/utils.py new file mode 100644 index 0000000..ed47a91 --- /dev/null +++ b/eval/claudeagent/utils.py @@ -0,0 +1,78 @@ +import json +import pandas as pd +from dataclasses import dataclass + + +def read_json(fn): + with open(fn, encoding="utf-8") as f: + return json.load(f) + + +def write_json(obj, fn): + with open(fn, 'w', encoding="utf-8") as f: + json.dump(obj, f, indent=2) + + +def format_join(join_pair): + """Format a [left, right] join pair for display.""" + left, right = join_pair + return f"{left.split('.')[0]} joins {right.split('.')[0]} on {left} = {right}" + + +def format_table(table_name, corpus_tables, use_instance: bool, corpus_markdowns=None, trim=False): + rows = corpus_tables[table_name]["example_rows"] + if trim: + for row in rows: + for i in range(len(row)): + if isinstance(row[i], str): + row[i] = row[i][:500] + + cols = corpus_tables[table_name]["column_names"] + + if corpus_markdowns is None: + df = pd.DataFrame(rows, columns=cols) + df_md = df.to_markdown(index=False) + else: + df_md = corpus_markdowns[table_name] + + table_string = [ + f'Table name: {table_name}', + f"Example table content:\n{df_md}", + ] + + if use_instance: + instances = corpus_tables[table_name]["example_columns"] + table_string.append("Top-10 most occurring values for each column:") + for col_idx, col_name in enumerate(cols): + _instance = " | ".join([str(x) for x in instances[col_idx]]) + table_string.append(f"{col_name}: {_instance}") + + return "\n".join(table_string) + + +def format_tables(tables, corpus_tables, use_instance, corpus_markdowns=None): + return "\n\n".join( + format_table(t, corpus_tables, use_instance, corpus_markdowns) for t in tables + ) + + +@dataclass +class EvalConfig: + gold_tables: bool + join_keys: bool + mapping: bool + knowledge: bool + decomp: bool + instances: bool = False + + +def system(content: str): + return {"role": "system", "content": content} + + +def user(content: str): + return {"role": "user", "content": content} + + +def assistant(content: str): + return {"role": "assistant", "content": content} diff --git a/eval/evaluate_ex_acc.py b/eval/evaluate_ex_acc.py index 4422c1b..e39b16d 100644 --- a/eval/evaluate_ex_acc.py +++ b/eval/evaluate_ex_acc.py @@ -11,10 +11,17 @@ from utils.ex_acc import get_mysql_credentials, execute_sql_with_timeout, compare_results from utils.utils import write_json +CANDIDATE_SEP = "-- ===CANDIDATE=== --" + + def main(): parser = argparse.ArgumentParser(description="Unified evaluation script for text-to-SQL baselines") parser.add_argument("--dataset", type=str, required=True, help="Dataset name for MySQL credentials, e.g. dw") parser.add_argument("--input_dir", type=str, required=True, help="Path to the unified output directory containing generated/ and gold/ subdirectories") + parser.add_argument("--multi", action="store_true", + help="Generated files may hold several candidate queries joined by " + f"'{CANDIDATE_SEP}'; score a question correct if ANY candidate matches " + "(also reports candidate-1-only accuracy)") args = parser.parse_args() generated_dir = Path(args.input_dir) / "generated" @@ -35,6 +42,7 @@ def main(): total_attempted = total_queries total_score = 0 + total_first_match = 0 nonempty_gold_total = 0 nonempty_gold_score = 0 @@ -61,50 +69,83 @@ def main(): with open(pred_sql_path, "r") as f: pred_sql = f.read().strip() - pred_df = None - pred_err = None - if pred_sql: - pred_df, pred_err = execute_sql_with_timeout(pred_sql, mysql_creds) - - if pred_df is None: - pred_df = pd.DataFrame() - - # Compare - match, msg = compare_results(pred_df, gold_df) + candidates = [pred_sql] + if args.multi and pred_sql: + candidates = [c.strip() for c in pred_sql.split(CANDIDATE_SEP) if c.strip()] or [""] + + # Score each candidate; question is correct if any candidate matches. + match, msg, pred_err, matched_idx = False, "No prediction", None, None + first_match = False + pred_empty = True + executed = False + for idx, cand in enumerate(candidates): + if not cand: + continue + executed = True + cand_df, cand_err = execute_sql_with_timeout(cand, mysql_creds) + if cand_df is None: + cand_df = pd.DataFrame() + cand_match, cand_msg = compare_results(cand_df, gold_df) + if idx == 0: + # candidate 1 = the model's best guess; report its stats as primary + msg, pred_err, pred_empty, first_match = cand_msg, cand_err, cand_df.empty, cand_match + if cand_match: + match, matched_idx = True, idx + 1 + if idx > 0: + msg = f"Match via candidate {idx + 1} (candidate 1: {msg})" + break + if not executed: + # No prediction at all: compare an empty result set against gold so an + # empty prediction still scores 1 iff gold is also empty. This matches + # the original scorer's "both empty -> match" semantics; without it the + # --multi refactor would silently zero every empty-pred/empty-gold case + # for all baselines scored by this shared script. + match, msg = compare_results(pd.DataFrame(), gold_df) + first_match = match score = 1 if match else 0 total_score += score + total_first_match += 1 if first_match else 0 gold_is_empty = gold_df.empty - + if not gold_is_empty: nonempty_gold_total += 1 nonempty_gold_score += score - - results.append({ + + entry = { "file": filename, "match": match, "score": score, "message": msg, "gold_empty": gold_is_empty, - "pred_empty": pred_df.empty, + "pred_empty": pred_empty, "gold_error": gold_err, "pred_error": pred_err - }) + } + if args.multi: + entry["n_candidates"] = len(candidates) + entry["matched_candidate"] = matched_idx + entry["candidate1_match"] = first_match + results.append(entry) acc_including_empty = (100 * total_score / total_attempted) if total_attempted > 0 else 0.0 if nonempty_gold_total > 0: acc_excluding_empty = 100 * nonempty_gold_score / nonempty_gold_total # Save results summary + metrics = { + "total_evaluated": total_attempted, + "exact_matches": total_score, + "accuracy_including_empty": acc_including_empty, + "nonempty_gold_total": nonempty_gold_total, + "nonempty_gold_score": nonempty_gold_score, + "accuracy_excluding_empty": acc_excluding_empty if nonempty_gold_total > 0 else None + } + if args.multi: + metrics["candidate1_matches"] = total_first_match + metrics["candidate1_accuracy"] = (100 * total_first_match / total_attempted) if total_attempted else 0.0 summary_data = { - "metrics": { - "total_evaluated": total_attempted, - "exact_matches": total_score, - "accuracy_including_empty": acc_including_empty, - "nonempty_gold_total": nonempty_gold_total, - "nonempty_gold_score": nonempty_gold_score, - "accuracy_excluding_empty": acc_excluding_empty if nonempty_gold_total > 0 else None - }, + "metrics": metrics, "details": sorted(results, key=lambda x: x["file"]) } diff --git a/eval/myagent/.gitignore b/eval/myagent/.gitignore new file mode 100644 index 0000000..1f9d07d --- /dev/null +++ b/eval/myagent/.gitignore @@ -0,0 +1,4 @@ +# Raw per-run generation artifacts (regenerable; can be large). The meaningful +# results are captured in RESULTS.md; full SQL outputs live under +# eval/unified-output/ (gitignored: contains gold SQL from the gated dataset). +output/ diff --git a/eval/myagent/README.md b/eval/myagent/README.md new file mode 100644 index 0000000..d684963 --- /dev/null +++ b/eval/myagent/README.md @@ -0,0 +1,100 @@ +# myagent — Codex-backed text-to-SQL method + +A drop-in BEAVER method folder (mirrors `fewshot/`) whose agent delegates SQL +generation to the local **Codex CLI** via `codex exec` (headless, one-shot). + +## Backend +`agent.py::run_agent` builds a single prompt from the question + schema (+ hints +for the active `--setting`), runs: + +``` +codex exec --sandbox read-only --skip-git-repo-check -C \ + -c model_reasoning_effort= -o "" +``` + +in a throwaway working directory (Codex never touches the repo), and reads the +final agent message back via `--output-last-message`. `…` / ```` ```sql ```` +wrappers are stripped by `clean_sql`. + +Uses your existing Codex login. **With a ChatGPT login, only Codex-supported +models work** (e.g. `gpt-5-codex`, `gpt-5.5`); arbitrary names like `gpt-5-mini` +are rejected — so BEAVER's `--model` is a label only and is *not* forwarded to +Codex. Pick the Codex model with `CODEX_MODEL`. + +### Config (env vars, optional) +| var | default | meaning | +|-----|---------|---------| +| `CODEX_MODEL` | account default | Codex model (`codex -m`) | +| `CODEX_REASONING_EFFORT` | `low` | minimal / low / medium / high / xhigh | +| `CODEX_SANDBOX` | `read-only` | sandbox policy | +| `CODEX_TIMEOUT` | `300` | per codex-call seconds | +| `CODEX_BIN` | `codex` | path to the codex binary | + +### Execution-guided self-fix (optional) +With `CODEX_SQL_FIX=1`, after generating, the agent runs its own SQL against the +live MySQL database; if it **fails to execute**, it feeds *only the database error* +back to Codex and asks for a corrected query, looping up to `CODEX_FIX_ATTEMPTS` +times. It is **gold-blind** — it never sees the correct/expected rows, only whether +its own query errors — and stops as soon as the query executes (rows may still be +wrong). This fixes syntax/execution errors, not semantic correctness. Execution is +read-only guarded (refuses non-SELECT statements) and time-capped. + +| var | default | meaning | +|-----|---------|---------| +| `CODEX_SQL_FIX` | `0` | `1` to enable the execute-and-fix loop | +| `CODEX_FIX_ATTEMPTS` | `2` | max correction rounds on execution error | +| `CODEX_FIX_TIMEOUT_MS` | `15000` | SELECT execution cap during the fix check | + +```bash +CODEX_SQL_FIX=1 CODEX_REASONING_EFFORT=high ./run.sh --dataset dw --setting 1 +``` +Needs the dataset's MySQL DB loaded + `MYSQL_*` creds (read from env or nearest `.env`). + +### Execution-guided explore / decompose / review (optional, gold-blind) +| var | default | meaning | +|-----|---------|---------| +| `CODEX_SQL_EXPLORE` | `0` | run read-only queries against the real tables, inspect the rows *its own* queries return, self-check, then finalize | +| `CODEX_EXPLORE_STEPS` | `4` | max exploratory query rounds | +| `CODEX_EXPLORE_ROWS` | `20` | rows returned per exploratory query | +| `CODEX_DECOMPOSE` | `0` | prepend guidance to self-decompose and validate each sub-step with SQL | +| `CODEX_REVIEW` | `0` | final subagent pass that reviews the answer vs the question for intent capture | + +All are gold-blind (DB access is mediated read-only by this process; the model +never sees gold rows). Compose freely, e.g. explore + final fix: +```bash +CODEX_SQL_EXPLORE=1 CODEX_SQL_FIX=1 CODEX_REASONING_EFFORT=high ./run.sh --dataset dw --setting 2 +``` + +> **Note (see RESULTS.md):** `explore/verify` helps (+3–4); but `CODEX_DECOMPOSE` +> and `CODEX_REVIEW` are **net-negative on `dw`** (self-decompose −5, subagent +> review −7 vs baseline) — a documented negative result, off by default. The +> best config is the simple one: hints + explore + fix. + +## Files +| file | role | edit? | +|------|------|-------| +| `agent.py` | Codex-backed `run_agent` (the seam) | swap backend here | +| `prompt.py` | builds per-question context + hints per `--setting` | rarely | +| `execute.py` | runs the agent in parallel (default 4 workers), resume support | no | +| `unify.py` | reshapes output into `unified-output/myagent//{generated,gold}` | no | +| `run.sh` | CLI wrapper: execute → unify | no | + +## Run +```bash +cd eval/myagent +./run.sh --dataset dw --setting 1 # setting 1/2 need no retrieval +# ./run.sh --dataset dw --setting 0 # setting 0 needs retrieve/retrieve.py first +# CODEX_MODEL=gpt-5-codex ./run.sh --dataset dw --setting 1 +``` + +## Score (from `eval/`) +```bash +RUN= +uv run python evaluate_ex_acc.py --dataset dw --input_dir unified-output/myagent/$RUN +uv run python evaluate_subtasks.py --dataset dw --model gpt-5-mini --input_dir unified-output/myagent/$RUN +``` +`evaluate_ex_acc.py` needs MySQL loaded + creds in `.env`. `evaluate_subtasks.py` +needs an LLM key (its `--model` is the *grader*, unrelated to the Codex backend). + +> Each `codex exec` runs ~30 s at `low` effort; keep `--num_workers` modest to +> avoid Codex rate limits. The full 100-question `dev_sampled` run is sized accordingly. diff --git a/eval/myagent/RESULTS.md b/eval/myagent/RESULTS.md new file mode 100644 index 0000000..8b37c41 --- /dev/null +++ b/eval/myagent/RESULTS.md @@ -0,0 +1,315 @@ +# myagent — Codex on BEAVER `dw`: results + +Evaluation of the Codex-backed agent (`agent.py`) on the BEAVER `dw` benchmark. +All numbers are **execution accuracy** (generated SQL's result set equals gold's) +on the standard 100-question `dev_sampled` set, scored against the live MySQL +`dw` database. Generation uses the local Codex CLI (model `gpt-5.5` via ChatGPT +login). Every enhancement below is **gold-blind** — the agent never sees the +expected/correct answer. + +> See `../claudeagent/RESULTS.md` for the Claude (`claude -p`) vs Codex +> head-to-head and the cross-model ensembling finding. + +## Scoreboard (`dw`, 100 questions, `high` reasoning effort) + +| Config | exec acc | correct | SQL errors | empty | mismatch | +|--------|:--------:|:-------:|:----------:|:-----:|:--------:| +| setting 1, one-shot (baseline) | 23% | 23 | 11 | 2 | 64 | +| setting 1 + self-fix | 23% | 23 | 0 | 5 | 72 | +| setting 1 + explore/verify | 26% | 26 | 5 | 3 | 66 | +| setting 2 + self-fix | 30% | 30 | 1 | 4 | 65 | +| setting 2 + explore/verify + fix | 34% | 34 | 0 | 2 | 64 | +| setting 2 + explore/verify + fix + style guide | 37% | 37 | 0 | — | — | +| **+ precedence fix + schema skill + 3 candidates (best guess)** | **39%** | **39** | 0 | — | — | +| *same run, pass@3 (any of 3 candidates matches)* | *49%* | *49* | — | — | — | + +5-question sanity samples (setting 1): low 20%, high 40%, xhigh 40%. +Full-100, setting 1: high 23%, xhigh 25% (within run-to-run noise). + +## Settings +- **setting 1** — hints: gold tables, column mapping, join keys. +- **setting 2** — setting 1 + domain knowledge + query decomposition. +- (setting 0 = retrieved tables only; not run here — needs `retrieve/`.) + +## Enhancements (all in `agent.py`, all gold-blind) +- **self-fix** (`CODEX_SQL_FIX=1`) — runs its own SQL read-only; on an execution + error, feeds back *only the DB error* and asks Codex to fix, looping up to + `CODEX_FIX_ATTEMPTS`. Stops once the query executes. → guarantees executable + output (SQL errors → 0); does not change correctness on its own. +- **explore/verify** (`CODEX_SQL_EXPLORE=1`) — the agent runs read-only queries + against the real tables (sample rows, counts, its candidate query) and inspects + the rows *its own* queries return, then self-checks and revises before + finalizing. Never shown gold rows. → +3–4 pts. +- **style guide** (`CODEX_STYLE_GUIDE=1`) — a "house style" prior appended to the + prompt: 8 conventions of this benchmark's reference SQL (see the style-guide + section below). Gold-blind per question, but fit to the dataset's conventions + via the failure taxonomy. → +3 net, and cracks 5 questions no prior run solved. + +## Findings +- **Lever ranking:** richer hints (+7) > explore/verify (+3–4) > self-fix + (robustness, ~0 acc) ≈ reasoning effort (within noise, high↔xhigh). +- Levers **stack independently**: setting 2 (30%) + explore/verify + fix → **34%**. +- **explore and fix are complementary**: explore improves correctness; the final + fix pass drives SQL errors to 0 (explore alone left 5). +- The dominant failure mode is **semantic** (~64 "values mismatch") at every + config — queries execute but return the wrong rows. Even with all 5 oracle + hints (setting 2), ~66% still miss, i.e. SQL *construction* on these enterprise + schemas is hard even when schema-linking is handed over. +- **Headroom:** the union of correct sets across runs (~39) exceeds any single + run, indicating self-consistency / majority-vote ensembling is the next lever. + +## Hint ablation: is the decomposition hint worth it? +Controlled ablation (Codex, high + explore/verify + fix), setting 2 with vs +without `--decomp` (all else identical: gold tables + mapping + join keys + +domain knowledge): + +| Config | exec acc | correct | +|--------|:--------:|:-------:| +| setting 2 (with decomp) | **34%** | 34 | +| setting 2 − decomp | 28% | 28 | + +Per-question, decomposition **helps 13, hurts 7 → net +6**. It is net-**positive** +despite occasionally backfiring. Don't drop it. + +## Failure analysis (setting 2 "values mismatch") +Two dissected mismatches — both cases where the *decomposition* hint embedded +gold-query scaffolding that contradicted the final ask, and the model baked it +into the answer: +- **dw_2933** — decomposition said "top 10 organizations"; the question asks "for + each organization". Agent added `LIMIT 10` → 10 rows vs gold's 154. +- **dw_104** — decomposition mentioned "a window of 2 preceding and 1 following"; + the question wants the *overall* average. Claude used a rolling + `AVG() OVER (... ROWS BETWEEN 2 PRECEDING AND 1 FOLLOWING)` (wrong per-row + deviations); Codex used `AVG() OVER ()` (overall) and got it right — a clean + backend-divergence case. + +Caveat (see ablation above): these are a real failure *mode* but a *minority* — +the ablation shows decomposition is net-positive overall. Hand-picked errors +identify modes; only the controlled run gives net impact. + +## Negative result: self-decomposition and subagent review hurt +Tested whether the agent's *own* reasoning could replace the provided +decomposition hint: drop `--decomp`, then have the agent self-decompose + +validate each sub-step with SQL (`CODEX_DECOMPOSE`), and/or add a final subagent +that reviews the answer vs the question for intent (`CODEX_REVIEW`). Isolation +(Codex, high, explore + fix, setting 2 minus decomp): + +| Config | exec acc | Δ vs baseline | +|--------|:--------:|:-------------:| +| neither (baseline, s2 − decomp) | 28% | — | +| + self-decompose only | 23% | −5 | +| + review only | 21% | −7 | +| + both | 21% | −7 | +| *(reference: s2 **with** decomp hint)* | *34%* | *+6* | + +Both components are **net-negative**. The **subagent review is the main culprit +(−7)**: gold-blind, it rewrites already-correct queries into wrong ones (adds more +errors than it removes). Self-decompose (−5) pushes toward more elaborate, fragile +constructions. `both = review-only` → review dominates once present. + +**Throughline of the study's three "more reasoning" hypotheses — all refuted by +controlled runs:** dropping the decomp hint hurt (−6); self-decomposing instead +hurt (−5); adding a review subagent hurt most (−7). On BEAVER `dw` the oracle +hints beat the agent's own reasoning, and self-critique without ground truth is +actively harmful. Winner stays the simple recipe: **hints + explore + fix (34%)**. + +## Failure taxonomy: why the misses miss +Categorized all 65 dossier-able failures of the best pre-style-guide run +(question + hints + gold + Codex/Claude best-run SQL, diffed per question): + +| Cause | n | What it is | +|-------|:-:|------------| +| underdetermined question | 28 | NL can't distinguish gold from pred: `COUNT` vs `COUNT(DISTINCT)`, LEFT-vs-INNER survivorship, unstated `>0` filters, `TERM_CODE` vs `EFFECTIVE_TERM_CODE`, running vs plain aggregates, how blocks combine | +| gold suspect | 18 | gold contradicts the question: `STDDEV_SAMP` despite "use STDDEV only" in the question text; "variance in 2022" with no year filter; dangling fan-out joins inflating sums | +| hint backfire | 10 | decomposition sub-questions contradict the final question ("top 10" vs "for each", "Chemistry only" vs "each department") and the model follows the hint | +| model error | 6 | mostly inter-block linkage in composed queries | +| evaluator artifact | 3 | formatting / extra display column | + +Only ~9% of failures are model SQL errors. In **46/65 (71%) both backends made +the identical non-gold choice** — convergent evidence the question or gold, not +the model, is the bottleneck (and a ceiling on what ensembling can recover). + +Structural signal: 50/100 questions were never solved by *any* of 11 full runs +(both backends, all configs). The 9 `base` (real, single-intent) questions solve +at 78% with zero never-solved; the 91 template-composed questions collapse +monotonically with sub-question count (0 parts → 0% never-solved; 4 → 65%). +Each composed sub-part plus the combination step is another independent chance +to diverge from gold's arbitrary convention. + +### Near-miss check: the evaluator is (almost) not the problem +Re-executed every failed gold/pred pair with looser matching (numeric +normalization, full column-permutation, drop-one-column projection): it rescues +exactly **1 question per run** (Codex dw_5330; Claude dw_427). The failures are +real result-set differences. Their shape is bimodal: ~40% share *zero* rows with +gold (one divergent aggregate column — a `STDDEV_SAMP` factor, a fan-out-inflated +`SUM` — poisons every tuple), ~20% overlap gold ≥90% (boundary-row conventions: +survivorship, rollup rows, LIMIT ties). Exact-set match hides how close misses are; +a partial-credit metric (row-F1) would separate the two modes. + +## Style guide: encode the house conventions → 37% (new best) +The taxonomy implies gold has a consistent *style*. `CODEX_STYLE_GUIDE=1` appends +8 gold-blind rules to the prompt (`_STYLE_GUIDE` in `agent.py`): (1) final +question overrides decomposition hints — no leaked top-k/filters/total rows; +(2) only the requested columns, in question order; (3) no DISTINCT inside +aggregates unless the question says "unique"; (4) "at least one related Y" = +fan-out join, not EXISTS; (5) LEFT JOIN when attaching secondary blocks; +(6) raw values — no ROUND/CAST/date parsing; (7) simplest window form; +(8) "more than" = `>`, "at least" = `>=`. + +Mechanism check on 10 targeted never-solved failures (`dev_styleguide10.json`), +best config, style arm vs same-config control re-run: **6/10 vs 0/10** — every +win maps 1:1 to its targeted rule. + +Full 100-question run: **37 vs 34** — gained 13 (incl. **5 from the never-solved +core**: dw_1133, dw_1970, dw_3310, dw_4588, dw_5136; cross-run union 50 → 55), +lost 10. Of the losses, 7 were flaky (solved ≤4/11 prior runs — churn); the +systematic backfire is **rule 5**: dw_3585 (previously 11/11) flipped INNER→LEFT +where gold uses INNER, and rule 3 cost dw_5668 where gold *does* use +`COUNT(DISTINCT)` — the golds are internally inconsistent on their own +conventions. Softening rule 5 (LEFT JOIN only when the question implies keeping +unmatched entities) is the obvious next tweak. + +Caveat for write-ups: the rules were derived from this benchmark's failure modes, +so this is a benchmark-adapted prior — gold-blind per question, but fit to the +dataset's house style. + +### Style guide iteration (v2/v3): what's durable and what's zero-sum +Targeted A/B on 12 questions (7 hint-backfires, 3 v1 regressions, 2 passing +guards): two changes are durable across independent runs — the **decomposition +precedence sentence** in `prompt.py` (question text overrides conflicting +sub-question filters/limits) and **rule 9** (use every hinted table/join key; +keep "redundant" bridge joins), which cracked never-solved dw_2933. **Rule 5's +direction (LEFT vs INNER when combining blocks) is zero-sum**: no wording wins +both dw_3310/dw_5136 (gold LEFT) and dw_3585 (gold INNER) — the NL is identical +in form ("for each X, show [stats A] and [stats B]") and gold is arbitrary; kept +default-LEFT because it targets never-solved questions. The 6 remaining +hint-backfire questions moved under no variant: each stacks a second gold-side +divergence (substituted sub-questions, fan-out inflation), so single fixes can't +reach them. + +## Schema skill: DB-profiled data facts → +1, and a diagnosis +`CODEX_SCHEMA_SKILL=1` injects per-question data facts profiled read-only from +the live DB (gold-blind: schema + data statistics only, never questions/gold) +from `dw_schema_skill.json`: table grains, value vocabularies with lookup-table +meanings (EO=Electronic options, RC=Recommended, ...), string-date columns +('DD-MON-YY' → MIN/MAX is lexicographic), TERM_CODE vs EFFECTIVE_TERM_CODE +differing in 78% of rows, dept names mapping to multiple codes, per-key fan-out +ratios. + +On 10 data-fact failures + 2 guards: **skill 4/12 vs control 3/12**, no +regressions. Only the pure-data-fact failure flipped (dw_140, lexicographic +dates — never solved by 13 prior runs). The facts visibly nudged the rest +(dw_5638 started hedging `IN ('RQ','EO')`; dw_779 started using +EFFECTIVE_TERM_CODE) without flipping them: the *menu* of values is learnable, +but gold's arbitrary NL→value assignment ("optional" = 'EO' alone; EFFECTIVE +only in one CTE) is not. Interventional confirmation that the residual wall is +gold conventions, not missing data knowledge. Keep the flag: free, +1, no +downside. Regenerate the skill with the profiler (scratchpad `profile_dw.py` +pattern) in ~3 min. + +## Multi-candidate: 3 answers spanning the ambiguity → pass@1 39%, pass@3 49% +`CODEX_N_CANDIDATES=3`: the final answer becomes `//` — +candidate 1 the best guess, candidates 2–3 flipping the choices the question +does not determine (COUNT distinctness, join survivorship, term-code column, +value mapping, window form). Candidates live in one .sql joined by +`-- ===CANDIDATE=== --`; score with `evaluate_ex_acc.py --multi` (any-match + +candidate-1 accuracy). + +Full-100 (style guide v3 + schema skill + N=3): + +| Metric | score | +|---|:---:| +| candidate 1 only (deployable single answer) | **39%** — new best | +| pass@3 (any candidate) | **49%** | + +Match histogram 39/9/1 (c1/c2/c3): the "second guess on the least-sure choice" +recovers 9 questions alone. Candidate 1 *improved* over the single-answer run +(37→39): articulating uncertainty doesn't hurt the primary. This one run cracked +8 of the original 50 never-solved and its pass@3 set approaches the all-time +union across ~15 runs (~56). The +10 pass@3−pass@1 band is the measured +coverable-ambiguity margin; converting it to real accuracy needs a gold-blind +selector over the executed result sets (self-agreement / cross-backend vote) — +the candidates already execute independently, so that selector is the next +cheap experiment. + +## Negative result #4: adversarial review still hurts — and now we know why +Rebuilt `CODEX_REVIEW` from the one-shot intent check (−7 above) into an +adversarial, tool-using reviewer: it gets the question + all N candidates with +their executed sample rows, probes with its OWN read-only queries (recompute one +group directly, check filter literals against the data, count rows across joins; +`CODEX_REVIEW_STEPS=4`), and may replace a candidate only on concrete +contradicting evidence — otherwise keep it byte-for-byte. Two measurements +against the frozen multi-candidate baseline (16 questions: 5 c1-pass, 5 +c2/3-pass, 6 fail): + +| Arm | pass@3 | Δ | +|-----|:------:|:--:| +| baseline (multi3, no review) | 10/16 | — | +| paired: reviewer applied to the *frozen* candidate bundles | 8/16 | **−2** | +| end-to-end: fresh generation with review in the loop | 6/16 | −4 | + +The conservatism worked: 13/16 bundles kept verbatim (the old reviewer rewrote +nearly everything). But **every intervention was harmful or neutral** — it broke +dw_3183 (c1-pass) and dw_287 (c2-pass) and fixed none of the 6 failures. + +The failure is structural, not prompt quality. dw_287 shows the mechanism: gold +silently re-applies filters the question never states; the *passing* candidate +mirrors gold's unstated filter, so an evidence-driven reviewer probes, finds the +candidate genuinely contradicts the question's plain text, and "corrects" it — +destroying the match. Gold-blind review optimizes question-faithfulness; the +residual scoring margin rewards gold-faithfulness; on exactly the questions +where a reviewer finds actionable evidence, the two point in opposite +directions. Meanwhile the errors probes can legitimately catch are already +eliminated upstream by explore + fix, leaving the reviewer only false positives +to act on. + +Fourth independently-measured negative for critique-style enhancements (naive +review −7, self-decompose −5, both −7, adversarial evidence-based review −2 +paired / −4 e2e): on BEAVER, self-critique without ground truth is +*systematically* anti-correlated with score at the margin where it acts. Ship +with `CODEX_REVIEW=0`. If ever revived, restrict the reviewer to *picking* +among candidates (reordering can't break a bundle) — never editing them. + +## GPT-5.6 rerun of the best config: same coverage, worse candidate ranking +Reran the best config (style guide v3 + schema skill + 3 candidates, setting 2, +high, explore + fix) unchanged on the new Codex default model (`gpt-5.6-sol`, +2026-07-10). Two results, one artifact: + +| Model | pass@1 (candidate 1) | pass@3 | +|-------|:--------------------:|:------:| +| gpt-5.5 (best run above) | **39%** | 49% | +| gpt-5.6, raw run | 30% | 47% | +| gpt-5.6, after entity fix | **33%** | **49%** | + +**Artifact:** gpt-5.6 HTML-escapes angle brackets inside `` spans on some +answers (`<>` for `<>`, `>` for `>`) — 8/100 predictions affected, 7 of +them 1064 syntax errors at eval time; gpt-5.5 produced 0. `clean_sql` in +`agent_common.py` now unescapes entities, and the 5.6 numbers above were +rescored on the same generations with the escaping undone (SQL errors → 0). + +**Reading:** pass@3 is identical to 5.5 (49 vs 49, symmetric churn: lost +dw_2878/3298/4570/5638, won dw_1570/2132/4771/779 — dw_779 finally flipped after +merely "hedging" under 5.5). The pass@1 gap is a **ranking regression**: 5.6 +lost candidate-1 on 7 questions and won 1, but 5 of the 7 losses still pass via +candidate 2/3 (match histogram c1/c2/c3: 5.5 = 39/9/1, 5.6 = 33/14/2). 5.6 +covers the same readings; it more often puts the gold-matching reading second. +Same conclusion as before, now model-robust: the pass@3−pass@1 band (here 16 +pts) is where the value is, and a gold-blind selector over executed result sets +remains the next experiment. Generation was noticeably faster (~28 min for 94 +questions, 4 workers). + +## Reproduce +```bash +# best config (single answer = candidate 1; pass@3 via --multi) +cd eval/myagent +CODEX_REASONING_EFFORT=high CODEX_SQL_EXPLORE=1 CODEX_SQL_FIX=1 \ +CODEX_STYLE_GUIDE=1 CODEX_SCHEMA_SKILL=1 CODEX_N_CANDIDATES=3 \ + ./run.sh --dataset dw --setting 2 + +# score (from eval/); drop --multi for single-candidate runs +cd .. && uv run python evaluate_ex_acc.py --dataset dw --multi \ + --input_dir unified-output/myagent/ +``` +Requires the `dw` MySQL DB loaded + `MYSQL_*` creds in `.env`. Per-run SQL outputs +and `summary_ex_acc.json` are written under `eval/unified-output/myagent/` +(gitignored — contains gold SQL from the gated dataset). diff --git a/eval/myagent/agent.py b/eval/myagent/agent.py new file mode 100644 index 0000000..16dd85c --- /dev/null +++ b/eval/myagent/agent.py @@ -0,0 +1,493 @@ +""" +============================================================================ + Codex-backed agent for BEAVER text-to-SQL. + Three generation modes (pick via env), all gold-blind: + * plain : one `codex exec` call -> SQL (default) + * self-fix : CODEX_SQL_FIX=1 -> run own SQL, fix execution errors + * explore+verify : CODEX_SQL_EXPLORE=1 -> run read-only queries against the + real tables, see the rows returned, self-check, finalize +============================================================================ + +The backend-agnostic machinery (response parsing, the read-only SQL guard, +mediated DB access, credential stripping, the explore protocol) lives in +eval/agent_common.py and is shared with eval/claudeagent. This file holds only +the Codex CLI invocation and the Codex-specific mode orchestration. + +NOTE on isolation: `codex exec --sandbox read-only` restricts writes and +network but NOT filesystem reads, so a determined model could still read files +by absolute path. This is defense-in-depth for a cooperative model, not a hard +security boundary — true isolation would require running the CLI in a +container without the data/ tree mounted. + + instance fields (built in prompt.py::build_instances): + id, question, db, tables, prompt (chat messages), hints, record + -> NOTE: do not read record['sql']; that is the gold answer. + +Config (env vars, all optional): + CODEX_MODEL Codex model (`codex -m`); unset -> account default. + CODEX_REASONING_EFFORT minimal|low|medium|high|xhigh (default: low) + CODEX_SANDBOX read-only|workspace-write|danger-full-access (default: read-only) + CODEX_TIMEOUT per codex-call seconds (default: 300) + CODEX_MAX_RETRIES retries for a failed/timed-out codex call (default: 2) + CODEX_BIN path to codex (default: codex) + + CODEX_SQL_FIX 1 -> fix execution errors via error feedback (default 0) + CODEX_FIX_ATTEMPTS max fix rounds (default 2) + + CODEX_SQL_EXPLORE 1 -> explore/verify loop (overrides SQL_FIX) (default 0) + CODEX_EXPLORE_STEPS max exploratory query rounds (default 4) + CODEX_EXPLORE_ROWS max rows returned per exploratory query (default 20) + CODEX_FIX_TIMEOUT_MS SELECT execution cap, ms (default 10000; keep <= the + scorer's QUERY_TIMEOUT so a query the agent verifies + as OK also passes scoring) + MYSQL_HOST/USER/PASSWORD DB creds (env or nearest .env) + + CODEX_DECOMPOSE 1 -> self-decompose the question and validate each + sub-step with SQL (requires CODEX_SQL_EXPLORE) + CODEX_REVIEW 1 -> adversarial review subagent: probes the executor's + candidate(s) with its OWN read-only queries (gold-blind) + and replaces a candidate only on concrete contradicting + evidence; keeps candidates byte-for-byte otherwise + CODEX_REVIEW_STEPS max reviewer probe rounds (default 4) + CODEX_STYLE_GUIDE 1 -> prepend the gold-blind reference-query house style + CODEX_SCHEMA_SKILL 1 -> inject profiled data facts (dw_schema_skill.json) + for the tables in play; profiled read-only from the + DB, gold-blind (see scratchpad profile_dw.py) + CODEX_N_CANDIDATES N>1 -> emit N candidate queries spanning plausible + readings of an ambiguous question, joined by the + '-- ===CANDIDATE=== --' marker in one .sql file; + score with evaluate_ex_acc.py --multi +""" +import os +import sys +import time +import shutil +import tempfile +import subprocess + +# Shared, backend-agnostic primitives (eval/agent_common.py). eval/ is this +# file's grandparent; add it to the path so the import resolves regardless of +# the working directory execute.py is launched from. +_EVAL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if _EVAL_DIR not in sys.path: + sys.path.append(_EVAL_DIR) + +from agent_common import ( # noqa: E402 + clean_sql, + render_prompt, + cli_env as _cli_env, + is_read_only as _is_read_only, + timed as _timed, + DBUnavailable as _DBUnavailable, + execute_sql as _execute_sql_raw, + query_preview as _query_preview_raw, + fix_prompt as _fix_prompt, + EXPLORE_PROTOCOL as _EXPLORE_PROTOCOL, + RUN_SQL_RE as _RUN_SQL_RE, +) + +CODEX_BIN = os.getenv("CODEX_BIN", "codex") +CODEX_MODEL = os.getenv("CODEX_MODEL") +CODEX_REASONING_EFFORT = os.getenv("CODEX_REASONING_EFFORT", "low") +CODEX_SANDBOX = os.getenv("CODEX_SANDBOX", "read-only") +CODEX_TIMEOUT = int(os.getenv("CODEX_TIMEOUT", "300")) +CODEX_MAX_RETRIES = int(os.getenv("CODEX_MAX_RETRIES", "2")) + +CODEX_SQL_FIX = os.getenv("CODEX_SQL_FIX", "0") not in ("0", "", "false", "False") +CODEX_FIX_ATTEMPTS = int(os.getenv("CODEX_FIX_ATTEMPTS", "2")) + +CODEX_SQL_EXPLORE = os.getenv("CODEX_SQL_EXPLORE", "0") not in ("0", "", "false", "False") +CODEX_EXPLORE_STEPS = int(os.getenv("CODEX_EXPLORE_STEPS", "4")) +CODEX_EXPLORE_ROWS = int(os.getenv("CODEX_EXPLORE_ROWS", "20")) +CODEX_FIX_TIMEOUT_MS = int(os.getenv("CODEX_FIX_TIMEOUT_MS", "10000")) + +# Self-decompose the question (and validate each sub-step with SQL) instead of +# relying on a provided decomposition hint. +CODEX_DECOMPOSE = os.getenv("CODEX_DECOMPOSE", "0") not in ("0", "", "false", "False") +# Final subagent pass: adversarial review of the executor's candidates. The +# reviewer probes with its OWN read-only queries (gold-blind) and replaces a +# candidate only on concrete contradicting evidence. +CODEX_REVIEW = os.getenv("CODEX_REVIEW", "0") not in ("0", "", "false", "False") +CODEX_REVIEW_STEPS = int(os.getenv("CODEX_REVIEW_STEPS", "4")) +# Gold-blind "house style" prior: conventions of this benchmark's reference SQL +# (derived from failure-mode analysis, not from any per-question gold answer). +CODEX_STYLE_GUIDE = os.getenv("CODEX_STYLE_GUIDE", "0") not in ("0", "", "false", "False") +# Schema skill: per-table data facts profiled read-only from the DB (gold-blind). +CODEX_SCHEMA_SKILL = os.getenv("CODEX_SCHEMA_SKILL", "0") not in ("0", "", "false", "False") +# Emit N candidate queries spanning the plausible readings of an ambiguous +# question (1 = classic single answer). Candidates are stored in one .sql file +# joined by _CANDIDATE_SEP; score with evaluate_ex_acc.py --multi. +CODEX_N_CANDIDATES = max(1, int(os.getenv("CODEX_N_CANDIDATES", "1"))) +_CANDIDATE_SEP = "\n-- ===CANDIDATE=== --\n" +_SCHEMA_SKILL_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "dw_schema_skill.json") +_schema_skill_cache = None + + +def _schema_skill_section(tables): + """Build the schema-facts block for the tables in play (plus global notes).""" + global _schema_skill_cache + if _schema_skill_cache is None: + import json + with open(_SCHEMA_SKILL_PATH, encoding="utf-8") as f: + _schema_skill_cache = json.load(f) + parts = [_schema_skill_cache["_global"]] + parts += [_schema_skill_cache[t] for t in tables if t in _schema_skill_cache] + return ( + "\n\n### Database facts (profiled read-only from the live dw database)\n" + + "\n\n".join(parts) + ) + + +# ----------------------- Codex CLI invocation ----------------------- + +def _codex_call(prompt: str) -> str: + """One headless `codex exec` call -> raw final message text. Retries a + failed/timed-out call up to CODEX_MAX_RETRIES times so a transient CLI + failure does not silently become an empty prediction.""" + last_err = None + for attempt in range(CODEX_MAX_RETRIES + 1): + workdir = tempfile.mkdtemp(prefix="codex_beaver_") + last_msg = os.path.join(workdir, "_last_message.txt") + cmd = [ + CODEX_BIN, "exec", "--sandbox", CODEX_SANDBOX, "--skip-git-repo-check", + "-C", workdir, "-c", f"model_reasoning_effort={CODEX_REASONING_EFFORT}", "-o", last_msg, + ] + if CODEX_MODEL: + cmd += ["-m", CODEX_MODEL] + cmd.append(prompt) + try: + proc = subprocess.run( + cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=CODEX_TIMEOUT, env=_cli_env(), + ) + raw = "" + if os.path.exists(last_msg): + with open(last_msg, encoding="utf-8") as f: + raw = f.read() + if raw.strip(): + return raw + if proc.returncode == 0: + return raw # rc==0 with no message: genuine empty answer, do not retry + tail = ((proc.stdout or "") + (proc.stderr or ""))[-800:].strip() + last_err = f"codex exec failed (rc={proc.returncode}): {tail}" + except subprocess.TimeoutExpired: + last_err = f"codex exec timed out after {CODEX_TIMEOUT}s" + finally: + shutil.rmtree(workdir, ignore_errors=True) + if attempt < CODEX_MAX_RETRIES: + time.sleep(2 * (attempt + 1)) + raise RuntimeError(last_err or "codex exec failed") + + +def _codex_generate(prompt: str) -> str: + return clean_sql(_codex_call(prompt)) + + +# DB access with this backend's execution timeout bound in (the primitives take +# the cap as a parameter; these thin wrappers keep the internal call sites tidy). +def _execute_sql(sql: str, db: str): + return _execute_sql_raw(sql, db, CODEX_FIX_TIMEOUT_MS) + + +def _query_preview(sql: str, db: str, max_rows: int): + return _query_preview_raw(sql, db, max_rows, CODEX_FIX_TIMEOUT_MS) + + +# ------------------------------- modes ------------------------------- + +def _fix_loop(base, db, sql): + """Given a candidate SQL, repair execution errors (error feedback only).""" + if not sql: + return sql + for _ in range(CODEX_FIX_ATTEMPTS): + try: + ok, err = _execute_sql(sql, db) + except _DBUnavailable as e: + # Can't verify (DB down / bad creds): do NOT rewrite a possibly-correct query. + print(f"DB unavailable; skipping fix loop, keeping candidate SQL: {e}") + break + if ok: + break + fixed = _codex_generate(_fix_prompt(base, db, sql, err)) + if not fixed or fixed == sql: + break + sql = fixed + return sql + + +_STYLE_GUIDE = """\ + +### Reference-query house style (follow unless the question explicitly says otherwise) +Your answer is scored by exact result-set match against a reference query written in a +rigid data-warehouse house style. Mirror these conventions even when an alternative +reading seems cleaner or more "correct": + +1. The final user question is the sole authority on the output. Decomposition + sub-questions are scaffolding for HOW to structure the query: if a sub-question + mentions a filter, top-k limit, ranking column, or extra total row that the final + question does not ask for, do NOT let it into the result. Never add LIMIT unless the + final question asks for a top/bottom subset. Never emit grand-total or subtotal rows + unless the final question asks for them. +2. Output exactly the columns the final question asks for, in the order it lists them — + no extra identifier, ranking, or ordering columns. +3. Aggregate over the raw join result. Do NOT add DISTINCT inside COUNT/SUM/AVG to + compensate for row duplication introduced by joins — duplicated rows are intended + weighting in this warehouse. Use COUNT(DISTINCT ...) only when the question + explicitly says "unique", "distinct", or "different". +4. Implement "X with at least one related Y" by JOINing Y directly (keeping the + resulting row multiplication in downstream aggregates), not via EXISTS/IN semi-joins. +5. When attaching a secondary block (stats, top-k membership, comparison values) to a + main block, use LEFT JOIN and keep non-matching rows with NULLs; do not INNER JOIN + away rows unless the question says to exclude them. +6. Keep raw values raw: no ROUND, no CAST, no STR_TO_DATE/DATE_FORMAT on string-typed + date columns (compare and MIN/MAX them as plain strings), no reformatting. +7. Window functions: use the simplest form — default frames (no explicit ROWS BETWEEN), + and ORDER BY inside OVER() only when the question asks for a running/cumulative value. +8. "more than" / "greater than" / "over" = strict >; "at least" / "no less than" = >= + (and symmetrically for "less than" vs "at most"). +9. Use EVERY provided table and EVERY provided join key. If a hinted table looks + redundant (contributes no output columns, or a more direct join key exists), join it + anyway along the hinted path and keep the row multiplication it causes — bridge + tables are part of the intended semantics, not noise to optimize away.\ +""" + + +_DECOMPOSE_GUIDANCE = """\ + +### Approach: decompose, then validate each part with SQL +Break the question into its sub-steps (filters, joins, groupings, aggregations, +top-k/window pieces). For EACH sub-step, write a small SQL query and RUN it to +confirm that piece returns sensible data — spot-check intermediate results (row +counts, sample values, distinct keys, ranges) so each part looks valid before you +compose them. Only after the parts check out, compose the full query, run it, and +confirm the final rows match the question's intent.\ +""" + + +_MULTI_ANSWER_INSTR = """\ + +### Final answer format: {n} candidate queries (this replaces the single format) +Enterprise questions like these often admit more than one defensible reading — the +question text may not pin down choices such as: COUNT vs COUNT(DISTINCT); INNER vs +LEFT JOIN when combining blocks (drop vs keep unmatched rows as NULLs); which of two +similar columns to use (e.g. TERM_CODE vs EFFECTIVE_TERM_CODE); which code value a +phrase maps to; a plain aggregate vs a running/windowed one; whether an auxiliary +filter also applies to a comparison population. + +Give your final answer as {n} COMPLETE, independently executable MySQL queries that +span the most plausible readings, each wrapped in its own numbered tag from to + (and matching ... ): +your best-guess query +second query, changing the choice you are LEAST sure about +... continue through , each changing a different uncertain choice. + +Rules: candidate 1 is your best guess. Candidates must differ SEMANTICALLY (different +rows/values), not by formatting or aliases. Each must return the same columns in the +same order. Do not change choices the question clearly determines.\ +""" + + +def _split_candidates(text: str): + """Extract .. ... blocks; fall back to a single block.""" + out = [] + for i in range(1, CODEX_N_CANDIDATES + 1): + tag, close = f"", f"" + if tag in text and close in text: + out.append(text.split(tag, 1)[1].split(close, 1)[0].strip()) + if not out: + single = clean_sql(text) + if single: + out = [single] + return out + + +def _parse_action(text: str): + """Return ('answer', sql) or ('run', query) from a model turn. + + A RUN_SQL request takes precedence over a bare '' *mention* (e.g. the + model restating the protocol); only a complete ... span is + treated as a final answer that overrides an accompanying RUN_SQL.""" + if not text: + return "answer", "" + m = _RUN_SQL_RE.search(text) + has_complete_ans = ("" in text and "" in text) or ( + "" in text and "" in text) + if m and not has_complete_ans: + q = text[m.end():].split(" 1: + return "answer", _CANDIDATE_SEP.join(_split_candidates(text)) + return "answer", clean_sql(text) + + +def _run_explore(base, db): + guidance = _DECOMPOSE_GUIDANCE if CODEX_DECOMPOSE else "" + multi = _MULTI_ANSWER_INSTR.format(n=CODEX_N_CANDIDATES) if CODEX_N_CANDIDATES > 1 else "" + transcript = base + guidance + _EXPLORE_PROTOCOL.format(db=db, steps=CODEX_EXPLORE_STEPS) + multi + queries_run = 0 + nudged = False + for step in range(CODEX_EXPLORE_STEPS): + resp = _codex_call(transcript) + action, payload = _parse_action(resp) + if action == "answer" and payload: + # Enforce the protocol's "verify first" rule with a single nudge. + if queries_run == 0 and not nudged and step < CODEX_EXPLORE_STEPS - 1: + nudged = True + transcript += ( + f"\n\n### Proposed answer (NOT yet verified)\n{payload}\n\n" + "You have not run any verification query. As required, run at least one " + "read-only RUN_SQL query to check this answer before giving ." + ) + continue + return payload + if action == "run" and payload: + queries_run += 1 + result = _query_preview(payload, db, CODEX_EXPLORE_ROWS) + transcript += ( + f"\n\n### Your query (step {step + 1})\n{payload}\n\n### Result\n{result}\n\n" + f"Run another RUN_SQL query, or output your final ...." + ) + else: + break + # Out of steps (or no parseable action): force a final answer. Do NOT fall + # back to the last exploratory probe — a DESCRIBE/COUNT probe is not an answer. + if CODEX_N_CANDIDATES > 1: + final = _codex_call( + transcript + "\n\nYou must now output ONLY your final answer in the " + f".. ... .. format.", + ) + return _CANDIDATE_SEP.join(_split_candidates(final)) + final = _codex_call( + transcript + "\n\nYou must now output ONLY your final answer as YOUR MYSQL QUERY.", + ) + return clean_sql(final) + + +_ADVERSARIAL_REVIEW_PROTOCOL = """\ + +### Your task: ADVERSARIAL review of the executor's candidate queries +Another agent (the executor) answered the question above with the {n} candidate +quer{ies} shown below, together with a sample of the rows each returns. Your job is +to try to REFUTE each candidate by running your OWN read-only queries — probes that +are DIFFERENT from the candidate itself, chosen to expose an error if one exists: +- recompute one group's aggregate directly from the base tables and compare it to + the candidate's value for that group; +- check that every literal the candidate filters on actually exists in that column + (SELECT DISTINCT / COUNT of the value); +- count rows before and after each join to detect unintended fan-out or dropped rows; +- check the output column count and order against what the question asks for; +- check for elements leaked from the decomposition hints that the final question + does not ask for (LIMIT/top-k, extra filters, grand-total rows, ranking columns). + +Merely re-running a candidate is NOT verification. + +Verdict rules — be adversarial about EVIDENCE, conservative about EDITS: +- You may REPLACE a candidate only when a probe produced CONCRETE CONTRADICTING + EVIDENCE: a recomputed number that disagrees, a filter literal that does not exist + in the data, a join provably dropping/multiplying rows against the question's + meaning, a column set that does not match the question. +- Style preferences, alternative readings of ambiguous phrasing, or "I would have + written it differently" are NOT evidence — in that case KEEP the candidate + byte-for-byte unchanged. +- The candidates deliberately span DIFFERENT plausible readings of the question's + ambiguities. Do NOT collapse them into one reading; refute a candidate only on + its own terms. + +Respond with EXACTLY ONE of the following each turn (nothing else): + +1) To run a read-only probe (SELECT/WITH/SHOW/DESCRIBE/EXPLAIN), output: +RUN_SQL: + + +2) When done, output the final candidate set — confirmed candidates copied +byte-for-byte, refuted ones replaced by your corrected query: +{tags} + +You have at most {steps} probes. Run at least one probe per candidate before finalizing.\ +""" + + +def _split_candidates_indexed(text: str, n: int): + """Extract blocks by index; None where a tag is absent.""" + out = [] + for i in range(1, n + 1): + tag, close = f"", f"" + if tag in text and close in text: + out.append(text.split(tag, 1)[1].split(close, 1)[0].strip() or None) + else: + out.append(None) + return out + + +def _review(question, sql, db): + """Adversarial review subagent (gold-blind): probes the executor's candidates + with its own read-only queries and replaces a candidate only on concrete + contradicting evidence. Works on the whole candidate bundle at once so probes + are shared and the deliberate spread across readings is preserved.""" + candidates = [c.strip() for c in sql.split(_CANDIDATE_SEP) if c.strip()] + if not candidates: + return sql + n = len(candidates) + ies = "y" if n == 1 else "ies" + tags = "\n".join(f"candidate {i}, confirmed or corrected" for i in range(1, n + 1)) + + shown = [] + for i, cand in enumerate(candidates, 1): + preview = _query_preview(cand, db, CODEX_EXPLORE_ROWS) + shown.append(f"#### Candidate {i}\n{cand}\n\n#### Candidate {i} result (sample)\n{preview}") + transcript = ( + f"A text-to-SQL executor was given this task:\n\nQUESTION:\n{question}\n\n" + + "\n\n".join(shown) + + _ADVERSARIAL_REVIEW_PROTOCOL.format(n=n, ies=ies, tags=tags, steps=CODEX_REVIEW_STEPS) + ) + for step in range(CODEX_REVIEW_STEPS): + resp = _codex_call(transcript) + m = _RUN_SQL_RE.search(resp) + has_final = "" in resp and "" in resp + if m and not has_final: + probe = resp[m.end():].split(" str: + # `model` is a run label only (used for the output-dir name); the backend + # model is selected by the CODEX_MODEL env var, not this argument. + base = render_prompt(instance) + if CODEX_STYLE_GUIDE: + base += _STYLE_GUIDE + if CODEX_SCHEMA_SKILL: + base += _schema_skill_section(instance.get("tables") or []) + db = instance.get("db") or "dw" + question = instance.get("question", "") + if CODEX_DECOMPOSE and not CODEX_SQL_EXPLORE: + print("warning: CODEX_DECOMPOSE requires CODEX_SQL_EXPLORE=1; ignoring it this run.") + if CODEX_SQL_EXPLORE: + sql = _run_explore(base, db) + elif CODEX_N_CANDIDATES > 1: + raw = _codex_call(base + _MULTI_ANSWER_INSTR.format(n=CODEX_N_CANDIDATES)) + sql = _CANDIDATE_SEP.join(_split_candidates(raw)) + else: + sql = _codex_generate(base) + if CODEX_REVIEW and sql: # adversarial review (handles single or multi bundle) + sql = _review(question, sql, db) + if CODEX_SQL_FIX and sql: # final error-repair pass (per candidate in multi mode) + if CODEX_N_CANDIDATES > 1 and _CANDIDATE_SEP in sql: + fixed = [_fix_loop(base, db, cand.strip()) for cand in sql.split(_CANDIDATE_SEP)] + sql = _CANDIDATE_SEP.join(c for c in fixed if c) + else: + sql = _fix_loop(base, db, sql) + return sql diff --git a/eval/myagent/dw_schema_skill.json b/eval/myagent/dw_schema_skill.json new file mode 100644 index 0000000..fc01307 --- /dev/null +++ b/eval/myagent/dw_schema_skill.json @@ -0,0 +1,37 @@ +{ + "_global": "Facts below were profiled read-only from the live `dw` database (schema + data\nstatistics only). They are facts about the data, not query instructions.\n- Date-like columns are STRING typed with format 'DD-MON-YY' (e.g. '01-FEB-10').\n MIN/MAX/ORDER BY on them is LEXICOGRAPHIC, not chronological.\n- Term codes are strings like '2022FA' (year + FA/SP/SU/JA). ACADEMIC_YEAR-style\n columns are numeric years.\n- Some text columns contain the literal string 'nan' as a missing-value marker\n (distinct from NULL); flagged per table below.\n- Many *_KEY/*_CODE/*_ID columns are non-unique; joining on them multiplies rows.\n Per-table \"fan-out\" notes give rows-per-distinct-value for such columns.", + "ACADEMIC_TERMS": "ACADEMIC_TERMS (144 rows)\n- unique per row: ACADEMIC_TERMS_KEY, TERM_CODE, TERM_DESCRIPTION, TERM_SELECTOR\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): TERM_START_DATE, TERM_END_DATE, LAST_DAY_OF_FINAL_EXAM, PRE_REGISTRATION_START_DAY, REGISTRATION_DAY, FIRST_DAY_OF_CLASSES, LAST_DAY_OF_CLASSES, ADD_DATE, DROP_DATE, GRADUATE_AWARD_START_DATE, GRADUATE_AWARD_END_DATE\n- IS_CURRENT_TERM values: N(143), Y(1)\n- IS_REGULAR_TERM values: N(72), Y(72)\n- TERM_STATUS_INDICATOR values: P(120), NULL(21), F(2), C(1)\n- TERM_STATUS values: Previous(120), Unspecified(21), Future(2), Current(1)", + "ACADEMIC_TERMS_ALL": "ACADEMIC_TERMS_ALL (300 rows)\n- unique per row: ACADEMIC_TERMS_KEY, TERM_CODE, TERM_DESCRIPTION, TERM_SELECTOR\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): TERM_START_DATE, TERM_END_DATE, LAST_DAY_OF_FINAL_EXAM, PRE_REGISTRATION_START_DAY, REGISTRATION_DAY, FIRST_DAY_OF_CLASSES, LAST_DAY_OF_CLASSES, GRADUATE_AWARD_START_DATE, GRADUATE_AWARD_END_DATE\n- IS_CURRENT_TERM values: N(299), Y(1)\n- TERM_STATUS_INDICATOR values: P(276), NULL(21), F(2), C(1)", + "ACADEMIC_TERM_PARAMETER": "ACADEMIC_TERM_PARAMETER (3 rows)\n- unique per row: TERM_PARAMETER, TERM_INDICATOR, TERM_CODE, TERM_DESCRIPTION\n- TERM_PARAMETER values: SIS_CURRENT_TERM(1), SIS_PREVIOUS_TERM(1), SIS_UPCOMING_TERM(1)\n- TERM_INDICATOR values: C(1), P(1), F(1)\n- TERM_CODE values: 2024SU(1), 2025FA(1), 2025JA(1)\n- TERM_DESCRIPTION values: Fall Term 2024-2025(1), Summer Term 2024(1), January Term 2024-2025(1)\n- TERM_START_DATE values: 03-SEP-24(1), 10-JUN-24(1), 06-JAN-25(1)\n- TERM_END_DATE values: 20-DEC-24(1), 20-AUG-24(1), 31-JAN-25(1)\n- TERM_LAST_DAY_BEFORE_NEXT_TERM values: 05-JAN-25(1), 02-SEP-24(1), 31-JAN-25(1)\n- IS_CURRENT_TERM values: N(2), Y(1)", + "CIP": "CIP (3,059 rows)\n- unique per row: PROGRAM_CODE\n- WAREHOUSE_LOAD_DATE values: 17-MAY-23(2142), 11-JUN-14(917)\n- VERSION values: 2020(2142), 1990(606), 2000(159), 2010(152)\n- fan-out (non-unique) keys: CATEGORY_CODE (~58.83 rows/value), FOUR_DIGIT_CODE (~5.26 rows/value)", + "CIS_COURSE_CATALOG": "CIS_COURSE_CATALOG (10,000 rows)\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): LAST_ACTIVITY_DATE\n- IS_OFFERED_FALL_TERM values: Y(7321), N(2679)\n- IS_OFFERED_IAP values: N(7206), Y(2794)\n- IS_OFFERED_SPRING_TERM values: Y(7440), N(2560)\n- IS_OFFERED_SUMMER_TERM values: N(7645), Y(2355)\n- ACADEMIC_YEAR values: 2008(1444), 2007(1429), 2006(1377), 2005(1369), 2009(1353), 2004(1185), 2003(1018), 2002(825)\n- IS_PRINTED_IN_BULLETIN values: Y(6376), N(3624)\n- EFFECTIVE_TERM_CODE: 25 distinct, e.g. 1989FA, 2002FA, 2007FA, 2004FA, 2005FA, 2003FA, 2006FA, 2001FA, 1997FA, 2008FA, 2000FA, 1994FA, 1996FA, 1999FA, 1998FA, ...\n- IS_VARIABLE_UNITS values: Y(6286), N(3714)\n- LECTURE_UNITS values: 0(6307), 3(2256), 2(888), 4(300), 1(176), 5(47), 6(26)\n- PREPARATION_UNITS values: 0(6348), 9(1443), 4(685), 6(599), 8(278), 3(190), 2(125), 5(104), 1(102), 7(68), 10(37), 12(13), 18(7), 11(1)\n- TOTAL_UNITS values: 0(6286), 12(1791), 6(943), 9(635), 3(162), 2(61), 4(56), 8(20), 15(15), 1(8), 18(7), 24(7), 5(5), 7(4)\n- DESIGN_UNITS values: 0(9953), 4(22), 6(17), 8(4), 3(4)\n- GRADE_TYPE values: L(7043), P(2957)\n- GRADE_TYPE_DESC values: Letter graded(7043), P/D/F(2957)\n- GRADE_RULE values: R(5454), N(3294), J(1232), T(20)\n- GRADE_RULE_DESC values: Can be repeated for credit(5454), Not repeatable for credit(3294), Continuing and Repeatable(1232), Continuing(20)\n- HGN_CODE values: H(4654), U(3772), G(1574)\n- HGN_DESC values: High Graduate(4654), Undergraduate(3772), Graduate(1574)\n- COMM_REQ_ATTRIBUTE values: NULL(9906), CIM(94)\n- COMM_REQ_ATTRIBUTE_DESC values: NULL(9906), Communication Intensive Major(94)\n- TUITION_ATTRIBUTE values: NULL(9507), RESH(478), NTRN(15)\n- TUITION_ATTRIBUTE_DESC values: NULL(9507), Pre-thesis Research Subject(478), Internship(15)\n- WRITE_REQ_ATTRIBUTE values: NULL(9991), WRT2(9)\n- WRITE_REQ_ATTRIBUTE_DESC values: NULL(9991), Writing Requirement, Phase II(9)\n- SUPERVISOR_ATTRIBUTE values: NULL(9147), UROP(508), THG(187), THU(158)\n- SUPERVISOR_ATTRIBUTE_DESC values: NULL(9147), UROP subject(508), Grad Thesis(187), Undergrad Thesis(158)\n- IS_OFFERED_THIS_YEAR values: Y(8903), NULL(726), N(371)\n- fan-out (non-unique) keys: SUBJECT_ID (~4.41 rows/value), SUBJECT_CODE (~243.9 rows/value), SOURCE_SUBJECT_ID (~4.41 rows/value), PRINT_SUBJECT_ID (~4.38 rows/value), DEPARTMENT_CODE (~212.77 rows/value), EFFECTIVE_TERM_CODE (~400.0 rows/value)\n- CAUTION: 2 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains", + "COURSE_CATALOG_SUBJECT_OFFERED": "COURSE_CATALOG_SUBJECT_OFFERED (10,000 rows)\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): LAST_ACTIVITY_DATE\n- ACADEMIC_YEAR: 24 distinct, e.g. 2024, 2021, 2025, 2023, 2022, 2020, 2019, 2017, 2014, 2016, 2018, 2010, 2015, 2009, 2013, ...\n- IS_PRINTED_IN_BULLETIN values: Y(9274), N(725), S(1)\n- IS_VARIABLE_UNITS values: N(7852), Y(2148)\n- LECTURE_UNITS values: 3(4113), 0(2477), 4(1436), 2(1000), 5(674), 1(239), 6(54), 9(6), 8(1)\n- LAB_UNITS: 18 distinct, e.g. 0, 2, 3, 1, 4, 6, 8, 12, 9, 7, 5, 10, 20, 16, 24, ...\n- PREPARATION_UNITS: 17 distinct, e.g. 9, 0, 8, 6, 7, 4, 3, 5, 2, 1, 10, 12, 11, 18, 15, ...\n- TOTAL_UNITS: 24 distinct, e.g. 12, 0, 6, 9, 3, 15, 1, 4, 18, 2, 21, 13, 16, 24, 20, ...\n- DESIGN_UNITS values: 0(9829), 4(73), 12(47), 6(28), 2(13), 3(5), 8(4), 9(1)\n- GRADE_TYPE values: L(8597), P(1403)\n- GRADE_TYPE_DESC values: Letter graded(8597), P/D/F(1403)\n- GRADE_RULE values: N(7092), R(2242), J(557), T(109)\n- GRADE_RULE_DESC values: Not repeatable for credit(7092), Can be repeated for credit(2242), Continuing and Repeatable(557), Continuing(109)\n- HGN_CODE values: U(5092), G(2824), H(2084)\n- HGN_DESC values: Undergraduate(5092), Graduate(2824), High Graduate(2084)\n- HGN_EXCEPT values: NULL(9975), (H except 18)(15), (H except XVIII)(6), (H except 2, 6, 8, 12, 13, 16, 18, 22)(2), (H except II, VI, VIII, XII, XIII, XVI, XVIII, XXII)(1), H except XVIII(1)\n- GIR_ATTRIBUTE: 18 distinct, e.g. None, HE, REST, LAB, LAB2, CAL2, CHEM, HD4, BIOL, PHY1, HD2, PHY2, HD3, RST2, CAL1, ...\n- GIR_ATTRIBUTE_DESC: 19 distinct, e.g. None, HASS Elective, Rest Elec in Sci & Tech, Institute Lab, Calculus II, Chemistry, HASS-D, Category 4, Biology, Physics I, HASS-D, Category 2, Physics II, HASS-D, Category 3, 1/2 Institute Lab, 1/2 Rest Elec in Sci & Tech, Calculus I, ...\n- COMM_REQ_ATTRIBUTE values: NULL(9147), CIM(469), CIH(347), CIHW(37)\n- COMM_REQ_ATTRIBUTE_DESC values: NULL(9147), Communication Intensive Major(469), Communication Intensive HASS(347), Communication Intensive Writing(37)\n- TUITION_ATTRIBUTE values: NULL(9614), RESH(288), NTRN(96), COOP(2)\n- TUITION_ATTRIBUTE_DESC values: NULL(9614), Pre-thesis Research Subject(288), Internship(96), Co-op Subject(2)\n- WRITE_REQ_ATTRIBUTE values: NULL(9993), WRT2(5), WRT1(2)\n- WRITE_REQ_ATTRIBUTE_DESC values: NULL(9993), Writing Requirement, Phase II(5), Writing Requirement, Phase I(2)\n- SUPERVISOR_ATTRIBUTE values: NULL(9393), UROP(214), INDP(208), THG(120), THU(65)\n- SUPERVISOR_ATTRIBUTE_DESC values: NULL(9393), UROP subject(214), Independent Study(208), Grad Thesis(120), Undergrad Thesis(65)\n- IS_OFFERED_THIS_YEAR values: Y(8911), N(958), NULL(131)\n- IS_OFFERED_FALL_TERM values: Y(6653), N(3347)\n- IS_OFFERED_IAP values: N(8512), Y(1488)\n- IS_OFFERED_SPRING_TERM values: Y(6860), N(3140)\n- IS_OFFERED_SUMMER_TERM values: N(8782), Y(1218)\n- HASS_ATTRIBUTE values: NULL(8972), HH(479), HS(298), HA(234), HE(9), HA,HH(8)\n- HASS_ATTRIBUTE_DESC values: NULL(8972), HASS Humanities(479), HASS Social Sciences(298), HASS Arts(234), HASS Elective(9), Arts + Humanities(8)\n- TERM_DURATION values: Full Term Subject(6724), NULL(2849), Second Half Term Subject(187), First Half Term Subject(175), Partial Term Subject(65)\n- GLOBAL_COUNTRIES: 20 distinct, e.g. None, China, France, Japan, United States of America, Spain, Developing Countries, Germany, Brazil|Uruguay|Vietnam|Russia|Australia, Mexico|Spain, Indonesia, China|India, United States of America|India|South Africa, Jordan, France|Russia|United Kingdom|Germany, ...\n- IS_MASTER_SECTION values: N(4682), Y(4239), NULL(1079)\n- IS_LECTURE_SECTION values: N(6106), Y(2815), NULL(1079)\n- IS_LAB_SECTION values: N(8445), NULL(1079), Y(476)\n- IS_RECITATION_SECTION values: N(7580), Y(1341), NULL(1079)\n- IS_DESIGN_SECTION values: N(8879), NULL(1079), Y(42)\n- fan-out (non-unique) keys: TERM_CODE (~105.26 rows/value), SUBJECT_ID (~2.01 rows/value), SUBJECT_CODE (~181.82 rows/value), SOURCE_SUBJECT_ID (~2.19 rows/value), PRINT_SUBJECT_ID (~1.98 rows/value), DEPARTMENT_CODE (~166.67 rows/value)\n- CAUTION: EFFECTIVE_TERM_CODE and TERM_CODE DIFFER in 6988/8921 rows (78%) \u2014 not interchangeable\n- CAUTION: 4 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains", + "FCLT_BUILDING": "FCLT_BUILDING (242 rows)\n- unique per row: FCLT_BUILDING_KEY, BUILDING_NUMBER, BUILDING_SORT\n- PARENT_BUILDING_NUMBER values: NULL(211), W61(10), 14(4), 62(3), 64(3), W85ABC(3), W85HJK(3), W85DE(2), W85FG(2), 42(1)\n- PARENT_BUILDING_NAME values: NULL(211), MACGREGOR HOUSE(10), HAYDEN MEMORIAL LIBRARY(4), ALUMNI HOUSES: MUNROE HAYDEN WOOD(3), EAST CAMPUS: WALCOTT BEMIS GOODALE(3), WESTGATE (ABC)(3), WESTGATE (HJK)(3), WESTGATE (DE)(2), WESTGATE (FG)(2), COGENERATION PLANT(1)\n- PARENT_BUILDING_NAME_LONG values: NULL(211), Frank S MacGregor House(10), Charles Hayden Memorial Library(4), Alumni Houses: Munroe Hayden Wood(3), Alumni Houses: Walcott Bemis Goodale(3), Westgate ABC(3), Westgate HJK(3), Westgate DE(2), Westgate FG(2), William R. Dickson Cogeneration Plant(1)\n- SITE values: MIT(198), BATES(14), HAY(12), LINC(9), END(2), DC(2), BOS(2), WILM(1), HOLYOKE(1), MED(1)\n- CAMPUS_SECTOR values: WEST(71), MAIN GROUP(60), OFFCAMPUS(44), EAST(25), NORTHWEST(22), NORTH(11), NORTHEAST(8), WESTWEST(1)\n- ACCESS_LEVEL_CODE values: 2(185), 1(47), 0(10)\n- ACCESS_LEVEL_NAME values: 2(185), 1(47), 0(10)\n- BUILDING_TYPE values: ACADEMIC(126), SERVICE(59), RESIDENT(57)\n- OWNERSHIP_TYPE values: OWNED(220), LEASED(22)\n- BUILDING_USE values: AER(124), DHOA(54), OTH(32), STAC(17), (NULL)(8), GAR(7)\n- OCCUPANCY_CLASS: 20 distinct, e.g. (NULL), UGB, UGR2, UGBA3, UGA3, UGU, UGF1, UGA3B, UGS2, UGR1A3, UGBI2, UGBR3, UGE, UGA1A3, UGBR1, ...\n- DATE_ACQUIRED: 25 distinct, e.g. None, 07/01/1963, 05/01/1961, 07/01/2016, 02/01/2016, 04/16/1982, 02/13/2024, 04/01/1980, 12/31/1945, 07/01/1997, 01/01/1960, 04/01/1966, 02/01/1966, 10/01/1958, 12/31/1947, ...\n- fan-out (non-unique) keys: ACCESS_LEVEL_CODE (~80.67 rows/value), COST_CENTER_CODE (~2.22 rows/value), COST_COLLECTOR_KEY (~2.22 rows/value)", + "FCLT_BUILDING_ADDRESS": "FCLT_BUILDING_ADDRESS (785 rows)\n- unique per row: FCLT_BUILDING_ADDRESS_KEY\n- POSTAL_CODE values: 2139(489), 2142(194), 1949(28), 1886(27), 2421(18), 1879(9), 2026(5), 2110(3), 2210(2), 20002(2), 20036(2), 2155(2), 1040(2), 1887(2)\n- ADDRESS_PURPOSE values: STREET(242), E911_1(240), MAIL(159), PARCL1(107), E911_2(14), PARCL2(12), PARCL3(3), E911_3(3), E911_4(1), E911_5(1), E911_6(1), DELIVERY(1), PARCL4(1)\n- STREET_NUMBER_SUFFIX values: NULL(759), R(26)\n- STREET_SUFFIX values: ST(295), AVE(188), NULL(130), DR(116), RD(36), SQ(11), DRIVE(5), AVENUE(2), CIR(2)\n- POST_DIRECTIONAL values: NULL(754), (Rear)(27), NE(2), NW(2)\n- CITY values: CAMBRIDGE(600), NULL(83), MIDDLETON(28), WESTFORD(27), LEXINGTON(18), TYNGSBOROUGH(9), DEDHAM(5), BOSTON(5), WASHINGTON(4), MEDFORD(2), HOLYOKE(2), WILMINGTON(2)\n- STATE values: MA(698), NULL(83), DC(4)\n- fan-out (non-unique) keys: POSTAL_CODE (~56.07 rows/value), FCLT_BUILDING_KEY (~3.24 rows/value), ADDRESS_CITY_ID (~6.54 rows/value)", + "FCLT_BUILDING_HIST": "FCLT_BUILDING_HIST (10,000 rows)\n- unique per row: FCLT_BUILDING_HIST_KEY\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): WAREHOUSE_LOAD_DATE\n- PARENT_BUILDING_NUMBER values: NULL(8434), W61(410), W70(287), 14(172), 62(129), 64(127), W85ABC(123), W85HJK(120), W85DE(80), W85FG(80), 42(38)\n- PARENT_BUILDING_NAME values: NULL(8434), MACGREGOR HOUSE(410), NEW HOUSE(287), HAYDEN MEMORIAL LIBRARY(172), ALUMNI HOUSES: MUNROE HAYDEN WOOD(129), EAST CAMPUS: WALCOTT BEMIS GOODALE(127), WESTGATE (ABC)(123), WESTGATE (HJK)(120), WESTGATE (DE)(80), WESTGATE (FG)(80), COGENERATION PLANT(38)\n- PARENT_BUILDING_NAME_LONG values: NULL(8434), Frank S MacGregor House(410), New West Campus Houses(287), Charles Hayden Memorial Library(172), Alumni Houses: Munroe Hayden Wood(129), Alumni Houses: Walcott Bemis Goodale(127), Westgate ABC(123), Westgate HJK(120), Westgate DE(80), Westgate FG(80), William R. Dickson Cogeneration Plant(38)\n- SITE values: MIT(8397), BATES(566), LINC(373), HAY(363), BOS(93), END(85), SOM(42), HOLYOKE(41), DC(40)\n- CAMPUS_SECTOR values: WEST(2912), MAIN GROUP(2585), OFFCAMPUS(1502), EAST(935), NORTHWEST(685), NULL(559), NORTH(436), NORTHEAST(261), WESTWEST(80), EASTEAST(45)\n- ACCESS_LEVEL_CODE values: 2(7572), 1(1762), 0(479), 3(187)\n- ACCESS_LEVEL_NAME values: 2(7572), 1(1762), 0(479), 3(187)\n- BUILDING_TYPE values: ACADEMIC(5057), RESIDENT(2563), SERVICE(2380)\n- OWNERSHIP_TYPE values: OWNED(9001), LEASED(999)\n- BUILDING_USE values: AER(5147), DHOA(2496), OTH(1248), STAC(729), (NULL)(212), GAR(168)\n- OCCUPANCY_CLASS: 25 distinct, e.g. (NULL), UGB, UGR2, UGBA3, UGA3, UGU, UGA3B, UGF1, UGS2, UGR1A3, UGBI2, UGHH, UGE, UGBR1, UGR4, ...\n- fan-out (non-unique) keys: FCLT_BUILDING_KEY (~39.22 rows/value), ACCESS_LEVEL_CODE (~2500.0 rows/value), COST_CENTER_CODE (~83.33 rows/value), COST_COLLECTOR_KEY (~83.33 rows/value)", + "FCLT_ORGANIZATION": "FCLT_ORGANIZATION (180 rows)\n- FCLT_MAJOR_ORG_KEY values: 230(89), 129(38), 163(35), 271(6), 267(3), 216(3), 105(1), 125(1), 210(1), 217(1), 224(1), 275(1)\n- MAJOR_ORG values: PROVST(89), CHNCLR(38), EXECVP(35), ZORG(6), VP-SCP(3), OTHMIT(3), ALL(1), CHAIRM(1), OFPRES(1), OTHNON(1), PRES(1), XXXXX(1)\n- ORGANIZATION_LEVEL values: 5(107), 4(50), 6(13), 3(7), 1(2), 2(1)\n- ASSIGNABLE values: 1(175), 0(5)\n- fan-out (non-unique) keys: FCLT_ORG_PARENT_KEY (~5.62 rows/value), FCLT_MAJOR_ORG_KEY (~15.0 rows/value)", + "FCLT_ORG_DLC_KEY": "FCLT_ORG_DLC_KEY (168 rows)\n- unique per row: FCLT_ORGANIZATION_KEY", + "FCLT_ROOMS": "FCLT_ROOMS (10,000 rows)\n- FCLT_MAJOR_USE_KEY values: 108(3140), 102(1524), 109(1355), 107(1313), 106(1230), 101(529), 104(283), 112(165), 110(131), 103(120), 111(82), 113(79), 105(49)\n- MAJOR_USE_DESC values: OFFICES(3140), CIRCULAT(1524), RESIDENT(1355), MECHANIC(1313), LABS(1230), BLDG SRV(529), GENERAL(283), SUPPORT(165), SPECIAL(131), CLASSRMS(120), STUDY(82), UNCLASS(79), HEALTH(49)\n- ROOM_FULL_NAME: 17 distinct, e.g. None, WOMENS LOCKER, DE ROTHSCHILD ROOM, NORTH LOBDELL BALCONY, WOMENS TEAM ROOM, CHU ROOM, LAN JEN, MENS LOCKER, COMPTON LOUNGE, HUNTINGTON HALL, KRESGE LOBBY, ENGINEERING CONFERENCE ROOM, STRATTON BALCONY, EXPERIMENTAL MEDIA FACILITY, PHILIPPE VILLERS (THE CUBE), GIVEN ROOM, SMALL DINING ROOM, ...\n- DEPT_CODE values: NULL(9965), 93700(17), 93300(9), 93400(4), 93600(4), 93800(1)\n- ACCESS_LEVEL values: 2(5455), 1(1741), 3(1566), 0(1238)\n- fan-out (non-unique) keys: FCLT_BUILDING_KEY (~49.5 rows/value), FCLT_FLOOR_KEY (~10.81 rows/value), FCLT_MAJOR_USE_KEY (~769.23 rows/value), FCLT_USE_KEY (~117.65 rows/value), FCLT_ORGANIZATION_KEY (~79.37 rows/value), DEPT_CODE (~2000.0 rows/value)", + "HR_ORG_UNIT": "HR_ORG_UNIT (641 rows)\n- unique per row: HR_ORG_UNIT_KEY, HR_ORG_UNIT_ID\n- HR_ORG_UNIT_LEVEL values: DEPARTMENTS(494), ORGANIZATION LEVEL(108), SUB DEPARTMENT(17), NULL(14), NON HIERARCHY ORG UNITS(4), TOP LEVEL(3), ALL MIT(1)\n- ORG_HIER_TOP_LEVEL_NAME values: Provost Area(464), Executive Vice President Area(142), NULL(21), President & Chair of the Corporation(12), Other Org Units(2)\n- ORG_HIER_ROOT_NAME values: MIT-All(640), NULL(1)\n- HR_ORG_LEVEL1_ID values: 10000000(627), NULL(14)\n- HR_ORG_LEVEL1_SORT values: 1(627), NULL(14)\n- HR_ORG_LEVEL1_NAME values: MIT-All(640), NULL(1)\n- HR_ORG_LEVEL2_ID values: 10000001(464), 10000002(142), NULL(15), 10000003(12), 19999000(8)\n- HR_ORG_LEVEL2_SORT values: 2(464), 3(142), NULL(15), 4(12), 623(8)\n- HR_ORG_LEVEL2_NAME values: Provost Area(464), Executive Vice President Area(142), NULL(15), President & Chair of the Corporation(12), Other Org Units(8)\n- WAREHOUSE_LOAD_DATE values: 03-DEC-24(640), 13-DEC-24(1)\n- fan-out (non-unique) keys: HR_ORG_LEVEL2_ID (~160.25 rows/value), HR_ORG_LEVEL3_ID (~12.33 rows/value), HR_ORG_LEVEL4_ID (~2.58 rows/value), HR_ORG_LEVEL5_ID (~2.9 rows/value), HR_ORG_LEVEL6_ID (~10.17 rows/value), HR_ORG_LEVEL7_ID (~16.87 rows/value)\n- CAUTION: 1 HR_DEPARTMENT_NAME values map to MULTIPLE HR_DEPARTMENT_CODEs \u2014 name and code are different grains\n- CAUTION: 1 HR_ORG_LEVEL1_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 5 HR_ORG_LEVEL2_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 27 HR_ORG_LEVEL3_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 57 HR_ORG_LEVEL4_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 17 HR_ORG_LEVEL5_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 9 HR_ORG_LEVEL6_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains\n- CAUTION: 1 HR_ORG_LEVEL7_NAME values map to MULTIPLE HR_ORG_UNIT_KEYs \u2014 name and code are different grains", + "IAP_SUBJECT_DETAIL": "IAP_SUBJECT_DETAIL (465 rows)\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): PREREG_DEADLINE, CREATE_DATE, LAST_ACTIVITY_DATE\n- ENROLLMENT_TYPE values: Advance sign-up required(310), No advance sign-up(106), Other(42), First come, first served (no advance sign-up)(7)\n- MAX_ENROLLMENT values: NULL(324), 30(46), 20(27), 200(24), 40(18), 60(15), 25(3), 12(3), 6(3), 8(2)\n- ATTENDANCE values: Participants welcome at individual sessions(249), Participants must attend all sessions(134), Other(82)\n- PREREQUISITES: 25 distinct, e.g. None, Interest in social media or music, none, Programming experience, deep learning environments, Instructor approval, Register for 16.687 (U-level; 3 units; graded PDF), Being stressed out and/or thinking of starting a company, Enthusiasm!, Working knowledge of one programming/scripting language. Lap, experience with GIS software, Experience of Java, SQL, and data structures recommended, MIT ID, Knowledge of French, 3.091 or equivalent background, Passion towards creating opportunities for children in URM, ...\n- FEE values: NULL(435), 10(24), 25(3), 168(2), 36(1)\n- FEE_REASON values: NULL(435), Class Registration(24), kit and fabricated parts(3), employee, $105 stu/postdoc/spouse, $136.50 trad/retiree(2), one lesson, packages of lessons have a discount per lesson(1)\n- IS_MULTIPLE_SESSION values: Y(426), N(39)\n- IS_CANCELLED values: N(463), Y(2)\n- fan-out (non-unique) keys: IAP_SUBJECT_CATEGORY_KEY (~10.11 rows/value), IAP_SUBJECT_SPONSOR_KEY (~6.84 rows/value), IAP_SUBJECT_SESSION_KEY (~3.27 rows/value), IAP_SUBJECT_PERSON_KEY (~3.27 rows/value)", + "LIBRARY_RESERVE_MATRL_DETAIL": "LIBRARY_RESERVE_MATRL_DETAIL (10,000 rows)\n- LIBRARY_MATERIAL_STATUS_KEY values: U(6663), N(1953), Y(1338), O(32), X(14)\n- TERM_CODE values: 2009FA(3311), 2009SP(2638), 2008SP(2463), 2010SP(619), 2008SU(600), 2009SU(256), 2011FA(92), 2009JA(21)\n- WAREHOUSE_LOAD_DATE values: 05-DEC-08(3311), 08-MAY-09(2638), 09-MAY-08(2463), 07-MAY-10(619), 01-AUG-08(600), 31-JUL-09(256), 18-NOV-10(92), 02-FEB-09(21)\n- fan-out (non-unique) keys: LIBRARY_COURSE_INSTRUCTOR_KEY (~6.2 rows/value), LIBRARY_SUBJECT_OFFERED_KEY (~6.45 rows/value), LIBRARY_MATERIAL_STATUS_KEY (~2000.0 rows/value), TERM_CODE (~1250.0 rows/value), SUBJECT_ID (~9.89 rows/value)\n- LIBRARY_MATERIAL_STATUS lookup mapping (LIBRARY_MATERIAL_STATUS_KEY -> meaning): U=Unknown, R=None, N=Non-Required Course Material, O=Reserve only, X=No Required Textbook, Y=Required Course Material", + "LIBRARY_SUBJECT_OFFERED": "LIBRARY_SUBJECT_OFFERED (10,000 rows)\n- unique per row: LIBRARY_SUBJECT_OFFERED_KEY\n- COURSE_NUMBER: 16 distinct, e.g. 15, 18, 21F, 2, 11, 21G, 1, 17, 16, 21H, 12, 14, 10, 21A, 20, ...\n- COURSE_NUMBER_SORT: 16 distinct, e.g. 15, 18, 21F, 2, 11, 21G, 1, 17, 16, 21H, 12, 14, 10, 21A, 20, ...\n- COURSE_NUMBER_DESC: 16 distinct, e.g. Management, Mathematics, Foreign Languages/Literatures, Mechanical Engineering, Urban Studies and Planning, Global Languages, Civil and Environmental Eng, Political Science, Aeronautics and Astronautics, History, Earth, Atmos, & Planetary Sci, Economics, Chemical Engineering, Anthropology, Prog in Applied Biological Sci, ...\n- OFFER_DEPT_CODE: 16 distinct, e.g. 15, 18, 21F, 2, 11, 21G, 1, 17, 16, 21H, 12, 14, 10, 21A, 20, ...\n- OFFER_DEPT_NAME: 16 distinct, e.g. Management, Mathematics, Global Studies & Languages, Mechanical Engineering, Urban Studies and Planning, Global Languages, Civil and Environmental Eng, Political Science, Aeronautics and Astronautics, History, Earth, Atmos & Planetary Sci, Economics, Chemical Engineering, Anthropology, Biological Engineering, ...\n- OFFER_SCHOOL_NAME values: Hum, Arts & Social Sciences(3607), Engineering(2679), Science(1601), Sloan School of Management(1407), Architecture and Planning(706)\n- fan-out (non-unique) keys: TERM_CODE (~192.31 rows/value), MASTER_SUBJECT_ID (~5.26 rows/value), SUBJECT_ID (~4.45 rows/value), OFFER_DEPT_CODE (~625.0 rows/value), RESPONSIBLE_FACULTY_MIT_ID (~6.83 rows/value)\n- CAUTION: 3 OFFER_SCHOOL_NAME values map to MULTIPLE OFFER_DEPT_CODEs \u2014 name and code are different grains", + "MASTER_DEPT_HIERARCHY": "MASTER_DEPT_HIERARCHY (310 rows)\n- unique per row: DLC_KEY, DLC_CODE\n- MASTER_DEPT_HIER_LEVEL_2_CODE values: D_PROVOST_AREA(234), D_EXECVP_AREA(52), D_PRES_AREA(7), D_OTHER_ORG(7), D_INST_REL_AREA(3), D_UNDEF_DEFUNCT(3), D_OBSOLETE(3), D_OUTSIDE_INST(1)\n- MASTER_DEPT_HIER_LEVEL_2_NAME values: Provost Area(234), Executive Vice President's Area(52), President's area(7), Outside organizations affiliated with MIT(7), Miscellaneous Institute Related(3), Undefined or defunct(3), Obsolete DLC codes(3), Other institutions outside of MIT(1)\n- MASTER_DEPT_HIER_LEVEL_3_CODE: 21 distinct, e.g. None, D_SCHOOL_ENG, D_VPRES, D_DSL, D_DUE, D_FINANCE_AREA, D_SCHOOL_SCI, D_OFC_PROVOST, D_SCHOOL_HUM, D_COLLEGE_COMPU, D_SCHOOL_ARCH, D_OVC_AREA, D_DL_AREA, D_HR_AREA, D_LIB_TR_PRESS, ...\n- MASTER_DEPT_HIER_LEVEL_3_NAME: 21 distinct, e.g. None, School of Engineering, VP Research, Dean for Student Life, Dean for Undergraduate Education, VP Finance Area, School of Science, Office of the Provost Area, School of Humanities & Social Science, Stephen A. Schwarzman College of Computing, School of Architecture & Planning, Office of the Vice Chancellor's Area, Digital Learning Area, HR Area, Libraries, Tech Review, and MIT Press, ...\n- MASTER_DEPT_HIER_LEVEL_4_CODE values: NULL(301), D_OSATT_AREA(6), D_SOURCING_AREA(3)\n- MASTER_DEPT_HIER_LEVEL_4_NAME values: NULL(301), Office of Strategic Alliances & Tech Transfer Area(6), Sourcing Area(3)\n- fan-out (non-unique) keys: MASTER_DEPT_HIER_LEVEL_2_CODE (~38.75 rows/value), MASTER_DEPT_HIER_LEVEL_3_CODE (~15.5 rows/value), MASTER_DEPT_HIER_LEVEL_4_CODE (~155.0 rows/value)\n- CAUTION: 1 DLC_NAME values map to MULTIPLE DLC_KEYs \u2014 name and code are different grains\n- CAUTION: 1 DLC_NAME values map to MULTIPLE DLC_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_1_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_2_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_1_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_3_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_1_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_4_CODEs \u2014 name and code are different grains\n- CAUTION: 2 MASTER_DEPT_HIER_LEVEL_2_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_3_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_3_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_2_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_4_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_2_CODEs \u2014 name and code are different grains\n- CAUTION: 1 MASTER_DEPT_HIER_LEVEL_4_NAME values map to MULTIPLE MASTER_DEPT_HIER_LEVEL_3_CODEs \u2014 name and code are different grains", + "MIT_STUDENT_DIRECTORY": "MIT_STUDENT_DIRECTORY (10,000 rows)\n- STUDENT_YEAR values: G(6275), 4(1041), 3(965), 2(924), 1(783), NULL(12)", + "SE_PERSON": "SE_PERSON (10,000 rows)\n- unique per row: MIT_ID\n- EMPLOYEE_TYPE values: Student(3515), Other Academic Group(2140), Admin Staff(1587), Sponsored Research Staff(979), Support Staff(662), Faculty(579), Service Staff(478), Medical(60)", + "SIS_ADMIN_DEPARTMENT": "SIS_ADMIN_DEPARTMENT (179 rows)\n- unique per row: SIS_ADMIN_DEPARTMENT_CODE\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): LAST_ACTIVITY_DATE\n- CAUTION: 17 SIS_ADMIN_DEPARTMENT_NAME values map to MULTIPLE SIS_ADMIN_DEPARTMENT_CODEs \u2014 name and code are different grains", + "SIS_COURSE_DESCRIPTION": "SIS_COURSE_DESCRIPTION (695 rows)\n- unique per row: SIS_COURSE_DESCRIPTION_KEY\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): LAST_ACTIVITY_DATE\n- SCHOOL_NAME values: Engineering(321), Science(106), Hum, Arts & Social Sciences(93), Architecture and Planning(46), MIT, academic(45), Sloan School of Management(42), Non-MIT(30), Schwarzman Coll of Comp(7), Whitaker Coll of HST; HST(5)\n- SCHOOL_NAME_IN_COMMENCEMENT_BK values: School of Engineering(321), School of Science(106), School of Humanities, Arts, and Social Sciences(93), NULL(75), School of Architecture and Planning(46), Sloan School of Management(42), Schwarzman College of Computing(7), Whitaker College of Health Sciences and Technology(5)\n- COURSE_LEVEL values: G(477), U(218)\n- IS_DEGREE_GRANTING values: Y(390), N(305)\n- DEFAULT_ULTIMATE_DEGREE values: NDG(292), SM(129), SB(114), DOC(99), MNG(24), NULL(11), MBA(7), MF(6), MAP(4), MD(3), HA(2), MCP(1), MBN(1), MA(1), DDM(1)\n- GRADAUTE_LEVEL values: NULL(410), Masters(179), Doctoral(106)\n- GRADUATE_LEVEL values: NULL(410), Masters(179), Doctoral(106)\n- fan-out (non-unique) keys: CIP_PROGRAM_CODE (~10.86 rows/value)", + "SIS_DEPARTMENT": "SIS_DEPARTMENT (128 rows)\n- unique per row: DEPARTMENT_FULL_NAME\n- string-date columns ('DD-MON-YY', lexicographic MIN/MAX): DEPARTMENT_LAST_ACTIVITY_DATE\n- SCHOOL_CODE values: Y(29), E(25), H(20), Z(16), X(13), S(9), A(5), M(5), T(3), W(2), NULL(1)\n- SCHOOL_NAME values: MIT, academic(29), Engineering(25), Hum, Arts & Social Sciences(20), Non-MIT(16), MIT, non-academic(13), Science(9), Sloan School of Management(5), Architecture and Planning(5), Whitaker Coll of HST; HST(3), Schwarzman Coll of Comp(2), Not Available(1)\n- IS_DEGREE_GRANTING values: Y(69), N(59)\n- SCHOOL_NAME_IN_COMMENCEMENT_BK values: NULL(59), School of Engineering(25), School of Humanities, Arts, and Social Sciences(20), School of Science(9), Sloan School of Management(5), School of Architecture and Planning(5), Whitaker College of Health Sciences and Technology(3), Schwarzman College of Computing(2)\n- DEPARTMENT_NAME_HISTORY values: NULL(115), Was in Sch of Eng until 12/31/2019(1), Civil Eng ( )(1), Anthropol/Archaeol until 1998-99(1), Music and Theater Arts(1), For Lang & Lit(1), associated with school of engineering through 5th week of 1998SP(1), was CAES(1), was TOX(1), was BEH(1), was Applied Biological Engineering(1), formerly ARC(1), formerly UAAO(1), Computational Design and Optimization(1)\n- fan-out (non-unique) keys: SCHOOL_CODE (~12.8 rows/value), DEPT_BUDGET_CODE (~2.17 rows/value), DLC_KEY (~1.8 rows/value)\n- CAUTION: 4 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains", + "SIS_SUBJECT_CODE": "SIS_SUBJECT_CODE (221 rows)\n- unique per row: COURSE_NUMBER, SUBJECT_CODE\n- SCHOOL_CODE values: Z(122), NULL(37), E(20), H(17), S(8), Y(7), A(4), W(3), M(2), T(1)\n- SCHOOL_NAME values: Non-MIT(122), NULL(37), Engineering(20), Hum, Arts & Social Sciences(17), Science(8), MIT, academic(7), Architecture and Planning(4), Schwarzman Coll of Comp(3), Sloan School of Management(2), Whitaker Coll of HST; HST(1)\n- fan-out (non-unique) keys: DEPARTMENT_CODE (~3.62 rows/value), SCHOOL_CODE (~24.56 rows/value)\n- CAUTION: 1 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains", + "SPACE_DETAIL": "SPACE_DETAIL (10,000 rows)\n- fan-out (non-unique) keys: BUILDING_KEY (~344.83 rows/value), FLOOR_KEY (~370.37 rows/value), SPACE_UNIT_KEY (~151.52 rows/value), SPACE_USAGE_KEY (~163.93 rows/value)", + "SPACE_FLOOR": "SPACE_FLOOR (49 rows)\n- unique per row: FLOOR_KEY, FLOOR, FLOOR_NAME", + "SPACE_SUPERVISOR_USAGE": "SPACE_SUPERVISOR_USAGE (2,135 rows)\n- unique per row: MIT_ID\n- DEPT_COUNT values: 1(2020), 2(100), 3(13), 4(2)\n- SQFT_PER_RES_VOL values: 0.0(2132), -18.0(1), 5.0(1), 1.0(1)", + "SPACE_UNIT": "SPACE_UNIT (150 rows)\n- unique per row: FCLT_ORGANIZATION_KEY, SPACE_UNIT_KEY, SPACE_UNIT_CODE", + "STUDENT_DEPARTMENT": "STUDENT_DEPARTMENT (79 rows)\n- unique per row: DEPARTMENT_CODE, DEPARTMENT_FULL_NAME\n- SCHOOL_CODE values: E(23), H(17), Z(14), S(8), Y(7), A(4), T(2), M(2), W(2)\n- SCHOOL_NAME values: Engineering(23), Hum, Arts & Social Sciences(17), Non-MIT(14), Science(8), MIT, academic(7), Architecture and Planning(4), Whitaker Coll of HST; HST(2), Sloan School of Management(2), Schwarzman Coll of Comp(2)\n- fan-out (non-unique) keys: SCHOOL_CODE (~8.78 rows/value)\n- CAUTION: 1 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains", + "SUBJECT_OFFERED": "SUBJECT_OFFERED (10,000 rows)\n- unique per row: SUBJECT_KEY, SUBJECT_OFFERED_SUMMARY_KEY, SUBJECT_SUMMARY_KEY\n- CLUSTER_LIST values: HAA.0000, HAA.0062, HAA.0074, HAA.0090, HAA.0094, HAA.0096, HAA.0104, HAA.0105, HAA.0107, HAA.0119, HAA.0120, HAA.0121, HAA.012(5361), HAA.0000, HAA.0023, HAA.0029, HAA.0031, HAA.0042, HAA.0062, HAA.0071, HAA.0074, HAA.0090, HAA.0094, HAA.0096, HAA.0104, HAA.010(2566), HAA.0000, HAA.0018, HAA.0021, HAA.0023, HAA.0025, HAA.0026, HAA.0029, HAA.0031, HAA.0033, HAA.0036, HAA.0042, HAA.0062, HAA.007(2069), NULL(4)\n- HGN_CODE values: N(8557), H(1365), G(78)\n- HGN_CODE_DESC values: Not for graduate credit(8557), Higher level graduate program(1365), Graduate program(78)\n- CLUSTER_ENROLLMENT_NUMBER values: 0(9996), NULL(4)\n- fan-out (non-unique) keys: MASTER_SUBJECT_KEY (~370.37 rows/value), COMPOSITE_SUBJECT_KEY (~370.37 rows/value), TERM_CODE (~370.37 rows/value), SUBJECT_ID (~4.81 rows/value), HGN_CODE (~3333.33 rows/value), SUBJECT_GROUPING_KEY (~370.37 rows/value)", + "SUBJECT_OFFERED_SUMMARY": "SUBJECT_OFFERED_SUMMARY (10,000 rows)\n- unique per row: SUBJECT_OFFERED_SUMMARY_KEY, SUBJECT_SUMMARY_KEY\n- CLUSTER_TYPE values: S(7412), NULL(2054), M(320), J(214)\n- CLUSTER_TYPE_DESC values: SWE: School-Wide Electives(7412), NULL(2054), Meeting Together(320), Joint subject(214)\n- HGN_CODE values: H(4334), N(3228), G(2429), NULL(9)\n- HGN_CODE_DESC values: Higher level graduate program(4334), Not for graduate credit(3228), Graduate program(2429), NULL(9)\n- OFFER_SCHOOL_NAME values: Non-MIT(7348), Engineering(1062), Science(595), Hum, Arts & Social Sciences(567), Sloan School of Management(216), Architecture and Planning(150), MIT, academic(57), Schwarzman Coll of Comp(4), Whitaker Coll of HST; HST(1)\n- TOTAL_UNITS: 16 distinct, e.g. 12, 1, 6, 3, 9, 15, 4, 21, 24, None, 7, 8, 18, 2, 0, ...\n- LECTURE_UNITS values: 0(6983), 3(1479), 12(454), 2(427), 4(292), 5(138), 1(115), 6(101), NULL(9), 24(1), 9(1)\n- LAB_UNITS values: 12(4360), 0(2743), 1(1723), 6(637), 3(256), 15(137), 2(52), 4(32), 21(24), 24(14), 8(10), NULL(9), 9(3)\n- PREPARATION_UNITS values: 0(7513), 9(1026), 6(297), 7(255), 8(241), 2(180), 3(177), 4(145), 1(69), 5(44), 10(40), NULL(9), 18(3), 12(1)\n- fan-out (non-unique) keys: COMPOSITE_SUBJECT_KEY (~3.68 rows/value), TERM_CODE (~81.3 rows/value), SUBJECT_ID (~2.43 rows/value), MASTER_SUBJECT_ID (~29.07 rows/value), HGN_CODE (~3333.33 rows/value), OFFER_DEPT_CODE (~196.08 rows/value)\n- CAUTION: 1 OFFER_DEPT_NAME values map to MULTIPLE OFFER_DEPT_CODEs \u2014 name and code are different grains\n- CAUTION: 6 OFFER_SCHOOL_NAME values map to MULTIPLE OFFER_DEPT_CODEs \u2014 name and code are different grains", + "SUBJECT_SUMMARY": "SUBJECT_SUMMARY (10,000 rows)\n- unique per row: SUBJECT_SUMMARY_KEY\n- CLUSTER_TYPE values: S(7360), NULL(1888), J(442), M(306), (4)\n- CLUSTER_TYPE_DESC values: SWE: School-Wide Electives(7360), NULL(1892), Joint subject(442), Meeting Together(306)\n- SCHOOL_CODE values: Z(7313), E(915), H(593), S(443), A(327), M(216), Y(81), T(60), NULL(42), W(8), X(2)\n- SCHOOL_NAME values: Non-MIT(7311), Engineering(975), Hum, Arts & Social Sciences(604), Science(454), Architecture and Planning(332), Sloan School of Management(220), MIT, academic(84), Schwarzman Coll of Comp(18), MIT, non-academic(1), Whitaker Coll of HST; HST(1)\n- TOTAL_UNITS: 23 distinct, e.g. 12, 1, 6, 9, 15, 3, None, 24, 4, 18, 13, 21, 2, 14, 16, ...\n- LECTURE_UNITS values: 0(7885), 3(1251), 2(313), 4(299), 5(85), 1(82), NULL(40), 12(23), 6(19), 9(2), 24(1)\n- LAB_UNITS: 23 distinct, e.g. 12, 1, 0, 6, 15, 3, 2, 4, None, 24, 8, 9, 7, 18, 21, ...\n- PREP_UNITS: 16 distinct, e.g. 0, 9, 8, 6, 4, 7, 3, 2, 5, 1, None, 10, 18, 12, 11, ...\n- DESIGN_UNITS values: 0(9947), NULL(43), 6(5), 4(3), 12(2)\n- fan-out (non-unique) keys: TERM_CODE (~83.33 rows/value), SUBJECT_ID (~1.9 rows/value), MASTER_SUBJECT_ID (~5.78 rows/value), ULT_MASTER_SUBJECT_ID (~5.79 rows/value), DEPARTMENT_CODE (~172.41 rows/value), SCHOOL_CODE (~1000.0 rows/value)\n- CAUTION: 3 DEPARTMENT_NAME values map to MULTIPLE DEPARTMENT_CODEs \u2014 name and code are different grains\n- CAUTION: 4 SCHOOL_NAME values map to MULTIPLE SCHOOL_CODEs \u2014 name and code are different grains", + "TIP_DETAIL": "TIP_DETAIL (10,000 rows)\n- TIP_MATERIAL_STATUS_KEY values: RQ(3441), NM(2249), EO(2110), RC(1843), U(197), PC(62), CL(50), NL(27), BR(10), NS(6), NB(4), (1)\n- fan-out (non-unique) keys: TIP_MATERIAL_STATUS_KEY (~833.33 rows/value), TERM_CODE (~158.73 rows/value), SUBJECT_ID (~2.76 rows/value)\n- TIP_MATERIAL_STATUS lookup mapping (TIP_MATERIAL_STATUS_KEY -> meaning): PC=Val-u option, NL=None, (blank)=None, NM=Course has no materials, EO=Electronic options, RQ=Required, NS=None, BR=Bookstore recommends, NB=None, RC=Recommended, CL=Go to class first, U=Unknown", + "TIP_SUBJECT_OFFERED": "TIP_SUBJECT_OFFERED (10,000 rows)\n- unique per row: TIP_SUBJECT_OFFERED_KEY\n- IS_NO_COURSE_MATERIAL values: NULL(7929), Y(1091), N(980)\n- OFFER_SCHOOL_NAME values: Engineering(3536), Hum, Arts & Social Sciences(2233), Science(1725), Architecture and Planning(1253), Sloan School of Management(818), MIT, academic(364), Schwarzman Coll of Comp(51), Whitaker Coll of HST; HST(16), MIT, non-academic(4)\n- fan-out (non-unique) keys: TERM_CODE (~79.37 rows/value), MASTER_SUBJECT_ID (~2.2 rows/value), SUBJECT_ID (~2.01 rows/value), OFFER_DEPT_CODE (~161.29 rows/value), RESPONSIBLE_FACULTY_MIT_ID (~4.05 rows/value)\n- CAUTION: 3 OFFER_DEPT_NAME values map to MULTIPLE OFFER_DEPT_CODEs \u2014 name and code are different grains\n- CAUTION: 7 OFFER_SCHOOL_NAME values map to MULTIPLE OFFER_DEPT_CODEs \u2014 name and code are different grains", + "ZPM_ROOMS_LOAD": "ZPM_ROOMS_LOAD (10,000 rows)\n- ACCESS_LEVEL values: 2(5692), 1(1519), 0(1458), 3(1331)\n- fan-out (non-unique) keys: SPACE_UNIT_CODE (~151.52 rows/value), HR_ORG_UNIT_ID (~151.52 rows/value)" +} \ No newline at end of file diff --git a/eval/myagent/execute.py b/eval/myagent/execute.py new file mode 100644 index 0000000..8846229 --- /dev/null +++ b/eval/myagent/execute.py @@ -0,0 +1,98 @@ +"""Generation driver for a custom agent. + +Builds one instance per question, runs your agent (agent.run_agent) over them in +parallel with resume support, and writes each prediction to + //predicted_0.sql (+ generation.log) +which unify.py then reshapes into the unified generated/ + gold/ layout. + +You normally do NOT edit this file — put your agent in agent.py. +""" +import os +import argparse +import traceback +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from prompt import build_instances +from utils import EvalConfig +from agent import run_agent + + +def _output_paths(output_dir, instance_id): + instance_dir = os.path.join(output_dir, instance_id) + return instance_dir, os.path.join(instance_dir, "predicted_0.sql"), os.path.join(instance_dir, "generation.log") + + +def _process(instance, model, output_dir): + instance_id = instance["id"] + instance_dir, sql_file, log_file = _output_paths(output_dir, instance_id) + try: + sql = run_agent(instance, model) or "" + log = f"Question:\n{instance['question']}\n\nGenerated SQL:\n{sql}" + except Exception as e: + sql = "" + log = f"Error: {e}\n{traceback.format_exc()}" + print(f"Error on {instance_id}: {e}") + + # Writes are inside their own guard so a single failed write (bad locale, + # disk full, permissions) fails just this instance instead of aborting the + # whole batch via fut.result(). + try: + os.makedirs(instance_dir, exist_ok=True) + with open(sql_file, "w", encoding="utf-8") as f: + f.write(sql) + with open(log_file, "w", encoding="utf-8") as f: + f.write(log) + except Exception as e: + print(f"Failed to write output for {instance_id}: {e}") + return instance_id, False + return instance_id, bool(sql) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, default="codex") + parser.add_argument("--dataset", type=str, default="dw") + parser.add_argument("--q_fn", type=str, default="dev_sampled") + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--data_dir", type=str, default="../../data") + parser.add_argument("--num_workers", type=int, default=4) + # --setting flags (set by run.sh) + parser.add_argument("--gold_tables", action="store_true") + parser.add_argument("--join_keys", action="store_true") + parser.add_argument("--mapping", action="store_true") + parser.add_argument("--knowledge", action="store_true") + parser.add_argument("--decomp", action="store_true") + args = parser.parse_args() + + eval_config = EvalConfig( + gold_tables=args.gold_tables, join_keys=args.join_keys, + mapping=args.mapping, knowledge=args.knowledge, decomp=args.decomp, + ) + print(eval_config) + + instances = build_instances(args.dataset, args.q_fn, eval_config, args.data_dir) + os.makedirs(args.output_dir, exist_ok=True) + + # Resume: skip instances that already have a non-empty prediction. + todo = [] + for inst in instances: + _, sql_file, _ = _output_paths(args.output_dir, inst["id"]) + if not os.path.exists(sql_file) or os.path.getsize(sql_file) == 0: + todo.append(inst) + + print(f"Output directory: {args.output_dir}") + if not todo: + print("All instances already processed.") + return + print(f"Running agent on {len(todo)} / {len(instances)} instances (model={args.model})...") + + with ThreadPoolExecutor(max_workers=args.num_workers) as ex: + futures = [ex.submit(_process, inst, args.model, args.output_dir) for inst in todo] + for fut in tqdm(futures, total=len(futures)): + fut.result() + + +if __name__ == "__main__": + main() diff --git a/eval/myagent/prompt.py b/eval/myagent/prompt.py new file mode 100644 index 0000000..e4037b3 --- /dev/null +++ b/eval/myagent/prompt.py @@ -0,0 +1,184 @@ +"""Build per-question instances for the agent. + +This mirrors the context/hint assembly used by the `fewshot` baseline +(`eval/fewshot/prompt.py`) so the --setting flag behaves identically, but it +returns a list of *aligned* instance dicts instead of two parallel lists. Each +instance carries both: + * a ready-to-send OpenAI-style chat `prompt` (system + few-shot example + user) + for agents that just want to call a chat model, and + * the structured fields (question, db, tables, and the active hint strings) for + agents that do their own multi-step reasoning / tool use. +""" +from pathlib import Path + +from utils import ( + read_json, format_tables, EvalConfig, format_join, system, user, assistant, +) + + +def get_mapping(mapping): + desc = [] + for sq in mapping: + cols_desc = [] + for col in mapping[sq]: + table_name, col = col.split(".") + cols_desc.append(f"column {col} in table {table_name.lower()}") + desc.append(f'"{sq}" in the user question refers to {", ".join(cols_desc)}') + return "\n".join(desc) + + +def get_join_keys(join_keys): + return "\n".join(format_join(jk) for jk in join_keys) + + +def get_knowledge(domain_knowledge): + return "\n".join(domain_knowledge) + + +def get_decomp(decomps): + return "\n".join(f"Subquery {i + 1}: {sq}" for i, sq in enumerate(decomps)) + + +def get_user_prompt(q, tables, corpus_tables, eval_config: EvalConfig): + desc = [ + format_tables(tables, corpus_tables, eval_config.instances), + f"User question: {q['question']}", + ] + if eval_config.mapping: + desc.append(f"Mapping:\n{get_mapping(q['column_mapping'])}") + if eval_config.join_keys: + desc.append(f"Join keys:\n{get_join_keys(q['join_keys'])}") + if eval_config.knowledge and get_knowledge(q['domain_knowledge']) != '': + desc.append(f"Domain knowledge:\n{get_knowledge(q['domain_knowledge'])}") + if eval_config.decomp and get_decomp(q["sub_questions"]) != "": + desc.append(f"Query decomposition:\n{get_decomp(q['sub_questions'])}") + return user("\n\n".join(desc)) + + +def get_retrieved_tables(dataset: str, data_dir="../../data"): + retrieved_fn = Path(f"{data_dir}/{dataset}/retrieval/retrieved_tables.json") + reranked_fn = Path(f"{data_dir}/{dataset}/retrieval/reranked_tables.json") + if not retrieved_fn.exists(): + raise FileNotFoundError( + f"No retrieved tables at {retrieved_fn}. At --setting 0 the agent gets the " + f"retrieved candidate tables; run `python retrieve/retrieve.py --dataset {dataset} ...` " + f"first, or use --setting 1/2 which inject gold tables and need no retrieval." + ) + if reranked_fn.exists(): + print(f"Loading reranked tables from {reranked_fn}") + return read_json(reranked_fn) + print(f"Loading retrieved tables from {retrieved_fn}") + return read_json(retrieved_fn) + + +def _build_instruction(eval_config, q_knowledge, q_decomp, q, structures): + db_type = "MySQL" + instruction = ["You are given a list of tables", "a user question"] + if eval_config.join_keys: + instruction.append("join keys among the provided tables") + if eval_config.mapping: + instruction.append("a mapping from information mentioned in the user question to columns in the provided tables") + if q_knowledge: + instruction.append("domain knowledge") + if q_decomp: + instruction.append("decomposition of the user question") + instruction[-1] = f"and {instruction[-1]}" + instruction = ", ".join(instruction) + ", " + + instruction += ( + f"your task is output a {db_type} SQL statement that can be used to answer the user " + f"question based on the provided information. You need to ensure that syntax and functions " + f"used in your SQL statement are appropriate for {db_type} database. If you are unable to " + f"determine the SQL statement, output None. " + ) + if eval_config.mapping: + instruction += "You should use the provided mapping to determine which columns and tables should be used in the SQL statement. " + if eval_config.join_keys: + instruction += "You should use the provided join keys to determine how to connect the tables in the SQL statement. " + if q_knowledge: + instruction += "You should use the provided domain knowledge to determine which tables, columns, and literals should be used in the SQL statement. " + if q_decomp: + instruction += ( + "You must answer each subquery individually and then combine them to form the complete " + "SQL statement. Each subquery you generate must be explicitly used in the final SQL " + "statement, without being simplified. " + "The subqueries are scaffolding for how to STRUCTURE the SQL. If a subquery mentions a " + "filter, top-k limit, ranking, or extra total row that the final user question does not " + "ask for, the final user question always takes precedence: keep the subquery's structure " + "but do not carry the conflicting filter, limit, or extra rows/columns into the result. " + ) + instruction += ( + "Below is the structure of the SQL statement with subqueries denoted. Each provided " + "subquery is used in the final SQL statement in such a structure." + ) + structure_name = q.get("detailed_category") + if structure_name and structure_name != 'real' and structure_name in structures: + structure = structures[structure_name] + instruction += f"\n\n{structure['structure']}" + instruction += f"\n\n{structure['subquery_decomposition']} " + + instruction += "The SQL statement need to be wrapped in tags." + return instruction + + +def build_instances(dataset: str, q_fn: str, eval_config: EvalConfig, data_dir: str = "../../data"): + """Return a list of aligned instance dicts, one per question that has table context. + + Each instance: + id : question id + question : natural-language question + db : target database name + tables : list of candidate table names provided to the agent + prompt : OpenAI-style chat messages (system + few-shot example + user) + hints : {mapping, join_keys, domain_knowledge, decomposition} active strings + record : full question/gold dict from .json (avoid peeking at gold `sql`) + """ + structures = read_json(f"{data_dir}/template_structure.json") + qs = read_json(f"{data_dir}/{dataset}/{q_fn}.json") + example = read_json(f"{data_dir}/{dataset}/example.json") + dev_tables = read_json(f"{data_dir}/{dataset}/dev_tables.json") + + retrieved_tables = None if eval_config.gold_tables else get_retrieved_tables(dataset, data_dir) + + example_prompt = [ + get_user_prompt(example, example["tables"], dev_tables, eval_config), + assistant(f"SQL: {example['sql']}"), + ] + + instances = [] + skipped = 0 + for q in qs: + q_id = q['id'] + q_knowledge = eval_config.knowledge and get_knowledge(q['domain_knowledge']) != '' + q_decomp = eval_config.decomp and get_decomp(q["sub_questions"]) != '' + + if eval_config.gold_tables: + tables = q["tables"] + else: + if q_id not in retrieved_tables: + skipped += 1 + continue + tables = retrieved_tables[q_id] + + instruction = _build_instruction(eval_config, q_knowledge, q_decomp, q, structures) + prompt = [system(instruction)] + example_prompt + [get_user_prompt(q, tables, dev_tables, eval_config)] + + instances.append({ + "id": q_id, + "question": q["question"], + "db": q.get("db", dataset), + "tables": tables, + "prompt": prompt, + "hints": { + "mapping": get_mapping(q['column_mapping']) if eval_config.mapping else None, + "join_keys": get_join_keys(q['join_keys']) if eval_config.join_keys else None, + "domain_knowledge": get_knowledge(q['domain_knowledge']) if q_knowledge else None, + "decomposition": get_decomp(q['sub_questions']) if q_decomp else None, + }, + "record": q, + }) + + if skipped: + print(f"Skipped {skipped} question(s) with no retrieved tables.") + print(f"#instances: {len(instances)}") + return instances diff --git a/eval/myagent/run.sh b/eval/myagent/run.sh new file mode 100755 index 0000000..346b6f5 --- /dev/null +++ b/eval/myagent/run.sh @@ -0,0 +1,115 @@ +#!/bin/bash +set -e + +# ============================================================================ +# Custom-agent generation pipeline for BEAVER +# Usage: ./run.sh --dataset dw --setting 1 +# +# Step 1 (execute.py): run your agent (agent.py) over every question +# Step 2 (unify.py): reshape outputs into unified-output/myagent// +# +# Then score with (from eval/): +# uv run python evaluate_ex_acc.py --dataset --input_dir unified-output/myagent/ +# uv run python evaluate_subtasks.py --dataset --model