From fa6dafbc0d962d9f2cc4905593fd47924280d992 Mon Sep 17 00:00:00 2001 From: sam-dembrane Date: Tue, 14 Jul 2026 11:22:39 +0200 Subject: [PATCH] =?UTF-8?q?feat(runtime):=20exa=5Fsearch=20FunctionTool=20?= =?UTF-8?q?=E2=80=94=20make=20the=20tool=20Flash=20hallucinates=20real?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 30 days of audit logs show Exa itself never failed (8/8 API calls healthy); every 'exa breaks' incident was agent-side: 1. The skill catalog advertises exa-search in every system prompt but no exa_search tool existed — Flash pattern-matched the name to a native tool and emitted calls the runner had no handler for, killing the whole session with exit 1 (3 crashes: 2026-07-09 x2, 2026-07-13; the last one misdiagnosed itself to the user). Sam journaled the root cause on 07-09 and re-hit it on 07-13 — prose can't patch a model prior, so the tool now exists with the exact name and query arg Flash already tries to call. 2. The skill's only curl template piped to jq, which isn't in the container — searches succeeded, got billed, and the output was eaten (2 occurrences). The tool formats results in Python; the rewritten skill bans jq by name and documents the python3 fallback. 3. Skill body drifted from the current Exa API (stale type enum, response fields promised unconditionally) — corrected against the 2026-07 docs. Registered in worker_tools (main/worker/pro_executor) and the parallel_workers fanout; mentor stays read-only. Live-verified against the real API: happy path + include_domains + 401-as-string (~$0.02). Diagnosis by a 4-agent research workflow, 2026-07-14 maintenance. Co-Authored-By: Claude Fable 5 --- src/runtime/adk_runner.py | 86 +++++++++++++++++++++++++++++ src/skills/exa-search/skill.md | 93 +++++++++++++++++++------------- tests/runtime/test_exa_search.py | 74 +++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 36 deletions(-) create mode 100644 tests/runtime/test_exa_search.py diff --git a/src/runtime/adk_runner.py b/src/runtime/adk_runner.py index ca76bfc..3ff496d 100644 --- a/src/runtime/adk_runner.py +++ b/src/runtime/adk_runner.py @@ -446,6 +446,90 @@ async def fetch_url(url: str) -> str: return f"error fetching {url}: {exc}" +def exa_search( + query: str, + num_results: int = 5, + include_domains: Optional[list[str]] = None, + category: Optional[str] = None, +) -> str: + """Semantic web search via the Exa API. Returns ranked candidate URLs + with snippet highlights — use it to DISCOVER a URL, then `fetch_url` + the best result to read the page. Costs real money (~$0.005-0.01 per + call): one query per task; if results miss, refine the query rather + than re-running variations. See src/skills/exa-search/skill.md. + + query: the search string — be specific (names, versions, domains). + num_results: how many results (default 5, capped at 10 here). + include_domains: optional, e.g. ["docs.anthropic.com"] to anchor on + known-authoritative sources. + category: optional index narrowing — one of: company, research paper, + news, personal site, financial report, people. + """ + # Exists because the skill catalog advertises `exa-search` in every + # session's system prompt, and with no matching tool registered Flash + # pattern-matched the name to a native tool and emitted `exa_search(...)` + # calls the runner had no handler for — an unhandled exception that + # killed the whole session (3 crashes, 2026-07-09..13). Name and `query` + # arg deliberately match what Flash already tried to call. + try: + import httpx + + api_key = os.environ.get("EXA_API_KEY") + if not api_key: + return "error: EXA_API_KEY not set in environment" + body: dict = { + "query": query, + "numResults": max(1, min(num_results, 10)), + "contents": {"highlights": True}, + } + if include_domains: + body["includeDomains"] = include_domains + if category: + body["category"] = category + resp = httpx.post( + "https://api.exa.ai/search", + headers={"x-api-key": api_key}, + json=body, + timeout=60.0, + ) + try: + data = resp.json() + except Exception: + return ( + f"error: Exa returned non-JSON (HTTP {resp.status_code}): " + f"{resp.text[:300]}" + ) + if resp.status_code != 200: + # Error body shape: {"requestId", "error", "tag"} — surface the + # tag so Sam can branch (429 = back off; 400 INVALID_REQUEST_BODY + # = fix params, e.g. category+excludeDomains conflict; 402 = + # out of credits). + return ( + f"error: Exa HTTP {resp.status_code}" + f" [{data.get('tag', 'no-tag')}]: {data.get('error', '(no message)')}" + ) + results = data.get("results") or [] + if not results: + return ( + f"no results for {query!r} — this is a valid empty response, " + "not an error. Refine the query (add a name/version/domain)." + ) + lines: list[str] = [] + for i, r in enumerate(results, 1): + highlight = ((r.get("highlights") or [""])[0]).replace("\n", " ").strip() + lines.append( + f"{i}. {r.get('title')}\n" + f" {r.get('url')}\n" + f" published: {r.get('publishedDate') or 'unknown'}\n" + f" {highlight[:400]}" + ) + cost = (data.get("costDollars") or {}).get("total") + lines.append(f"(exa cost this query: ${cost})") + return "\n".join(lines) + except Exception as exc: + return f"error: exa_search failed: {type(exc).__name__}: {exc}" + + # ─── Worker agent instruction ────────────────────────────────────────────────── @@ -736,6 +820,7 @@ async def run_one(task: str, idx: int) -> str: FunctionTool(func=grep), FunctionTool(func=glob_files), FunctionTool(func=fetch_url), + FunctionTool(func=exa_search), ], ) svc = InMemorySessionService() @@ -1526,6 +1611,7 @@ async def run(self, request: AgentRunRequest) -> AgentRunResult: FunctionTool(func=grep), FunctionTool(func=glob_files), FunctionTool(func=fetch_url), + FunctionTool(func=exa_search), ] worker_instruction = _load_worker_instruction() diff --git a/src/skills/exa-search/skill.md b/src/skills/exa-search/skill.md index 2859bfc..3425ea3 100644 --- a/src/skills/exa-search/skill.md +++ b/src/skills/exa-search/skill.md @@ -1,75 +1,96 @@ --- name: exa-search -description: How to find candidate URLs about a topic via semantic search before fetching them. Uses the Exa AI API. Reach for this when Sam doesn't already have a URL. -when_to_use: When Sam needs to find URLs about a topic (a paper, a doc, a vendor's API, a library) and doesn't already have the specific URL. NOT when Sam already has a URL — use `fetch_url` for that. NOT when the info lives in our own repo / Linear / Slack — use `read_file`, `grep`, or the Linear MCP. +description: When and how to use the `exa_search` runtime tool — semantic web search that finds candidate URLs about a topic before fetching them. Reach for this when Sam doesn't already have a URL. +when_to_use: When Sam needs to find URLs about a topic (a paper, a doc, a vendor's API, a library) and doesn't already have the specific URL — call the native `exa_search` tool (it IS registered in the runtime). NOT when Sam already has a URL — use `fetch_url`. NOT when the info lives in our own repos, Linear, or Slack — use `read_file`/`grep`, the Linear API via curl, or the Slack API. --- # Skill: exa-search -Exa is a semantic search API — it returns ranked candidate URLs for a query, with snippets/highlights from each page. Use it to *discover* URLs; use `fetch_url` to actually read one. +Exa is a semantic search API. **`exa_search` is a native runtime tool** — call it +directly like `fetch_url`; do not curl the API for ordinary searches. It returns +ranked candidate URLs with snippet highlights. Use it to *discover* URLs; use +`fetch_url` to actually read one. -## When to use this skill +## When to use -The trigger is **you don't know the URL yet**. Examples: +The trigger is **you don't know the URL yet**: - "find the latest Vertex AI Gemini 3.5 quota docs" - "look up Anthropic's pricing page" - "is there a paper on inverted-worker LLM architectures?" -- "find the OpenCode docs on `apply_patch`" -## When NOT to use this skill +## When NOT to use | Situation | Use instead | |---|---| | You already have the URL | `fetch_url` | | The info is in `Dembrane/sam` | `read_file` / `grep` / `glob_files` | | The info is in another Dembrane repo | clone + `read_file` / `grep` | -| The info is in Linear (issues, comments, projects) | Linear MCP tools | +| The info is in Linear (issues, comments, projects) | Linear API via bash + curl (see `linear-issue-workflow`) | | The info is in Slack (a thread, a canvas, a file) | Slack API via `bash` + curl | -Mixing this skill with `fetch_url` for the *same* task is the standard flow: Exa finds the URL, `fetch_url` gets the content. +The standard flow: `exa_search` finds the URL → `fetch_url` reads it. ## How to use it -The API key lives in env as `EXA_API_KEY`. Endpoint and auth: +``` +exa_search(query="Vertex AI Gemini 3.5 Flash EU multi-region quota docs", + num_results=5, + include_domains=["cloud.google.com"]) # optional +``` + +- `query` (required) — be specific: names, versions, error strings, domains. +- `num_results` — default 5; the tool caps at 10. Refine, don't widen. +- `include_domains` — anchor on known-authoritative sources early; better + signal-to-cost than a generic search. +- `category` — one of `company`, `research paper`, `news`, `personal site`, + `financial report`, `people`. Caution: `company`/`people` combined with + date filters or `excludeDomains` is rejected by Exa (400). + +The tool returns numbered results (title, URL, publishedDate, highlight +snippet) plus the actual dollar cost of the query. An empty result set comes +back as a "no results" message — that's a valid answer, not an error; sharpen +the query. + +## Curl fallback (advanced params only) + +Only when you need something the tool doesn't expose (`type: deep`, +`startPublishedDate`, `contents.text`, the `/contents` or `/answer` +endpoints). The key is in env as `EXA_API_KEY`. +**`jq` is NOT installed in this container — never pipe to it.** +Use `python3 -m json.tool`, or write the response to a file and parse +with `python3`. ```bash curl -s -X POST 'https://api.exa.ai/search' \ -H "x-api-key: $EXA_API_KEY" \ -H 'Content-Type: application/json' \ - -d '{ - "query": "Vertex AI Gemini 3.5 Flash EU multi-region quota docs", - "numResults": 5, - "contents": {"highlights": true} - }' | jq '.results[] | {title, url, snippet: .highlights[0]}' + -d '{"query": "...", "numResults": 5, "type": "deep", "contents": {"highlights": true}}' \ + | python3 -m json.tool ``` -Key params: - -- `query` (required) — your search string. Make it specific. -- `numResults` (default 10, range 1-100) — keep it ≤5 unless you genuinely need a wider net. -- `contents.highlights: true` — returns relevant snippets per result so you can pick without fetching every URL. -- `type` — `auto` (default), `fast`, `deep`, `instant`. `auto` is right for most cases. Use `deep` only when the first `auto` pass missed. -- `includeDomains` / `excludeDomains` — `["docs.cloud.google.com"]` shape. Use to anchor on known-authoritative sources. -- `category` — `research paper`, `news`, `company`, `people`, etc. Narrows the search index. - -Response shape (the parts that matter): - -- `results[]` — each has `title`, `url`, `publishedDate`, `author`, `text`, `highlights[]`, `summary` -- `costDollars` — actual cost of THIS query. Log it in the journal if it surprises you. +Current `type` values: `instant`, `fast`, `auto` (default), `deep-lite`, +`deep`, `deep-reasoning` ($$$, 12–50s). Response fields `text` / `summary` +only appear if you requested them under `contents`. Errors come back as +`{"requestId", "error", "tag"}` — branch on `tag`. Rate limit: 10 QPS on +/search; back off exponentially on 429. ## Cost discipline -Exa charges per query. Cheap (cents) but not free — and the budget compounds if you fan out. +Exa charges per query (~$0.005–0.01 for a normal search). Cheap but not free. - **One query per task.** Don't loop. Don't run 5 variations. -- **One target URL per query result set.** Pick the most relevant from `highlights`, then `fetch_url`. Don't fetch every result. -- **Refine, don't paginate.** If `numResults=5` doesn't surface the right page, your query is too vague — make it more specific. Asking for more results rarely fixes a vague query. -- **Use `includeDomains` early.** When you know the source family (Anthropic docs, Google Cloud docs, GitHub), include them. Better signal-to-cost ratio than a generic search. -- **Journal the cost** if a query was unusual (e.g., `deep` type). Pattern detection lives in daily-maintenance §1 — if Exa cost shows up as friction across multiple sessions, that's a signal to add a rule. +- **One target URL per result set.** Pick the best from the highlights, then + `fetch_url` it. Don't fetch every result. +- **Refine, don't paginate.** If 5 results don't surface the right page, the + query is too vague. +- **Journal the cost** if a query was unusual (e.g. a `deep` type via curl). + Pattern detection lives in daily-maintenance §1 — if Exa cost shows up as + friction across multiple sessions, that's a signal to add a rule. ## Then what -`exa-search` returns URLs + snippets. Pick the most relevant. Then `fetch_url` it to get the actual page content. Don't try to synthesize from the snippets alone — they're a clue, not the content. - -If none of the results look right, the query was wrong. Re-query with a sharper specific (a function name, a version number, a domain). +Pick the most relevant result and `fetch_url` it. Don't synthesize from +snippets alone — they're a clue, not the content. If nothing looks right, +the query was wrong: re-query with a sharper specific (function name, +version number, domain). diff --git a/tests/runtime/test_exa_search.py b/tests/runtime/test_exa_search.py new file mode 100644 index 0000000..e49cac3 --- /dev/null +++ b/tests/runtime/test_exa_search.py @@ -0,0 +1,74 @@ +"""Tests for the `exa_search` runtime tool. + +Regression coverage for the 2026-07-09..13 crash mode: the skill catalog +advertised `exa-search` while no `exa_search` FunctionTool existed, so the +model emitted calls the runner had no handler for and the whole session +died. These tests pin the tool's existence, its registration, and its +never-raise error contract. No network calls — the live happy path is +covered by the pre-merge verification plan in the PR. +""" +from __future__ import annotations + +import importlib +import inspect + + +def _adk_runner(): + from src.runtime import adk_runner + importlib.reload(adk_runner) + return adk_runner + + +def test_exa_search_exists_with_expected_signature(tmp_sam_home): + adk_runner = _adk_runner() + params = inspect.signature(adk_runner.exa_search).parameters + # `query` first and required — it's the arg shape the model already + # emitted during the crash incidents. + assert list(params)[0] == "query" + assert params["query"].default is inspect.Parameter.empty + assert params["num_results"].default == 5 + + +def test_exa_search_missing_key_returns_error_string(tmp_sam_home, monkeypatch): + adk_runner = _adk_runner() + monkeypatch.delenv("EXA_API_KEY", raising=False) + + result = adk_runner.exa_search("anything") + + assert result == "error: EXA_API_KEY not set in environment" + + +def test_exa_search_never_raises_on_connection_failure(tmp_sam_home, monkeypatch): + # Tool-error contract: failures come back as strings the model can read, + # never exceptions (an exception here is exactly the old crash mode). + adk_runner = _adk_runner() + monkeypatch.setenv("EXA_API_KEY", "test-key") + import httpx + + def boom(*args, **kwargs): + raise httpx.ConnectError("no route to host") + + monkeypatch.setattr(httpx, "post", boom) + + result = adk_runner.exa_search("anything") + + assert result.startswith("error: exa_search failed: ConnectError") + + +def test_exa_search_wraps_as_function_tool(tmp_sam_home): + from google.adk.tools import FunctionTool + + adk_runner = _adk_runner() + tool = FunctionTool(func=adk_runner.exa_search) + assert tool.name == "exa_search" + + +def test_exa_search_registered_in_worker_tools_and_fanout(tmp_sam_home): + # Guard both registration sites: the shared worker_tools list (flows to + # main, worker, pro_executor) and the parallel_workers fanout branch. + from pathlib import Path + + source = Path("src/runtime/adk_runner.py").read_text() + assert source.count("FunctionTool(func=exa_search)") == 2 + worker_tools_block = source.split("worker_tools = [")[1].split("]")[0] + assert "exa_search" in worker_tools_block