From cdf80c2cc1f761d32a037fb0c06e8024c24a92b6 Mon Sep 17 00:00:00 2001 From: Emos21 Date: Wed, 29 Jul 2026 17:25:29 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20search=5Fweb=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=EF=BC=8C=E6=8E=A5=E5=85=A5=20You.com=20?= =?UTF-8?q?=E8=81=94=E7=BD=91=E6=A3=80=E7=B4=A2=E4=BD=9C=E4=B8=BA=E5=B8=A6?= =?UTF-8?q?=E5=BC=95=E7=94=A8=E7=9A=84=E8=A1=A5=E5=85=85=E6=9D=A5=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 配置 YDC_API_KEY 后，对话 Agent 多出 search_web 工具，把公开网页作为本地 知识库之外的补充来源；结果复用现有引用管线（source=标题，snippet 含来源 URL）， 回答须标注 URL。未配置 Key 时不挂载该工具，行为完全不变。 - infra/websearch/youcom_client.py：外部 I/O 层（AGENTS 规则 9），httpx 调用 You.com Search API（X-API-Key + 团队约定 User-Agent），归一化 web/news、 count 夹取与总数截断；网络/状态码/解析异常统一返回空结果，绝不抛出。 - usecases/chat_agent_tools.py：search_web 工具与系统提示按 Key 可用性挂载， 写入 ctx.last_citations（合成 web:{i} chunk_id，URL 随 snippet 持久化）。 - settings.py：新增 YDC_API_KEY 与 YDC_WEB_SEARCH_COUNT。 - tests：infra 客户端 8 例（映射/鉴权头/夹取截断/各类降级/Key 不泄露）+ Agent 工具 3 例（无 Key 不挂载、写入 Web 引用、空查询）。 - .env.example / backend/README：同步 YDC_API_KEY 与联网检索说明。 --- backend/.env.example | 9 ++ backend/README.md | 4 + backend/docpaws/infra/websearch/__init__.py | 7 + .../docpaws/infra/websearch/youcom_client.py | 104 ++++++++++++ backend/docpaws/settings.py | 5 + backend/docpaws/usecases/chat_agent_tools.py | 58 ++++++- backend/requirements.txt | 1 + backend/tests/test_chat_agent_search_web.py | 68 ++++++++ backend/tests/test_youcom_websearch.py | 151 ++++++++++++++++++ 9 files changed, 405 insertions(+), 2 deletions(-) create mode 100644 backend/docpaws/infra/websearch/__init__.py create mode 100644 backend/docpaws/infra/websearch/youcom_client.py create mode 100644 backend/tests/test_chat_agent_search_web.py create mode 100644 backend/tests/test_youcom_websearch.py diff --git a/backend/.env.example b/backend/.env.example index f184be4..75d6853 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -54,6 +54,15 @@ RETRIEVAL_CACHE_TTL_SECONDS=600 # =========================================== RETRIEVAL_MAX_DISTANCE=0 +# =========================================== +# 联网检索（可选）：You.com Search API +# 配置后 Agent 多出 search_web 工具，可把公开网页作为本地知识库之外的 +# 补充来源（结果带来源 URL 引用）；留空则不挂载该工具，行为完全不变。 +# 获取 Key：https://you.com/platform/api-keys +# =========================================== +YDC_API_KEY= +YDC_WEB_SEARCH_COUNT=5 + # =========================================== # 索引任务超时（秒） # =========================================== diff --git a/backend/README.md b/backend/README.md index ce92c03..6f13dd6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -85,3 +85,7 @@ python ../eval/run_rag_eval.py ## 检索阈值与拒答 `.env` 可选配置 `RETRIEVAL_MAX_DISTANCE`（FAISS L2，越小越相似；`0` 表示不按距离过滤，仅无结果时拒答）。未过阈值或检索为空时返回：「未检索到足够相关内容，无法基于文档回答。」Agent 流在 `run_agent_stream` 前会做预检（统计/列文档类问题除外）。 + +## 联网检索（可选） + +配置 `.env` 的 `YDC_API_KEY`（You.com Search API）后，对话 Agent 会多出一个 `search_web` 工具：把公开网页作为本地知识库之外的补充来源，用于最新信息或本地文档未覆盖的公开知识，结果带来源 URL 引用（复用现有引用管线）。未配置 Key 时不挂载该工具，行为完全不变。客户端在 `docpaws/infra/websearch/youcom_client.py`（外部 I/O 层），工具在 `docpaws/usecases/chat_agent_tools.py`。任何网络/状态码/解析异常都会降级为空结果，不影响本地检索链路。 diff --git a/backend/docpaws/infra/websearch/__init__.py b/backend/docpaws/infra/websearch/__init__.py new file mode 100644 index 0000000..65638cb --- /dev/null +++ b/backend/docpaws/infra/websearch/__init__.py @@ -0,0 +1,7 @@ +"""联网检索基础设施：You.com Search API 客户端（外部 I/O 层）。""" +from docpaws.infra.websearch.youcom_client import ( + web_search_available, + youcom_web_search, +) + +__all__ = ["web_search_available", "youcom_web_search"] diff --git a/backend/docpaws/infra/websearch/youcom_client.py b/backend/docpaws/infra/websearch/youcom_client.py new file mode 100644 index 0000000..da940cc --- /dev/null +++ b/backend/docpaws/infra/websearch/youcom_client.py @@ -0,0 +1,104 @@ +"""You.com Search API 客户端。 + +外部 I/O 层（见 AGENTS.md 规则 9）：调用 You.com Search API +（GET https://ydc-index.io/v1/search，X-API-Key 鉴权），把网页与新闻结果 +归一化为 [{title, url, snippet}]，供 usecases 层的 search_web 工具编排。 + +API Key 取自 settings.YDC_API_KEY（环境变量 YDC_API_KEY，团队约定名）。 +任何网络 / 状态码 / 解析异常都会被捕获并返回空结果，绝不向上抛出。 +""" +from __future__ import annotations + +import logging + +import httpx + +from docpaws.settings import settings + +logger = logging.getLogger(__name__) + +# You.com Search API 端点（团队约定） +YOUCOM_ENDPOINT = "https://ydc-index.io/v1/search" +# 团队约定：对 You.com 主机的请求需带此 User-Agent（lowercased owner-repo slug） +YOUCOM_USER_AGENT = "youdotcom-integration/biao994-docpaws" + +MAX_COUNT = 20 +REQUEST_TIMEOUT = 15.0 + + +def web_search_available() -> bool: + """是否已配置可用的 You.com API Key。""" + return bool(settings.YDC_API_KEY) + + +def youcom_web_search(query: str, count: int | None = None) -> list[dict]: + """调用 You.com Search API，返回归一化结果 [{title, url, snippet}]。 + + 未配置 Key、网络异常、非 2xx、解析失败时统一返回 []（记录日志，绝不抛出）。 + """ + api_key = settings.YDC_API_KEY + if not api_key: + logger.warning("web_search: 未配置 YDC_API_KEY，跳过联网检索") + return [] + + try: + n = int(count) if count is not None else int(settings.YDC_WEB_SEARCH_COUNT) + except (TypeError, ValueError): + n = int(settings.YDC_WEB_SEARCH_COUNT) + n = max(1, min(n, MAX_COUNT)) + + try: + with httpx.Client(timeout=REQUEST_TIMEOUT) as client: + resp = client.get( + YOUCOM_ENDPOINT, + params={"query": query, "count": n}, + headers={"X-API-Key": api_key, "User-Agent": YOUCOM_USER_AGENT}, + ) + except httpx.HTTPError as exc: + logger.error("You.com 联网检索请求失败: %s", exc) + return [] + + if resp.status_code != 200: + # 不回显响应体，避免泄露账号信息；401 鉴权 / 429 限流 / 5xx 服务端 + logger.error("You.com 联网检索返回状态码 %s", resp.status_code) + return [] + + try: + payload = resp.json() + except ValueError as exc: + logger.error("You.com 响应解析失败: %s", exc) + return [] + + return _normalize(payload, n) + + +def _normalize(payload: dict, n: int) -> list[dict]: + """把 You.com 响应归一化为 [{title, url, snippet}]。 + + 响应结构 {"results": {"web": [...], "news": [...]}}；news 可能缺失， + 每条结果除 url/title/description/snippets 外字段均视为可选，防御式读取。 + web 与 news 各返回最多 n 条，按合并总数截断，与 count 语义对齐。 + """ + if not isinstance(payload, dict): + return [] + results_node = payload.get("results") + if not isinstance(results_node, dict): + return [] + items: list[dict] = [] + for bucket in ("web", "news"): + entries = results_node.get(bucket) + if not isinstance(entries, list): + continue + for entry in entries: + if not isinstance(entry, dict): + continue + snippets = entry.get("snippets") + snippet_text = "" + if isinstance(snippets, list): + snippet_text = " ".join(s for s in snippets if isinstance(s, str)) + items.append({ + "title": entry.get("title") or "", + "url": entry.get("url") or "", + "snippet": entry.get("description") or snippet_text, + }) + return items[:n] diff --git a/backend/docpaws/settings.py b/backend/docpaws/settings.py index 88c7c28..00b6c6d 100644 --- a/backend/docpaws/settings.py +++ b/backend/docpaws/settings.py @@ -98,5 +98,10 @@ def is_sqlite(self) -> bool: # PDF 卡片缩略图（首页 WebP，最长边像素） THUMBNAIL_MAX_WIDTH = int(os.getenv("THUMBNAIL_MAX_WIDTH", "400")) + # You.com 联网检索（可选）：作为本地知识库之外的补充来源。 + # 团队约定环境变量名固定为 YDC_API_KEY；留空则 search_web 工具不可用。 + YDC_API_KEY = os.getenv("YDC_API_KEY", "") + YDC_WEB_SEARCH_COUNT = int(os.getenv("YDC_WEB_SEARCH_COUNT", "5")) + settings = Settings() diff --git a/backend/docpaws/usecases/chat_agent_tools.py b/backend/docpaws/usecases/chat_agent_tools.py index 5b258ee..9c118bc 100644 --- a/backend/docpaws/usecases/chat_agent_tools.py +++ b/backend/docpaws/usecases/chat_agent_tools.py @@ -12,6 +12,7 @@ from langchain_core.tools import tool from sqlmodel import Session +from docpaws.infra.websearch import web_search_available, youcom_web_search from docpaws.usecases.chat_scope import ( count_documents_in_scope, document_title_for_id, @@ -50,7 +51,7 @@ def build_agent_system_prompt(ctx: AgentToolContext) -> str: scope_type=ctx.scope_type, scope_id=ctx.scope_id, ) - return f"""你是 DocPaws 智能文档助手。当前对话已锁定在以下范围，不可切换知识库或文件夹： + base = f"""你是 DocPaws 智能文档助手。当前对话已锁定在以下范围，不可切换知识库或文件夹： {scope_label} ## 工具（按问题类型选择，勿向用户暴露工具名） @@ -69,6 +70,17 @@ def build_agent_system_prompt(ctx: AgentToolContext) -> str: - 回答使用中文，简洁准确；若工具无结果，如实说明 - 凡内容类问题必须先调 query_knowledge_base，禁止不查资料直接答 """ + if web_search_available(): + base += """ +## 联网检索（本地知识库之外的补充来源） +6. **search_web** — 用 You.com 联网检索公开网页。仅在问题涉及最新信息、时效性内容， + 或本地文档明显未覆盖的公开知识时使用；结果非本地文档，回答须显式标注来源 URL。 + +## 联网规则 +- 始终优先本地文档（query_knowledge_base）；仅当本地无法回答或需要最新信息时才用 search_web +- search_web 的结果来自互联网，须在回答中标注对应来源 URL +""" + return base def build_chat_agent_tools(ctx: AgentToolContext) -> list: @@ -236,10 +248,52 @@ def search_documents(keyword: str) -> str: parts ) - return [ + @tool + def search_web(query: str) -> str: + """用 You.com 联网检索公开网页，作为本地知识库之外的补充来源。 + + 适合最新信息、时效性内容或本地文档未覆盖的公开知识；结果非本地文档， + 回答须显式标注来源 URL。 + """ + q = (query or "").strip() + if not q: + return "查询不能为空。" + + results = youcom_web_search(q) + if not results: + return f"未检索到与「{q}」相关的联网结果。" + + citations: list[dict] = [] + lines: list[str] = [] + for i, item in enumerate(results): + title = item.get("title") or "网页结果" + url = item.get("url") or "" + snippet = item.get("snippet") or "" + lines.append(f"[{i + 1}] {title}\n{snippet}\n来源: {url}") + citation_snippet = snippet + ("\n" + url if url else "") + citations.append({ + # 联网结果无本地 chunk，用合成 id 占位（非本地文档） + "chunk_id": f"web:{i}", + "document_id": "", + "page_no": None, + "source": title, + # snippet 兜底为标题，保证非空（空 snippet 在历史还原时会被丢弃） + "snippet": citation_snippet.strip() or title, + }) + + ctx.last_citations = citations + ctx.last_hit_chunks = [] + body = "\n\n".join(lines) + return "联网检索结果（请基于以下内容作答并标注来源 URL）：\n\n" + body + + tools = [ count_scope_documents, list_scope_documents, lookup_scope_document, query_knowledge_base, search_documents, ] + # 仅在配置了 YDC_API_KEY 时挂载联网检索工具，未配置时行为完全不变 + if web_search_available(): + tools.append(search_web) + return tools diff --git a/backend/requirements.txt b/backend/requirements.txt index c4b4908..6c8328a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,6 +13,7 @@ python-dotenv==1.2.1 pymupdf==1.25.3 Pillow>=10.0.0 fastapi==0.115.0 +httpx==0.28.1 python-multipart==0.0.29 itsdangerous>=2.1.0 uvicorn[standard]==0.31.1 diff --git a/backend/tests/test_chat_agent_search_web.py b/backend/tests/test_chat_agent_search_web.py new file mode 100644 index 0000000..5d7424c --- /dev/null +++ b/backend/tests/test_chat_agent_search_web.py @@ -0,0 +1,68 @@ +"""search_web Agent 工具：仅在配置 Key 时挂载，返回联网结果并写入 Web 引用。""" +from docpaws.settings import settings +from docpaws.usecases.chat_agent_tools import ( + AgentToolContext, + build_chat_agent_tools, +) + + +def _ctx() -> AgentToolContext: + return AgentToolContext( + session=None, + kb_id="kb1", + scope_type="kb", + scope_id=None, + vectorstore=None, + metadata_filter=None, + search_k=5, + cache_redis=None, + artifact_id="art", + scope_token="tok", + model_name="m", + ) + + +def _find(tools, name): + for t in tools: + if getattr(t, "name", None) == name: + return t + return None + + +def test_search_web_absent_without_key(monkeypatch): + monkeypatch.setattr(settings, "YDC_API_KEY", "") + tools = build_chat_agent_tools(_ctx()) + assert _find(tools, "search_web") is None + + +def test_search_web_sets_web_citations(monkeypatch): + monkeypatch.setattr(settings, "YDC_API_KEY", "k") + fake = [ + {"title": "标题A", "url": "https://e.com/a", "snippet": "描述A"}, + {"title": "标题B", "url": "https://e.com/b", "snippet": "描述B"}, + ] + monkeypatch.setattr( + "docpaws.usecases.chat_agent_tools.youcom_web_search", + lambda q, count=None: fake, + ) + ctx = _ctx() + tool = _find(build_chat_agent_tools(ctx), "search_web") + assert tool is not None + + out = tool.invoke({"query": "最新进展"}) + assert "标题A" in out and "https://e.com/a" in out + assert "来源: https://e.com/a" in out + + assert len(ctx.last_citations) == 2 + c0 = ctx.last_citations[0] + assert c0["chunk_id"] == "web:0" + assert c0["document_id"] == "" + assert c0["source"] == "标题A" + assert "https://e.com/a" in c0["snippet"] # URL 随 snippet 保留，可持久化 + assert ctx.last_hit_chunks == [] + + +def test_search_web_empty_query(monkeypatch): + monkeypatch.setattr(settings, "YDC_API_KEY", "k") + tool = _find(build_chat_agent_tools(_ctx()), "search_web") + assert "不能为空" in tool.invoke({"query": " "}) diff --git a/backend/tests/test_youcom_websearch.py b/backend/tests/test_youcom_websearch.py new file mode 100644 index 0000000..a0f0c8f --- /dev/null +++ b/backend/tests/test_youcom_websearch.py @@ -0,0 +1,151 @@ +"""infra/websearch/youcom_client 单元测试（离线，不触网、不需要真实 Key）。 + +用 httpx.MockTransport 打桩 You.com Search API，覆盖：来源映射、鉴权头与 +User-Agent、count 夹取与总数截断、未配置 Key、各类错误（非 2xx / 网络异常 / +解析失败）优雅降级、异常响应结构不崩溃、Key 不泄露。 + +标准库 unittest，pytest 亦可收集；可 `python tests/test_youcom_websearch.py` 直跑。 +""" +import unittest +from unittest import mock + +import httpx + +from docpaws.infra.websearch import youcom_client as yc +from docpaws.settings import settings + +UA = "youdotcom-integration/biao994-docpaws" + +SAMPLE = { + "results": { + "web": [ + {"url": "https://e.com/a", "title": "标题A", "description": "描述A", + "snippets": ["片段A1", "片段A2"]}, + {"url": "https://e.com/b", "title": "标题B", "snippets": ["片段B1"]}, + ], + "news": [ + {"url": "https://e.com/n", "title": "新闻N", "description": "新闻描述"}, + ], + }, + "metadata": {"query": "t"}, +} + + +def _mock_httpx(handler): + """返回一个把 MockTransport 注入 httpx.Client 的替身工厂。""" + real_client = httpx.Client + + def factory(*_args, **kwargs): + return real_client( + transport=httpx.MockTransport(handler), timeout=kwargs.get("timeout", 15) + ) + return factory + + +class YouComClientTest(unittest.TestCase): + + def setUp(self): + self._key_patch = mock.patch.object(settings, "YDC_API_KEY", "secret-key") + self._key_patch.start() + + def tearDown(self): + self._key_patch.stop() + + def test_available_reflects_key(self): + self.assertTrue(yc.web_search_available()) + with mock.patch.object(settings, "YDC_API_KEY", ""): + self.assertFalse(yc.web_search_available()) + + def test_no_key_returns_empty_without_calling(self): + with mock.patch.object(settings, "YDC_API_KEY", ""): + with mock.patch.object(yc.httpx, "Client") as client: + self.assertEqual(yc.youcom_web_search("q"), []) + client.assert_not_called() + + def test_maps_web_and_news_with_headers(self): + captured = {} + + def handler(request): + captured["headers"] = request.headers + captured["params"] = dict(request.url.params) + return httpx.Response(200, json=SAMPLE) + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + out = yc.youcom_web_search("测试", count=5) + + self.assertEqual(len(out), 3) + self.assertEqual(out[0], { + "title": "标题A", "url": "https://e.com/a", "snippet": "描述A"}) + self.assertEqual(out[1]["snippet"], "片段B1") # description 缺失回退 snippet + self.assertEqual(out[2]["title"], "新闻N") # news 也纳入 + self.assertEqual(captured["headers"]["X-API-Key"], "secret-key") + self.assertEqual(captured["headers"]["User-Agent"], UA) + self.assertEqual(captured["params"]["query"], "测试") + self.assertEqual(captured["params"]["count"], "5") + + def test_count_clamped_and_total_truncated(self): + body = {"results": { + "web": [{"url": f"u{i}", "title": f"T{i}"} for i in range(4)], + "news": [{"url": "un", "title": "N"}], + }} + seen = {} + + def handler(request): + seen["count"] = request.url.params.get("count") + return httpx.Response(200, json=body) + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + yc.youcom_web_search("q", count=99) # 99 → MAX_COUNT(20) + self.assertEqual(seen["count"], "20") + out = yc.youcom_web_search("q", count=3) # web+news=5 → 截断为 3 + self.assertEqual(len(out), 3) + + def test_non_200_returns_empty_no_key_leak(self): + def handler(request): + return httpx.Response(500, text="server boom secret-key") + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + out = yc.youcom_web_search("q") + self.assertEqual(out, []) + self.assertNotIn("secret-key", str(out)) + + def test_network_error_returns_empty(self): + def handler(request): + raise httpx.ConnectError("boom") + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + self.assertEqual(yc.youcom_web_search("q"), []) + + def test_bad_json_returns_empty(self): + def handler(request): + return httpx.Response(200, text="not-json{{") + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + self.assertEqual(yc.youcom_web_search("q"), []) + + def test_non_dict_payloads_return_empty(self): + # 顶层为 list/str，或 results 为非 dict：不得抛出，统一返回 [] + for body in ('[1, 2, 3]', '"oops"', '{"results": "oops"}', '{"results": [1]}'): + def handler(request, _b=body): + return httpx.Response(200, text=_b, headers={"content-type": "application/json"}) + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + self.assertEqual(yc.youcom_web_search("q"), [], body) + + def test_hostile_shapes_no_crash(self): + body = {"results": { + "web": ["str", 42, None, {"title": None, "url": None, + "snippets": [None, "有效片段"]}], + "news": "not-a-list", + }} + + def handler(request): + return httpx.Response(200, json=body) + + with mock.patch.object(yc.httpx, "Client", _mock_httpx(handler)): + out = yc.youcom_web_search("q") + self.assertEqual(len(out), 1) # 非 dict / 非法项被跳过 + self.assertEqual(out[0]["snippet"], "有效片段") + + +if __name__ == "__main__": + unittest.main()