Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions backend/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@ RETRIEVAL_CACHE_TTL_SECONDS=600
# ===========================================
RETRIEVAL_MAX_DISTANCE=0

# ===========================================
# 联网检索(可选):You.com Search API
# 配置后 Agent 多出 search_web 工具,可把公开网页作为本地知识库之外的
# 补充来源(结果带来源 URL 引用);留空则不挂载该工具,行为完全不变。
# 获取 Key:https://you.com/platform/api-keys
# ===========================================
YDC_API_KEY=
YDC_WEB_SEARCH_COUNT=5

# ===========================================
# 索引任务超时(秒)
# ===========================================
Expand Down
4 changes: 4 additions & 0 deletions backend/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,7 @@ python ../eval/run_rag_eval.py
## 检索阈值与拒答

`.env` 可选配置 `RETRIEVAL_MAX_DISTANCE`(FAISS L2,越小越相似;`0` 表示不按距离过滤,仅无结果时拒答)。未过阈值或检索为空时返回:「未检索到足够相关内容,无法基于文档回答。」Agent 流在 `run_agent_stream` 前会做预检(统计/列文档类问题除外)。

## 联网检索(可选)

配置 `.env` 的 `YDC_API_KEY`(You.com Search API)后,对话 Agent 会多出一个 `search_web` 工具:把公开网页作为本地知识库之外的补充来源,用于最新信息或本地文档未覆盖的公开知识,结果带来源 URL 引用(复用现有引用管线)。未配置 Key 时不挂载该工具,行为完全不变。客户端在 `docpaws/infra/websearch/youcom_client.py`(外部 I/O 层),工具在 `docpaws/usecases/chat_agent_tools.py`。任何网络/状态码/解析异常都会降级为空结果,不影响本地检索链路。
7 changes: 7 additions & 0 deletions backend/docpaws/infra/websearch/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
"""联网检索基础设施:You.com Search API 客户端(外部 I/O 层)。"""
from docpaws.infra.websearch.youcom_client import (
web_search_available,
youcom_web_search,
)

__all__ = ["web_search_available", "youcom_web_search"]
104 changes: 104 additions & 0 deletions backend/docpaws/infra/websearch/youcom_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
"""You.com Search API 客户端。

外部 I/O 层(见 AGENTS.md 规则 9):调用 You.com Search API
(GET https://ydc-index.io/v1/search,X-API-Key 鉴权),把网页与新闻结果
归一化为 [{title, url, snippet}],供 usecases 层的 search_web 工具编排。

API Key 取自 settings.YDC_API_KEY(环境变量 YDC_API_KEY,团队约定名)。
任何网络 / 状态码 / 解析异常都会被捕获并返回空结果,绝不向上抛出。
"""
from __future__ import annotations

import logging

import httpx

from docpaws.settings import settings

logger = logging.getLogger(__name__)

# You.com Search API 端点(团队约定)
YOUCOM_ENDPOINT = "https://ydc-index.io/v1/search"
# 团队约定:对 You.com 主机的请求需带此 User-Agent(lowercased owner-repo slug)
YOUCOM_USER_AGENT = "youdotcom-integration/biao994-docpaws"

MAX_COUNT = 20
REQUEST_TIMEOUT = 15.0


def web_search_available() -> bool:
"""是否已配置可用的 You.com API Key。"""
return bool(settings.YDC_API_KEY)


def youcom_web_search(query: str, count: int | None = None) -> list[dict]:
"""调用 You.com Search API,返回归一化结果 [{title, url, snippet}]。

未配置 Key、网络异常、非 2xx、解析失败时统一返回 [](记录日志,绝不抛出)。
"""
api_key = settings.YDC_API_KEY
if not api_key:
logger.warning("web_search: 未配置 YDC_API_KEY,跳过联网检索")
return []

try:
n = int(count) if count is not None else int(settings.YDC_WEB_SEARCH_COUNT)
except (TypeError, ValueError):
n = int(settings.YDC_WEB_SEARCH_COUNT)
n = max(1, min(n, MAX_COUNT))

try:
with httpx.Client(timeout=REQUEST_TIMEOUT) as client:
resp = client.get(
YOUCOM_ENDPOINT,
params={"query": query, "count": n},
headers={"X-API-Key": api_key, "User-Agent": YOUCOM_USER_AGENT},
)
except httpx.HTTPError as exc:
logger.error("You.com 联网检索请求失败: %s", exc)
return []

if resp.status_code != 200:
# 不回显响应体,避免泄露账号信息;401 鉴权 / 429 限流 / 5xx 服务端
logger.error("You.com 联网检索返回状态码 %s", resp.status_code)
return []

try:
payload = resp.json()
except ValueError as exc:
logger.error("You.com 响应解析失败: %s", exc)
return []

return _normalize(payload, n)


def _normalize(payload: dict, n: int) -> list[dict]:
"""把 You.com 响应归一化为 [{title, url, snippet}]。

响应结构 {"results": {"web": [...], "news": [...]}};news 可能缺失,
每条结果除 url/title/description/snippets 外字段均视为可选,防御式读取。
web 与 news 各返回最多 n 条,按合并总数截断,与 count 语义对齐。
"""
if not isinstance(payload, dict):
return []
results_node = payload.get("results")
if not isinstance(results_node, dict):
return []
items: list[dict] = []
for bucket in ("web", "news"):
entries = results_node.get(bucket)
if not isinstance(entries, list):
continue
for entry in entries:
if not isinstance(entry, dict):
continue
snippets = entry.get("snippets")
snippet_text = ""
if isinstance(snippets, list):
snippet_text = " ".join(s for s in snippets if isinstance(s, str))
items.append({
"title": entry.get("title") or "",
"url": entry.get("url") or "",
"snippet": entry.get("description") or snippet_text,
})
return items[:n]
5 changes: 5 additions & 0 deletions backend/docpaws/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,5 +98,10 @@ def is_sqlite(self) -> bool:
# PDF 卡片缩略图(首页 WebP,最长边像素)
THUMBNAIL_MAX_WIDTH = int(os.getenv("THUMBNAIL_MAX_WIDTH", "400"))

# You.com 联网检索(可选):作为本地知识库之外的补充来源。
# 团队约定环境变量名固定为 YDC_API_KEY;留空则 search_web 工具不可用。
YDC_API_KEY = os.getenv("YDC_API_KEY", "")
YDC_WEB_SEARCH_COUNT = int(os.getenv("YDC_WEB_SEARCH_COUNT", "5"))


settings = Settings()
58 changes: 56 additions & 2 deletions backend/docpaws/usecases/chat_agent_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from langchain_core.tools import tool
from sqlmodel import Session

from docpaws.infra.websearch import web_search_available, youcom_web_search
from docpaws.usecases.chat_scope import (
count_documents_in_scope,
document_title_for_id,
Expand Down Expand Up @@ -50,7 +51,7 @@ def build_agent_system_prompt(ctx: AgentToolContext) -> str:
scope_type=ctx.scope_type,
scope_id=ctx.scope_id,
)
return f"""你是 DocPaws 智能文档助手。当前对话已锁定在以下范围,不可切换知识库或文件夹:
base = f"""你是 DocPaws 智能文档助手。当前对话已锁定在以下范围,不可切换知识库或文件夹:
{scope_label}

## 工具(按问题类型选择,勿向用户暴露工具名)
Expand All @@ -69,6 +70,17 @@ def build_agent_system_prompt(ctx: AgentToolContext) -> str:
- 回答使用中文,简洁准确;若工具无结果,如实说明
- 凡内容类问题必须先调 query_knowledge_base,禁止不查资料直接答
"""
if web_search_available():
base += """
## 联网检索(本地知识库之外的补充来源)
6. **search_web** — 用 You.com 联网检索公开网页。仅在问题涉及最新信息、时效性内容,
或本地文档明显未覆盖的公开知识时使用;结果非本地文档,回答须显式标注来源 URL。

## 联网规则
- 始终优先本地文档(query_knowledge_base);仅当本地无法回答或需要最新信息时才用 search_web
- search_web 的结果来自互联网,须在回答中标注对应来源 URL
"""
return base


def build_chat_agent_tools(ctx: AgentToolContext) -> list:
Expand Down Expand Up @@ -236,10 +248,52 @@ def search_documents(keyword: str) -> str:
parts
)

return [
@tool
def search_web(query: str) -> str:
"""用 You.com 联网检索公开网页,作为本地知识库之外的补充来源。

适合最新信息、时效性内容或本地文档未覆盖的公开知识;结果非本地文档,
回答须显式标注来源 URL。
"""
q = (query or "").strip()
if not q:
return "查询不能为空。"

results = youcom_web_search(q)
if not results:
return f"未检索到与「{q}」相关的联网结果。"

citations: list[dict] = []
lines: list[str] = []
for i, item in enumerate(results):
title = item.get("title") or "网页结果"
url = item.get("url") or ""
snippet = item.get("snippet") or ""
lines.append(f"[{i + 1}] {title}\n{snippet}\n来源: {url}")
citation_snippet = snippet + ("\n" + url if url else "")
citations.append({
# 联网结果无本地 chunk,用合成 id 占位(非本地文档)
"chunk_id": f"web:{i}",
"document_id": "",
"page_no": None,
"source": title,
# snippet 兜底为标题,保证非空(空 snippet 在历史还原时会被丢弃)
"snippet": citation_snippet.strip() or title,
})

ctx.last_citations = citations
ctx.last_hit_chunks = []
body = "\n\n".join(lines)
return "联网检索结果(请基于以下内容作答并标注来源 URL):\n\n" + body

tools = [
count_scope_documents,
list_scope_documents,
lookup_scope_document,
query_knowledge_base,
search_documents,
]
# 仅在配置了 YDC_API_KEY 时挂载联网检索工具,未配置时行为完全不变
if web_search_available():
tools.append(search_web)
return tools
1 change: 1 addition & 0 deletions backend/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ python-dotenv==1.2.1
pymupdf==1.25.3
Pillow>=10.0.0
fastapi==0.115.0
httpx==0.28.1
python-multipart==0.0.29
itsdangerous>=2.1.0
uvicorn[standard]==0.31.1
Expand Down
68 changes: 68 additions & 0 deletions backend/tests/test_chat_agent_search_web.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
"""search_web Agent 工具:仅在配置 Key 时挂载,返回联网结果并写入 Web 引用。"""
from docpaws.settings import settings
from docpaws.usecases.chat_agent_tools import (
AgentToolContext,
build_chat_agent_tools,
)


def _ctx() -> AgentToolContext:
return AgentToolContext(
session=None,
kb_id="kb1",
scope_type="kb",
scope_id=None,
vectorstore=None,
metadata_filter=None,
search_k=5,
cache_redis=None,
artifact_id="art",
scope_token="tok",
model_name="m",
)


def _find(tools, name):
for t in tools:
if getattr(t, "name", None) == name:
return t
return None


def test_search_web_absent_without_key(monkeypatch):
monkeypatch.setattr(settings, "YDC_API_KEY", "")
tools = build_chat_agent_tools(_ctx())
assert _find(tools, "search_web") is None


def test_search_web_sets_web_citations(monkeypatch):
monkeypatch.setattr(settings, "YDC_API_KEY", "k")
fake = [
{"title": "标题A", "url": "https://e.com/a", "snippet": "描述A"},
{"title": "标题B", "url": "https://e.com/b", "snippet": "描述B"},
]
monkeypatch.setattr(
"docpaws.usecases.chat_agent_tools.youcom_web_search",
lambda q, count=None: fake,
)
ctx = _ctx()
tool = _find(build_chat_agent_tools(ctx), "search_web")
assert tool is not None

out = tool.invoke({"query": "最新进展"})
assert "标题A" in out and "https://e.com/a" in out
assert "来源: https://e.com/a" in out

assert len(ctx.last_citations) == 2
c0 = ctx.last_citations[0]
assert c0["chunk_id"] == "web:0"
assert c0["document_id"] == ""
assert c0["source"] == "标题A"
assert "https://e.com/a" in c0["snippet"] # URL 随 snippet 保留,可持久化
assert ctx.last_hit_chunks == []


def test_search_web_empty_query(monkeypatch):
monkeypatch.setattr(settings, "YDC_API_KEY", "k")
tool = _find(build_chat_agent_tools(_ctx()), "search_web")
assert "不能为空" in tool.invoke({"query": " "})
Loading