T4: AnySearch 联网检索接入 (anysearch_client trust_env直连 NO_PROXY, kb_router use_web RRF融合, chat_context 自动注入联网上下文, admin 支持 anysearch 服务类型, 新增 POST /api/knowledge/web-search 端点)
This commit is contained in:
1 parent
be7d792dc2
commit
3c9d9518b1
5 files changed
+3175
-7
No files matched your search
@@ -0,0 +1,118 @@
|
|||||||
|
"""
|
||||||
|
AnySearch 联网检索客户端
|
||||||
|
======================
|
||||||
|
调用 AnySearch 网页/领域搜索 API (https://api.anysearch.com/v1/search),
|
||||||
|
为 AI 聊天注入实时联网检索结果,亦可被后端端点 / Agent 工具复用。
|
||||||
|
|
||||||
|
约束:平台容器存在智能路由代理,对外部 API 偶发重置;本客户端使用
|
||||||
|
trust_env=False 直连 (等价于 NO_PROXY=api.anysearch.com),规避代理重置。
|
||||||
|
失败一律返回空/降级结果,保证聊天链路不中断。
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from typing import List, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger("aiflow.anysearch")
|
||||||
|
|
||||||
|
DEFAULT_API_URL = "https://api.anysearch.com/v1/search"
|
||||||
|
DEFAULT_ZONE = "cn"
|
||||||
|
REQUEST_TIMEOUT = 8.0
|
||||||
|
|
||||||
|
|
||||||
|
async def anysearch_search(
|
||||||
|
api_key: str,
|
||||||
|
api_url: str = DEFAULT_API_URL,
|
||||||
|
query: str = "",
|
||||||
|
top_k: int = 5,
|
||||||
|
zone: str = DEFAULT_ZONE,
|
||||||
|
freshness: Optional[str] = None,
|
||||||
|
timeout: float = REQUEST_TIMEOUT,
|
||||||
|
) -> List[Dict]:
|
||||||
|
"""调用 AnySearch 搜索,返回结构化结果列表。
|
||||||
|
|
||||||
|
每个结果:
|
||||||
|
{
|
||||||
|
"content": snippet 摘要文本,
|
||||||
|
"score": 0.9..0.5 递减,
|
||||||
|
"source": "web_anysearch",
|
||||||
|
"metadata": {"url", "title", "source_domain", "zone"}
|
||||||
|
}
|
||||||
|
失败时返回 [] (不抛异常)。
|
||||||
|
"""
|
||||||
|
if not api_key:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
import httpx
|
||||||
|
except ImportError:
|
||||||
|
logger.warning("httpx not available for AnySearch")
|
||||||
|
return []
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"query": query,
|
||||||
|
"zone": zone,
|
||||||
|
"top_k": max(1, min(int(top_k), 20)),
|
||||||
|
}
|
||||||
|
if freshness:
|
||||||
|
payload["freshness"] = freshness
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Authorization": f"Bearer {api_key}",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
# trust_env=False -> 忽略容器内 HTTP(S)_PROXY,直连 api.anysearch.com
|
||||||
|
async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client:
|
||||||
|
resp = await client.post(api_url, json=payload, headers=headers)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
logger.warning(f"AnySearch HTTP {resp.status_code}: {resp.text[:200]}")
|
||||||
|
return []
|
||||||
|
data = resp.json()
|
||||||
|
if data.get("code") != 0:
|
||||||
|
logger.warning(f"AnySearch biz error: {data.get('message')}")
|
||||||
|
return []
|
||||||
|
results = (data.get("data") or {}).get("results") or []
|
||||||
|
hits = []
|
||||||
|
for idx, r in enumerate(results[:top_k]):
|
||||||
|
title = r.get("title", "") or ""
|
||||||
|
url = r.get("url", "") or ""
|
||||||
|
snippet = r.get("snippet") or r.get("content") or ""
|
||||||
|
if not snippet and not title:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
domain = urlparse(url).netloc
|
||||||
|
except Exception:
|
||||||
|
domain = ""
|
||||||
|
hits.append({
|
||||||
|
"content": snippet,
|
||||||
|
"score": max(0.4, 0.9 - idx * 0.08),
|
||||||
|
"source": "web_anysearch",
|
||||||
|
"metadata": {
|
||||||
|
"url": url,
|
||||||
|
"title": title,
|
||||||
|
"source_domain": domain,
|
||||||
|
"zone": zone,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return hits
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"AnySearch search failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
async def anysearch_health(api_key: str, api_url: str = DEFAULT_API_URL, timeout: float = REQUEST_TIMEOUT) -> Dict:
|
||||||
|
"""连通性探测:返回 {"status": "connected"/"disconnected", "detail": ...}。"""
|
||||||
|
if not api_key:
|
||||||
|
return {"status": "disconnected", "detail": "未配置 API Key", "connected": False, "usable": False}
|
||||||
|
try:
|
||||||
|
import httpx
|
||||||
|
async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client:
|
||||||
|
resp = await client.post(
|
||||||
|
api_url,
|
||||||
|
json={"query": "ping", "zone": DEFAULT_ZONE, "top_k": 1},
|
||||||
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
|
||||||
|
)
|
||||||
|
if resp.status_code == 200:
|
||||||
|
return {"status": "connected", "detail": "AnySearch 连接正常", "connected": True, "usable": True}
|
||||||
|
return {"status": "error", "detail": f"HTTP {resp.status_code}", "connected": False, "usable": False}
|
||||||
|
except Exception as e:
|
||||||
|
return {"status": "error", "detail": str(e)[:200], "connected": False, "usable": False}
|
||||||
@@ -32,7 +32,7 @@ logger = logging.getLogger("aiflow")
|
|||||||
|
|
||||||
class ExternalServiceConfig(BaseModel):
|
class ExternalServiceConfig(BaseModel):
|
||||||
name: str = Field(min_length=1, max_length=200)
|
name: str = Field(min_length=1, max_length=200)
|
||||||
service_type: str = Field(..., description="dify")
|
service_type: str = Field(..., description="dify / anysearch")
|
||||||
api_url: str = Field(min_length=1, max_length=500)
|
api_url: str = Field(min_length=1, max_length=500)
|
||||||
api_key: Optional[str] = None
|
api_key: Optional[str] = None
|
||||||
config: Optional[Dict[str, Any]] = None
|
config: Optional[Dict[str, Any]] = None
|
||||||
@@ -738,12 +738,15 @@ def _get_external_services_table(db: Session):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict:
|
async def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict:
|
||||||
"""测试外部知识库服务连接状态(当前仅支持 Dify)"""
|
"""测试外部知识库服务连接状态(支持 Dify / AnySearch)"""
|
||||||
try:
|
try:
|
||||||
if service_type == "dify":
|
if service_type == "dify":
|
||||||
from ...dify_kb import dify_test
|
from ...dify_kb import dify_test
|
||||||
return dify_test(api_url, api_key)
|
return dify_test(api_url, api_key)
|
||||||
|
if service_type == "anysearch":
|
||||||
|
from ...anysearch_client import anysearch_health
|
||||||
|
return await anysearch_health(api_key or "", api_url or "https://api.anysearch.com/v1/search")
|
||||||
else:
|
else:
|
||||||
# 未知类型(如历史遗留的 fastgpt/ragflow),按通用方式探测可达性
|
# 未知类型(如历史遗留的 fastgpt/ragflow),按通用方式探测可达性
|
||||||
headers = {}
|
headers = {}
|
||||||
@@ -781,6 +784,7 @@ async def list_external_services(
|
|||||||
"items": services,
|
"items": services,
|
||||||
"supported_types": [
|
"supported_types": [
|
||||||
{"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理(RAG)"},
|
{"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理(RAG)"},
|
||||||
|
{"type": "anysearch", "name": "AnySearch 联网检索", "description": "实时联网网页/领域搜索,为 AI 聊天注入最新网络信息"},
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -794,12 +798,12 @@ async def create_external_service(
|
|||||||
):
|
):
|
||||||
"""创建新的外部知识库服务连接配置"""
|
"""创建新的外部知识库服务连接配置"""
|
||||||
# 验证服务类型(当前仅支持 Dify)
|
# 验证服务类型(当前仅支持 Dify)
|
||||||
valid_types = ["dify"]
|
valid_types = ["dify", "anysearch"]
|
||||||
if req.service_type not in valid_types:
|
if req.service_type not in valid_types:
|
||||||
raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}")
|
raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}")
|
||||||
|
|
||||||
# 检查连接
|
# 检查连接
|
||||||
conn_result = _check_service_connection(req.service_type, req.api_url, req.api_key or "")
|
conn_result = await _check_service_connection(req.service_type, req.api_url, req.api_key or "")
|
||||||
|
|
||||||
service_id = str(uuid.uuid4())
|
service_id = str(uuid.uuid4())
|
||||||
config_json = json.dumps(req.config or {}, ensure_ascii=False)
|
config_json = json.dumps(req.config or {}, ensure_ascii=False)
|
||||||
@@ -916,7 +920,7 @@ async def update_external_service(
|
|||||||
new_key = req.api_key or row[4]
|
new_key = req.api_key or row[4]
|
||||||
if req.api_url or req.api_key:
|
if req.api_url or req.api_key:
|
||||||
service_type = row[2]
|
service_type = row[2]
|
||||||
conn_result = _check_service_connection(service_type, new_url, new_key)
|
conn_result = await _check_service_connection(service_type, new_url, new_key)
|
||||||
updates["status"] = conn_result.get("status", "disconnected")
|
updates["status"] = conn_result.get("status", "disconnected")
|
||||||
|
|
||||||
updates["updated_at"] = "NOW()"
|
updates["updated_at"] = "NOW()"
|
||||||
@@ -999,7 +1003,7 @@ async def check_external_service_status(
|
|||||||
if not row:
|
if not row:
|
||||||
raise HTTPException(status_code=404, detail="外部知识库服务不存在")
|
raise HTTPException(status_code=404, detail="外部知识库服务不存在")
|
||||||
|
|
||||||
conn_result = _check_service_connection(row[0], row[1], row[2])
|
conn_result = await _check_service_connection(row[0], row[1], row[2])
|
||||||
|
|
||||||
# 更新状态
|
# 更新状态
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1483,6 +1483,46 @@ async def tree_search_endpoint(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class WebSearchRequest(BaseModel):
|
||||||
|
query: str = Field(min_length=1, max_length=500)
|
||||||
|
top_k: int = 5
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/web-search", summary="AnySearch 联网检索")
|
||||||
|
async def web_search_endpoint(
|
||||||
|
req: WebSearchRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
current_user: User = Depends(get_current_user),
|
||||||
|
):
|
||||||
|
"""联网检索(AnySearch):返回实时网页结果,供前端 / 智能体引用。"""
|
||||||
|
try:
|
||||||
|
from sqlalchemy import text
|
||||||
|
import json as _json
|
||||||
|
from ..anysearch_client import anysearch_search
|
||||||
|
row = db.execute(text(
|
||||||
|
"SELECT api_key, api_url, config FROM external_kb_services "
|
||||||
|
"WHERE service_type='anysearch' AND is_enabled=1 LIMIT 1"
|
||||||
|
)).fetchone()
|
||||||
|
if not row:
|
||||||
|
return {"code": 0, "data": {"results": [], "enabled": False, "message": "未配置 AnySearch 联网检索服务"}}
|
||||||
|
api_key = row[0] or ""
|
||||||
|
api_url = row[1] or "https://api.anysearch.com/v1/search"
|
||||||
|
cfg = row[2]
|
||||||
|
if isinstance(cfg, str):
|
||||||
|
try:
|
||||||
|
cfg = _json.loads(cfg)
|
||||||
|
except Exception:
|
||||||
|
cfg = {}
|
||||||
|
cfg = cfg or {}
|
||||||
|
zone = cfg.get("zone", "cn")
|
||||||
|
freshness = cfg.get("freshness")
|
||||||
|
hits = await anysearch_search(api_key, api_url, req.query, top_k=req.top_k, zone=zone, freshness=freshness)
|
||||||
|
return {"code": 0, "data": {"results": hits, "enabled": True}}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"web-search endpoint failed: {e}")
|
||||||
|
return {"code": 1, "message": str(e), "data": {"results": []}}
|
||||||
|
|
||||||
|
|
||||||
def generate_document_excerpt(doc, query: str) -> str:
|
def generate_document_excerpt(doc, query: str) -> str:
|
||||||
"""根据文档类型生成模拟的原文摘录"""
|
"""根据文档类型生成模拟的原文摘录"""
|
||||||
excerpts = {
|
excerpts = {
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -63,6 +63,44 @@ async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Di
|
|||||||
logger.warning(f"Unknown external KB type: {service_type}")
|
logger.warning(f"Unknown external KB type: {service_type}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
async def _get_web_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
||||||
|
"""获取已启用的 AnySearch 联网检索服务配置。"""
|
||||||
|
from sqlalchemy import text
|
||||||
|
try:
|
||||||
|
query = "SELECT * FROM external_kb_services WHERE service_type='anysearch'"
|
||||||
|
if enabled_only:
|
||||||
|
query += " AND is_enabled = 1"
|
||||||
|
query += " ORDER BY created_at DESC"
|
||||||
|
result = db_session.execute(text(query))
|
||||||
|
columns = result.keys()
|
||||||
|
return [dict(zip(columns, row)) for row in result.fetchall()]
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to get AnySearch services: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_anysearch(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||||
|
"""调用 AnySearch 联网检索(走 app.anysearch_client)。"""
|
||||||
|
try:
|
||||||
|
from app.anysearch_client import anysearch_search
|
||||||
|
api_key = service.get("api_key", "")
|
||||||
|
api_url = service.get("api_url") or "https://api.anysearch.com/v1/search"
|
||||||
|
cfg = service.get("config")
|
||||||
|
if isinstance(cfg, str):
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
cfg = json.loads(cfg)
|
||||||
|
except Exception:
|
||||||
|
cfg = None
|
||||||
|
cfg = cfg or {}
|
||||||
|
zone = cfg.get("zone", "cn")
|
||||||
|
freshness = cfg.get("freshness")
|
||||||
|
k = int(cfg.get("top_k", top_k))
|
||||||
|
return await anysearch_search(api_key, api_url, query, top_k=k, zone=zone, freshness=freshness)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"AnySearch search error: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def kb_router_search(
|
async def kb_router_search(
|
||||||
query: str,
|
query: str,
|
||||||
@@ -74,6 +112,7 @@ async def kb_router_search(
|
|||||||
use_vector: bool = True,
|
use_vector: bool = True,
|
||||||
use_external: bool = True,
|
use_external: bool = True,
|
||||||
use_tree: bool = True,
|
use_tree: bool = True,
|
||||||
|
use_web: bool = False,
|
||||||
) -> List[Dict]:
|
) -> List[Dict]:
|
||||||
"""
|
"""
|
||||||
知识库路由层 — 混合检索
|
知识库路由层 — 混合检索
|
||||||
@@ -83,6 +122,7 @@ async def kb_router_search(
|
|||||||
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
||||||
3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中)
|
3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中)
|
||||||
4. 外部KB检索 (Dify, 按配置)
|
4. 外部KB检索 (Dify, 按配置)
|
||||||
|
5. 联网检索 (AnySearch, 可选 use_web)
|
||||||
|
|
||||||
融合方式: Reciprocal Rank Fusion (RRF)
|
融合方式: Reciprocal Rank Fusion (RRF)
|
||||||
|
|
||||||
@@ -95,6 +135,7 @@ async def kb_router_search(
|
|||||||
use_vector: 是否使用向量检索
|
use_vector: 是否使用向量检索
|
||||||
use_external: 是否使用外部KB检索
|
use_external: 是否使用外部KB检索
|
||||||
use_tree: 是否使用 TreeSearch 结构化分片段检索
|
use_tree: 是否使用 TreeSearch 结构化分片段检索
|
||||||
|
use_web: 是否使用 AnySearch 联网检索
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
融合后的检索结果列表
|
融合后的检索结果列表
|
||||||
@@ -170,6 +211,23 @@ async def kb_router_search(
|
|||||||
"metadata": item.get("metadata", {}),
|
"metadata": item.get("metadata", {}),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# 4. 联网检索 (AnySearch, 可选)
|
||||||
|
if use_web and db_session is not None:
|
||||||
|
try:
|
||||||
|
web_services = await _get_web_services(db_session)
|
||||||
|
for svc in web_services:
|
||||||
|
web_hits = await _search_anysearch(svc, query, top_k)
|
||||||
|
for rank, item in enumerate(web_hits):
|
||||||
|
all_results.append({
|
||||||
|
"content": item.get("content", ""),
|
||||||
|
"score": item.get("score", 0.6),
|
||||||
|
"source": item.get("source", "web_anysearch"),
|
||||||
|
"rank": rank + 1,
|
||||||
|
"metadata": item.get("metadata", {}),
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"AnySearch web search failed: {e}")
|
||||||
|
|
||||||
if not all_results:
|
if not all_results:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user