feat(kb): 接入 Dify 知识库,移除 FastGPT;AI 聊天产出+用户文档自动入库量化
- 新增 app/dify_kb.py Dify v1 客户端(连通测试/检索/文本入库) - kb_router 仅保留 Dify 外部检索分支 - knowledge_admin 去掉 FastGPT/RAGFlow,新增 dify 批量入库端点 - chat.py 注入 Dify RAG 上下文 + 助手回答 fire-and-forget 入库 - 前端 Knowledge.tsx 改为 Dify 单服务 UI(可通讯/可用验证+入库按钮) - DB: external_kb_services 删除 fastgpt,qihuai 设为 https+connected
This commit is contained in:
1 parent
d3a890eb18
commit
a1669374e6
5 files changed
+3708
-3491
No files matched your search
+25
-117
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
知识库路由层
|
||||
============
|
||||
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(FastGPT/Dify/RAGFlow)。
|
||||
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
|
||||
根据知识库配置自动选择最优检索策略。
|
||||
|
||||
使用方式:
|
||||
@@ -9,17 +9,14 @@
|
||||
results = await kb_router_search(query, kb_id, top_k=5)
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import httpx
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
logger = logging.getLogger("aiflow.kb_router")
|
||||
|
||||
|
||||
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
||||
"""获取已配置的外部KB服务"""
|
||||
"""获取已配置的外部KB服务(仅 dify)"""
|
||||
from sqlalchemy import text
|
||||
try:
|
||||
query = "SELECT * FROM external_kb_services"
|
||||
@@ -34,123 +31,34 @@ async def _get_external_services(db_session, enabled_only: bool = True) -> List[
|
||||
return []
|
||||
|
||||
|
||||
async def _search_fastgpt(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
||||
"""搜索 FastGPT 知识库"""
|
||||
async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||
"""检索 Dify 知识库(走 app.dify_kb 客户端)"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{api_url.rstrip('/')}/api/v1/dataset/search",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"similarity": 0.5,
|
||||
"limit": top_k,
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
results = []
|
||||
for item in data.get("data", {}).get("list", []):
|
||||
results.append({
|
||||
"content": item.get("q", item.get("content", "")),
|
||||
"score": float(item.get("score", 0.5)),
|
||||
"source": f"FastGPT: {item.get('collectionId', '')}",
|
||||
"metadata": item,
|
||||
})
|
||||
return results
|
||||
else:
|
||||
logger.warning(f"FastGPT search returned {resp.status_code}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"FastGPT search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def _search_dify(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
||||
"""搜索 Dify 知识库"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{api_url.rstrip('/')}/v1/datasets/retrieval",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"query": query,
|
||||
"retrieval_mode": "semantic",
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
results = []
|
||||
for item in data.get("records", []):
|
||||
segment = item.get("segment", {})
|
||||
results.append({
|
||||
"content": segment.get("content", ""),
|
||||
"score": float(item.get("score", 0.5)),
|
||||
"source": f"Dify: {segment.get('id', '')}",
|
||||
"metadata": item,
|
||||
})
|
||||
return results
|
||||
else:
|
||||
logger.warning(f"Dify search returned {resp.status_code}")
|
||||
return []
|
||||
from app.dify_kb import dify_retrieve
|
||||
api_url = service.get("api_url", "")
|
||||
api_key = service.get("api_key", "")
|
||||
dataset_name = None
|
||||
cfg = service.get("config")
|
||||
if isinstance(cfg, str):
|
||||
try:
|
||||
import json
|
||||
cfg = json.loads(cfg)
|
||||
except Exception:
|
||||
cfg = None
|
||||
if isinstance(cfg, dict):
|
||||
dataset_name = cfg.get("dataset_name")
|
||||
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
|
||||
return hits
|
||||
except Exception as e:
|
||||
logger.error(f"Dify search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def _search_ragflow(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
||||
"""搜索 RAGFlow 知识库"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{api_url.rstrip('/')}/api/v1/retrieval",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"question": query,
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
results = []
|
||||
for item in data.get("data", {}).get("chunks", []):
|
||||
results.append({
|
||||
"content": item.get("content", ""),
|
||||
"score": float(item.get("similarity", 0.5)),
|
||||
"source": f"RAGFlow: {item.get('document_id', '')}",
|
||||
"metadata": item,
|
||||
})
|
||||
return results
|
||||
else:
|
||||
logger.warning(f"RAGFlow search returned {resp.status_code}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"RAGFlow search error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||
"""路由到对应的外部KB服务"""
|
||||
"""路由到对应的外部KB服务(当前仅支持 dify)"""
|
||||
service_type = service.get("service_type", "")
|
||||
api_url = service.get("api_url", "")
|
||||
api_key = service.get("api_key", "")
|
||||
|
||||
if service_type == "fastgpt":
|
||||
return await _search_fastgpt(api_url, api_key, query, top_k)
|
||||
elif service_type == "dify":
|
||||
return await _search_dify(api_url, api_key, query, top_k)
|
||||
elif service_type == "ragflow":
|
||||
return await _search_ragflow(api_url, api_key, query, top_k)
|
||||
if service_type == "dify":
|
||||
return await _search_dify(service, query, top_k)
|
||||
else:
|
||||
logger.warning(f"Unknown external KB type: {service_type}")
|
||||
return []
|
||||
@@ -171,7 +79,7 @@ async def kb_router_search(
|
||||
检索策略:
|
||||
1. 本地 BM25 检索 (MySQL, 已有)
|
||||
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
||||
3. 外部KB检索 (FastGPT/Dify/RAGFlow, 按配置)
|
||||
3. 外部KB检索 (Dify, 按配置)
|
||||
|
||||
融合方式: Reciprocal Rank Fusion (RRF)
|
||||
|
||||
@@ -218,7 +126,7 @@ async def kb_router_search(
|
||||
except Exception as e:
|
||||
logger.warning(f"Vector search failed: {e}")
|
||||
|
||||
# 3. 外部KB检索
|
||||
# 3. 外部KB检索 (Dify)
|
||||
if use_external and db_session:
|
||||
services = await _get_external_services(db_session)
|
||||
for service in services:
|
||||
@@ -228,7 +136,7 @@ async def kb_router_search(
|
||||
all_results.append({
|
||||
"content": item.get("content", ""),
|
||||
"score": item.get("score", 0.5),
|
||||
"source": item.get("source", f"external_{service['service_type']}"),
|
||||
"source": item.get("source", "external_dify"),
|
||||
"rank": rank + 1,
|
||||
"metadata": item.get("metadata", {}),
|
||||
})
|
||||
|
||||
Reference in new issue
Block a user