feat(kb): 接入 Dify 知识库,移除 FastGPT;AI 聊天产出+用户文档自动入库量化

- 新增 app/dify_kb.py Dify v1 客户端(连通测试/检索/文本入库)
- kb_router 仅保留 Dify 外部检索分支
- knowledge_admin 去掉 FastGPT/RAGFlow,新增 dify 批量入库端点
- chat.py 注入 Dify RAG 上下文 + 助手回答 fire-and-forget 入库
- 前端 Knowledge.tsx 改为 Dify 单服务 UI(可通讯/可用验证+入库按钮)
- DB: external_kb_services 删除 fastgpt,qihuai 设为 https+connected
This commit is contained in:
QihuAI Design committed 2026-07-14 20:56:53 +08:00
1 parent d3a890eb18
commit a1669374e6
5 files changed
+541 -324

No files matched your search

+127 -183
View File
@@ -32,7 +32,7 @@ logger = logging.getLogger("aiflow")
class ExternalServiceConfig(BaseModel): class ExternalServiceConfig(BaseModel):
name: str = Field(min_length=1, max_length=200) name: str = Field(min_length=1, max_length=200)
service_type: str = Field(..., description="fastgpt / dify / ragflow") service_type: str = Field(..., description="dify")
api_url: str = Field(min_length=1, max_length=500) api_url: str = Field(min_length=1, max_length=500)
api_key: Optional[str] = None api_key: Optional[str] = None
config: Optional[Dict[str, Any]] = None config: Optional[Dict[str, Any]] = None
@@ -739,48 +739,27 @@ def _get_external_services_table(db: Session):
def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict: def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict:
"""测试外部知识库服务连接状态""" """测试外部知识库服务连接状态(当前仅支持 Dify)"""
try: try:
if service_type == "dify":
from ...dify_kb import dify_test
return dify_test(api_url, api_key)
else:
# 未知类型(如历史遗留的 fastgpt/ragflow),按通用方式探测可达性
headers = {} headers = {}
if api_key: if api_key:
headers["Authorization"] = f"Bearer {api_key}" headers["Authorization"] = f"Bearer {api_key}"
if service_type == "fastgpt":
# FastGPT 健康检查
check_url = f"{api_url.rstrip('/')}/api/v1/status"
resp = httpx.get(check_url, headers=headers, timeout=10)
if resp.status_code == 200:
return {"connected": True, "status": "connected", "detail": "FastGPT 连接成功"}
else:
return {"connected": False, "status": "error", "detail": f"FastGPT 返回状态码 {resp.status_code}"}
elif service_type == "dify":
# Dify 健康检查
check_url = f"{api_url.rstrip('/')}/v1/datasets"
resp = httpx.get(check_url, headers=headers, timeout=10)
if resp.status_code in [200, 401]: # 401说明服务存在但需要认证
return {"connected": True, "status": "connected", "detail": "Dify 连接成功"}
else:
return {"connected": False, "status": "error", "detail": f"Dify 返回状态码 {resp.status_code}"}
elif service_type == "ragflow":
# RAGFlow 健康检查
check_url = f"{api_url.rstrip('/')}/api/v1/datasets"
resp = httpx.get(check_url, headers=headers, timeout=10)
if resp.status_code in [200, 401]:
return {"connected": True, "status": "connected", "detail": "RAGFlow 连接成功"}
else:
return {"connected": False, "status": "error", "detail": f"RAGFlow 返回状态码 {resp.status_code}"}
else:
# 通用检查
resp = httpx.get(api_url, headers=headers, timeout=10) resp = httpx.get(api_url, headers=headers, timeout=10)
return {"connected": resp.status_code < 500, "status": "connected" if resp.status_code < 500 else "error"} return {
"connected": resp.status_code < 500,
"usable": resp.status_code < 500,
"status": "connected" if resp.status_code < 500 else "error",
"detail": f"通用探测返回状态码 {resp.status_code}",
}
except httpx.TimeoutException: except httpx.TimeoutException:
return {"connected": False, "status": "timeout", "detail": "连接超时"} return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时"}
except Exception as e: except Exception as e:
return {"connected": False, "status": "error", "detail": str(e)} return {"connected": False, "usable": False, "status": "error", "detail": str(e)}
@router.get("/external/services", summary="获取外部知识库服务列表") @router.get("/external/services", summary="获取外部知识库服务列表")
@@ -801,9 +780,7 @@ async def list_external_services(
"total": len(services), "total": len(services),
"items": services, "items": services,
"supported_types": [ "supported_types": [
{"type": "fastgpt", "name": "FastGPT", "description": "开源 AI 知识库平台,支持多种向量模型"}, {"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理(RAG)"},
{"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理"},
{"type": "ragflow", "name": "RAGFlow", "description": "基于深度文档理解的开源 RAG 引擎"},
] ]
} }
} }
@@ -816,8 +793,8 @@ async def create_external_service(
db: Session = Depends(get_db), db: Session = Depends(get_db),
): ):
"""创建新的外部知识库服务连接配置""" """创建新的外部知识库服务连接配置"""
# 验证服务类型 # 验证服务类型(当前仅支持 Dify)
valid_types = ["fastgpt", "dify", "ragflow"] valid_types = ["dify"]
if req.service_type not in valid_types: if req.service_type not in valid_types:
raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}") raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}")
@@ -1082,163 +1059,30 @@ async def sync_knowledge_base(
if api_key: if api_key:
headers["Authorization"] = f"Bearer {api_key}" headers["Authorization"] = f"Bearer {api_key}"
if service_type == "fastgpt" and req.sync_direction == "push": if service_type == "dify" and req.sync_direction == "push":
# 向 FastGPT 推送知识库数据 # 向 Dify 推送知识库数据(使用 app.dify_kb 客户端,按 dataset_name 解析)
# 1. 在 FastGPT 创建/获取对应的 dataset from ...dify_kb import dify_ingest_text, resolve_dataset_id
ext_kb_id = getattr(kb, "external_id", None) dataset_name = service_config.get("dataset_name") if isinstance(service_config, dict) else None
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
if not ext_kb_id: if not ds_id:
# 创建新 dataset sync_result["detail"] = "无法解析 Dify 知识库 ID(请检查 API Key 是否绑定到指定数据集)"
create_resp = httpx.post(
f"{api_url.rstrip('/')}/api/v1/datasets",
headers=headers,
json={"name": kb.name, "description": kb.description or ""},
timeout=30,
)
if create_resp.status_code == 200:
ext_data = create_resp.json()
ext_kb_id = ext_data.get("data", {}).get("id") or ext_data.get("id")
# 更新 KB 的 external_id
try:
setattr(kb, "external_id", ext_kb_id)
except Exception:
db.execute(
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
{"eid": ext_kb_id, "kid": kb_id}
)
else:
sync_result["detail"] = f"FastGPT 创建 dataset 失败: {create_resp.status_code}"
sync_result["synced"] = False sync_result["synced"] = False
else:
if ext_kb_id:
# 推送文档内容到 FastGPT
docs = db.query(KnowledgeDocument).filter( docs = db.query(KnowledgeDocument).filter(
KnowledgeDocument.knowledge_base_id == kb_id, KnowledgeDocument.knowledge_base_id == kb_id,
KnowledgeDocument.status == "completed" KnowledgeDocument.status == "completed"
).all() ).all()
synced_count = 0 synced_count = 0
for doc in docs: for doc in docs:
content = doc.content or "" content = doc.content or ""
if not content: if not content:
continue continue
if dify_ingest_text(api_url, api_key, content[:50000], name=doc.filename, dataset_name=dataset_name):
# 分段推送
push_resp = httpx.post(
f"{api_url.rstrip('/')}/api/v1/datasets/{ext_kb_id}/documents",
headers=headers,
json={
"name": doc.filename,
"content": content[:50000], # 限制单次推送大小
},
timeout=30,
)
if push_resp.status_code in [200, 201]:
synced_count += 1 synced_count += 1
sync_result["synced"] = True
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 FastGPT"
sync_result["items_synced"] = synced_count
elif service_type == "dify" and req.sync_direction == "push":
# 向 Dify 推送知识库数据
ext_kb_id = getattr(kb, "external_id", None)
if not ext_kb_id:
create_resp = httpx.post(
f"{api_url.rstrip('/')}/v1/datasets",
headers=headers,
json={"name": kb.name},
timeout=30,
)
if create_resp.status_code in [200, 201]:
ext_data = create_resp.json()
ext_kb_id = ext_data.get("id")
try:
setattr(kb, "external_id", ext_kb_id)
except Exception:
db.execute(
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
{"eid": ext_kb_id, "kid": kb_id}
)
if ext_kb_id:
docs = db.query(KnowledgeDocument).filter(
KnowledgeDocument.knowledge_base_id == kb_id,
KnowledgeDocument.status == "completed"
).all()
synced_count = 0
for doc in docs:
content = doc.content or ""
if not content:
continue
# Dify 文档创建
doc_resp = httpx.post(
f"{api_url.rstrip('/')}/v1/datasets/{ext_kb_id}/document/create_by_text",
headers=headers,
json={
"name": doc.filename,
"text": content[:50000],
"indexing_technique": "high_quality",
},
timeout=30,
)
if doc_resp.status_code in [200, 201]:
synced_count += 1
sync_result["synced"] = True sync_result["synced"] = True
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify" sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify"
sync_result["items_synced"] = synced_count sync_result["items_synced"] = synced_count
elif service_type == "ragflow" and req.sync_direction == "push":
# 向 RAGFlow 推送知识库数据
ext_kb_id = getattr(kb, "external_id", None)
if not ext_kb_id:
create_resp = httpx.post(
f"{api_url.rstrip('/')}/api/v1/datasets",
headers=headers,
json={"name": kb.name},
timeout=30,
)
if create_resp.status_code in [200, 201]:
ext_data = create_resp.json()
ext_kb_id = ext_data.get("id")
try:
setattr(kb, "external_id", ext_kb_id)
except Exception:
db.execute(
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
{"eid": ext_kb_id, "kid": kb_id}
)
if ext_kb_id:
docs = db.query(KnowledgeDocument).filter(
KnowledgeDocument.knowledge_base_id == kb_id,
KnowledgeDocument.status == "completed"
).all()
synced_count = 0
for doc in docs:
content = doc.content or ""
if not content:
continue
doc_resp = httpx.post(
f"{api_url.rstrip('/')}/api/v1/datasets/{ext_kb_id}/documents",
headers=headers,
json={"name": doc.filename, "content": content[:50000]},
timeout=30,
)
if doc_resp.status_code in [200, 201]:
synced_count += 1
sync_result["synced"] = True
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 RAGFlow"
sync_result["items_synced"] = synced_count
elif req.sync_direction == "pull": elif req.sync_direction == "pull":
# 从外部服务拉取知识库数据 # 从外部服务拉取知识库数据
sync_result["detail"] = f"从 {service_type} 拉取数据(暂未实现完整拉取逻辑)" sync_result["detail"] = f"从 {service_type} 拉取数据(暂未实现完整拉取逻辑)"
@@ -1288,6 +1132,106 @@ async def sync_knowledge_base(
return {"code": 0, "data": sync_result} return {"code": 0, "data": sync_result}
# ============================================================
# 平台内容批量入库到 Dify(AI聊天产出 + 用户上传文档)
# ============================================================
@router.post("/external/dify/ingest", summary="将平台内容批量入库到 Dify 知识库")
async def ingest_platform_to_dify(
admin_user=Depends(get_current_admin),
db: Session = Depends(get_db),
):
"""把平台中「可量化的内容」推送到 Dify 知识库(qihuai):
- AI 聊天的助手回答(文本、非工具/非媒体、长度 > 120)
- 用户上传的文档资产(assets, category='file' 且含文本内容)
返回各来源的入库数量。
"""
from ...dify_kb import dify_ingest_text, resolve_dataset_id
# 读取已启用的 Dify 服务
try:
row = db.execute(
text("SELECT api_url, api_key, config FROM external_kb_services "
"WHERE service_type='dify' AND is_enabled=1 LIMIT 1")
).fetchone()
except Exception:
row = None
if not row:
raise HTTPException(status_code=400, detail="未配置已启用的 Dify 知识库服务,请先添加 Dify 服务并测试连接")
api_url, api_key, config_raw = row
dataset_name = None
if config_raw:
try:
cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw)
dataset_name = cfg.get("dataset_name")
except Exception:
pass
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
if not ds_id:
raise HTTPException(status_code=400, detail="无法解析 Dify 知识库 ID(请检查 API Key 与 dataset_name)")
stats = {"chat_answers": 0, "user_docs": 0, "skipped": 0, "failed": 0}
# 1. AI 聊天助手回答
try:
from ...models import Message
answers = db.query(Message).filter(
Message.role == "assistant"
).order_by(Message.created_at.asc()).limit(2000).all()
for m in answers:
content = getattr(m, "content", None) or ""
content = content.strip()
# 跳过媒体/工具类消息与过短内容
if len(content) < 120:
stats["skipped"] += 1
continue
if content.startswith("[") and ("MEDIA" in content or "IMAGE" in content or "VIDEO" in content):
stats["skipped"] += 1
continue
conv_id = getattr(m, "conversation_id", None) or "unknown"
if dify_ingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{m.id}", dataset_name=dataset_name):
stats["chat_answers"] += 1
else:
stats["failed"] += 1
except Exception as e:
logger.error(f"ingest chat answers failed: {e}")
# 2. 用户上传的文档资产
try:
from ...models import Asset
assets = db.query(Asset).filter(
getattr(Asset, "category", "file") == "file"
).order_by(getattr(Asset, "created_at", text("id")).asc()).limit(2000).all()
for a in assets:
content = getattr(a, "content", None) or ""
content = content.strip()
if len(content) < 50:
stats["skipped"] += 1
continue
fname = getattr(a, "filename", None) or getattr(a, "name", None) or "asset"
if dify_ingest_text(api_url, api_key, content[:50000], name=f"asset_{a.id}_{fname}", dataset_name=dataset_name):
stats["user_docs"] += 1
else:
stats["failed"] += 1
except Exception as e:
logger.error(f"ingest assets failed: {e}")
LogService.log_audit(
admin_id=admin_user.id,
admin_name=admin_user.username,
action="ingest_platform_to_dify",
target_type="external_kb_service",
target_id=ds_id,
detail=f"平台内容入库 Dify: 聊天回答 {stats['chat_answers']} 条, 用户文档 {stats['user_docs']} 条, 跳过 {stats['skipped']}, 失败 {stats['failed']}",
)
return {
"code": 0,
"message": f"入库完成:聊天回答 {stats['chat_answers']} 条,用户文档 {stats['user_docs']} 条",
"data": stats,
}
# ============================================================ # ============================================================
# 批量操作 # 批量操作
# ============================================================ # ============================================================
+66 -14
View File
@@ -490,14 +490,13 @@ def _retrieve_kb_context(db: Session, kb_ids: list, query: str, max_chars: int =
2. 构建文档列表(content+title+metadata) 2. 构建文档列表(content+title+metadata)
3. BM25检索最相关的段落 3. BM25检索最相关的段落
4. 拼接为上下文注入system prompt 4. 拼接为上下文注入system prompt
同时叠加 Dify 外部知识库(RAG)检索结果。
""" """
if not kb_ids: local_context = ""
return ""
if kb_ids:
kbs = db.query(KnowledgeBase).filter(KnowledgeBase.id.in_(kb_ids)).all() kbs = db.query(KnowledgeBase).filter(KnowledgeBase.id.in_(kb_ids)).all()
if not kbs: if kbs:
return ""
# 收集所有文档段落 # 收集所有文档段落
# Stage C.1: 优先使用 KnowledgeDocumentChunk 表的预分段数据 # Stage C.1: 优先使用 KnowledgeDocumentChunk 表的预分段数据
all_segments = [] all_segments = []
@@ -554,20 +553,59 @@ def _retrieve_kb_context(db: Session, kb_ids: list, query: str, max_chars: int =
if not all_segments: if not all_segments:
# 退化: 使用原始朴素拼接 # 退化: 使用原始朴素拼接
kb_names = ", ".join([k.name for k in kbs]) kb_names = ", ".join([k.name for k in kbs])
return f"参考知识库: {kb_names}" local_context = f"参考知识库: {kb_names}"
else:
# BM25检索最相关的段落 # BM25检索最相关的段落
top_segments = _bm25_search(query, all_segments, top_k=8) top_segments = _bm25_search(query, all_segments, top_k=8)
# 拼接上下文
context_parts = [seg["content"] for seg in top_segments] context_parts = [seg["content"] for seg in top_segments]
context = "\n".join(context_parts) local_context = "\n".join(context_parts)
if len(local_context) > max_chars:
local_context = local_context[:max_chars] + "...(内容截断)"
# 限制总长度 # 叠加 Dify 外部知识库(RAG)检索
if len(context) > max_chars: dify_context = ""
context = context[:max_chars] + "...(内容截断)" try:
from app.dify_kb import get_dify_kb_context
dify_context = get_dify_kb_context(db, query, top_k=5)
except Exception as e:
logger.warning(f"Dify KB context retrieval failed: {e}")
return context parts = []
if local_context:
parts.append(local_context)
if dify_context:
parts.append("【Dify 知识库检索结果】\n" + dify_context)
return "\n\n".join(parts)
async def _ingest_answer_to_dify(conv_id: str, msg_id: str, content: str):
"""后台任务:将助手回答入库到 Dify 知识库(量化后可供 RAG 检索复用)。"""
try:
from app.dify_kb import aingest_text
from app.database import SessionLocal
from sqlalchemy import text as _t
db = SessionLocal()
try:
row = db.execute(_t(
"SELECT api_url, api_key, config FROM external_kb_services "
"WHERE service_type='dify' AND is_enabled=1 LIMIT 1"
)).fetchone()
finally:
db.close()
if not row:
return
api_url, api_key, config_raw = row
dataset_name = None
if config_raw:
try:
import json
cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw)
dataset_name = cfg.get("dataset_name")
except Exception:
pass
await aingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{msg_id}", dataset_name=dataset_name)
except Exception as e:
logger.warning(f"后台入库 Dify 失败: {e}")
# ============================================================ # ============================================================
@@ -1203,6 +1241,20 @@ async def send_message(
) )
db.add(ai_msg) db.add(ai_msg)
# 后台异步:将实质性文本回答入库到 Dify 知识库(量化后可供 RAG 复用)
_ingest_text = (ai_content or "").strip()
if len(_ingest_text) > 120 and not (
_ingest_text.startswith("[MEDIA")
or _ingest_text.startswith("data:")
or _ingest_text.startswith("http")
or "<img" in _ingest_text
or "![]( " in _ingest_text
):
try:
asyncio.create_task(_ingest_answer_to_dify(conv_id, ai_msg.id, _ingest_text))
except Exception:
pass
# 更新used_skills # 更新used_skills
used_skills = [] used_skills = []
if skills: if skills:
+248
View File
@@ -0,0 +1,248 @@
"""
Dify 知识库客户端
================
封装与 Dify 知识库(v1 API)的交互:连通性测试、检索(RAG)、文本入库。
设计用于替换原 FastGPT 外部知识库对接。
已验证的接入点(2026-07-14 实测):
- Base: https://dify.qihuyun.com/v1
(直接 HTTP 访问会被 nginx 301 跳转到 HTTPS,必须 HTTPS 直连,
且内网自签证书需 verify=False)
- Auth: Bearer {api_key}
- GET /v1/datasets 列出知识库(可据此解析 dataset id)
- POST /v1/datasets/{id}/document/create-by-text 文本入库
- POST /v1/datasets/{id}/document/create-by-file 文件入库(multipart)
- POST /v1/datasets/{id}/retrieve 检索(RAG)
retrieve 必需完整的 retrieval_model 字典,否则返回 400。
"""
import logging
import threading
import time
import httpx
logger = logging.getLogger("aiflow.dify_kb")
# dataset id 缓存: key -> (dataset_id, ts)
_dataset_cache: dict = {}
_dataset_cache_lock = threading.Lock()
_DATASET_CACHE_TTL = 3600
def _normalize_base(api_url: str) -> str:
"""统一 base url -> https 形式,且不带 /v1 后缀。"""
u = (api_url or "").strip()
if not u:
u = "https://dify.qihuyun.com/v1"
u = u.rstrip("/")
if u.endswith("/v1"):
u = u[: -len("/v1")]
if u.startswith("http://"):
u = "https://" + u[len("http://"):]
return u
def dify_test(api_url: str, api_key: str) -> dict:
"""验证 Dify 知识库「可通讯 + 可用」。
返回:
{
"connected": bool,
"usable": bool,
"status": "connected" | "auth_error" | "timeout" | "error",
"detail": str,
"datasets": [{"id":..., "name":...}, ...]
}
"""
base = _normalize_base(api_url)
headers = {"Authorization": f"Bearer {api_key}"}
try:
with httpx.Client(base_url=base, timeout=15, verify=False) as client:
r = client.get("/v1/datasets", headers=headers)
if r.status_code == 200:
data = r.json()
datasets = data.get("data", []) or []
names = [d.get("name") for d in datasets]
# 能列出知识库即代表鉴权通过、可读取 —— 视为可用。
return {
"connected": True,
"usable": True,
"status": "connected",
"detail": f"Dify 连接成功,发现 {len(datasets)} 个知识库: {', '.join([n for n in names if n]) or '无'}",
"datasets": [{"id": d.get("id"), "name": d.get("name")} for d in datasets],
}
elif r.status_code == 401:
return {
"connected": True,
"usable": False,
"status": "auth_error",
"detail": "已连通 Dify,但 API Key 鉴权失败(401),请检查 API Key。",
"datasets": [],
}
else:
return {
"connected": False,
"usable": False,
"status": "error",
"detail": f"Dify 返回状态码 {r.status_code}",
"datasets": [],
}
except httpx.TimeoutException:
return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时", "datasets": []}
except Exception as e:
return {"connected": False, "usable": False, "status": "error", "detail": str(e), "datasets": []}
def resolve_dataset_id(api_url: str, api_key: str, dataset_name: str = None) -> str:
"""解析 dataset id(优先按名称匹配,否则取列表第一个),带缓存。"""
base = _normalize_base(api_url)
headers = {"Authorization": f"Bearer {api_key}"}
cache_key = f"{base}|{api_key}|{dataset_name}"
now = time.time()
with _dataset_cache_lock:
c = _dataset_cache.get(cache_key)
if c and now - c[1] < _DATASET_CACHE_TTL:
return c[0]
ds_id = None
try:
with httpx.Client(base_url=base, timeout=15, verify=False) as client:
r = client.get("/v1/datasets", headers=headers)
if r.status_code == 200:
datasets = r.json().get("data", []) or []
if dataset_name:
match = [d for d in datasets if d.get("name") == dataset_name]
if match:
ds_id = match[0].get("id")
if not ds_id and datasets:
ds_id = datasets[0].get("id")
except Exception as e:
logger.error(f"resolve_dataset_id failed: {e}")
if ds_id:
with _dataset_cache_lock:
_dataset_cache[cache_key] = (ds_id, now)
return ds_id
def dify_retrieve(api_url, api_key, query, top_k=5, dataset_name=None) -> list:
"""检索,返回 [{content, score, source, metadata}]。"""
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
if not ds_id:
return []
base = _normalize_base(api_url)
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
payload = {
"query": query,
"retrieval_model": {
"search_method": "semantic_search",
"reranking_enable": False,
"score_threshold_enabled": False,
"top_k": top_k,
},
}
try:
with httpx.Client(base_url=base, timeout=30, verify=False) as client:
r = client.post(f"/v1/datasets/{ds_id}/retrieve", headers=headers, json=payload)
if r.status_code == 200:
data = r.json()
out = []
for item in data.get("records", []):
seg = item.get("segment", {})
out.append({
"content": seg.get("content", ""),
"score": float(item.get("score", 0.5)),
"source": f"Dify: {seg.get('document_id', '')}",
"metadata": item,
})
return out
else:
logger.warning(f"Dify retrieve returned {r.status_code}: {r.text[:200]}")
return []
except Exception as e:
logger.error(f"Dify retrieve error: {e}")
return []
def dify_ingest_text(api_url, api_key, text, name=None, dataset_name=None) -> bool:
"""文本入库(create-by-text)。返回是否成功。"""
if not text or not text.strip():
return False
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
if not ds_id:
return False
base = _normalize_base(api_url)
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
payload = {
"name": name or f"doc-{int(time.time())}",
"text": text,
"indexing_technique": "high_quality",
}
try:
with httpx.Client(base_url=base, timeout=60, verify=False) as client:
r = client.post(f"/v1/datasets/{ds_id}/document/create-by-text", headers=headers, json=payload)
if r.status_code in (200, 201):
return True
else:
logger.warning(f"Dify ingest text returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Dify ingest text error: {e}")
return False
async def aingest_text(api_url, api_key, text, name=None, dataset_name=None) -> bool:
"""异步文本入库(供 chat 实时落库使用)。"""
if not text or not text.strip():
return False
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
if not ds_id:
return False
base = _normalize_base(api_url)
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
payload = {
"name": name or f"doc-{int(time.time())}",
"text": text,
"indexing_technique": "high_quality",
}
try:
async with httpx.AsyncClient(base_url=base, timeout=60, verify=False) as client:
r = await client.post(f"/v1/datasets/{ds_id}/document/create-by-text", headers=headers, json=payload)
if r.status_code in (200, 201):
return True
else:
logger.warning(f"Dify aingest returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Dify aingest error: {e}")
return False
def get_dify_kb_context(db, query: str, top_k: int = 5) -> str:
"""读取已启用的 Dify 外部服务,检索并返回拼接后的上下文文本(供 chat 注入)。"""
from sqlalchemy import text as _text
try:
result = db.execute(_text(
"SELECT api_url, api_key, config FROM external_kb_services "
"WHERE service_type='dify' AND is_enabled=1 AND status='connected' LIMIT 1"
))
row = result.fetchone()
if not row:
return ""
api_url, api_key, config_raw = row
dataset_name = None
if config_raw:
try:
cfg = config_raw if isinstance(config_raw, dict) else __import__("json").loads(config_raw)
dataset_name = cfg.get("dataset_name")
except Exception:
pass
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
if not hits:
return ""
parts = []
for i, h in enumerate(hits, 1):
content = (h.get("content") or "").strip()
if content:
parts.append(f"[{i}] {content}")
return "\n\n".join(parts)
except Exception as e:
logger.error(f"get_dify_kb_context error: {e}")
return ""
+24 -116
View File
@@ -1,7 +1,7 @@
""" """
知识库路由层 知识库路由层
============ ============
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(FastGPT/Dify/RAGFlow)。 将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
根据知识库配置自动选择最优检索策略。 根据知识库配置自动选择最优检索策略。
使用方式: 使用方式:
@@ -9,17 +9,14 @@
results = await kb_router_search(query, kb_id, top_k=5) results = await kb_router_search(query, kb_id, top_k=5)
""" """
import os
import json
import logging import logging
import httpx
from typing import List, Dict, Optional from typing import List, Dict, Optional
logger = logging.getLogger("aiflow.kb_router") logger = logging.getLogger("aiflow.kb_router")
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]: async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
"""获取已配置的外部KB服务""" """获取已配置的外部KB服务(仅 dify)"""
from sqlalchemy import text from sqlalchemy import text
try: try:
query = "SELECT * FROM external_kb_services" query = "SELECT * FROM external_kb_services"
@@ -34,123 +31,34 @@ async def _get_external_services(db_session, enabled_only: bool = True) -> List[
return [] return []
async def _search_fastgpt(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]: async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
"""搜索 FastGPT 知识库""" """检索 Dify 知识库(走 app.dify_kb 客户端)"""
try: try:
async with httpx.AsyncClient(timeout=30) as client: from app.dify_kb import dify_retrieve
resp = await client.post( api_url = service.get("api_url", "")
f"{api_url.rstrip('/')}/api/v1/dataset/search", api_key = service.get("api_key", "")
headers={ dataset_name = None
"Authorization": f"Bearer {api_key}", cfg = service.get("config")
"Content-Type": "application/json", if isinstance(cfg, str):
},
json={
"query": query,
"similarity": 0.5,
"limit": top_k,
},
)
if resp.status_code == 200:
data = resp.json()
results = []
for item in data.get("data", {}).get("list", []):
results.append({
"content": item.get("q", item.get("content", "")),
"score": float(item.get("score", 0.5)),
"source": f"FastGPT: {item.get('collectionId', '')}",
"metadata": item,
})
return results
else:
logger.warning(f"FastGPT search returned {resp.status_code}")
return []
except Exception as e:
logger.error(f"FastGPT search error: {e}")
return []
async def _search_dify(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
"""搜索 Dify 知识库"""
try: try:
async with httpx.AsyncClient(timeout=30) as client: import json
resp = await client.post( cfg = json.loads(cfg)
f"{api_url.rstrip('/')}/v1/datasets/retrieval", except Exception:
headers={ cfg = None
"Authorization": f"Bearer {api_key}", if isinstance(cfg, dict):
"Content-Type": "application/json", dataset_name = cfg.get("dataset_name")
}, hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
json={ return hits
"query": query,
"retrieval_mode": "semantic",
"top_k": top_k,
},
)
if resp.status_code == 200:
data = resp.json()
results = []
for item in data.get("records", []):
segment = item.get("segment", {})
results.append({
"content": segment.get("content", ""),
"score": float(item.get("score", 0.5)),
"source": f"Dify: {segment.get('id', '')}",
"metadata": item,
})
return results
else:
logger.warning(f"Dify search returned {resp.status_code}")
return []
except Exception as e: except Exception as e:
logger.error(f"Dify search error: {e}") logger.error(f"Dify search error: {e}")
return [] return []
async def _search_ragflow(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
"""搜索 RAGFlow 知识库"""
try:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
f"{api_url.rstrip('/')}/api/v1/retrieval",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"question": query,
"top_k": top_k,
},
)
if resp.status_code == 200:
data = resp.json()
results = []
for item in data.get("data", {}).get("chunks", []):
results.append({
"content": item.get("content", ""),
"score": float(item.get("similarity", 0.5)),
"source": f"RAGFlow: {item.get('document_id', '')}",
"metadata": item,
})
return results
else:
logger.warning(f"RAGFlow search returned {resp.status_code}")
return []
except Exception as e:
logger.error(f"RAGFlow search error: {e}")
return []
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]: async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
"""路由到对应的外部KB服务""" """路由到对应的外部KB服务(当前仅支持 dify)"""
service_type = service.get("service_type", "") service_type = service.get("service_type", "")
api_url = service.get("api_url", "") if service_type == "dify":
api_key = service.get("api_key", "") return await _search_dify(service, query, top_k)
if service_type == "fastgpt":
return await _search_fastgpt(api_url, api_key, query, top_k)
elif service_type == "dify":
return await _search_dify(api_url, api_key, query, top_k)
elif service_type == "ragflow":
return await _search_ragflow(api_url, api_key, query, top_k)
else: else:
logger.warning(f"Unknown external KB type: {service_type}") logger.warning(f"Unknown external KB type: {service_type}")
return [] return []
@@ -171,7 +79,7 @@ async def kb_router_search(
检索策略: 检索策略:
1. 本地 BM25 检索 (MySQL, 已有) 1. 本地 BM25 检索 (MySQL, 已有)
2. 向量语义检索 (PostgreSQL+pgvector, 新增) 2. 向量语义检索 (PostgreSQL+pgvector, 新增)
3. 外部KB检索 (FastGPT/Dify/RAGFlow, 按配置) 3. 外部KB检索 (Dify, 按配置)
融合方式: Reciprocal Rank Fusion (RRF) 融合方式: Reciprocal Rank Fusion (RRF)
@@ -218,7 +126,7 @@ async def kb_router_search(
except Exception as e: except Exception as e:
logger.warning(f"Vector search failed: {e}") logger.warning(f"Vector search failed: {e}")
# 3. 外部KB检索 # 3. 外部KB检索 (Dify)
if use_external and db_session: if use_external and db_session:
services = await _get_external_services(db_session) services = await _get_external_services(db_session)
for service in services: for service in services:
@@ -228,7 +136,7 @@ async def kb_router_search(
all_results.append({ all_results.append({
"content": item.get("content", ""), "content": item.get("content", ""),
"score": item.get("score", 0.5), "score": item.get("score", 0.5),
"source": item.get("source", f"external_{service['service_type']}"), "source": item.get("source", "external_dify"),
"rank": rank + 1, "rank": rank + 1,
"metadata": item.get("metadata", {}), "metadata": item.get("metadata", {}),
}) })
+76 -11
View File
@@ -66,7 +66,10 @@ const Knowledge: React.FC = () => {
// External Services // External Services
const [extServices, setExtServices] = useState<ExternalService[]>([]); const [extServices, setExtServices] = useState<ExternalService[]>([]);
const [showExtModal, setShowExtModal] = useState(false); const [showExtModal, setShowExtModal] = useState(false);
const [extForm, setExtForm] = useState({ name: '', service_type: 'fastgpt', api_url: '', api_key: '' }); const [extForm, setExtForm] = useState({ name: 'Dify 知识库', service_type: 'dify', api_url: 'https://dify.qihuyun.com/v1', api_key: '', dataset_name: 'qihuai' });
const [extMsg, setExtMsg] = useState('');
const [ingesting, setIngesting] = useState(false);
const [ingestResult, setIngestResult] = useState<any>(null);
// KB Modal // KB Modal
const [showModal, setShowModal] = useState(false); const [showModal, setShowModal] = useState(false);
@@ -210,7 +213,19 @@ const Knowledge: React.FC = () => {
}; };
const handleCreateExtService = async () => { const handleCreateExtService = async () => {
try { const res = await adminAPI.createAdminKnowledgeExtService(extForm); if (res.code === 0) { setShowExtModal(false); setExtForm({ name: '', service_type: 'fastgpt', api_url: '', api_key: '' }); fetchExtServices(); } } catch (e) {} try {
const payload = {
...extForm,
config: { dataset_name: extForm.dataset_name || 'qihuai' },
};
const res = await adminAPI.createAdminKnowledgeExtService(payload);
if (res.code === 0) {
setShowExtModal(false);
setExtForm({ name: 'Dify 知识库', service_type: 'dify', api_url: 'https://dify.qihuyun.com/v1', api_key: '', dataset_name: 'qihuai' });
setExtMsg('');
fetchExtServices();
}
} catch (e: any) { setExtMsg(`添加失败: ${e?.message || '未知错误'}`); }
}; };
const handleDeleteExtService = async (id: string) => { const handleDeleteExtService = async (id: string) => {
@@ -219,7 +234,38 @@ const Knowledge: React.FC = () => {
}; };
const handleCheckExtStatus = async (id: string) => { const handleCheckExtStatus = async (id: string) => {
try { const res = await adminAPI.checkAdminKnowledgeExtStatus(id); if (res.code === 0) { alert(`连接状态: ${res.data.connected ? '已连接' : '未连接'}\n详情: ${res.data.detail || '无'}`); fetchExtServices(); } } catch (e) {} try {
const res = await adminAPI.checkAdminKnowledgeExtStatus(id);
if (res.code === 0) {
const d = res.data || {};
const statusText = d.connected ? (d.usable ? '可通讯且可用 ✅' : '可通讯但不可用 ⚠️') : '不可通讯 ❌';
alert(`连接状态: ${statusText}\n\n详情: ${d.detail || '无'}`);
fetchExtServices();
}
} catch (e) {}
};
const handleIngestPlatform = async () => {
if (!confirm('将把平台「AI聊天回答 + 用户上传文档」中有价值的内容批量入库到 Dify 知识库(量化后可被 AI 聊天检索复用)。确认执行?')) return;
setIngesting(true); setIngestResult(null);
try {
const token = localStorage.getItem('aiflow_token');
const res = await fetch(`/app/api/admin/knowledge/external/dify/ingest`, {
method: 'POST',
headers: { 'Authorization': `Bearer ${token}` },
});
const data = await res.json();
if (data.code === 0) {
setIngestResult(data.data);
setExtMsg(`✅ ${data.message}`);
} else {
setExtMsg(`❌ ${data.message || '入库失败'}`);
}
} catch (e: any) {
setExtMsg(`❌ 入库失败: ${e?.message || '网络错误'}`);
} finally {
setIngesting(false);
}
}; };
const statusBadge = (status: string) => { const statusBadge = (status: string) => {
@@ -487,7 +533,7 @@ const Knowledge: React.FC = () => {
{extServices.map(svc => ( {extServices.map(svc => (
<tr key={svc.id}> <tr key={svc.id}>
<td>{svc.name}</td> <td>{svc.name}</td>
<td><span style={{ fontSize: 12, padding: '2px 6px', background: '#F3F4FF', color: '#8B5CF6', borderRadius: 3 }}>{svc.service_type === 'fastgpt' ? 'FastGPT' : svc.service_type === 'dify' ? 'Dify' : 'RAGFlow'}</span></td> <td><span style={{ fontSize: 12, padding: '2px 6px', background: '#F3F4FF', color: '#8B5CF6', borderRadius: 3 }}>Dify</span></td>
<td style={{ fontSize: 11, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis' }}>{svc.api_url}</td> <td style={{ fontSize: 11, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis' }}>{svc.api_url}</td>
<td>{statusBadge(svc.status)}</td> <td>{statusBadge(svc.status)}</td>
<td> <td>
@@ -501,17 +547,36 @@ const Knowledge: React.FC = () => {
</div> </div>
)} )}
<div style={{ borderTop: '1px solid #E5E7EB', paddingTop: 16 }}> <div style={{ borderTop: '1px solid #E5E7EB', paddingTop: 16 }}>
<h4 style={{ margin: '0 0 12px 0', fontSize: 14 }}>添加新服务</h4> <h4 style={{ margin: '0 0 12px 0', fontSize: 14 }}>配置 Dify 知识库</h4>
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12 }}> <div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12 }}>
<div><label style={labelStyle}>名称</label><input value={extForm.name} onChange={e => setExtForm({ ...extForm, name: e.target.value })} placeholder="如: 生产环境 FastGPT" style={fullInput} /></div> <div><label style={labelStyle}>名称</label><input value={extForm.name} onChange={e => setExtForm({ ...extForm, name: e.target.value })} placeholder="如: 生产环境 Dify" style={fullInput} /></div>
<div><label style={labelStyle}>类型</label><select value={extForm.service_type} onChange={e => setExtForm({ ...extForm, service_type: e.target.value })} style={fullInput}><option value="fastgpt">FastGPT</option><option value="dify">Dify</option><option value="ragflow">RAGFlow</option></select></div> <div><label style={labelStyle}>知识库名称(Dataset)</label><input value={extForm.dataset_name} onChange={e => setExtForm({ ...extForm, dataset_name: e.target.value })} placeholder="如: qihuai" style={fullInput} /></div>
</div> </div>
<div style={{ marginTop: 12 }}><label style={labelStyle}>API URL</label><input value={extForm.api_url} onChange={e => setExtForm({ ...extForm, api_url: e.target.value })} placeholder="https://your-instance.com/api/v1" style={fullInput} /></div> <div style={{ marginTop: 12 }}><label style={labelStyle}>API URL</label><input value={extForm.api_url} onChange={e => setExtForm({ ...extForm, api_url: e.target.value })} placeholder="https://dify.qihuyun.com/v1" style={fullInput} /></div>
<div style={{ marginTop: 12 }}><label style={labelStyle}>API Key</label><input type="password" value={extForm.api_key} onChange={e => setExtForm({ ...extForm, api_key: e.target.value })} placeholder="Bearer Token 或 API Key" style={fullInput} /></div> <div style={{ marginTop: 12 }}><label style={labelStyle}>API Key(dataset 密钥)</label><input type="password" value={extForm.api_key} onChange={e => setExtForm({ ...extForm, api_key: e.target.value })} placeholder="dataset-xxxx" style={fullInput} /></div>
<div style={{ display: 'flex', gap: 12, marginTop: 16 }}>
<button onClick={handleCreateExtService} style={btnStyle('#8B5CF6')}>添加服务</button> {extMsg && (
<div style={{ marginTop: 12, fontSize: 13, color: extMsg.startsWith('✅') ? '#16A34A' : '#DC2626', padding: '6px 12px', background: extMsg.startsWith('✅') ? '#DCFCE7' : '#FEE2E2', borderRadius: 6 }}>
{extMsg}
</div>
)}
<div style={{ display: 'flex', gap: 12, marginTop: 16, flexWrap: 'wrap' }}>
<button onClick={handleCreateExtService} style={btnStyle('#8B5CF6')}>保存并测试连接</button>
<button onClick={handleIngestPlatform} disabled={ingesting} style={btnStyle(ingesting ? '#D1D5DB' : '#219B5E')}>
{ingesting ? '入库中...' : '入库平台内容'}
</button>
<button onClick={() => setShowExtModal(false)} style={btnStyle('#6B7280')}>关闭</button> <button onClick={() => setShowExtModal(false)} style={btnStyle('#6B7280')}>关闭</button>
</div> </div>
{ingestResult && (
<div style={{ marginTop: 12, fontSize: 13, padding: 12, background: '#F0FDF4', border: '1px solid #BBF7D0', borderRadius: 8 }}>
<div style={{ fontWeight: 600, marginBottom: 6 }}>入库结果</div>
<div>AI 聊天回答:<b>{ingestResult.chat_answers}</b> 条</div>
<div>用户上传文档:<b>{ingestResult.user_docs}</b> 条</div>
<div>跳过(过短/空):{ingestResult.skipped} 条 · 失败:{ingestResult.failed} 条</div>
</div>
)}
</div> </div>
</div> </div>
</div> </div>