feat(kb): 接入 Dify 知识库,移除 FastGPT;AI 聊天产出+用户文档自动入库量化
- 新增 app/dify_kb.py Dify v1 客户端(连通测试/检索/文本入库) - kb_router 仅保留 Dify 外部检索分支 - knowledge_admin 去掉 FastGPT/RAGFlow,新增 dify 批量入库端点 - chat.py 注入 Dify RAG 上下文 + 助手回答 fire-and-forget 入库 - 前端 Knowledge.tsx 改为 Dify 单服务 UI(可通讯/可用验证+入库按钮) - DB: external_kb_services 删除 fastgpt,qihuai 设为 https+connected
This commit is contained in:
1 parent
d3a890eb18
commit
a1669374e6
5 files changed
+595
-378
No files matched your search
@@ -32,7 +32,7 @@ logger = logging.getLogger("aiflow")
|
|||||||
|
|
||||||
class ExternalServiceConfig(BaseModel):
|
class ExternalServiceConfig(BaseModel):
|
||||||
name: str = Field(min_length=1, max_length=200)
|
name: str = Field(min_length=1, max_length=200)
|
||||||
service_type: str = Field(..., description="fastgpt / dify / ragflow")
|
service_type: str = Field(..., description="dify")
|
||||||
api_url: str = Field(min_length=1, max_length=500)
|
api_url: str = Field(min_length=1, max_length=500)
|
||||||
api_key: Optional[str] = None
|
api_key: Optional[str] = None
|
||||||
config: Optional[Dict[str, Any]] = None
|
config: Optional[Dict[str, Any]] = None
|
||||||
@@ -739,48 +739,27 @@ def _get_external_services_table(db: Session):
|
|||||||
|
|
||||||
|
|
||||||
def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict:
|
def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict:
|
||||||
"""测试外部知识库服务连接状态"""
|
"""测试外部知识库服务连接状态(当前仅支持 Dify)"""
|
||||||
try:
|
try:
|
||||||
headers = {}
|
if service_type == "dify":
|
||||||
if api_key:
|
from ...dify_kb import dify_test
|
||||||
headers["Authorization"] = f"Bearer {api_key}"
|
return dify_test(api_url, api_key)
|
||||||
|
|
||||||
if service_type == "fastgpt":
|
|
||||||
# FastGPT 健康检查
|
|
||||||
check_url = f"{api_url.rstrip('/')}/api/v1/status"
|
|
||||||
resp = httpx.get(check_url, headers=headers, timeout=10)
|
|
||||||
if resp.status_code == 200:
|
|
||||||
return {"connected": True, "status": "connected", "detail": "FastGPT 连接成功"}
|
|
||||||
else:
|
|
||||||
return {"connected": False, "status": "error", "detail": f"FastGPT 返回状态码 {resp.status_code}"}
|
|
||||||
|
|
||||||
elif service_type == "dify":
|
|
||||||
# Dify 健康检查
|
|
||||||
check_url = f"{api_url.rstrip('/')}/v1/datasets"
|
|
||||||
resp = httpx.get(check_url, headers=headers, timeout=10)
|
|
||||||
if resp.status_code in [200, 401]: # 401说明服务存在但需要认证
|
|
||||||
return {"connected": True, "status": "connected", "detail": "Dify 连接成功"}
|
|
||||||
else:
|
|
||||||
return {"connected": False, "status": "error", "detail": f"Dify 返回状态码 {resp.status_code}"}
|
|
||||||
|
|
||||||
elif service_type == "ragflow":
|
|
||||||
# RAGFlow 健康检查
|
|
||||||
check_url = f"{api_url.rstrip('/')}/api/v1/datasets"
|
|
||||||
resp = httpx.get(check_url, headers=headers, timeout=10)
|
|
||||||
if resp.status_code in [200, 401]:
|
|
||||||
return {"connected": True, "status": "connected", "detail": "RAGFlow 连接成功"}
|
|
||||||
else:
|
|
||||||
return {"connected": False, "status": "error", "detail": f"RAGFlow 返回状态码 {resp.status_code}"}
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# 通用检查
|
# 未知类型(如历史遗留的 fastgpt/ragflow),按通用方式探测可达性
|
||||||
|
headers = {}
|
||||||
|
if api_key:
|
||||||
|
headers["Authorization"] = f"Bearer {api_key}"
|
||||||
resp = httpx.get(api_url, headers=headers, timeout=10)
|
resp = httpx.get(api_url, headers=headers, timeout=10)
|
||||||
return {"connected": resp.status_code < 500, "status": "connected" if resp.status_code < 500 else "error"}
|
return {
|
||||||
|
"connected": resp.status_code < 500,
|
||||||
|
"usable": resp.status_code < 500,
|
||||||
|
"status": "connected" if resp.status_code < 500 else "error",
|
||||||
|
"detail": f"通用探测返回状态码 {resp.status_code}",
|
||||||
|
}
|
||||||
except httpx.TimeoutException:
|
except httpx.TimeoutException:
|
||||||
return {"connected": False, "status": "timeout", "detail": "连接超时"}
|
return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时"}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return {"connected": False, "status": "error", "detail": str(e)}
|
return {"connected": False, "usable": False, "status": "error", "detail": str(e)}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/external/services", summary="获取外部知识库服务列表")
|
@router.get("/external/services", summary="获取外部知识库服务列表")
|
||||||
@@ -801,9 +780,7 @@ async def list_external_services(
|
|||||||
"total": len(services),
|
"total": len(services),
|
||||||
"items": services,
|
"items": services,
|
||||||
"supported_types": [
|
"supported_types": [
|
||||||
{"type": "fastgpt", "name": "FastGPT", "description": "开源 AI 知识库平台,支持多种向量模型"},
|
{"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理(RAG)"},
|
||||||
{"type": "dify", "name": "Dify", "description": "LLM 应用开发平台,内置知识库管理"},
|
|
||||||
{"type": "ragflow", "name": "RAGFlow", "description": "基于深度文档理解的开源 RAG 引擎"},
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -816,8 +793,8 @@ async def create_external_service(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
"""创建新的外部知识库服务连接配置"""
|
"""创建新的外部知识库服务连接配置"""
|
||||||
# 验证服务类型
|
# 验证服务类型(当前仅支持 Dify)
|
||||||
valid_types = ["fastgpt", "dify", "ragflow"]
|
valid_types = ["dify"]
|
||||||
if req.service_type not in valid_types:
|
if req.service_type not in valid_types:
|
||||||
raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}")
|
raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}")
|
||||||
|
|
||||||
@@ -1082,163 +1059,30 @@ async def sync_knowledge_base(
|
|||||||
if api_key:
|
if api_key:
|
||||||
headers["Authorization"] = f"Bearer {api_key}"
|
headers["Authorization"] = f"Bearer {api_key}"
|
||||||
|
|
||||||
if service_type == "fastgpt" and req.sync_direction == "push":
|
if service_type == "dify" and req.sync_direction == "push":
|
||||||
# 向 FastGPT 推送知识库数据
|
# 向 Dify 推送知识库数据(使用 app.dify_kb 客户端,按 dataset_name 解析)
|
||||||
# 1. 在 FastGPT 创建/获取对应的 dataset
|
from ...dify_kb import dify_ingest_text, resolve_dataset_id
|
||||||
ext_kb_id = getattr(kb, "external_id", None)
|
dataset_name = service_config.get("dataset_name") if isinstance(service_config, dict) else None
|
||||||
|
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
|
||||||
if not ext_kb_id:
|
if not ds_id:
|
||||||
# 创建新 dataset
|
sync_result["detail"] = "无法解析 Dify 知识库 ID(请检查 API Key 是否绑定到指定数据集)"
|
||||||
create_resp = httpx.post(
|
sync_result["synced"] = False
|
||||||
f"{api_url.rstrip('/')}/api/v1/datasets",
|
else:
|
||||||
headers=headers,
|
|
||||||
json={"name": kb.name, "description": kb.description or ""},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if create_resp.status_code == 200:
|
|
||||||
ext_data = create_resp.json()
|
|
||||||
ext_kb_id = ext_data.get("data", {}).get("id") or ext_data.get("id")
|
|
||||||
# 更新 KB 的 external_id
|
|
||||||
try:
|
|
||||||
setattr(kb, "external_id", ext_kb_id)
|
|
||||||
except Exception:
|
|
||||||
db.execute(
|
|
||||||
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
|
|
||||||
{"eid": ext_kb_id, "kid": kb_id}
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
sync_result["detail"] = f"FastGPT 创建 dataset 失败: {create_resp.status_code}"
|
|
||||||
sync_result["synced"] = False
|
|
||||||
|
|
||||||
if ext_kb_id:
|
|
||||||
# 推送文档内容到 FastGPT
|
|
||||||
docs = db.query(KnowledgeDocument).filter(
|
docs = db.query(KnowledgeDocument).filter(
|
||||||
KnowledgeDocument.knowledge_base_id == kb_id,
|
KnowledgeDocument.knowledge_base_id == kb_id,
|
||||||
KnowledgeDocument.status == "completed"
|
KnowledgeDocument.status == "completed"
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
synced_count = 0
|
synced_count = 0
|
||||||
for doc in docs:
|
for doc in docs:
|
||||||
content = doc.content or ""
|
content = doc.content or ""
|
||||||
if not content:
|
if not content:
|
||||||
continue
|
continue
|
||||||
|
if dify_ingest_text(api_url, api_key, content[:50000], name=doc.filename, dataset_name=dataset_name):
|
||||||
# 分段推送
|
|
||||||
push_resp = httpx.post(
|
|
||||||
f"{api_url.rstrip('/')}/api/v1/datasets/{ext_kb_id}/documents",
|
|
||||||
headers=headers,
|
|
||||||
json={
|
|
||||||
"name": doc.filename,
|
|
||||||
"content": content[:50000], # 限制单次推送大小
|
|
||||||
},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if push_resp.status_code in [200, 201]:
|
|
||||||
synced_count += 1
|
synced_count += 1
|
||||||
|
|
||||||
sync_result["synced"] = True
|
|
||||||
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 FastGPT"
|
|
||||||
sync_result["items_synced"] = synced_count
|
|
||||||
|
|
||||||
elif service_type == "dify" and req.sync_direction == "push":
|
|
||||||
# 向 Dify 推送知识库数据
|
|
||||||
ext_kb_id = getattr(kb, "external_id", None)
|
|
||||||
|
|
||||||
if not ext_kb_id:
|
|
||||||
create_resp = httpx.post(
|
|
||||||
f"{api_url.rstrip('/')}/v1/datasets",
|
|
||||||
headers=headers,
|
|
||||||
json={"name": kb.name},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if create_resp.status_code in [200, 201]:
|
|
||||||
ext_data = create_resp.json()
|
|
||||||
ext_kb_id = ext_data.get("id")
|
|
||||||
try:
|
|
||||||
setattr(kb, "external_id", ext_kb_id)
|
|
||||||
except Exception:
|
|
||||||
db.execute(
|
|
||||||
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
|
|
||||||
{"eid": ext_kb_id, "kid": kb_id}
|
|
||||||
)
|
|
||||||
|
|
||||||
if ext_kb_id:
|
|
||||||
docs = db.query(KnowledgeDocument).filter(
|
|
||||||
KnowledgeDocument.knowledge_base_id == kb_id,
|
|
||||||
KnowledgeDocument.status == "completed"
|
|
||||||
).all()
|
|
||||||
|
|
||||||
synced_count = 0
|
|
||||||
for doc in docs:
|
|
||||||
content = doc.content or ""
|
|
||||||
if not content:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Dify 文档创建
|
|
||||||
doc_resp = httpx.post(
|
|
||||||
f"{api_url.rstrip('/')}/v1/datasets/{ext_kb_id}/document/create_by_text",
|
|
||||||
headers=headers,
|
|
||||||
json={
|
|
||||||
"name": doc.filename,
|
|
||||||
"text": content[:50000],
|
|
||||||
"indexing_technique": "high_quality",
|
|
||||||
},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if doc_resp.status_code in [200, 201]:
|
|
||||||
synced_count += 1
|
|
||||||
|
|
||||||
sync_result["synced"] = True
|
sync_result["synced"] = True
|
||||||
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify"
|
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify"
|
||||||
sync_result["items_synced"] = synced_count
|
sync_result["items_synced"] = synced_count
|
||||||
|
|
||||||
elif service_type == "ragflow" and req.sync_direction == "push":
|
|
||||||
# 向 RAGFlow 推送知识库数据
|
|
||||||
ext_kb_id = getattr(kb, "external_id", None)
|
|
||||||
|
|
||||||
if not ext_kb_id:
|
|
||||||
create_resp = httpx.post(
|
|
||||||
f"{api_url.rstrip('/')}/api/v1/datasets",
|
|
||||||
headers=headers,
|
|
||||||
json={"name": kb.name},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if create_resp.status_code in [200, 201]:
|
|
||||||
ext_data = create_resp.json()
|
|
||||||
ext_kb_id = ext_data.get("id")
|
|
||||||
try:
|
|
||||||
setattr(kb, "external_id", ext_kb_id)
|
|
||||||
except Exception:
|
|
||||||
db.execute(
|
|
||||||
text("UPDATE knowledge_bases SET external_id = :eid WHERE id = :kid"),
|
|
||||||
{"eid": ext_kb_id, "kid": kb_id}
|
|
||||||
)
|
|
||||||
|
|
||||||
if ext_kb_id:
|
|
||||||
docs = db.query(KnowledgeDocument).filter(
|
|
||||||
KnowledgeDocument.knowledge_base_id == kb_id,
|
|
||||||
KnowledgeDocument.status == "completed"
|
|
||||||
).all()
|
|
||||||
|
|
||||||
synced_count = 0
|
|
||||||
for doc in docs:
|
|
||||||
content = doc.content or ""
|
|
||||||
if not content:
|
|
||||||
continue
|
|
||||||
|
|
||||||
doc_resp = httpx.post(
|
|
||||||
f"{api_url.rstrip('/')}/api/v1/datasets/{ext_kb_id}/documents",
|
|
||||||
headers=headers,
|
|
||||||
json={"name": doc.filename, "content": content[:50000]},
|
|
||||||
timeout=30,
|
|
||||||
)
|
|
||||||
if doc_resp.status_code in [200, 201]:
|
|
||||||
synced_count += 1
|
|
||||||
|
|
||||||
sync_result["synced"] = True
|
|
||||||
sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 RAGFlow"
|
|
||||||
sync_result["items_synced"] = synced_count
|
|
||||||
|
|
||||||
elif req.sync_direction == "pull":
|
elif req.sync_direction == "pull":
|
||||||
# 从外部服务拉取知识库数据
|
# 从外部服务拉取知识库数据
|
||||||
sync_result["detail"] = f"从 {service_type} 拉取数据(暂未实现完整拉取逻辑)"
|
sync_result["detail"] = f"从 {service_type} 拉取数据(暂未实现完整拉取逻辑)"
|
||||||
@@ -1288,6 +1132,106 @@ async def sync_knowledge_base(
|
|||||||
return {"code": 0, "data": sync_result}
|
return {"code": 0, "data": sync_result}
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 平台内容批量入库到 Dify(AI聊天产出 + 用户上传文档)
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
@router.post("/external/dify/ingest", summary="将平台内容批量入库到 Dify 知识库")
|
||||||
|
async def ingest_platform_to_dify(
|
||||||
|
admin_user=Depends(get_current_admin),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
"""把平台中「可量化的内容」推送到 Dify 知识库(qihuai):
|
||||||
|
- AI 聊天的助手回答(文本、非工具/非媒体、长度 > 120)
|
||||||
|
- 用户上传的文档资产(assets, category='file' 且含文本内容)
|
||||||
|
返回各来源的入库数量。
|
||||||
|
"""
|
||||||
|
from ...dify_kb import dify_ingest_text, resolve_dataset_id
|
||||||
|
|
||||||
|
# 读取已启用的 Dify 服务
|
||||||
|
try:
|
||||||
|
row = db.execute(
|
||||||
|
text("SELECT api_url, api_key, config FROM external_kb_services "
|
||||||
|
"WHERE service_type='dify' AND is_enabled=1 LIMIT 1")
|
||||||
|
).fetchone()
|
||||||
|
except Exception:
|
||||||
|
row = None
|
||||||
|
if not row:
|
||||||
|
raise HTTPException(status_code=400, detail="未配置已启用的 Dify 知识库服务,请先添加 Dify 服务并测试连接")
|
||||||
|
api_url, api_key, config_raw = row
|
||||||
|
dataset_name = None
|
||||||
|
if config_raw:
|
||||||
|
try:
|
||||||
|
cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw)
|
||||||
|
dataset_name = cfg.get("dataset_name")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
|
||||||
|
if not ds_id:
|
||||||
|
raise HTTPException(status_code=400, detail="无法解析 Dify 知识库 ID(请检查 API Key 与 dataset_name)")
|
||||||
|
|
||||||
|
stats = {"chat_answers": 0, "user_docs": 0, "skipped": 0, "failed": 0}
|
||||||
|
|
||||||
|
# 1. AI 聊天助手回答
|
||||||
|
try:
|
||||||
|
from ...models import Message
|
||||||
|
answers = db.query(Message).filter(
|
||||||
|
Message.role == "assistant"
|
||||||
|
).order_by(Message.created_at.asc()).limit(2000).all()
|
||||||
|
for m in answers:
|
||||||
|
content = getattr(m, "content", None) or ""
|
||||||
|
content = content.strip()
|
||||||
|
# 跳过媒体/工具类消息与过短内容
|
||||||
|
if len(content) < 120:
|
||||||
|
stats["skipped"] += 1
|
||||||
|
continue
|
||||||
|
if content.startswith("[") and ("MEDIA" in content or "IMAGE" in content or "VIDEO" in content):
|
||||||
|
stats["skipped"] += 1
|
||||||
|
continue
|
||||||
|
conv_id = getattr(m, "conversation_id", None) or "unknown"
|
||||||
|
if dify_ingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{m.id}", dataset_name=dataset_name):
|
||||||
|
stats["chat_answers"] += 1
|
||||||
|
else:
|
||||||
|
stats["failed"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"ingest chat answers failed: {e}")
|
||||||
|
|
||||||
|
# 2. 用户上传的文档资产
|
||||||
|
try:
|
||||||
|
from ...models import Asset
|
||||||
|
assets = db.query(Asset).filter(
|
||||||
|
getattr(Asset, "category", "file") == "file"
|
||||||
|
).order_by(getattr(Asset, "created_at", text("id")).asc()).limit(2000).all()
|
||||||
|
for a in assets:
|
||||||
|
content = getattr(a, "content", None) or ""
|
||||||
|
content = content.strip()
|
||||||
|
if len(content) < 50:
|
||||||
|
stats["skipped"] += 1
|
||||||
|
continue
|
||||||
|
fname = getattr(a, "filename", None) or getattr(a, "name", None) or "asset"
|
||||||
|
if dify_ingest_text(api_url, api_key, content[:50000], name=f"asset_{a.id}_{fname}", dataset_name=dataset_name):
|
||||||
|
stats["user_docs"] += 1
|
||||||
|
else:
|
||||||
|
stats["failed"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"ingest assets failed: {e}")
|
||||||
|
|
||||||
|
LogService.log_audit(
|
||||||
|
admin_id=admin_user.id,
|
||||||
|
admin_name=admin_user.username,
|
||||||
|
action="ingest_platform_to_dify",
|
||||||
|
target_type="external_kb_service",
|
||||||
|
target_id=ds_id,
|
||||||
|
detail=f"平台内容入库 Dify: 聊天回答 {stats['chat_answers']} 条, 用户文档 {stats['user_docs']} 条, 跳过 {stats['skipped']}, 失败 {stats['failed']}",
|
||||||
|
)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"code": 0,
|
||||||
|
"message": f"入库完成:聊天回答 {stats['chat_answers']} 条,用户文档 {stats['user_docs']} 条",
|
||||||
|
"data": stats,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# 批量操作
|
# 批量操作
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|||||||
+116
-64
@@ -490,84 +490,122 @@ def _retrieve_kb_context(db: Session, kb_ids: list, query: str, max_chars: int =
|
|||||||
2. 构建文档列表(content+title+metadata)
|
2. 构建文档列表(content+title+metadata)
|
||||||
3. BM25检索最相关的段落
|
3. BM25检索最相关的段落
|
||||||
4. 拼接为上下文注入system prompt
|
4. 拼接为上下文注入system prompt
|
||||||
|
同时叠加 Dify 外部知识库(RAG)检索结果。
|
||||||
"""
|
"""
|
||||||
if not kb_ids:
|
local_context = ""
|
||||||
return ""
|
|
||||||
|
|
||||||
kbs = db.query(KnowledgeBase).filter(KnowledgeBase.id.in_(kb_ids)).all()
|
if kb_ids:
|
||||||
if not kbs:
|
kbs = db.query(KnowledgeBase).filter(KnowledgeBase.id.in_(kb_ids)).all()
|
||||||
return ""
|
if kbs:
|
||||||
|
# 收集所有文档段落
|
||||||
|
# Stage C.1: 优先使用 KnowledgeDocumentChunk 表的预分段数据
|
||||||
|
all_segments = []
|
||||||
|
for kb in kbs:
|
||||||
|
# 先查chunk表
|
||||||
|
chunks = db.query(KnowledgeDocumentChunk).filter(
|
||||||
|
KnowledgeDocumentChunk.knowledge_base_id == kb.id
|
||||||
|
).order_by(KnowledgeDocumentChunk.chunk_index).limit(30).all()
|
||||||
|
|
||||||
# 收集所有文档段落
|
if chunks:
|
||||||
# Stage C.1: 优先使用 KnowledgeDocumentChunk 表的预分段数据
|
# 使用预分段数据
|
||||||
all_segments = []
|
for chunk in chunks:
|
||||||
for kb in kbs:
|
if chunk.content and len(chunk.content.strip()) >= 20:
|
||||||
# 先查chunk表
|
|
||||||
chunks = db.query(KnowledgeDocumentChunk).filter(
|
|
||||||
KnowledgeDocumentChunk.knowledge_base_id == kb.id
|
|
||||||
).order_by(KnowledgeDocumentChunk.chunk_index).limit(30).all()
|
|
||||||
|
|
||||||
if chunks:
|
|
||||||
# 使用预分段数据
|
|
||||||
for chunk in chunks:
|
|
||||||
if chunk.content and len(chunk.content.strip()) >= 20:
|
|
||||||
all_segments.append({
|
|
||||||
"content": "[{}/{}/段{}]: {}".format(
|
|
||||||
kb.name, chunk.document_id[:8], chunk.chunk_index, chunk.content[:500]
|
|
||||||
),
|
|
||||||
"raw_content": chunk.content,
|
|
||||||
"kb_name": kb.name,
|
|
||||||
})
|
|
||||||
else:
|
|
||||||
# 退化: 使用原始文档分段
|
|
||||||
docs = db.query(KnowledgeDocument).filter(
|
|
||||||
KnowledgeDocument.knowledge_base_id == kb.id,
|
|
||||||
KnowledgeDocument.status == "completed"
|
|
||||||
).order_by(KnowledgeDocument.updated_at.desc()).limit(20).all()
|
|
||||||
|
|
||||||
for doc in docs:
|
|
||||||
content = doc.content or ""
|
|
||||||
if not content:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 如果文档较长, 分段处理 (每段~800字)
|
|
||||||
if len(content) > 800:
|
|
||||||
paragraphs = re.split(r'\n\n+', content)
|
|
||||||
for pi, para in enumerate(paragraphs):
|
|
||||||
if len(para.strip()) >= 20:
|
|
||||||
all_segments.append({
|
all_segments.append({
|
||||||
"content": "[{}/{}/段{}]: {}".format(
|
"content": "[{}/{}/段{}]: {}".format(
|
||||||
kb.name, doc.name, pi+1, para.strip()[:500]
|
kb.name, chunk.document_id[:8], chunk.chunk_index, chunk.content[:500]
|
||||||
),
|
),
|
||||||
"raw_content": para.strip(),
|
"raw_content": chunk.content,
|
||||||
|
"kb_name": kb.name,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
# 退化: 使用原始文档分段
|
||||||
|
docs = db.query(KnowledgeDocument).filter(
|
||||||
|
KnowledgeDocument.knowledge_base_id == kb.id,
|
||||||
|
KnowledgeDocument.status == "completed"
|
||||||
|
).order_by(KnowledgeDocument.updated_at.desc()).limit(20).all()
|
||||||
|
|
||||||
|
for doc in docs:
|
||||||
|
content = doc.content or ""
|
||||||
|
if not content:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 如果文档较长, 分段处理 (每段~800字)
|
||||||
|
if len(content) > 800:
|
||||||
|
paragraphs = re.split(r'\n\n+', content)
|
||||||
|
for pi, para in enumerate(paragraphs):
|
||||||
|
if len(para.strip()) >= 20:
|
||||||
|
all_segments.append({
|
||||||
|
"content": "[{}/{}/段{}]: {}".format(
|
||||||
|
kb.name, doc.name, pi+1, para.strip()[:500]
|
||||||
|
),
|
||||||
|
"raw_content": para.strip(),
|
||||||
|
"kb_name": kb.name,
|
||||||
|
"doc_name": doc.name,
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
all_segments.append({
|
||||||
|
"content": "[{}/{}]: {}".format(kb.name, doc.name, content[:500]),
|
||||||
|
"raw_content": content,
|
||||||
"kb_name": kb.name,
|
"kb_name": kb.name,
|
||||||
"doc_name": doc.name,
|
"doc_name": doc.name,
|
||||||
})
|
})
|
||||||
else:
|
|
||||||
all_segments.append({
|
|
||||||
"content": "[{}/{}]: {}".format(kb.name, doc.name, content[:500]),
|
|
||||||
"raw_content": content,
|
|
||||||
"kb_name": kb.name,
|
|
||||||
"doc_name": doc.name,
|
|
||||||
})
|
|
||||||
|
|
||||||
if not all_segments:
|
if not all_segments:
|
||||||
# 退化: 使用原始朴素拼接
|
# 退化: 使用原始朴素拼接
|
||||||
kb_names = ", ".join([k.name for k in kbs])
|
kb_names = ", ".join([k.name for k in kbs])
|
||||||
return f"参考知识库: {kb_names}"
|
local_context = f"参考知识库: {kb_names}"
|
||||||
|
else:
|
||||||
|
# BM25检索最相关的段落
|
||||||
|
top_segments = _bm25_search(query, all_segments, top_k=8)
|
||||||
|
context_parts = [seg["content"] for seg in top_segments]
|
||||||
|
local_context = "\n".join(context_parts)
|
||||||
|
if len(local_context) > max_chars:
|
||||||
|
local_context = local_context[:max_chars] + "...(内容截断)"
|
||||||
|
|
||||||
# BM25检索最相关的段落
|
# 叠加 Dify 外部知识库(RAG)检索
|
||||||
top_segments = _bm25_search(query, all_segments, top_k=8)
|
dify_context = ""
|
||||||
|
try:
|
||||||
|
from app.dify_kb import get_dify_kb_context
|
||||||
|
dify_context = get_dify_kb_context(db, query, top_k=5)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Dify KB context retrieval failed: {e}")
|
||||||
|
|
||||||
# 拼接上下文
|
parts = []
|
||||||
context_parts = [seg["content"] for seg in top_segments]
|
if local_context:
|
||||||
context = "\n".join(context_parts)
|
parts.append(local_context)
|
||||||
|
if dify_context:
|
||||||
|
parts.append("【Dify 知识库检索结果】\n" + dify_context)
|
||||||
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
# 限制总长度
|
|
||||||
if len(context) > max_chars:
|
|
||||||
context = context[:max_chars] + "...(内容截断)"
|
|
||||||
|
|
||||||
return context
|
async def _ingest_answer_to_dify(conv_id: str, msg_id: str, content: str):
|
||||||
|
"""后台任务:将助手回答入库到 Dify 知识库(量化后可供 RAG 检索复用)。"""
|
||||||
|
try:
|
||||||
|
from app.dify_kb import aingest_text
|
||||||
|
from app.database import SessionLocal
|
||||||
|
from sqlalchemy import text as _t
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
row = db.execute(_t(
|
||||||
|
"SELECT api_url, api_key, config FROM external_kb_services "
|
||||||
|
"WHERE service_type='dify' AND is_enabled=1 LIMIT 1"
|
||||||
|
)).fetchone()
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
if not row:
|
||||||
|
return
|
||||||
|
api_url, api_key, config_raw = row
|
||||||
|
dataset_name = None
|
||||||
|
if config_raw:
|
||||||
|
try:
|
||||||
|
import json
|
||||||
|
cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw)
|
||||||
|
dataset_name = cfg.get("dataset_name")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
await aingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{msg_id}", dataset_name=dataset_name)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"后台入库 Dify 失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
@@ -1203,6 +1241,20 @@ async def send_message(
|
|||||||
)
|
)
|
||||||
db.add(ai_msg)
|
db.add(ai_msg)
|
||||||
|
|
||||||
|
# 后台异步:将实质性文本回答入库到 Dify 知识库(量化后可供 RAG 复用)
|
||||||
|
_ingest_text = (ai_content or "").strip()
|
||||||
|
if len(_ingest_text) > 120 and not (
|
||||||
|
_ingest_text.startswith("[MEDIA")
|
||||||
|
or _ingest_text.startswith("data:")
|
||||||
|
or _ingest_text.startswith("http")
|
||||||
|
or "<img" in _ingest_text
|
||||||
|
or ":
|
||||||
|
try:
|
||||||
|
asyncio.create_task(_ingest_answer_to_dify(conv_id, ai_msg.id, _ingest_text))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# 更新used_skills
|
# 更新used_skills
|
||||||
used_skills = []
|
used_skills = []
|
||||||
if skills:
|
if skills:
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
"""
|
||||||
|
Dify 知识库客户端
|
||||||
|
================
|
||||||
|
封装与 Dify 知识库(v1 API)的交互:连通性测试、检索(RAG)、文本入库。
|
||||||
|
设计用于替换原 FastGPT 外部知识库对接。
|
||||||
|
|
||||||
|
已验证的接入点(2026-07-14 实测):
|
||||||
|
- Base: https://dify.qihuyun.com/v1
|
||||||
|
(直接 HTTP 访问会被 nginx 301 跳转到 HTTPS,必须 HTTPS 直连,
|
||||||
|
且内网自签证书需 verify=False)
|
||||||
|
- Auth: Bearer {api_key}
|
||||||
|
- GET /v1/datasets 列出知识库(可据此解析 dataset id)
|
||||||
|
- POST /v1/datasets/{id}/document/create-by-text 文本入库
|
||||||
|
- POST /v1/datasets/{id}/document/create-by-file 文件入库(multipart)
|
||||||
|
- POST /v1/datasets/{id}/retrieve 检索(RAG)
|
||||||
|
retrieve 必需完整的 retrieval_model 字典,否则返回 400。
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
logger = logging.getLogger("aiflow.dify_kb")
|
||||||
|
|
||||||
|
# dataset id 缓存: key -> (dataset_id, ts)
|
||||||
|
_dataset_cache: dict = {}
|
||||||
|
_dataset_cache_lock = threading.Lock()
|
||||||
|
_DATASET_CACHE_TTL = 3600
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_base(api_url: str) -> str:
|
||||||
|
"""统一 base url -> https 形式,且不带 /v1 后缀。"""
|
||||||
|
u = (api_url or "").strip()
|
||||||
|
if not u:
|
||||||
|
u = "https://dify.qihuyun.com/v1"
|
||||||
|
u = u.rstrip("/")
|
||||||
|
if u.endswith("/v1"):
|
||||||
|
u = u[: -len("/v1")]
|
||||||
|
if u.startswith("http://"):
|
||||||
|
u = "https://" + u[len("http://"):]
|
||||||
|
return u
|
||||||
|
|
||||||
|
|
||||||
|
def dify_test(api_url: str, api_key: str) -> dict:
|
||||||
|
"""验证 Dify 知识库「可通讯 + 可用」。
|
||||||
|
|
||||||
|
返回:
|
||||||
|
{
|
||||||
|
"connected": bool,
|
||||||
|
"usable": bool,
|
||||||
|
"status": "connected" | "auth_error" | "timeout" | "error",
|
||||||
|
"detail": str,
|
||||||
|
"datasets": [{"id":..., "name":...}, ...]
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
base = _normalize_base(api_url)
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}"}
|
||||||
|
try:
|
||||||
|
with httpx.Client(base_url=base, timeout=15, verify=False) as client:
|
||||||
|
r = client.get("/v1/datasets", headers=headers)
|
||||||
|
if r.status_code == 200:
|
||||||
|
data = r.json()
|
||||||
|
datasets = data.get("data", []) or []
|
||||||
|
names = [d.get("name") for d in datasets]
|
||||||
|
# 能列出知识库即代表鉴权通过、可读取 —— 视为可用。
|
||||||
|
return {
|
||||||
|
"connected": True,
|
||||||
|
"usable": True,
|
||||||
|
"status": "connected",
|
||||||
|
"detail": f"Dify 连接成功,发现 {len(datasets)} 个知识库: {', '.join([n for n in names if n]) or '无'}",
|
||||||
|
"datasets": [{"id": d.get("id"), "name": d.get("name")} for d in datasets],
|
||||||
|
}
|
||||||
|
elif r.status_code == 401:
|
||||||
|
return {
|
||||||
|
"connected": True,
|
||||||
|
"usable": False,
|
||||||
|
"status": "auth_error",
|
||||||
|
"detail": "已连通 Dify,但 API Key 鉴权失败(401),请检查 API Key。",
|
||||||
|
"datasets": [],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
return {
|
||||||
|
"connected": False,
|
||||||
|
"usable": False,
|
||||||
|
"status": "error",
|
||||||
|
"detail": f"Dify 返回状态码 {r.status_code}",
|
||||||
|
"datasets": [],
|
||||||
|
}
|
||||||
|
except httpx.TimeoutException:
|
||||||
|
return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时", "datasets": []}
|
||||||
|
except Exception as e:
|
||||||
|
return {"connected": False, "usable": False, "status": "error", "detail": str(e), "datasets": []}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_dataset_id(api_url: str, api_key: str, dataset_name: str = None) -> str:
|
||||||
|
"""解析 dataset id(优先按名称匹配,否则取列表第一个),带缓存。"""
|
||||||
|
base = _normalize_base(api_url)
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}"}
|
||||||
|
cache_key = f"{base}|{api_key}|{dataset_name}"
|
||||||
|
now = time.time()
|
||||||
|
with _dataset_cache_lock:
|
||||||
|
c = _dataset_cache.get(cache_key)
|
||||||
|
if c and now - c[1] < _DATASET_CACHE_TTL:
|
||||||
|
return c[0]
|
||||||
|
ds_id = None
|
||||||
|
try:
|
||||||
|
with httpx.Client(base_url=base, timeout=15, verify=False) as client:
|
||||||
|
r = client.get("/v1/datasets", headers=headers)
|
||||||
|
if r.status_code == 200:
|
||||||
|
datasets = r.json().get("data", []) or []
|
||||||
|
if dataset_name:
|
||||||
|
match = [d for d in datasets if d.get("name") == dataset_name]
|
||||||
|
if match:
|
||||||
|
ds_id = match[0].get("id")
|
||||||
|
if not ds_id and datasets:
|
||||||
|
ds_id = datasets[0].get("id")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"resolve_dataset_id failed: {e}")
|
||||||
|
if ds_id:
|
||||||
|
with _dataset_cache_lock:
|
||||||
|
_dataset_cache[cache_key] = (ds_id, now)
|
||||||
|
return ds_id
|
||||||
|
|
||||||
|
|
||||||
|
def dify_retrieve(api_url, api_key, query, top_k=5, dataset_name=None) -> list:
|
||||||
|
"""检索,返回 [{content, score, source, metadata}]。"""
|
||||||
|
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
|
||||||
|
if not ds_id:
|
||||||
|
return []
|
||||||
|
base = _normalize_base(api_url)
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||||
|
payload = {
|
||||||
|
"query": query,
|
||||||
|
"retrieval_model": {
|
||||||
|
"search_method": "semantic_search",
|
||||||
|
"reranking_enable": False,
|
||||||
|
"score_threshold_enabled": False,
|
||||||
|
"top_k": top_k,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with httpx.Client(base_url=base, timeout=30, verify=False) as client:
|
||||||
|
r = client.post(f"/v1/datasets/{ds_id}/retrieve", headers=headers, json=payload)
|
||||||
|
if r.status_code == 200:
|
||||||
|
data = r.json()
|
||||||
|
out = []
|
||||||
|
for item in data.get("records", []):
|
||||||
|
seg = item.get("segment", {})
|
||||||
|
out.append({
|
||||||
|
"content": seg.get("content", ""),
|
||||||
|
"score": float(item.get("score", 0.5)),
|
||||||
|
"source": f"Dify: {seg.get('document_id', '')}",
|
||||||
|
"metadata": item,
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
else:
|
||||||
|
logger.warning(f"Dify retrieve returned {r.status_code}: {r.text[:200]}")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Dify retrieve error: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def dify_ingest_text(api_url, api_key, text, name=None, dataset_name=None) -> bool:
|
||||||
|
"""文本入库(create-by-text)。返回是否成功。"""
|
||||||
|
if not text or not text.strip():
|
||||||
|
return False
|
||||||
|
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
|
||||||
|
if not ds_id:
|
||||||
|
return False
|
||||||
|
base = _normalize_base(api_url)
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||||
|
payload = {
|
||||||
|
"name": name or f"doc-{int(time.time())}",
|
||||||
|
"text": text,
|
||||||
|
"indexing_technique": "high_quality",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with httpx.Client(base_url=base, timeout=60, verify=False) as client:
|
||||||
|
r = client.post(f"/v1/datasets/{ds_id}/document/create-by-text", headers=headers, json=payload)
|
||||||
|
if r.status_code in (200, 201):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
logger.warning(f"Dify ingest text returned {r.status_code}: {r.text[:200]}")
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Dify ingest text error: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def aingest_text(api_url, api_key, text, name=None, dataset_name=None) -> bool:
|
||||||
|
"""异步文本入库(供 chat 实时落库使用)。"""
|
||||||
|
if not text or not text.strip():
|
||||||
|
return False
|
||||||
|
ds_id = resolve_dataset_id(api_url, api_key, dataset_name)
|
||||||
|
if not ds_id:
|
||||||
|
return False
|
||||||
|
base = _normalize_base(api_url)
|
||||||
|
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
|
||||||
|
payload = {
|
||||||
|
"name": name or f"doc-{int(time.time())}",
|
||||||
|
"text": text,
|
||||||
|
"indexing_technique": "high_quality",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(base_url=base, timeout=60, verify=False) as client:
|
||||||
|
r = await client.post(f"/v1/datasets/{ds_id}/document/create-by-text", headers=headers, json=payload)
|
||||||
|
if r.status_code in (200, 201):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
logger.warning(f"Dify aingest returned {r.status_code}: {r.text[:200]}")
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Dify aingest error: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_dify_kb_context(db, query: str, top_k: int = 5) -> str:
|
||||||
|
"""读取已启用的 Dify 外部服务,检索并返回拼接后的上下文文本(供 chat 注入)。"""
|
||||||
|
from sqlalchemy import text as _text
|
||||||
|
try:
|
||||||
|
result = db.execute(_text(
|
||||||
|
"SELECT api_url, api_key, config FROM external_kb_services "
|
||||||
|
"WHERE service_type='dify' AND is_enabled=1 AND status='connected' LIMIT 1"
|
||||||
|
))
|
||||||
|
row = result.fetchone()
|
||||||
|
if not row:
|
||||||
|
return ""
|
||||||
|
api_url, api_key, config_raw = row
|
||||||
|
dataset_name = None
|
||||||
|
if config_raw:
|
||||||
|
try:
|
||||||
|
cfg = config_raw if isinstance(config_raw, dict) else __import__("json").loads(config_raw)
|
||||||
|
dataset_name = cfg.get("dataset_name")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
|
||||||
|
if not hits:
|
||||||
|
return ""
|
||||||
|
parts = []
|
||||||
|
for i, h in enumerate(hits, 1):
|
||||||
|
content = (h.get("content") or "").strip()
|
||||||
|
if content:
|
||||||
|
parts.append(f"[{i}] {content}")
|
||||||
|
return "\n\n".join(parts)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"get_dify_kb_context error: {e}")
|
||||||
|
return ""
|
||||||
+25
-117
@@ -1,7 +1,7 @@
|
|||||||
"""
|
"""
|
||||||
知识库路由层
|
知识库路由层
|
||||||
============
|
============
|
||||||
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(FastGPT/Dify/RAGFlow)。
|
将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。
|
||||||
根据知识库配置自动选择最优检索策略。
|
根据知识库配置自动选择最优检索策略。
|
||||||
|
|
||||||
使用方式:
|
使用方式:
|
||||||
@@ -9,17 +9,14 @@
|
|||||||
results = await kb_router_search(query, kb_id, top_k=5)
|
results = await kb_router_search(query, kb_id, top_k=5)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
import httpx
|
|
||||||
from typing import List, Dict, Optional
|
from typing import List, Dict, Optional
|
||||||
|
|
||||||
logger = logging.getLogger("aiflow.kb_router")
|
logger = logging.getLogger("aiflow.kb_router")
|
||||||
|
|
||||||
|
|
||||||
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]:
|
||||||
"""获取已配置的外部KB服务"""
|
"""获取已配置的外部KB服务(仅 dify)"""
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
try:
|
try:
|
||||||
query = "SELECT * FROM external_kb_services"
|
query = "SELECT * FROM external_kb_services"
|
||||||
@@ -34,123 +31,34 @@ async def _get_external_services(db_session, enabled_only: bool = True) -> List[
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def _search_fastgpt(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||||
"""搜索 FastGPT 知识库"""
|
"""检索 Dify 知识库(走 app.dify_kb 客户端)"""
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=30) as client:
|
from app.dify_kb import dify_retrieve
|
||||||
resp = await client.post(
|
api_url = service.get("api_url", "")
|
||||||
f"{api_url.rstrip('/')}/api/v1/dataset/search",
|
api_key = service.get("api_key", "")
|
||||||
headers={
|
dataset_name = None
|
||||||
"Authorization": f"Bearer {api_key}",
|
cfg = service.get("config")
|
||||||
"Content-Type": "application/json",
|
if isinstance(cfg, str):
|
||||||
},
|
try:
|
||||||
json={
|
import json
|
||||||
"query": query,
|
cfg = json.loads(cfg)
|
||||||
"similarity": 0.5,
|
except Exception:
|
||||||
"limit": top_k,
|
cfg = None
|
||||||
},
|
if isinstance(cfg, dict):
|
||||||
)
|
dataset_name = cfg.get("dataset_name")
|
||||||
if resp.status_code == 200:
|
hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name)
|
||||||
data = resp.json()
|
return hits
|
||||||
results = []
|
|
||||||
for item in data.get("data", {}).get("list", []):
|
|
||||||
results.append({
|
|
||||||
"content": item.get("q", item.get("content", "")),
|
|
||||||
"score": float(item.get("score", 0.5)),
|
|
||||||
"source": f"FastGPT: {item.get('collectionId', '')}",
|
|
||||||
"metadata": item,
|
|
||||||
})
|
|
||||||
return results
|
|
||||||
else:
|
|
||||||
logger.warning(f"FastGPT search returned {resp.status_code}")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"FastGPT search error: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
async def _search_dify(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
|
||||||
"""搜索 Dify 知识库"""
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=30) as client:
|
|
||||||
resp = await client.post(
|
|
||||||
f"{api_url.rstrip('/')}/v1/datasets/retrieval",
|
|
||||||
headers={
|
|
||||||
"Authorization": f"Bearer {api_key}",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
json={
|
|
||||||
"query": query,
|
|
||||||
"retrieval_mode": "semantic",
|
|
||||||
"top_k": top_k,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if resp.status_code == 200:
|
|
||||||
data = resp.json()
|
|
||||||
results = []
|
|
||||||
for item in data.get("records", []):
|
|
||||||
segment = item.get("segment", {})
|
|
||||||
results.append({
|
|
||||||
"content": segment.get("content", ""),
|
|
||||||
"score": float(item.get("score", 0.5)),
|
|
||||||
"source": f"Dify: {segment.get('id', '')}",
|
|
||||||
"metadata": item,
|
|
||||||
})
|
|
||||||
return results
|
|
||||||
else:
|
|
||||||
logger.warning(f"Dify search returned {resp.status_code}")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Dify search error: {e}")
|
logger.error(f"Dify search error: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
async def _search_ragflow(api_url: str, api_key: str, query: str, top_k: int = 5) -> List[Dict]:
|
|
||||||
"""搜索 RAGFlow 知识库"""
|
|
||||||
try:
|
|
||||||
async with httpx.AsyncClient(timeout=30) as client:
|
|
||||||
resp = await client.post(
|
|
||||||
f"{api_url.rstrip('/')}/api/v1/retrieval",
|
|
||||||
headers={
|
|
||||||
"Authorization": f"Bearer {api_key}",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
json={
|
|
||||||
"question": query,
|
|
||||||
"top_k": top_k,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if resp.status_code == 200:
|
|
||||||
data = resp.json()
|
|
||||||
results = []
|
|
||||||
for item in data.get("data", {}).get("chunks", []):
|
|
||||||
results.append({
|
|
||||||
"content": item.get("content", ""),
|
|
||||||
"score": float(item.get("similarity", 0.5)),
|
|
||||||
"source": f"RAGFlow: {item.get('document_id', '')}",
|
|
||||||
"metadata": item,
|
|
||||||
})
|
|
||||||
return results
|
|
||||||
else:
|
|
||||||
logger.warning(f"RAGFlow search returned {resp.status_code}")
|
|
||||||
return []
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"RAGFlow search error: {e}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]:
|
||||||
"""路由到对应的外部KB服务"""
|
"""路由到对应的外部KB服务(当前仅支持 dify)"""
|
||||||
service_type = service.get("service_type", "")
|
service_type = service.get("service_type", "")
|
||||||
api_url = service.get("api_url", "")
|
if service_type == "dify":
|
||||||
api_key = service.get("api_key", "")
|
return await _search_dify(service, query, top_k)
|
||||||
|
|
||||||
if service_type == "fastgpt":
|
|
||||||
return await _search_fastgpt(api_url, api_key, query, top_k)
|
|
||||||
elif service_type == "dify":
|
|
||||||
return await _search_dify(api_url, api_key, query, top_k)
|
|
||||||
elif service_type == "ragflow":
|
|
||||||
return await _search_ragflow(api_url, api_key, query, top_k)
|
|
||||||
else:
|
else:
|
||||||
logger.warning(f"Unknown external KB type: {service_type}")
|
logger.warning(f"Unknown external KB type: {service_type}")
|
||||||
return []
|
return []
|
||||||
@@ -171,7 +79,7 @@ async def kb_router_search(
|
|||||||
检索策略:
|
检索策略:
|
||||||
1. 本地 BM25 检索 (MySQL, 已有)
|
1. 本地 BM25 检索 (MySQL, 已有)
|
||||||
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
2. 向量语义检索 (PostgreSQL+pgvector, 新增)
|
||||||
3. 外部KB检索 (FastGPT/Dify/RAGFlow, 按配置)
|
3. 外部KB检索 (Dify, 按配置)
|
||||||
|
|
||||||
融合方式: Reciprocal Rank Fusion (RRF)
|
融合方式: Reciprocal Rank Fusion (RRF)
|
||||||
|
|
||||||
@@ -218,7 +126,7 @@ async def kb_router_search(
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Vector search failed: {e}")
|
logger.warning(f"Vector search failed: {e}")
|
||||||
|
|
||||||
# 3. 外部KB检索
|
# 3. 外部KB检索 (Dify)
|
||||||
if use_external and db_session:
|
if use_external and db_session:
|
||||||
services = await _get_external_services(db_session)
|
services = await _get_external_services(db_session)
|
||||||
for service in services:
|
for service in services:
|
||||||
@@ -228,7 +136,7 @@ async def kb_router_search(
|
|||||||
all_results.append({
|
all_results.append({
|
||||||
"content": item.get("content", ""),
|
"content": item.get("content", ""),
|
||||||
"score": item.get("score", 0.5),
|
"score": item.get("score", 0.5),
|
||||||
"source": item.get("source", f"external_{service['service_type']}"),
|
"source": item.get("source", "external_dify"),
|
||||||
"rank": rank + 1,
|
"rank": rank + 1,
|
||||||
"metadata": item.get("metadata", {}),
|
"metadata": item.get("metadata", {}),
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -66,7 +66,10 @@ const Knowledge: React.FC = () => {
|
|||||||
// External Services
|
// External Services
|
||||||
const [extServices, setExtServices] = useState<ExternalService[]>([]);
|
const [extServices, setExtServices] = useState<ExternalService[]>([]);
|
||||||
const [showExtModal, setShowExtModal] = useState(false);
|
const [showExtModal, setShowExtModal] = useState(false);
|
||||||
const [extForm, setExtForm] = useState({ name: '', service_type: 'fastgpt', api_url: '', api_key: '' });
|
const [extForm, setExtForm] = useState({ name: 'Dify 知识库', service_type: 'dify', api_url: 'https://dify.qihuyun.com/v1', api_key: '', dataset_name: 'qihuai' });
|
||||||
|
const [extMsg, setExtMsg] = useState('');
|
||||||
|
const [ingesting, setIngesting] = useState(false);
|
||||||
|
const [ingestResult, setIngestResult] = useState<any>(null);
|
||||||
|
|
||||||
// KB Modal
|
// KB Modal
|
||||||
const [showModal, setShowModal] = useState(false);
|
const [showModal, setShowModal] = useState(false);
|
||||||
@@ -210,7 +213,19 @@ const Knowledge: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const handleCreateExtService = async () => {
|
const handleCreateExtService = async () => {
|
||||||
try { const res = await adminAPI.createAdminKnowledgeExtService(extForm); if (res.code === 0) { setShowExtModal(false); setExtForm({ name: '', service_type: 'fastgpt', api_url: '', api_key: '' }); fetchExtServices(); } } catch (e) {}
|
try {
|
||||||
|
const payload = {
|
||||||
|
...extForm,
|
||||||
|
config: { dataset_name: extForm.dataset_name || 'qihuai' },
|
||||||
|
};
|
||||||
|
const res = await adminAPI.createAdminKnowledgeExtService(payload);
|
||||||
|
if (res.code === 0) {
|
||||||
|
setShowExtModal(false);
|
||||||
|
setExtForm({ name: 'Dify 知识库', service_type: 'dify', api_url: 'https://dify.qihuyun.com/v1', api_key: '', dataset_name: 'qihuai' });
|
||||||
|
setExtMsg('');
|
||||||
|
fetchExtServices();
|
||||||
|
}
|
||||||
|
} catch (e: any) { setExtMsg(`添加失败: ${e?.message || '未知错误'}`); }
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDeleteExtService = async (id: string) => {
|
const handleDeleteExtService = async (id: string) => {
|
||||||
@@ -219,7 +234,38 @@ const Knowledge: React.FC = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const handleCheckExtStatus = async (id: string) => {
|
const handleCheckExtStatus = async (id: string) => {
|
||||||
try { const res = await adminAPI.checkAdminKnowledgeExtStatus(id); if (res.code === 0) { alert(`连接状态: ${res.data.connected ? '已连接' : '未连接'}\n详情: ${res.data.detail || '无'}`); fetchExtServices(); } } catch (e) {}
|
try {
|
||||||
|
const res = await adminAPI.checkAdminKnowledgeExtStatus(id);
|
||||||
|
if (res.code === 0) {
|
||||||
|
const d = res.data || {};
|
||||||
|
const statusText = d.connected ? (d.usable ? '可通讯且可用 ✅' : '可通讯但不可用 ⚠️') : '不可通讯 ❌';
|
||||||
|
alert(`连接状态: ${statusText}\n\n详情: ${d.detail || '无'}`);
|
||||||
|
fetchExtServices();
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleIngestPlatform = async () => {
|
||||||
|
if (!confirm('将把平台「AI聊天回答 + 用户上传文档」中有价值的内容批量入库到 Dify 知识库(量化后可被 AI 聊天检索复用)。确认执行?')) return;
|
||||||
|
setIngesting(true); setIngestResult(null);
|
||||||
|
try {
|
||||||
|
const token = localStorage.getItem('aiflow_token');
|
||||||
|
const res = await fetch(`/app/api/admin/knowledge/external/dify/ingest`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Authorization': `Bearer ${token}` },
|
||||||
|
});
|
||||||
|
const data = await res.json();
|
||||||
|
if (data.code === 0) {
|
||||||
|
setIngestResult(data.data);
|
||||||
|
setExtMsg(`✅ ${data.message}`);
|
||||||
|
} else {
|
||||||
|
setExtMsg(`❌ ${data.message || '入库失败'}`);
|
||||||
|
}
|
||||||
|
} catch (e: any) {
|
||||||
|
setExtMsg(`❌ 入库失败: ${e?.message || '网络错误'}`);
|
||||||
|
} finally {
|
||||||
|
setIngesting(false);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const statusBadge = (status: string) => {
|
const statusBadge = (status: string) => {
|
||||||
@@ -487,7 +533,7 @@ const Knowledge: React.FC = () => {
|
|||||||
{extServices.map(svc => (
|
{extServices.map(svc => (
|
||||||
<tr key={svc.id}>
|
<tr key={svc.id}>
|
||||||
<td>{svc.name}</td>
|
<td>{svc.name}</td>
|
||||||
<td><span style={{ fontSize: 12, padding: '2px 6px', background: '#F3F4FF', color: '#8B5CF6', borderRadius: 3 }}>{svc.service_type === 'fastgpt' ? 'FastGPT' : svc.service_type === 'dify' ? 'Dify' : 'RAGFlow'}</span></td>
|
<td><span style={{ fontSize: 12, padding: '2px 6px', background: '#F3F4FF', color: '#8B5CF6', borderRadius: 3 }}>Dify</span></td>
|
||||||
<td style={{ fontSize: 11, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis' }}>{svc.api_url}</td>
|
<td style={{ fontSize: 11, maxWidth: 200, overflow: 'hidden', textOverflow: 'ellipsis' }}>{svc.api_url}</td>
|
||||||
<td>{statusBadge(svc.status)}</td>
|
<td>{statusBadge(svc.status)}</td>
|
||||||
<td>
|
<td>
|
||||||
@@ -501,17 +547,36 @@ const Knowledge: React.FC = () => {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<div style={{ borderTop: '1px solid #E5E7EB', paddingTop: 16 }}>
|
<div style={{ borderTop: '1px solid #E5E7EB', paddingTop: 16 }}>
|
||||||
<h4 style={{ margin: '0 0 12px 0', fontSize: 14 }}>添加新服务</h4>
|
<h4 style={{ margin: '0 0 12px 0', fontSize: 14 }}>配置 Dify 知识库</h4>
|
||||||
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12 }}>
|
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: 12 }}>
|
||||||
<div><label style={labelStyle}>名称</label><input value={extForm.name} onChange={e => setExtForm({ ...extForm, name: e.target.value })} placeholder="如: 生产环境 FastGPT" style={fullInput} /></div>
|
<div><label style={labelStyle}>名称</label><input value={extForm.name} onChange={e => setExtForm({ ...extForm, name: e.target.value })} placeholder="如: 生产环境 Dify" style={fullInput} /></div>
|
||||||
<div><label style={labelStyle}>类型</label><select value={extForm.service_type} onChange={e => setExtForm({ ...extForm, service_type: e.target.value })} style={fullInput}><option value="fastgpt">FastGPT</option><option value="dify">Dify</option><option value="ragflow">RAGFlow</option></select></div>
|
<div><label style={labelStyle}>知识库名称(Dataset)</label><input value={extForm.dataset_name} onChange={e => setExtForm({ ...extForm, dataset_name: e.target.value })} placeholder="如: qihuai" style={fullInput} /></div>
|
||||||
</div>
|
</div>
|
||||||
<div style={{ marginTop: 12 }}><label style={labelStyle}>API URL</label><input value={extForm.api_url} onChange={e => setExtForm({ ...extForm, api_url: e.target.value })} placeholder="https://your-instance.com/api/v1" style={fullInput} /></div>
|
<div style={{ marginTop: 12 }}><label style={labelStyle}>API URL</label><input value={extForm.api_url} onChange={e => setExtForm({ ...extForm, api_url: e.target.value })} placeholder="https://dify.qihuyun.com/v1" style={fullInput} /></div>
|
||||||
<div style={{ marginTop: 12 }}><label style={labelStyle}>API Key</label><input type="password" value={extForm.api_key} onChange={e => setExtForm({ ...extForm, api_key: e.target.value })} placeholder="Bearer Token 或 API Key" style={fullInput} /></div>
|
<div style={{ marginTop: 12 }}><label style={labelStyle}>API Key(dataset 密钥)</label><input type="password" value={extForm.api_key} onChange={e => setExtForm({ ...extForm, api_key: e.target.value })} placeholder="dataset-xxxx" style={fullInput} /></div>
|
||||||
<div style={{ display: 'flex', gap: 12, marginTop: 16 }}>
|
|
||||||
<button onClick={handleCreateExtService} style={btnStyle('#8B5CF6')}>添加服务</button>
|
{extMsg && (
|
||||||
|
<div style={{ marginTop: 12, fontSize: 13, color: extMsg.startsWith('✅') ? '#16A34A' : '#DC2626', padding: '6px 12px', background: extMsg.startsWith('✅') ? '#DCFCE7' : '#FEE2E2', borderRadius: 6 }}>
|
||||||
|
{extMsg}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div style={{ display: 'flex', gap: 12, marginTop: 16, flexWrap: 'wrap' }}>
|
||||||
|
<button onClick={handleCreateExtService} style={btnStyle('#8B5CF6')}>保存并测试连接</button>
|
||||||
|
<button onClick={handleIngestPlatform} disabled={ingesting} style={btnStyle(ingesting ? '#D1D5DB' : '#219B5E')}>
|
||||||
|
{ingesting ? '入库中...' : '入库平台内容'}
|
||||||
|
</button>
|
||||||
<button onClick={() => setShowExtModal(false)} style={btnStyle('#6B7280')}>关闭</button>
|
<button onClick={() => setShowExtModal(false)} style={btnStyle('#6B7280')}>关闭</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{ingestResult && (
|
||||||
|
<div style={{ marginTop: 12, fontSize: 13, padding: 12, background: '#F0FDF4', border: '1px solid #BBF7D0', borderRadius: 8 }}>
|
||||||
|
<div style={{ fontWeight: 600, marginBottom: 6 }}>入库结果</div>
|
||||||
|
<div>AI 聊天回答:<b>{ingestResult.chat_answers}</b> 条</div>
|
||||||
|
<div>用户上传文档:<b>{ingestResult.user_docs}</b> 条</div>
|
||||||
|
<div>跳过(过短/空):{ingestResult.skipped} 条 · 失败:{ingestResult.failed} 条</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in new issue
Block a user