diff --git a/backend/python/app/anysearch_client.py b/backend/python/app/anysearch_client.py new file mode 100644 index 0000000..294d262 --- /dev/null +++ b/backend/python/app/anysearch_client.py @@ -0,0 +1,118 @@ +""" +AnySearch 联网检索客户端 +====================== +调用 AnySearch 网页/领域搜索 API (https://api.anysearch.com/v1/search)， +为 AI 聊天注入实时联网检索结果，亦可被后端端点 / Agent 工具复用。 + +约束：平台容器存在智能路由代理，对外部 API 偶发重置；本客户端使用 +trust_env=False 直连 (等价于 NO_PROXY=api.anysearch.com)，规避代理重置。 +失败一律返回空/降级结果，保证聊天链路不中断。 +""" +import logging +from typing import List, Dict, Optional + +logger = logging.getLogger("aiflow.anysearch") + +DEFAULT_API_URL = "https://api.anysearch.com/v1/search" +DEFAULT_ZONE = "cn" +REQUEST_TIMEOUT = 8.0 + + +async def anysearch_search( + api_key: str, + api_url: str = DEFAULT_API_URL, + query: str = "", + top_k: int = 5, + zone: str = DEFAULT_ZONE, + freshness: Optional[str] = None, + timeout: float = REQUEST_TIMEOUT, +) -> List[Dict]: + """调用 AnySearch 搜索，返回结构化结果列表。 + + 每个结果: + { + "content": snippet 摘要文本, + "score": 0.9..0.5 递减, + "source": "web_anysearch", + "metadata": {"url", "title", "source_domain", "zone"} + } + 失败时返回 [] (不抛异常)。 + """ + if not api_key: + return [] + try: + import httpx + except ImportError: + logger.warning("httpx not available for AnySearch") + return [] + + payload = { + "query": query, + "zone": zone, + "top_k": max(1, min(int(top_k), 20)), + } + if freshness: + payload["freshness"] = freshness + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + } + try: + # trust_env=False -> 忽略容器内 HTTP(S)_PROXY，直连 api.anysearch.com + async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client: + resp = await client.post(api_url, json=payload, headers=headers) + if resp.status_code != 200: + logger.warning(f"AnySearch HTTP {resp.status_code}: {resp.text[:200]}") + return [] + data = resp.json() + if data.get("code") != 0: + logger.warning(f"AnySearch biz error: {data.get('message')}") + return [] + results = (data.get("data") or {}).get("results") or [] + hits = [] + for idx, r in enumerate(results[:top_k]): + title = r.get("title", "") or "" + url = r.get("url", "") or "" + snippet = r.get("snippet") or r.get("content") or "" + if not snippet and not title: + continue + try: + from urllib.parse import urlparse + domain = urlparse(url).netloc + except Exception: + domain = "" + hits.append({ + "content": snippet, + "score": max(0.4, 0.9 - idx * 0.08), + "source": "web_anysearch", + "metadata": { + "url": url, + "title": title, + "source_domain": domain, + "zone": zone, + }, + }) + return hits + except Exception as e: + logger.warning(f"AnySearch search failed: {e}") + return [] + + +async def anysearch_health(api_key: str, api_url: str = DEFAULT_API_URL, timeout: float = REQUEST_TIMEOUT) -> Dict: + """连通性探测：返回 {"status": "connected"/"disconnected", "detail": ...}。""" + if not api_key: + return {"status": "disconnected", "detail": "未配置 API Key", "connected": False, "usable": False} + try: + import httpx + async with httpx.AsyncClient(trust_env=False, timeout=timeout) as client: + resp = await client.post( + api_url, + json={"query": "ping", "zone": DEFAULT_ZONE, "top_k": 1}, + headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}, + ) + if resp.status_code == 200: + return {"status": "connected", "detail": "AnySearch 连接正常", "connected": True, "usable": True} + return {"status": "error", "detail": f"HTTP {resp.status_code}", "connected": False, "usable": False} + except Exception as e: + return {"status": "error", "detail": str(e)[:200], "connected": False, "usable": False} diff --git a/backend/python/app/api/admin/knowledge_admin.py b/backend/python/app/api/admin/knowledge_admin.py index 79580cf..99f3d9d 100644 --- a/backend/python/app/api/admin/knowledge_admin.py +++ b/backend/python/app/api/admin/knowledge_admin.py @@ -1,1274 +1,1278 @@ -""" -知识库管理 API（管理员） -Phase 2.4: AI PaaS Knowledge Base Management + External KB Integration -""" -from fastapi import APIRouter, Depends, HTTPException, Query, Body -from sqlalchemy import or_, func, desc, asc, text -from sqlalchemy.orm import Session -from pydantic import BaseModel, Field -from typing import Optional, List, Dict, Any -from datetime import datetime -import uuid -import json -import logging -import httpx - -from ...database import get_db -from ...models import ( - KnowledgeBase, KnowledgeDocument, KnowledgeDirectory, - KnowledgeGraphNode, KnowledgeGraphEdge, KnowledgeGraphBuildLog, - User, gen_uuid -) -from ...auth import get_current_admin -from ...middleware.log_service import LogService - -router = APIRouter() -logger = logging.getLogger("aiflow") - - -# ============================================================ -# Pydantic 模型 -# ============================================================ - -class ExternalServiceConfig(BaseModel): - name: str = Field(min_length=1, max_length=200) - service_type: str = Field(..., description="dify") - api_url: str = Field(min_length=1, max_length=500) - api_key: Optional[str] = None - config: Optional[Dict[str, Any]] = None - - -class ExternalServiceUpdate(BaseModel): - name: Optional[str] = None - api_url: Optional[str] = None - api_key: Optional[str] = None - config: Optional[Dict[str, Any]] = None - is_enabled: Optional[bool] = None - - -class KBCreate(BaseModel): - name: str = Field(min_length=1, max_length=200) - description: Optional[str] = None - is_official: bool = False - external_service_id: Optional[str] = None - external_config: Optional[Dict[str, Any]] = None - - -class KBUpdate(BaseModel): - name: Optional[str] = None - description: Optional[str] = None - is_official: Optional[bool] = None - status: Optional[str] = None # active / inactive / archived - external_service_id: Optional[str] = None - external_config: Optional[Dict[str, Any]] = None - - -class SyncRequest(BaseModel): - external_service_id: Optional[str] = None # override KB's linked service - sync_direction: str = "pull" # pull / push / bidirectional - options: Optional[Dict[str, Any]] = None - - -# ============================================================ -# 辅助函数 -# ============================================================ - -def kb_to_dict(kb: KnowledgeBase, db: Session = None) -> dict: - """将 KnowledgeBase 转换为字典""" - result = { - "id": kb.id, - "user_id": kb.user_id, - "name": kb.name, - "description": kb.description, - "directory_path": kb.directory_path, - "document_count": kb.document_count or 0, - "knowledge_graph_count": kb.knowledge_graph_count or 0, - "created_at": kb.created_at.isoformat() if kb.created_at else None, - "updated_at": kb.updated_at.isoformat() if kb.updated_at else None, - } - - # 新增字段（Phase 2.4） - # 通过 getattr 安全获取，因为旧记录可能没有这些字段 - result["is_official"] = getattr(kb, "is_official", False) - result["status"] = getattr(kb, "status", "active") - result["external_service_id"] = getattr(kb, "external_service_id", None) - result["external_id"] = getattr(kb, "external_id", None) - result["external_sync_status"] = getattr(kb, "external_sync_status", "none") - result["external_config"] = None - ext_config_raw = getattr(kb, "external_config", None) - if ext_config_raw: - try: - result["external_config"] = json.loads(ext_config_raw) if isinstance(ext_config_raw, str) else ext_config_raw - except Exception: - result["external_config"] = {} - - # 查询关联用户名 - if db and kb.user_id: - user = db.query(User).filter(User.id == kb.user_id).first() - result["owner_name"] = user.username if user else "未知" - - # 计算存储占用 - if db: - total_size = db.query(func.sum(KnowledgeDocument.file_size)).filter( - KnowledgeDocument.knowledge_base_id == kb.id - ).scalar() or 0 - result["total_storage_bytes"] = int(total_size) - result["total_storage_mb"] = round(int(total_size) / (1024 * 1024), 2) - - return result - - -def doc_to_dict(doc: KnowledgeDocument) -> dict: - """将 KnowledgeDocument 转换为字典""" - return { - "id": doc.id, - "knowledge_base_id": doc.knowledge_base_id, - "directory_id": doc.directory_id, - "filename": doc.filename, - "file_type": doc.file_type, - "file_size": doc.file_size or 0, - "file_path": doc.file_path, - "status": doc.status, - "chunk_count": doc.chunk_count or 0, - "error_message": doc.error_message, - "created_at": doc.created_at.isoformat() if doc.created_at else None, - "updated_at": doc.updated_at.isoformat() if doc.updated_at else None, - } - - -def format_file_size(size_bytes: int) -> str: - """格式化文件大小""" - if size_bytes < 1024: - return f"{size_bytes} B" - elif size_bytes < 1024 * 1024: - return f"{size_bytes / 1024:.1f} KB" - elif size_bytes < 1024 * 1024 * 1024: - return f"{size_bytes / (1024 * 1024):.1f} MB" - else: - return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB" - - -# ============================================================ -# 统计概览 -# ============================================================ - -@router.get("/stats/overview", summary="知识库统计概览") -async def get_kb_stats( - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取知识库全局统计数据""" - # 基础统计 - total_kb = db.query(func.count(KnowledgeBase.id)).scalar() or 0 - total_docs = db.query(func.count(KnowledgeDocument.id)).scalar() or 0 - total_storage = db.query(func.sum(KnowledgeDocument.file_size)).scalar() or 0 - total_nodes = db.query(func.count(KnowledgeGraphNode.id)).scalar() or 0 - total_edges = db.query(func.count(KnowledgeGraphEdge.id)).scalar() or 0 - - # 官方 vs 用户知识库 - official_kb = 0 - user_kb = 0 - try: - official_kb = db.query(func.count(KnowledgeBase.id)).filter( - getattr(KnowledgeBase, "is_official", False) == True - ).scalar() or 0 - except Exception: - official_kb = 0 - user_kb = total_kb - official_kb - - # 文档状态分布 - doc_status_dist = {} - statuses = ["uploading", "processing", "completed", "failed"] - for s in statuses: - cnt = db.query(func.count(KnowledgeDocument.id)).filter( - KnowledgeDocument.status == s - ).scalar() or 0 - doc_status_dist[s] = cnt - - # 文件类型分布 - file_type_dist = {} - type_rows = db.query( - KnowledgeDocument.file_type, - func.count(KnowledgeDocument.id) - ).group_by(KnowledgeDocument.file_type).all() - for ft, cnt in type_rows: - file_type_dist[ft] = cnt - - # 最近7天创建的知识库数 - from datetime import timedelta - week_ago = datetime.now() - timedelta(days=7) - recent_kb = db.query(func.count(KnowledgeBase.id)).filter( - KnowledgeBase.created_at >= week_ago - ).scalar() or 0 - - # 活跃知识库（有文档的知识库） - active_kb = db.query(func.count(func.distinct(KnowledgeDocument.knowledge_base_id))).scalar() or 0 - - # 知识图谱统计 - kb_with_graph = db.query(func.count(func.distinct(KnowledgeGraphNode.knowledge_base_id))).scalar() or 0 - - # 外部集成统计 - external_kb = 0 - try: - external_kb = db.query(func.count(KnowledgeBase.id)).filter( - getattr(KnowledgeBase, "external_service_id", None) != None, - getattr(KnowledgeBase, "external_service_id", None) != "" - ).scalar() or 0 - except Exception: - external_kb = 0 - - return { - "code": 0, - "data": { - "total_knowledge_bases": total_kb, - "total_documents": total_docs, - "total_storage_bytes": int(total_storage), - "total_storage_display": format_file_size(int(total_storage)), - "official_kb": official_kb, - "user_kb": user_kb, - "active_kb": active_kb, - "kb_with_graph": kb_with_graph, - "external_kb": external_kb, - "recent_kb_7d": recent_kb, - "total_graph_nodes": int(total_nodes), - "total_graph_edges": int(total_edges), - "doc_status_distribution": doc_status_dist, - "file_type_distribution": file_type_dist, - } - } - - -# ============================================================ -# 知识库 CRUD -# ============================================================ - -@router.get("/", summary="获取知识库列表") -async def list_knowledge_bases( - keyword: str = Query(None, description="搜索关键词"), - status: str = Query(None, description="状态: active/inactive/archived"), - is_official: bool = Query(None, description="是否官方知识库"), - has_external: bool = Query(None, description="是否有外部集成"), - page: int = Query(1, ge=1), - page_size: int = Query(15, ge=1, le=100), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取所有知识库列表（分页、筛选）""" - query = db.query(KnowledgeBase) - - if keyword: - query = query.filter( - or_( - KnowledgeBase.name.ilike(f"%{keyword}%"), - KnowledgeBase.description.ilike(f"%{keyword}%"), - ) - ) - if status: - try: - query = query.filter(getattr(KnowledgeBase, "status", None) == status) - except Exception: - pass # 字段可能不存在 - if is_official is not None: - try: - query = query.filter(getattr(KnowledgeBase, "is_official", None) == is_official) - except Exception: - pass - - total = query.count() - kbs = query.order_by(desc(KnowledgeBase.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() - - return { - "code": 0, - "data": { - "total": total, - "page": page, - "page_size": page_size, - "items": [kb_to_dict(kb, db) for kb in kbs], - } - } - - -@router.get("/{kb_id}", summary="获取知识库详情") -async def get_knowledge_base( - kb_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取知识库详细信息，包含文档和图谱数据""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - result = kb_to_dict(kb, db) - - # 文档统计 - doc_stats = db.query( - KnowledgeDocument.status, - func.count(KnowledgeDocument.id), - func.sum(KnowledgeDocument.file_size) - ).filter(KnowledgeDocument.knowledge_base_id == kb_id).group_by( - KnowledgeDocument.status - ).all() - - result["doc_stats"] = { - s: {"count": int(cnt), "size": int(sz or 0)} - for s, cnt, sz in doc_stats - } - - # 图谱统计 - graph_nodes = db.query(func.count(KnowledgeGraphNode.id)).filter( - KnowledgeGraphNode.knowledge_base_id == kb_id - ).scalar() or 0 - graph_edges = db.query(func.count(KnowledgeGraphEdge.id)).filter( - KnowledgeGraphEdge.knowledge_base_id == kb_id - ).scalar() or 0 - result["graph_stats"] = { - "nodes": int(graph_nodes), - "edges": int(graph_edges), - } - - # 外部服务信息 - ext_service_id = getattr(kb, "external_service_id", None) - if ext_service_id: - # 查询外部服务配置 - try: - ext_service = db.query(ExternalKBService).filter( - ExternalKBService.id == ext_service_id - ).first() - if ext_service: - result["external_service"] = { - "id": ext_service.id, - "name": ext_service.name, - "service_type": ext_service.service_type, - "api_url": ext_service.api_url, - "status": getattr(ext_service, "status", "unknown"), - } - except Exception: - result["external_service"] = {"id": ext_service_id, "note": "外部服务表尚未初始化"} - - return {"code": 0, "data": result} - - -@router.post("/", summary="创建知识库") -async def create_knowledge_base( - req: KBCreate, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """创建新的知识库（支持官方平台知识库）""" - # 检查名称唯一性 - existing = db.query(KnowledgeBase).filter(KnowledgeBase.name == req.name).first() - if existing: - raise HTTPException(status_code=400, detail="知识库名称已存在") - - kb = KnowledgeBase( - id=str(uuid.uuid4()), - user_id=admin_user.id, - name=req.name, - description=req.description or "", - directory_path="/", - document_count=0, - knowledge_graph_count=0, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - - # Phase 2.4 新增字段 - try: - setattr(kb, "is_official", req.is_official) - setattr(kb, "status", "active") - if req.external_service_id: - setattr(kb, "external_service_id", req.external_service_id) - if req.external_config: - setattr(kb, "external_config", json.dumps(req.external_config, ensure_ascii=False)) - setattr(kb, "external_sync_status", "none") - except Exception: - logger.warning("KnowledgeBase 新增字段设置失败，可能表结构尚未更新") - - db.add(kb) - db.commit() - db.refresh(kb) - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="create_knowledge_base", - target_type="knowledge_base", - target_id=kb.id, - detail=f"创建知识库: {kb.name} (官方={req.is_official})", - ) - - return {"code": 0, "message": "知识库创建成功", "data": kb_to_dict(kb, db)} - - -@router.put("/{kb_id}", summary="更新知识库") -async def update_knowledge_base( - kb_id: str, - req: KBUpdate, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """更新知识库配置""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - if req.name is not None: - dup = db.query(KnowledgeBase).filter( - KnowledgeBase.name == req.name, - KnowledgeBase.id != kb_id - ).first() - if dup: - raise HTTPException(status_code=400, detail="知识库名称已存在") - kb.name = req.name - - if req.description is not None: - kb.description = req.description - - # Phase 2.4 新增字段更新 - try: - if req.is_official is not None: - setattr(kb, "is_official", req.is_official) - if req.status is not None: - setattr(kb, "status", req.status) - if req.external_service_id is not None: - setattr(kb, "external_service_id", req.external_service_id) - if req.external_config is not None: - setattr(kb, "external_config", json.dumps(req.external_config, ensure_ascii=False)) - except Exception: - logger.warning("KnowledgeBase 新增字段更新失败") - - kb.updated_at = datetime.now() - db.commit() - db.refresh(kb) - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="update_knowledge_base", - target_type="knowledge_base", - target_id=kb_id, - detail=f"更新知识库: {kb.name}", - ) - - return {"code": 0, "message": "知识库已更新", "data": kb_to_dict(kb, db)} - - -@router.delete("/{kb_id}", summary="删除知识库") -async def delete_knowledge_base( - kb_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """删除知识库及其所有文档""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - kb_name = kb.name - - # 删除关联文档 - docs = db.query(KnowledgeDocument).filter( - KnowledgeDocument.knowledge_base_id == kb_id - ).all() - for d in docs: - db.delete(d) - - # 删除关联图谱 - nodes = db.query(KnowledgeGraphNode).filter( - KnowledgeGraphNode.knowledge_base_id == kb_id - ).all() - for n in nodes: - db.delete(n) - - edges = db.query(KnowledgeGraphEdge).filter( - KnowledgeGraphEdge.knowledge_base_id == kb_id - ).all() - for e in edges: - db.delete(e) - - build_logs = db.query(KnowledgeGraphBuildLog).filter( - KnowledgeGraphBuildLog.knowledge_base_id == kb_id - ).all() - for bl in build_logs: - db.delete(bl) - - # 删除关联目录 - dirs = db.query(KnowledgeDirectory).filter( - KnowledgeDirectory.knowledge_base_id == kb_id - ).all() - for d in dirs: - db.delete(d) - - # 删除知识库本身 - db.delete(kb) - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="delete_knowledge_base", - target_type="knowledge_base", - target_id=kb_id, - detail=f"删除知识库: {kb_name} (含 {len(docs)} 文档, {len(nodes)} 节点, {len(edges)} 边)", - ) - - return {"code": 0, "message": "知识库已删除"} - - -@router.put("/{kb_id}/toggle", summary="启用/禁用知识库") -async def toggle_knowledge_base( - kb_id: str, - is_active: bool = Query(..., description="是否启用"), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """启用或禁用知识库""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - new_status = "active" if is_active else "inactive" - try: - setattr(kb, "status", new_status) - except Exception: - pass - - kb.updated_at = datetime.now() - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="toggle_knowledge_base", - target_type="knowledge_base", - target_id=kb_id, - detail=f"{'启用' if is_active else '禁用'}知识库: {kb.name}", - ) - - return { - "code": 0, - "message": f"知识库已{'启用' if is_active else '禁用'}", - "data": kb_to_dict(kb, db), - } - - -# ============================================================ -# 文档管理 -# ============================================================ - -@router.get("/{kb_id}/documents", summary="获取知识库文档列表") -async def get_kb_documents( - kb_id: str, - status: str = Query(None, description="文档状态筛选"), - file_type: str = Query(None, description="文件类型筛选"), - keyword: str = Query(None, description="文件名搜索"), - page: int = Query(1, ge=1), - page_size: int = Query(20, ge=1, le=100), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取知识库中的文档列表""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - query = db.query(KnowledgeDocument).filter( - KnowledgeDocument.knowledge_base_id == kb_id - ) - - if status: - query = query.filter(KnowledgeDocument.status == status) - if file_type: - query = query.filter(KnowledgeDocument.file_type == file_type) - if keyword: - query = query.filter(KnowledgeDocument.filename.ilike(f"%{keyword}%")) - - total = query.count() - docs = query.order_by(desc(KnowledgeDocument.created_at)).offset( - (page - 1) * page_size - ).limit(page_size).all() - - return { - "code": 0, - "data": { - "total": total, - "page": page, - "page_size": page_size, - "items": [doc_to_dict(d) for d in docs], - } - } - - -@router.delete("/{kb_id}/documents/{doc_id}", summary="删除知识库文档") -async def delete_kb_document( - kb_id: str, - doc_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """删除知识库中的单个文档""" - doc = db.query(KnowledgeDocument).filter( - KnowledgeDocument.id == doc_id, - KnowledgeDocument.knowledge_base_id == kb_id, - ).first() - if not doc: - raise HTTPException(status_code=404, detail="文档不存在") - - filename = doc.filename - db.delete(doc) - - # 更新知识库文档计数 - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if kb: - kb.document_count = db.query(func.count(KnowledgeDocument.id)).filter( - KnowledgeDocument.knowledge_base_id == kb_id - ).scalar() or 0 - kb.updated_at = datetime.now() - - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="delete_knowledge_document", - target_type="knowledge_document", - target_id=doc_id, - detail=f"删除文档: {filename} (知识库: {kb.name})", - ) - - return {"code": 0, "message": "文档已删除"} - - -# ============================================================ -# 知识图谱管理 -# ============================================================ - -@router.get("/{kb_id}/graph/stats", summary="获取知识图谱统计") -async def get_kb_graph_stats( - kb_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取知识库的知识图谱统计信息""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - nodes = db.query(func.count(KnowledgeGraphNode.id)).filter( - KnowledgeGraphNode.knowledge_base_id == kb_id - ).scalar() or 0 - - edges = db.query(func.count(KnowledgeGraphEdge.id)).filter( - KnowledgeGraphEdge.knowledge_base_id == kb_id - ).scalar() or 0 - - # 最近构建日志 - recent_logs = db.query(KnowledgeGraphBuildLog).filter( - KnowledgeGraphBuildLog.knowledge_base_id == kb_id - ).order_by(desc(KnowledgeGraphBuildLog.created_at)).limit(5).all() - - # 节点类型分布 - node_types = db.query( - KnowledgeGraphNode.label, - func.count(KnowledgeGraphNode.id) - ).filter( - KnowledgeGraphNode.knowledge_base_id == kb_id - ).group_by(KnowledgeGraphNode.label).limit(20).all() - - return { - "code": 0, - "data": { - "total_nodes": int(nodes), - "total_edges": int(edges), - "node_type_distribution": [ - {"label": lbl, "count": int(cnt)} for lbl, cnt in node_types - ], - "recent_build_logs": [ - { - "id": bl.id, - "status": getattr(bl, "status", "unknown"), - "created_at": bl.created_at.isoformat() if bl.created_at else None, - } - for bl in recent_logs - ], - } - } - - -# ============================================================ -# 外部知识库服务管理 -# ============================================================ - -# ExternalKBService 模型定义（如果表不存在会在初始化时创建） -# 这里用原生 SQL 操作，确保即使 ORM 模型未注册也能工作 - -def _get_external_services_table(db: Session): - """安全地获取外部知识库服务列表（兼容表不存在的情况）""" - try: - # 尝试使用 ORM 模型查询 - from ...models import ExternalKBService - return db.query(ExternalKBService).all() - except (AttributeError, ImportError): - # ORM 模型不存在，用原生 SQL - try: - result = db.execute( - text("SELECT id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at FROM external_kb_services ORDER BY created_at DESC") - ) - rows = result.fetchall() - return [ - { - "id": r[0], - "name": r[1], - "service_type": r[2], - "api_url": r[3], - "api_key": r[4], - "config": json.loads(r[5]) if r[5] else {}, - "is_enabled": bool(r[6]), - "status": r[7], - "created_at": r[8].isoformat() if r[8] else None, - "updated_at": r[9].isoformat() if r[9] else None, - } - for r in rows - ] - except Exception as e: - logger.warning(f"查询外部知识库服务表失败: {e}") - return [] - - -def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict: - """测试外部知识库服务连接状态（当前仅支持 Dify）""" - try: - if service_type == "dify": - from ...dify_kb import dify_test - return dify_test(api_url, api_key) - else: - # 未知类型（如历史遗留的 fastgpt/ragflow），按通用方式探测可达性 - headers = {} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - resp = httpx.get(api_url, headers=headers, timeout=10) - return { - "connected": resp.status_code < 500, - "usable": resp.status_code < 500, - "status": "connected" if resp.status_code < 500 else "error", - "detail": f"通用探测返回状态码 {resp.status_code}", - } - except httpx.TimeoutException: - return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时"} - except Exception as e: - return {"connected": False, "usable": False, "status": "error", "detail": str(e)} - - -@router.get("/external/services", summary="获取外部知识库服务列表") -async def list_external_services( - service_type: str = Query(None, description="服务类型: fastgpt/dify/ragflow"), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """获取所有外部知识库服务配置""" - services = _get_external_services_table(db) - - if service_type: - services = [s for s in services if s.get("service_type") == service_type] - - return { - "code": 0, - "data": { - "total": len(services), - "items": services, - "supported_types": [ - {"type": "dify", "name": "Dify", "description": "LLM 应用开发平台，内置知识库管理（RAG）"}, - ] - } - } - - -@router.post("/external/services", summary="创建外部知识库服务配置") -async def create_external_service( - req: ExternalServiceConfig, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """创建新的外部知识库服务连接配置""" - # 验证服务类型（当前仅支持 Dify） - valid_types = ["dify"] - if req.service_type not in valid_types: - raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}") - - # 检查连接 - conn_result = _check_service_connection(req.service_type, req.api_url, req.api_key or "") - - service_id = str(uuid.uuid4()) - config_json = json.dumps(req.config or {}, ensure_ascii=False) - conn_status = conn_result.get("status", "disconnected") - - # 使用原生 SQL 插入（兼容 ORM 模型可能未注册） - try: - db.execute( - text(f"INSERT INTO external_kb_services (id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at) " - f"VALUES (:id, :name, :stype, :url, :key, :cfg, 1, :status, NOW(), NOW())"), - { - "id": service_id, - "name": req.name, - "stype": req.service_type, - "url": req.api_url, - "key": req.api_key or "", - "cfg": config_json, - "status": conn_status, - } - ) - db.commit() - except Exception as e: - # 表可能不存在，尝试创建 - logger.warning(f"插入外部服务失败: {e}, 尝试创建表") - db.execute(text(""" - CREATE TABLE IF NOT EXISTS external_kb_services ( - id VARCHAR(36) PRIMARY KEY, - name VARCHAR(200) NOT NULL, - service_type VARCHAR(20) NOT NULL, - api_url VARCHAR(500) NOT NULL, - api_key VARCHAR(255) DEFAULT '', - config JSON DEFAULT NULL, - is_enabled BOOLEAN DEFAULT TRUE, - status VARCHAR(20) DEFAULT 'disconnected', - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 - """)) - db.commit() - # 再次尝试插入 - db.execute( - text(f"INSERT INTO external_kb_services (id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at) " - f"VALUES (:id, :name, :stype, :url, :key, :cfg, 1, :status, NOW(), NOW())"), - { - "id": service_id, - "name": req.name, - "stype": req.service_type, - "url": req.api_url, - "key": req.api_key or "", - "cfg": config_json, - "status": conn_status, - } - ) - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="create_external_kb_service", - target_type="external_kb_service", - target_id=service_id, - detail=f"创建外部知识库服务: {req.name} ({req.service_type})", - ) - - return { - "code": 0, - "message": "外部知识库服务配置已创建", - "data": { - "id": service_id, - "name": req.name, - "service_type": req.service_type, - "api_url": req.api_url, - "connection": conn_result, - } - } - - -@router.put("/external/services/{service_id}", summary="更新外部知识库服务配置") -async def update_external_service( - service_id: str, - req: ExternalServiceUpdate, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """更新外部知识库服务配置""" - # 查找现有服务 - try: - result = db.execute( - text("SELECT id, name, service_type, api_url, api_key FROM external_kb_services WHERE id = :sid"), - {"sid": service_id} - ) - row = result.fetchone() - except Exception: - raise HTTPException(status_code=404, detail="外部知识库服务不存在或表未创建") - - if not row: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - # 构建更新字段 - updates = {} - if req.name is not None: - updates["name"] = req.name - if req.api_url is not None: - updates["api_url"] = req.api_url - if req.api_key is not None: - updates["api_key"] = req.api_key - if req.config is not None: - updates["config"] = json.dumps(req.config, ensure_ascii=False) - if req.is_enabled is not None: - updates["is_enabled"] = 1 if req.is_enabled else 0 - - # 如果 URL 或 API Key 变了，重新测试连接 - new_url = req.api_url or row[3] - new_key = req.api_key or row[4] - if req.api_url or req.api_key: - service_type = row[2] - conn_result = _check_service_connection(service_type, new_url, new_key) - updates["status"] = conn_result.get("status", "disconnected") - - updates["updated_at"] = "NOW()" - - # 执行更新 - set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys() if k != "updated_at"]) - set_clause += ", updated_at = NOW()" - params = {k: v for k, v in updates.items() if k != "updated_at"} - params["sid"] = service_id - - db.execute( - text(f"UPDATE external_kb_services SET {set_clause} WHERE id = :sid"), - params - ) - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="update_external_kb_service", - target_type="external_kb_service", - target_id=service_id, - detail=f"更新外部知识库服务: {row[1]}", - ) - - return {"code": 0, "message": "外部知识库服务配置已更新"} - - -@router.delete("/external/services/{service_id}", summary="删除外部知识库服务配置") -async def delete_external_service( - service_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """删除外部知识库服务配置""" - try: - result = db.execute( - text("SELECT name FROM external_kb_services WHERE id = :sid"), - {"sid": service_id} - ) - row = result.fetchone() - except Exception: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - if not row: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - name = row[0] - db.execute(text("DELETE FROM external_kb_services WHERE id = :sid"), {"sid": service_id}) - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="delete_external_kb_service", - target_type="external_kb_service", - target_id=service_id, - detail=f"删除外部知识库服务: {name}", - ) - - return {"code": 0, "message": "外部知识库服务配置已删除"} - - -@router.get("/external/services/{service_id}/status", summary="检查外部知识库服务连接状态") -async def check_external_service_status( - service_id: str, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """实时检查外部知识库服务的连接状态""" - try: - result = db.execute( - text("SELECT service_type, api_url, api_key FROM external_kb_services WHERE id = :sid"), - {"sid": service_id} - ) - row = result.fetchone() - except Exception: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - if not row: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - conn_result = _check_service_connection(row[0], row[1], row[2]) - - # 更新状态 - try: - db.execute( - text("UPDATE external_kb_services SET status = :status, updated_at = NOW() WHERE id = :sid"), - {"status": conn_result.get("status", "disconnected"), "sid": service_id} - ) - db.commit() - except Exception: - pass - - return {"code": 0, "data": conn_result} - - -# ============================================================ -# 知识库同步（与外部服务） -# ============================================================ - -@router.post("/external/sync/{kb_id}", summary="同步知识库与外部服务") -async def sync_knowledge_base( - kb_id: str, - req: SyncRequest, - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """将知识库与外部知识库服务进行数据同步""" - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 确定同步目标服务 - ext_service_id = req.external_service_id or getattr(kb, "external_service_id", None) - if not ext_service_id: - raise HTTPException(status_code=400, detail="未指定外部知识库服务，请先关联或传入 external_service_id") - - # 查询外部服务配置 - try: - result = db.execute( - text("SELECT name, service_type, api_url, api_key, config FROM external_kb_services WHERE id = :sid"), - {"sid": ext_service_id} - ) - row = result.fetchone() - except Exception: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - if not row: - raise HTTPException(status_code=404, detail="外部知识库服务不存在") - - service_name, service_type, api_url, api_key, config_raw = row - service_config = json.loads(config_raw) if config_raw else {} - - # 执行同步 - sync_result = {"synced": False, "detail": "", "items_synced": 0} - - try: - headers = {} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - - if service_type == "dify" and req.sync_direction == "push": - # 向 Dify 推送知识库数据（使用 app.dify_kb 客户端，按 dataset_name 解析） - from ...dify_kb import dify_ingest_text, resolve_dataset_id - dataset_name = service_config.get("dataset_name") if isinstance(service_config, dict) else None - ds_id = resolve_dataset_id(api_url, api_key, dataset_name) - if not ds_id: - sync_result["detail"] = "无法解析 Dify 知识库 ID（请检查 API Key 是否绑定到指定数据集）" - sync_result["synced"] = False - else: - docs = db.query(KnowledgeDocument).filter( - KnowledgeDocument.knowledge_base_id == kb_id, - KnowledgeDocument.status == "completed" - ).all() - synced_count = 0 - for doc in docs: - content = doc.content or "" - if not content: - continue - if dify_ingest_text(api_url, api_key, content[:50000], name=doc.filename, dataset_name=dataset_name): - synced_count += 1 - sync_result["synced"] = True - sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify" - sync_result["items_synced"] = synced_count - - elif req.sync_direction == "pull": - # 从外部服务拉取知识库数据 - sync_result["detail"] = f"从 {service_type} 拉取数据（暂未实现完整拉取逻辑）" - sync_result["synced"] = True - sync_result["items_synced"] = 0 - - elif req.sync_direction == "bidirectional": - sync_result["detail"] = f"双向同步（暂未实现完整双向同步逻辑）" - sync_result["synced"] = True - sync_result["items_synced"] = 0 - - # 更新同步状态 - try: - setattr(kb, "external_sync_status", "synced" if sync_result["synced"] else "error") - except Exception: - db.execute( - text("UPDATE knowledge_bases SET external_sync_status = :es WHERE id = :kid"), - {"es": "synced" if sync_result["synced"] else "error", "kid": kb_id} - ) - - kb.updated_at = datetime.now() - db.commit() - - except Exception as e: - logger.error(f"知识库同步失败: {e}") - sync_result["detail"] = f"同步异常: {str(e)}" - sync_result["synced"] = False - - try: - setattr(kb, "external_sync_status", "error") - except Exception: - db.execute( - text("UPDATE knowledge_bases SET external_sync_status = 'error' WHERE id = :kid"), - {"kid": kb_id} - ) - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="sync_knowledge_base", - target_type="knowledge_base", - target_id=kb_id, - detail=f"同步知识库 '{kb.name}' → {service_name} ({service_type}), 方向={req.sync_direction}, 结果={sync_result['detail']}", - ) - - return {"code": 0, "data": sync_result} - - -# ============================================================ -# 平台内容批量入库到 Dify（AI聊天产出 + 用户上传文档） -# ============================================================ - -@router.post("/external/dify/ingest", summary="将平台内容批量入库到 Dify 知识库") -async def ingest_platform_to_dify( - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """把平台中「可量化的内容」推送到 Dify 知识库（qihuai）: - - AI 聊天的助手回答（文本、非工具/非媒体、长度 > 120） - - 用户上传的文档资产（assets, category='file' 且含文本内容） - 返回各来源的入库数量。 - """ - from ...dify_kb import dify_ingest_text, resolve_dataset_id - - # 读取已启用的 Dify 服务 - try: - row = db.execute( - text("SELECT api_url, api_key, config FROM external_kb_services " - "WHERE service_type='dify' AND is_enabled=1 LIMIT 1") - ).fetchone() - except Exception: - row = None - if not row: - raise HTTPException(status_code=400, detail="未配置已启用的 Dify 知识库服务，请先添加 Dify 服务并测试连接") - api_url, api_key, config_raw = row - dataset_name = None - if config_raw: - try: - cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw) - dataset_name = cfg.get("dataset_name") - except Exception: - pass - ds_id = resolve_dataset_id(api_url, api_key, dataset_name) - if not ds_id: - raise HTTPException(status_code=400, detail="无法解析 Dify 知识库 ID（请检查 API Key 与 dataset_name）") - - stats = {"chat_answers": 0, "user_docs": 0, "skipped": 0, "failed": 0} - - # 1. AI 聊天助手回答 - try: - from ...models import Message - answers = db.query(Message).filter( - Message.role == "assistant" - ).order_by(Message.created_at.asc()).limit(2000).all() - for m in answers: - content = getattr(m, "content", None) or "" - content = content.strip() - # 跳过媒体/工具类消息与过短内容 - if len(content) < 120: - stats["skipped"] += 1 - continue - if content.startswith("[") and ("MEDIA" in content or "IMAGE" in content or "VIDEO" in content): - stats["skipped"] += 1 - continue - conv_id = getattr(m, "conversation_id", None) or "unknown" - if dify_ingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{m.id}", dataset_name=dataset_name): - stats["chat_answers"] += 1 - else: - stats["failed"] += 1 - except Exception as e: - logger.error(f"ingest chat answers failed: {e}") - - # 2. 用户上传的文档资产 - try: - from ...models import Asset - assets = db.query(Asset).filter( - getattr(Asset, "category", "file") == "file" - ).order_by(getattr(Asset, "created_at", text("id")).asc()).limit(2000).all() - for a in assets: - content = getattr(a, "content", None) or "" - content = content.strip() - if len(content) < 50: - stats["skipped"] += 1 - continue - fname = getattr(a, "filename", None) or getattr(a, "name", None) or "asset" - if dify_ingest_text(api_url, api_key, content[:50000], name=f"asset_{a.id}_{fname}", dataset_name=dataset_name): - stats["user_docs"] += 1 - else: - stats["failed"] += 1 - except Exception as e: - logger.error(f"ingest assets failed: {e}") - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="ingest_platform_to_dify", - target_type="external_kb_service", - target_id=ds_id, - detail=f"平台内容入库 Dify: 聊天回答 {stats['chat_answers']} 条, 用户文档 {stats['user_docs']} 条, 跳过 {stats['skipped']}, 失败 {stats['failed']}", - ) - - return { - "code": 0, - "message": f"入库完成：聊天回答 {stats['chat_answers']} 条，用户文档 {stats['user_docs']} 条", - "data": stats, - } - - -# ============================================================ -# 批量操作 -# ============================================================ - -@router.post("/batch/delete", summary="批量删除知识库") -async def batch_delete_kbs( - kb_ids: List[str] = Body(..., description="知识库ID列表"), - admin_user=Depends(get_current_admin), - db: Session = Depends(get_db), -): - """批量删除知识库""" - deleted = 0 - failed = 0 - for kb_id in kb_ids: - kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() - if not kb: - failed += 1 - continue - - # 删除关联数据 - db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == kb_id).delete() - db.query(KnowledgeGraphNode).filter(KnowledgeGraphNode.knowledge_base_id == kb_id).delete() - db.query(KnowledgeGraphEdge).filter(KnowledgeGraphEdge.knowledge_base_id == kb_id).delete() - db.query(KnowledgeGraphBuildLog).filter(KnowledgeGraphBuildLog.knowledge_base_id == kb_id).delete() - db.query(KnowledgeDirectory).filter(KnowledgeDirectory.knowledge_base_id == kb_id).delete() - db.delete(kb) - deleted += 1 - - db.commit() - - LogService.log_audit( - admin_id=admin_user.id, - admin_name=admin_user.username, - action="batch_delete_knowledge_bases", - target_type="knowledge_base", - target_id=",".join(kb_ids[:10]), - detail=f"批量删除知识库: 成功 {deleted}, 失败 {failed}", - ) - - return {"code": 0, "data": {"deleted": deleted, "failed": failed}} +""" +知识库管理 API（管理员） +Phase 2.4: AI PaaS Knowledge Base Management + External KB Integration +""" +from fastapi import APIRouter, Depends, HTTPException, Query, Body +from sqlalchemy import or_, func, desc, asc, text +from sqlalchemy.orm import Session +from pydantic import BaseModel, Field +from typing import Optional, List, Dict, Any +from datetime import datetime +import uuid +import json +import logging +import httpx + +from ...database import get_db +from ...models import ( + KnowledgeBase, KnowledgeDocument, KnowledgeDirectory, + KnowledgeGraphNode, KnowledgeGraphEdge, KnowledgeGraphBuildLog, + User, gen_uuid +) +from ...auth import get_current_admin +from ...middleware.log_service import LogService + +router = APIRouter() +logger = logging.getLogger("aiflow") + + +# ============================================================ +# Pydantic 模型 +# ============================================================ + +class ExternalServiceConfig(BaseModel): + name: str = Field(min_length=1, max_length=200) + service_type: str = Field(..., description="dify / anysearch") + api_url: str = Field(min_length=1, max_length=500) + api_key: Optional[str] = None + config: Optional[Dict[str, Any]] = None + + +class ExternalServiceUpdate(BaseModel): + name: Optional[str] = None + api_url: Optional[str] = None + api_key: Optional[str] = None + config: Optional[Dict[str, Any]] = None + is_enabled: Optional[bool] = None + + +class KBCreate(BaseModel): + name: str = Field(min_length=1, max_length=200) + description: Optional[str] = None + is_official: bool = False + external_service_id: Optional[str] = None + external_config: Optional[Dict[str, Any]] = None + + +class KBUpdate(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + is_official: Optional[bool] = None + status: Optional[str] = None # active / inactive / archived + external_service_id: Optional[str] = None + external_config: Optional[Dict[str, Any]] = None + + +class SyncRequest(BaseModel): + external_service_id: Optional[str] = None # override KB's linked service + sync_direction: str = "pull" # pull / push / bidirectional + options: Optional[Dict[str, Any]] = None + + +# ============================================================ +# 辅助函数 +# ============================================================ + +def kb_to_dict(kb: KnowledgeBase, db: Session = None) -> dict: + """将 KnowledgeBase 转换为字典""" + result = { + "id": kb.id, + "user_id": kb.user_id, + "name": kb.name, + "description": kb.description, + "directory_path": kb.directory_path, + "document_count": kb.document_count or 0, + "knowledge_graph_count": kb.knowledge_graph_count or 0, + "created_at": kb.created_at.isoformat() if kb.created_at else None, + "updated_at": kb.updated_at.isoformat() if kb.updated_at else None, + } + + # 新增字段（Phase 2.4） + # 通过 getattr 安全获取，因为旧记录可能没有这些字段 + result["is_official"] = getattr(kb, "is_official", False) + result["status"] = getattr(kb, "status", "active") + result["external_service_id"] = getattr(kb, "external_service_id", None) + result["external_id"] = getattr(kb, "external_id", None) + result["external_sync_status"] = getattr(kb, "external_sync_status", "none") + result["external_config"] = None + ext_config_raw = getattr(kb, "external_config", None) + if ext_config_raw: + try: + result["external_config"] = json.loads(ext_config_raw) if isinstance(ext_config_raw, str) else ext_config_raw + except Exception: + result["external_config"] = {} + + # 查询关联用户名 + if db and kb.user_id: + user = db.query(User).filter(User.id == kb.user_id).first() + result["owner_name"] = user.username if user else "未知" + + # 计算存储占用 + if db: + total_size = db.query(func.sum(KnowledgeDocument.file_size)).filter( + KnowledgeDocument.knowledge_base_id == kb.id + ).scalar() or 0 + result["total_storage_bytes"] = int(total_size) + result["total_storage_mb"] = round(int(total_size) / (1024 * 1024), 2) + + return result + + +def doc_to_dict(doc: KnowledgeDocument) -> dict: + """将 KnowledgeDocument 转换为字典""" + return { + "id": doc.id, + "knowledge_base_id": doc.knowledge_base_id, + "directory_id": doc.directory_id, + "filename": doc.filename, + "file_type": doc.file_type, + "file_size": doc.file_size or 0, + "file_path": doc.file_path, + "status": doc.status, + "chunk_count": doc.chunk_count or 0, + "error_message": doc.error_message, + "created_at": doc.created_at.isoformat() if doc.created_at else None, + "updated_at": doc.updated_at.isoformat() if doc.updated_at else None, + } + + +def format_file_size(size_bytes: int) -> str: + """格式化文件大小""" + if size_bytes < 1024: + return f"{size_bytes} B" + elif size_bytes < 1024 * 1024: + return f"{size_bytes / 1024:.1f} KB" + elif size_bytes < 1024 * 1024 * 1024: + return f"{size_bytes / (1024 * 1024):.1f} MB" + else: + return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB" + + +# ============================================================ +# 统计概览 +# ============================================================ + +@router.get("/stats/overview", summary="知识库统计概览") +async def get_kb_stats( + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取知识库全局统计数据""" + # 基础统计 + total_kb = db.query(func.count(KnowledgeBase.id)).scalar() or 0 + total_docs = db.query(func.count(KnowledgeDocument.id)).scalar() or 0 + total_storage = db.query(func.sum(KnowledgeDocument.file_size)).scalar() or 0 + total_nodes = db.query(func.count(KnowledgeGraphNode.id)).scalar() or 0 + total_edges = db.query(func.count(KnowledgeGraphEdge.id)).scalar() or 0 + + # 官方 vs 用户知识库 + official_kb = 0 + user_kb = 0 + try: + official_kb = db.query(func.count(KnowledgeBase.id)).filter( + getattr(KnowledgeBase, "is_official", False) == True + ).scalar() or 0 + except Exception: + official_kb = 0 + user_kb = total_kb - official_kb + + # 文档状态分布 + doc_status_dist = {} + statuses = ["uploading", "processing", "completed", "failed"] + for s in statuses: + cnt = db.query(func.count(KnowledgeDocument.id)).filter( + KnowledgeDocument.status == s + ).scalar() or 0 + doc_status_dist[s] = cnt + + # 文件类型分布 + file_type_dist = {} + type_rows = db.query( + KnowledgeDocument.file_type, + func.count(KnowledgeDocument.id) + ).group_by(KnowledgeDocument.file_type).all() + for ft, cnt in type_rows: + file_type_dist[ft] = cnt + + # 最近7天创建的知识库数 + from datetime import timedelta + week_ago = datetime.now() - timedelta(days=7) + recent_kb = db.query(func.count(KnowledgeBase.id)).filter( + KnowledgeBase.created_at >= week_ago + ).scalar() or 0 + + # 活跃知识库（有文档的知识库） + active_kb = db.query(func.count(func.distinct(KnowledgeDocument.knowledge_base_id))).scalar() or 0 + + # 知识图谱统计 + kb_with_graph = db.query(func.count(func.distinct(KnowledgeGraphNode.knowledge_base_id))).scalar() or 0 + + # 外部集成统计 + external_kb = 0 + try: + external_kb = db.query(func.count(KnowledgeBase.id)).filter( + getattr(KnowledgeBase, "external_service_id", None) != None, + getattr(KnowledgeBase, "external_service_id", None) != "" + ).scalar() or 0 + except Exception: + external_kb = 0 + + return { + "code": 0, + "data": { + "total_knowledge_bases": total_kb, + "total_documents": total_docs, + "total_storage_bytes": int(total_storage), + "total_storage_display": format_file_size(int(total_storage)), + "official_kb": official_kb, + "user_kb": user_kb, + "active_kb": active_kb, + "kb_with_graph": kb_with_graph, + "external_kb": external_kb, + "recent_kb_7d": recent_kb, + "total_graph_nodes": int(total_nodes), + "total_graph_edges": int(total_edges), + "doc_status_distribution": doc_status_dist, + "file_type_distribution": file_type_dist, + } + } + + +# ============================================================ +# 知识库 CRUD +# ============================================================ + +@router.get("/", summary="获取知识库列表") +async def list_knowledge_bases( + keyword: str = Query(None, description="搜索关键词"), + status: str = Query(None, description="状态: active/inactive/archived"), + is_official: bool = Query(None, description="是否官方知识库"), + has_external: bool = Query(None, description="是否有外部集成"), + page: int = Query(1, ge=1), + page_size: int = Query(15, ge=1, le=100), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取所有知识库列表（分页、筛选）""" + query = db.query(KnowledgeBase) + + if keyword: + query = query.filter( + or_( + KnowledgeBase.name.ilike(f"%{keyword}%"), + KnowledgeBase.description.ilike(f"%{keyword}%"), + ) + ) + if status: + try: + query = query.filter(getattr(KnowledgeBase, "status", None) == status) + except Exception: + pass # 字段可能不存在 + if is_official is not None: + try: + query = query.filter(getattr(KnowledgeBase, "is_official", None) == is_official) + except Exception: + pass + + total = query.count() + kbs = query.order_by(desc(KnowledgeBase.created_at)).offset( + (page - 1) * page_size + ).limit(page_size).all() + + return { + "code": 0, + "data": { + "total": total, + "page": page, + "page_size": page_size, + "items": [kb_to_dict(kb, db) for kb in kbs], + } + } + + +@router.get("/{kb_id}", summary="获取知识库详情") +async def get_knowledge_base( + kb_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取知识库详细信息，包含文档和图谱数据""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + result = kb_to_dict(kb, db) + + # 文档统计 + doc_stats = db.query( + KnowledgeDocument.status, + func.count(KnowledgeDocument.id), + func.sum(KnowledgeDocument.file_size) + ).filter(KnowledgeDocument.knowledge_base_id == kb_id).group_by( + KnowledgeDocument.status + ).all() + + result["doc_stats"] = { + s: {"count": int(cnt), "size": int(sz or 0)} + for s, cnt, sz in doc_stats + } + + # 图谱统计 + graph_nodes = db.query(func.count(KnowledgeGraphNode.id)).filter( + KnowledgeGraphNode.knowledge_base_id == kb_id + ).scalar() or 0 + graph_edges = db.query(func.count(KnowledgeGraphEdge.id)).filter( + KnowledgeGraphEdge.knowledge_base_id == kb_id + ).scalar() or 0 + result["graph_stats"] = { + "nodes": int(graph_nodes), + "edges": int(graph_edges), + } + + # 外部服务信息 + ext_service_id = getattr(kb, "external_service_id", None) + if ext_service_id: + # 查询外部服务配置 + try: + ext_service = db.query(ExternalKBService).filter( + ExternalKBService.id == ext_service_id + ).first() + if ext_service: + result["external_service"] = { + "id": ext_service.id, + "name": ext_service.name, + "service_type": ext_service.service_type, + "api_url": ext_service.api_url, + "status": getattr(ext_service, "status", "unknown"), + } + except Exception: + result["external_service"] = {"id": ext_service_id, "note": "外部服务表尚未初始化"} + + return {"code": 0, "data": result} + + +@router.post("/", summary="创建知识库") +async def create_knowledge_base( + req: KBCreate, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """创建新的知识库（支持官方平台知识库）""" + # 检查名称唯一性 + existing = db.query(KnowledgeBase).filter(KnowledgeBase.name == req.name).first() + if existing: + raise HTTPException(status_code=400, detail="知识库名称已存在") + + kb = KnowledgeBase( + id=str(uuid.uuid4()), + user_id=admin_user.id, + name=req.name, + description=req.description or "", + directory_path="/", + document_count=0, + knowledge_graph_count=0, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + # Phase 2.4 新增字段 + try: + setattr(kb, "is_official", req.is_official) + setattr(kb, "status", "active") + if req.external_service_id: + setattr(kb, "external_service_id", req.external_service_id) + if req.external_config: + setattr(kb, "external_config", json.dumps(req.external_config, ensure_ascii=False)) + setattr(kb, "external_sync_status", "none") + except Exception: + logger.warning("KnowledgeBase 新增字段设置失败，可能表结构尚未更新") + + db.add(kb) + db.commit() + db.refresh(kb) + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="create_knowledge_base", + target_type="knowledge_base", + target_id=kb.id, + detail=f"创建知识库: {kb.name} (官方={req.is_official})", + ) + + return {"code": 0, "message": "知识库创建成功", "data": kb_to_dict(kb, db)} + + +@router.put("/{kb_id}", summary="更新知识库") +async def update_knowledge_base( + kb_id: str, + req: KBUpdate, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """更新知识库配置""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + if req.name is not None: + dup = db.query(KnowledgeBase).filter( + KnowledgeBase.name == req.name, + KnowledgeBase.id != kb_id + ).first() + if dup: + raise HTTPException(status_code=400, detail="知识库名称已存在") + kb.name = req.name + + if req.description is not None: + kb.description = req.description + + # Phase 2.4 新增字段更新 + try: + if req.is_official is not None: + setattr(kb, "is_official", req.is_official) + if req.status is not None: + setattr(kb, "status", req.status) + if req.external_service_id is not None: + setattr(kb, "external_service_id", req.external_service_id) + if req.external_config is not None: + setattr(kb, "external_config", json.dumps(req.external_config, ensure_ascii=False)) + except Exception: + logger.warning("KnowledgeBase 新增字段更新失败") + + kb.updated_at = datetime.now() + db.commit() + db.refresh(kb) + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="update_knowledge_base", + target_type="knowledge_base", + target_id=kb_id, + detail=f"更新知识库: {kb.name}", + ) + + return {"code": 0, "message": "知识库已更新", "data": kb_to_dict(kb, db)} + + +@router.delete("/{kb_id}", summary="删除知识库") +async def delete_knowledge_base( + kb_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """删除知识库及其所有文档""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + kb_name = kb.name + + # 删除关联文档 + docs = db.query(KnowledgeDocument).filter( + KnowledgeDocument.knowledge_base_id == kb_id + ).all() + for d in docs: + db.delete(d) + + # 删除关联图谱 + nodes = db.query(KnowledgeGraphNode).filter( + KnowledgeGraphNode.knowledge_base_id == kb_id + ).all() + for n in nodes: + db.delete(n) + + edges = db.query(KnowledgeGraphEdge).filter( + KnowledgeGraphEdge.knowledge_base_id == kb_id + ).all() + for e in edges: + db.delete(e) + + build_logs = db.query(KnowledgeGraphBuildLog).filter( + KnowledgeGraphBuildLog.knowledge_base_id == kb_id + ).all() + for bl in build_logs: + db.delete(bl) + + # 删除关联目录 + dirs = db.query(KnowledgeDirectory).filter( + KnowledgeDirectory.knowledge_base_id == kb_id + ).all() + for d in dirs: + db.delete(d) + + # 删除知识库本身 + db.delete(kb) + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="delete_knowledge_base", + target_type="knowledge_base", + target_id=kb_id, + detail=f"删除知识库: {kb_name} (含 {len(docs)} 文档, {len(nodes)} 节点, {len(edges)} 边)", + ) + + return {"code": 0, "message": "知识库已删除"} + + +@router.put("/{kb_id}/toggle", summary="启用/禁用知识库") +async def toggle_knowledge_base( + kb_id: str, + is_active: bool = Query(..., description="是否启用"), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """启用或禁用知识库""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + new_status = "active" if is_active else "inactive" + try: + setattr(kb, "status", new_status) + except Exception: + pass + + kb.updated_at = datetime.now() + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="toggle_knowledge_base", + target_type="knowledge_base", + target_id=kb_id, + detail=f"{'启用' if is_active else '禁用'}知识库: {kb.name}", + ) + + return { + "code": 0, + "message": f"知识库已{'启用' if is_active else '禁用'}", + "data": kb_to_dict(kb, db), + } + + +# ============================================================ +# 文档管理 +# ============================================================ + +@router.get("/{kb_id}/documents", summary="获取知识库文档列表") +async def get_kb_documents( + kb_id: str, + status: str = Query(None, description="文档状态筛选"), + file_type: str = Query(None, description="文件类型筛选"), + keyword: str = Query(None, description="文件名搜索"), + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取知识库中的文档列表""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + query = db.query(KnowledgeDocument).filter( + KnowledgeDocument.knowledge_base_id == kb_id + ) + + if status: + query = query.filter(KnowledgeDocument.status == status) + if file_type: + query = query.filter(KnowledgeDocument.file_type == file_type) + if keyword: + query = query.filter(KnowledgeDocument.filename.ilike(f"%{keyword}%")) + + total = query.count() + docs = query.order_by(desc(KnowledgeDocument.created_at)).offset( + (page - 1) * page_size + ).limit(page_size).all() + + return { + "code": 0, + "data": { + "total": total, + "page": page, + "page_size": page_size, + "items": [doc_to_dict(d) for d in docs], + } + } + + +@router.delete("/{kb_id}/documents/{doc_id}", summary="删除知识库文档") +async def delete_kb_document( + kb_id: str, + doc_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """删除知识库中的单个文档""" + doc = db.query(KnowledgeDocument).filter( + KnowledgeDocument.id == doc_id, + KnowledgeDocument.knowledge_base_id == kb_id, + ).first() + if not doc: + raise HTTPException(status_code=404, detail="文档不存在") + + filename = doc.filename + db.delete(doc) + + # 更新知识库文档计数 + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if kb: + kb.document_count = db.query(func.count(KnowledgeDocument.id)).filter( + KnowledgeDocument.knowledge_base_id == kb_id + ).scalar() or 0 + kb.updated_at = datetime.now() + + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="delete_knowledge_document", + target_type="knowledge_document", + target_id=doc_id, + detail=f"删除文档: {filename} (知识库: {kb.name})", + ) + + return {"code": 0, "message": "文档已删除"} + + +# ============================================================ +# 知识图谱管理 +# ============================================================ + +@router.get("/{kb_id}/graph/stats", summary="获取知识图谱统计") +async def get_kb_graph_stats( + kb_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取知识库的知识图谱统计信息""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + nodes = db.query(func.count(KnowledgeGraphNode.id)).filter( + KnowledgeGraphNode.knowledge_base_id == kb_id + ).scalar() or 0 + + edges = db.query(func.count(KnowledgeGraphEdge.id)).filter( + KnowledgeGraphEdge.knowledge_base_id == kb_id + ).scalar() or 0 + + # 最近构建日志 + recent_logs = db.query(KnowledgeGraphBuildLog).filter( + KnowledgeGraphBuildLog.knowledge_base_id == kb_id + ).order_by(desc(KnowledgeGraphBuildLog.created_at)).limit(5).all() + + # 节点类型分布 + node_types = db.query( + KnowledgeGraphNode.label, + func.count(KnowledgeGraphNode.id) + ).filter( + KnowledgeGraphNode.knowledge_base_id == kb_id + ).group_by(KnowledgeGraphNode.label).limit(20).all() + + return { + "code": 0, + "data": { + "total_nodes": int(nodes), + "total_edges": int(edges), + "node_type_distribution": [ + {"label": lbl, "count": int(cnt)} for lbl, cnt in node_types + ], + "recent_build_logs": [ + { + "id": bl.id, + "status": getattr(bl, "status", "unknown"), + "created_at": bl.created_at.isoformat() if bl.created_at else None, + } + for bl in recent_logs + ], + } + } + + +# ============================================================ +# 外部知识库服务管理 +# ============================================================ + +# ExternalKBService 模型定义（如果表不存在会在初始化时创建） +# 这里用原生 SQL 操作，确保即使 ORM 模型未注册也能工作 + +def _get_external_services_table(db: Session): + """安全地获取外部知识库服务列表（兼容表不存在的情况）""" + try: + # 尝试使用 ORM 模型查询 + from ...models import ExternalKBService + return db.query(ExternalKBService).all() + except (AttributeError, ImportError): + # ORM 模型不存在，用原生 SQL + try: + result = db.execute( + text("SELECT id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at FROM external_kb_services ORDER BY created_at DESC") + ) + rows = result.fetchall() + return [ + { + "id": r[0], + "name": r[1], + "service_type": r[2], + "api_url": r[3], + "api_key": r[4], + "config": json.loads(r[5]) if r[5] else {}, + "is_enabled": bool(r[6]), + "status": r[7], + "created_at": r[8].isoformat() if r[8] else None, + "updated_at": r[9].isoformat() if r[9] else None, + } + for r in rows + ] + except Exception as e: + logger.warning(f"查询外部知识库服务表失败: {e}") + return [] + + +async def _check_service_connection(service_type: str, api_url: str, api_key: str) -> dict: + """测试外部知识库服务连接状态（支持 Dify / AnySearch）""" + try: + if service_type == "dify": + from ...dify_kb import dify_test + return dify_test(api_url, api_key) + if service_type == "anysearch": + from ...anysearch_client import anysearch_health + return await anysearch_health(api_key or "", api_url or "https://api.anysearch.com/v1/search") + else: + # 未知类型（如历史遗留的 fastgpt/ragflow），按通用方式探测可达性 + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + resp = httpx.get(api_url, headers=headers, timeout=10) + return { + "connected": resp.status_code < 500, + "usable": resp.status_code < 500, + "status": "connected" if resp.status_code < 500 else "error", + "detail": f"通用探测返回状态码 {resp.status_code}", + } + except httpx.TimeoutException: + return {"connected": False, "usable": False, "status": "timeout", "detail": "连接超时"} + except Exception as e: + return {"connected": False, "usable": False, "status": "error", "detail": str(e)} + + +@router.get("/external/services", summary="获取外部知识库服务列表") +async def list_external_services( + service_type: str = Query(None, description="服务类型: fastgpt/dify/ragflow"), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """获取所有外部知识库服务配置""" + services = _get_external_services_table(db) + + if service_type: + services = [s for s in services if s.get("service_type") == service_type] + + return { + "code": 0, + "data": { + "total": len(services), + "items": services, + "supported_types": [ + {"type": "dify", "name": "Dify", "description": "LLM 应用开发平台，内置知识库管理（RAG）"}, + {"type": "anysearch", "name": "AnySearch 联网检索", "description": "实时联网网页/领域搜索，为 AI 聊天注入最新网络信息"}, + ] + } + } + + +@router.post("/external/services", summary="创建外部知识库服务配置") +async def create_external_service( + req: ExternalServiceConfig, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """创建新的外部知识库服务连接配置""" + # 验证服务类型（当前仅支持 Dify） + valid_types = ["dify", "anysearch"] + if req.service_type not in valid_types: + raise HTTPException(status_code=400, detail=f"不支持的服务类型: {req.service_type}") + + # 检查连接 + conn_result = await _check_service_connection(req.service_type, req.api_url, req.api_key or "") + + service_id = str(uuid.uuid4()) + config_json = json.dumps(req.config or {}, ensure_ascii=False) + conn_status = conn_result.get("status", "disconnected") + + # 使用原生 SQL 插入（兼容 ORM 模型可能未注册） + try: + db.execute( + text(f"INSERT INTO external_kb_services (id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at) " + f"VALUES (:id, :name, :stype, :url, :key, :cfg, 1, :status, NOW(), NOW())"), + { + "id": service_id, + "name": req.name, + "stype": req.service_type, + "url": req.api_url, + "key": req.api_key or "", + "cfg": config_json, + "status": conn_status, + } + ) + db.commit() + except Exception as e: + # 表可能不存在，尝试创建 + logger.warning(f"插入外部服务失败: {e}, 尝试创建表") + db.execute(text(""" + CREATE TABLE IF NOT EXISTS external_kb_services ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(200) NOT NULL, + service_type VARCHAR(20) NOT NULL, + api_url VARCHAR(500) NOT NULL, + api_key VARCHAR(255) DEFAULT '', + config JSON DEFAULT NULL, + is_enabled BOOLEAN DEFAULT TRUE, + status VARCHAR(20) DEFAULT 'disconnected', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """)) + db.commit() + # 再次尝试插入 + db.execute( + text(f"INSERT INTO external_kb_services (id, name, service_type, api_url, api_key, config, is_enabled, status, created_at, updated_at) " + f"VALUES (:id, :name, :stype, :url, :key, :cfg, 1, :status, NOW(), NOW())"), + { + "id": service_id, + "name": req.name, + "stype": req.service_type, + "url": req.api_url, + "key": req.api_key or "", + "cfg": config_json, + "status": conn_status, + } + ) + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="create_external_kb_service", + target_type="external_kb_service", + target_id=service_id, + detail=f"创建外部知识库服务: {req.name} ({req.service_type})", + ) + + return { + "code": 0, + "message": "外部知识库服务配置已创建", + "data": { + "id": service_id, + "name": req.name, + "service_type": req.service_type, + "api_url": req.api_url, + "connection": conn_result, + } + } + + +@router.put("/external/services/{service_id}", summary="更新外部知识库服务配置") +async def update_external_service( + service_id: str, + req: ExternalServiceUpdate, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """更新外部知识库服务配置""" + # 查找现有服务 + try: + result = db.execute( + text("SELECT id, name, service_type, api_url, api_key FROM external_kb_services WHERE id = :sid"), + {"sid": service_id} + ) + row = result.fetchone() + except Exception: + raise HTTPException(status_code=404, detail="外部知识库服务不存在或表未创建") + + if not row: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + # 构建更新字段 + updates = {} + if req.name is not None: + updates["name"] = req.name + if req.api_url is not None: + updates["api_url"] = req.api_url + if req.api_key is not None: + updates["api_key"] = req.api_key + if req.config is not None: + updates["config"] = json.dumps(req.config, ensure_ascii=False) + if req.is_enabled is not None: + updates["is_enabled"] = 1 if req.is_enabled else 0 + + # 如果 URL 或 API Key 变了，重新测试连接 + new_url = req.api_url or row[3] + new_key = req.api_key or row[4] + if req.api_url or req.api_key: + service_type = row[2] + conn_result = await _check_service_connection(service_type, new_url, new_key) + updates["status"] = conn_result.get("status", "disconnected") + + updates["updated_at"] = "NOW()" + + # 执行更新 + set_clause = ", ".join([f"{k} = :{k}" for k in updates.keys() if k != "updated_at"]) + set_clause += ", updated_at = NOW()" + params = {k: v for k, v in updates.items() if k != "updated_at"} + params["sid"] = service_id + + db.execute( + text(f"UPDATE external_kb_services SET {set_clause} WHERE id = :sid"), + params + ) + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="update_external_kb_service", + target_type="external_kb_service", + target_id=service_id, + detail=f"更新外部知识库服务: {row[1]}", + ) + + return {"code": 0, "message": "外部知识库服务配置已更新"} + + +@router.delete("/external/services/{service_id}", summary="删除外部知识库服务配置") +async def delete_external_service( + service_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """删除外部知识库服务配置""" + try: + result = db.execute( + text("SELECT name FROM external_kb_services WHERE id = :sid"), + {"sid": service_id} + ) + row = result.fetchone() + except Exception: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + if not row: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + name = row[0] + db.execute(text("DELETE FROM external_kb_services WHERE id = :sid"), {"sid": service_id}) + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="delete_external_kb_service", + target_type="external_kb_service", + target_id=service_id, + detail=f"删除外部知识库服务: {name}", + ) + + return {"code": 0, "message": "外部知识库服务配置已删除"} + + +@router.get("/external/services/{service_id}/status", summary="检查外部知识库服务连接状态") +async def check_external_service_status( + service_id: str, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """实时检查外部知识库服务的连接状态""" + try: + result = db.execute( + text("SELECT service_type, api_url, api_key FROM external_kb_services WHERE id = :sid"), + {"sid": service_id} + ) + row = result.fetchone() + except Exception: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + if not row: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + conn_result = await _check_service_connection(row[0], row[1], row[2]) + + # 更新状态 + try: + db.execute( + text("UPDATE external_kb_services SET status = :status, updated_at = NOW() WHERE id = :sid"), + {"status": conn_result.get("status", "disconnected"), "sid": service_id} + ) + db.commit() + except Exception: + pass + + return {"code": 0, "data": conn_result} + + +# ============================================================ +# 知识库同步（与外部服务） +# ============================================================ + +@router.post("/external/sync/{kb_id}", summary="同步知识库与外部服务") +async def sync_knowledge_base( + kb_id: str, + req: SyncRequest, + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """将知识库与外部知识库服务进行数据同步""" + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 确定同步目标服务 + ext_service_id = req.external_service_id or getattr(kb, "external_service_id", None) + if not ext_service_id: + raise HTTPException(status_code=400, detail="未指定外部知识库服务，请先关联或传入 external_service_id") + + # 查询外部服务配置 + try: + result = db.execute( + text("SELECT name, service_type, api_url, api_key, config FROM external_kb_services WHERE id = :sid"), + {"sid": ext_service_id} + ) + row = result.fetchone() + except Exception: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + if not row: + raise HTTPException(status_code=404, detail="外部知识库服务不存在") + + service_name, service_type, api_url, api_key, config_raw = row + service_config = json.loads(config_raw) if config_raw else {} + + # 执行同步 + sync_result = {"synced": False, "detail": "", "items_synced": 0} + + try: + headers = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + if service_type == "dify" and req.sync_direction == "push": + # 向 Dify 推送知识库数据（使用 app.dify_kb 客户端，按 dataset_name 解析） + from ...dify_kb import dify_ingest_text, resolve_dataset_id + dataset_name = service_config.get("dataset_name") if isinstance(service_config, dict) else None + ds_id = resolve_dataset_id(api_url, api_key, dataset_name) + if not ds_id: + sync_result["detail"] = "无法解析 Dify 知识库 ID（请检查 API Key 是否绑定到指定数据集）" + sync_result["synced"] = False + else: + docs = db.query(KnowledgeDocument).filter( + KnowledgeDocument.knowledge_base_id == kb_id, + KnowledgeDocument.status == "completed" + ).all() + synced_count = 0 + for doc in docs: + content = doc.content or "" + if not content: + continue + if dify_ingest_text(api_url, api_key, content[:50000], name=doc.filename, dataset_name=dataset_name): + synced_count += 1 + sync_result["synced"] = True + sync_result["detail"] = f"已推送 {synced_count}/{len(docs)} 文档到 Dify" + sync_result["items_synced"] = synced_count + + elif req.sync_direction == "pull": + # 从外部服务拉取知识库数据 + sync_result["detail"] = f"从 {service_type} 拉取数据（暂未实现完整拉取逻辑）" + sync_result["synced"] = True + sync_result["items_synced"] = 0 + + elif req.sync_direction == "bidirectional": + sync_result["detail"] = f"双向同步（暂未实现完整双向同步逻辑）" + sync_result["synced"] = True + sync_result["items_synced"] = 0 + + # 更新同步状态 + try: + setattr(kb, "external_sync_status", "synced" if sync_result["synced"] else "error") + except Exception: + db.execute( + text("UPDATE knowledge_bases SET external_sync_status = :es WHERE id = :kid"), + {"es": "synced" if sync_result["synced"] else "error", "kid": kb_id} + ) + + kb.updated_at = datetime.now() + db.commit() + + except Exception as e: + logger.error(f"知识库同步失败: {e}") + sync_result["detail"] = f"同步异常: {str(e)}" + sync_result["synced"] = False + + try: + setattr(kb, "external_sync_status", "error") + except Exception: + db.execute( + text("UPDATE knowledge_bases SET external_sync_status = 'error' WHERE id = :kid"), + {"kid": kb_id} + ) + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="sync_knowledge_base", + target_type="knowledge_base", + target_id=kb_id, + detail=f"同步知识库 '{kb.name}' → {service_name} ({service_type}), 方向={req.sync_direction}, 结果={sync_result['detail']}", + ) + + return {"code": 0, "data": sync_result} + + +# ============================================================ +# 平台内容批量入库到 Dify（AI聊天产出 + 用户上传文档） +# ============================================================ + +@router.post("/external/dify/ingest", summary="将平台内容批量入库到 Dify 知识库") +async def ingest_platform_to_dify( + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """把平台中「可量化的内容」推送到 Dify 知识库（qihuai）: + - AI 聊天的助手回答（文本、非工具/非媒体、长度 > 120） + - 用户上传的文档资产（assets, category='file' 且含文本内容） + 返回各来源的入库数量。 + """ + from ...dify_kb import dify_ingest_text, resolve_dataset_id + + # 读取已启用的 Dify 服务 + try: + row = db.execute( + text("SELECT api_url, api_key, config FROM external_kb_services " + "WHERE service_type='dify' AND is_enabled=1 LIMIT 1") + ).fetchone() + except Exception: + row = None + if not row: + raise HTTPException(status_code=400, detail="未配置已启用的 Dify 知识库服务，请先添加 Dify 服务并测试连接") + api_url, api_key, config_raw = row + dataset_name = None + if config_raw: + try: + cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw) + dataset_name = cfg.get("dataset_name") + except Exception: + pass + ds_id = resolve_dataset_id(api_url, api_key, dataset_name) + if not ds_id: + raise HTTPException(status_code=400, detail="无法解析 Dify 知识库 ID（请检查 API Key 与 dataset_name）") + + stats = {"chat_answers": 0, "user_docs": 0, "skipped": 0, "failed": 0} + + # 1. AI 聊天助手回答 + try: + from ...models import Message + answers = db.query(Message).filter( + Message.role == "assistant" + ).order_by(Message.created_at.asc()).limit(2000).all() + for m in answers: + content = getattr(m, "content", None) or "" + content = content.strip() + # 跳过媒体/工具类消息与过短内容 + if len(content) < 120: + stats["skipped"] += 1 + continue + if content.startswith("[") and ("MEDIA" in content or "IMAGE" in content or "VIDEO" in content): + stats["skipped"] += 1 + continue + conv_id = getattr(m, "conversation_id", None) or "unknown" + if dify_ingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{m.id}", dataset_name=dataset_name): + stats["chat_answers"] += 1 + else: + stats["failed"] += 1 + except Exception as e: + logger.error(f"ingest chat answers failed: {e}") + + # 2. 用户上传的文档资产 + try: + from ...models import Asset + assets = db.query(Asset).filter( + getattr(Asset, "category", "file") == "file" + ).order_by(getattr(Asset, "created_at", text("id")).asc()).limit(2000).all() + for a in assets: + content = getattr(a, "content", None) or "" + content = content.strip() + if len(content) < 50: + stats["skipped"] += 1 + continue + fname = getattr(a, "filename", None) or getattr(a, "name", None) or "asset" + if dify_ingest_text(api_url, api_key, content[:50000], name=f"asset_{a.id}_{fname}", dataset_name=dataset_name): + stats["user_docs"] += 1 + else: + stats["failed"] += 1 + except Exception as e: + logger.error(f"ingest assets failed: {e}") + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="ingest_platform_to_dify", + target_type="external_kb_service", + target_id=ds_id, + detail=f"平台内容入库 Dify: 聊天回答 {stats['chat_answers']} 条, 用户文档 {stats['user_docs']} 条, 跳过 {stats['skipped']}, 失败 {stats['failed']}", + ) + + return { + "code": 0, + "message": f"入库完成：聊天回答 {stats['chat_answers']} 条，用户文档 {stats['user_docs']} 条", + "data": stats, + } + + +# ============================================================ +# 批量操作 +# ============================================================ + +@router.post("/batch/delete", summary="批量删除知识库") +async def batch_delete_kbs( + kb_ids: List[str] = Body(..., description="知识库ID列表"), + admin_user=Depends(get_current_admin), + db: Session = Depends(get_db), +): + """批量删除知识库""" + deleted = 0 + failed = 0 + for kb_id in kb_ids: + kb = db.query(KnowledgeBase).filter(KnowledgeBase.id == kb_id).first() + if not kb: + failed += 1 + continue + + # 删除关联数据 + db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == kb_id).delete() + db.query(KnowledgeGraphNode).filter(KnowledgeGraphNode.knowledge_base_id == kb_id).delete() + db.query(KnowledgeGraphEdge).filter(KnowledgeGraphEdge.knowledge_base_id == kb_id).delete() + db.query(KnowledgeGraphBuildLog).filter(KnowledgeGraphBuildLog.knowledge_base_id == kb_id).delete() + db.query(KnowledgeDirectory).filter(KnowledgeDirectory.knowledge_base_id == kb_id).delete() + db.delete(kb) + deleted += 1 + + db.commit() + + LogService.log_audit( + admin_id=admin_user.id, + admin_name=admin_user.username, + action="batch_delete_knowledge_bases", + target_type="knowledge_base", + target_id=",".join(kb_ids[:10]), + detail=f"批量删除知识库: 成功 {deleted}, 失败 {failed}", + ) + + return {"code": 0, "data": {"deleted": deleted, "failed": failed}} diff --git a/backend/python/app/api/knowledge.py b/backend/python/app/api/knowledge.py index 13c2ee2..fe7625b 100644 --- a/backend/python/app/api/knowledge.py +++ b/backend/python/app/api/knowledge.py @@ -1,1515 +1,1555 @@ -import asyncio -from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File, Form -from sqlalchemy.orm import Session -from sqlalchemy import and_, desc, asc -from pydantic import BaseModel, Field -from typing import Optional, List -import uuid -import os -import json -import re -import logging -import httpx -from datetime import datetime - -from ..database import get_db -from ..models import ( - User, KnowledgeBase, KnowledgeDocument, KnowledgeDirectory, Asset, - KnowledgeGraphNode, KnowledgeGraphEdge, KnowledgeGraphBuildLog, - AIModel, ModelProvider, - gen_uuid -) -from ..auth import get_current_user -from ..middleware.log_service import LogService - -router = APIRouter() -logger = logging.getLogger("aiflow") - -# 知识库文件存储根目录 -KNOWLEDGE_STORAGE_DIR = os.environ.get("KNOWLEDGE_STORAGE_DIR", "/data/share/project/data/knowledge") - - -class CreateKnowledgeBaseRequest(BaseModel): - name: str = Field(min_length=1, max_length=200) - description: Optional[str] = None - icon: Optional[str] = None - - -class UpdateKnowledgeBaseRequest(BaseModel): - name: Optional[str] = None - description: Optional[str] = None - icon: Optional[str] = None - status: Optional[str] = None - - -class CreateDocumentRequest(BaseModel): - filename: str - file_size: Optional[int] = None - file_type: Optional[str] = None - - -class CreateDirectoryRequest(BaseModel): - name: str = Field(min_length=1, max_length=200) - parent_id: Optional[str] = None - knowledge_base_id: Optional[str] = None - - -class KnowledgeQARequest(BaseModel): - base_id: str - question: str = Field(min_length=1, max_length=5000) - - -@router.get("/bases") -def list_knowledge_bases( - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - bases = ( - db.query(KnowledgeBase) - .filter(KnowledgeBase.user_id == current_user.id) - .order_by(desc(KnowledgeBase.updated_at)) - .all() - ) - - # 如果用户没有知识库，自动创建两个默认知识库（每个用户只创建一次） - if not bases: - # 检查是否已存在名为"默认知识库"或"知识图谱库"的知识库（兜底检查） - existing_default = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.user_id == current_user.id, KnowledgeBase.name == "默认知识库") - ).first() - existing_graph = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.user_id == current_user.id, KnowledgeBase.name == "知识图谱库") - ).first() - - if not existing_default: - default_kb = KnowledgeBase( - id=gen_uuid(), - user_id=current_user.id, - name="默认知识库", - description="联网采集、直接上传的内容默认存储", - directory_path="/", - document_count=0, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - db.add(default_kb) - LogService.log_operation( - user_id=current_user.id, username=current_user.username, action="auto_create", - target="knowledge_base", target_id=default_kb.id, - detail="自动创建默认知识库", - ) - - if not existing_graph: - graph_kb = KnowledgeBase( - id=gen_uuid(), - user_id=current_user.id, - name="知识图谱库", - description="对话转知识图谱、构建知识图谱的默认存储", - directory_path="/", - document_count=0, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - db.add(graph_kb) - LogService.log_operation( - user_id=current_user.id, username=current_user.username, action="auto_create", - target="knowledge_base", target_id=graph_kb.id, - detail="自动创建知识图谱库", - ) - - db.commit() - - # 重新查询 - bases = ( - db.query(KnowledgeBase) - .filter(KnowledgeBase.user_id == current_user.id) - .order_by(desc(KnowledgeBase.updated_at)) - .all() - ) - - # 批量查询所有知识库的真实图谱节点数（从 knowledge_graph_nodes 表统计） - kb_ids = [kb.id for kb in bases] - graph_node_counts = {} - if kb_ids: - from sqlalchemy import func as sa_func - node_stats = db.query( - KnowledgeGraphNode.knowledge_base_id, - sa_func.count(KnowledgeGraphNode.id) - ).filter( - KnowledgeGraphNode.knowledge_base_id.in_(kb_ids) - ).group_by(KnowledgeGraphNode.knowledge_base_id).all() - graph_node_counts = {row[0]: row[1] for row in node_stats} - - result = [] - for kb in bases: - # 使用真实图谱数据，如果图谱表中有数据则用它，否则用旧字段（可能残留旧值） - real_node_count = graph_node_counts.get(kb.id) - if real_node_count is not None and real_node_count > 0: - display_graph_count = real_node_count - else: - display_graph_count = 0 # 清零旧残留值 - - result.append({ - "id": kb.id, - "name": kb.name, - "description": kb.description, - "document_count": kb.document_count or 0, - "knowledge_graph_count": display_graph_count, - "created_at": str(kb.created_at) if kb.created_at else None, - "updated_at": str(kb.updated_at) if kb.updated_at else None, - }) - - # 计算存储空间信息 - # 存储已用：所有知识库文档的真实文件大小总和 - if bases: - from sqlalchemy import func as sa_func - total_size_result = db.query(sa_func.sum(KnowledgeDocument.file_size)).filter( - KnowledgeDocument.knowledge_base_id.in_([kb.id for kb in bases]) - ).scalar() - storage_used = total_size_result or 0 - else: - storage_used = 0 - - # 根据用户会员等级计算存储限额 - membership_level = current_user.membership_level or 0 - storage_limits = {0: 1 * 1024 * 1024 * 1024, 2: 10 * 1024 * 1024 * 1024, 3: 100 * 1024 * 1024 * 1024} - storage_limit = storage_limits.get(membership_level, 1 * 1024 * 1024 * 1024) - - return { - "code": 0, - "data": { - "bases": result, - "storage_used": storage_used, - "storage_limit": storage_limit, - "membership_level": membership_level, - }, - "message": "ok", - } - - -@router.post("/bases") -def create_knowledge_base( - req: CreateKnowledgeBaseRequest, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = KnowledgeBase( - id=gen_uuid(), - user_id=current_user.id, - name=req.name, - description=req.description, - directory_path="/", - document_count=0, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - db.add(kb) - db.commit() - db.refresh(kb) - - LogService.log_operation( - user_id=current_user.id, username=current_user.username, action="create", - target="knowledge_base", target_id=kb.id, - detail=f"创建知识库 {kb.name}", - ) - - return { - "code": 0, - "data": { - "id": kb.id, "name": kb.name, "description": kb.description, - "created_at": str(kb.created_at), - }, - "message": "创建成功", - } - - -@router.post("/bases/{base_id}/build-graph") -def build_knowledge_graph( - base_id: str, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """构建知识图谱：调用LLM分析文档内容提取实体和关系，持久化到数据库""" - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 获取知识库中的已完成文档 - documents = db.query(KnowledgeDocument).filter( - and_( - KnowledgeDocument.knowledge_base_id == base_id, - KnowledgeDocument.status == "completed", - ) - ).all() - - if not documents: - raise HTTPException(status_code=400, detail="知识库中没有已完成的文档，请先上传文档并等待处理完成") - - # 获取LLM配置（获取默认启用的模型及其provider信息） - llm_config = _get_llm_config(db) - if not llm_config: - raise HTTPException(status_code=500, detail="没有可用的AI模型，请先在后台管理中配置模型") - - api_url, api_key, model_name, timeout = llm_config - - # 先清理旧图谱数据 - db.query(KnowledgeGraphEdge).filter( - KnowledgeGraphEdge.knowledge_base_id == base_id - ).delete() - db.query(KnowledgeGraphNode).filter( - KnowledgeGraphNode.knowledge_base_id == base_id - ).delete() - db.query(KnowledgeGraphBuildLog).filter( - KnowledgeGraphBuildLog.knowledge_base_id == base_id - ).delete() - db.flush() - - total_nodes = 0 - total_edges = 0 - - for doc in documents: - # 读取文档文件内容 - file_content = _read_document_content(doc) - - if not file_content or len(file_content.strip()) < 10: - logger.warning(f"文档 {doc.filename} 内容为空或过短，跳过图谱构建") - # 记录构建日志 - build_log = KnowledgeGraphBuildLog( - id=gen_uuid(), - knowledge_base_id=base_id, - document_id=doc.id, - node_count=0, - edge_count=0, - status="completed", - created_at=datetime.now(), - ) - db.add(build_log) - continue - - # 调用LLM提取实体和关系 - entities, relations = _extract_entities_with_llm( - api_url, api_key, model_name, timeout, doc.filename, file_content - ) - - # 将提取结果写入数据库 - doc_node_count, doc_edge_count = _persist_graph_data( - db, base_id, doc.id, entities, relations - ) - total_nodes += doc_node_count - total_edges += doc_edge_count - - # 记录构建日志 - build_log = KnowledgeGraphBuildLog( - id=gen_uuid(), - knowledge_base_id=base_id, - document_id=doc.id, - node_count=doc_node_count, - edge_count=doc_edge_count, - status="completed", - created_at=datetime.now(), - ) - db.add(build_log) - - db.commit() - - # 更新知识库的图谱计数 - kb.knowledge_graph_count = total_nodes - kb.updated_at = datetime.now() - db.commit() - - LogService.log_operation( - user_id=current_user.id, username=current_user.username, action="build_graph", - target="knowledge_base", target_id=base_id, - detail=f"为知识库 {kb.name} 构建知识图谱，共 {total_nodes} 节点、{total_edges} 边", - ) - - # 构建返回数据 - graph_data = _build_graph_response(db, base_id, total_nodes, total_edges, len(documents)) - - return { - "code": 0, - "data": graph_data, - "message": f"知识图谱构建完成，共 {total_nodes} 个节点、{total_edges} 条边", - } - - -def _get_llm_config(db: Session): - """获取LLM配置：优先默认模型，否则使用第一个启用的模型""" - logger.info("[图谱构建] 开始查找LLM配置...") - ai_model = db.query(AIModel).filter( - AIModel.is_default == True, - AIModel.is_enabled == True, - ).first() - - if not ai_model: - ai_model = db.query(AIModel).filter( - AIModel.is_enabled == True, - AIModel.endpoint.isnot(None), - ).first() - logger.info(f"[图谱构建] 无默认模型，尝试使用第一个启用模型: {ai_model.name if ai_model else 'None'}") - - if not ai_model: - logger.error("[图谱构建] 没有可用的AI模型！") - return None - - api_key = ai_model.api_key - api_url = ai_model.endpoint - - # 如果模型本身没有独立的api_key/endpoint，从provider获取 - if (not api_key or not api_url) and ai_model.provider_id: - provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() - if provider: - if not api_key: - api_key = provider.api_key - if not api_url and provider.api_base_url: - api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" - - if not api_url or not api_key: - logger.error(f"[图谱构建] LLM配置不完整: api_url={api_url}, api_key={'***' if api_key else 'None'}") - return None - - logger.info(f"[图谱构建] 找到LLM配置: model={ai_model.name}, url={api_url[:50]}..., timeout={ai_model.timeout or 60}") - return (api_url, api_key, ai_model.name, ai_model.timeout or 60) - - -def _read_document_content(doc: KnowledgeDocument) -> str: - """从磁盘读取文档的实际内容""" - storage_dir = os.environ.get("KNOWLEDGE_STORAGE_DIR", "/data/share/project/data/knowledge") - file_path = os.path.join(storage_dir, doc.knowledge_base_id, f"{doc.id}.{doc.file_type}") - - logger.info(f"[图谱构建] 读取文档: {doc.filename}, id={doc.id}, type={doc.file_type}, path={file_path}, status={doc.status}") - - if not os.path.exists(file_path): - # 尝试其他可能的路径 - alt_path = os.path.join(storage_dir, doc.knowledge_base_id, f"{doc.id}.{doc.file_type.lower()}") - if os.path.exists(alt_path): - file_path = alt_path - logger.info(f"[图谱构建] 使用备用路径: {file_path}") - else: - # 尝试直接读file_path字段 - if doc.file_path and os.path.exists(doc.file_path): - file_path = doc.file_path - logger.info(f"[图谱构建] 使用数据库file_path: {file_path}") - else: - logger.warning(f"[图谱构建] 文档文件不存在: {file_path} (备用: {alt_path}, 数据库file_path: {doc.file_path})") - return "" - - try: - with open(file_path, "r", encoding="utf-8") as f: - content = f.read() - return content - except Exception as e: - logger.error(f"读取文档文件失败: {file_path}, error={e}") - return "" - - -def _extract_entities_with_llm( - api_url: str, api_key: str, model_name: str, timeout: int, - doc_filename: str, file_content: str -): - """调用LLM分析文档内容，提取实体和关系""" - # 限制内容长度（防止token溢出） - max_content_len = 8000 - if len(file_content) > max_content_len: - truncated_content = file_content[:max_content_len] + "\n\n...(内容已截断)" - else: - truncated_content = file_content - - prompt = f"""分析以下文档内容，识别所有命名实体（人物、组织、地点、产品、技术术语等）和它们之间的关系。 - -文档标题：{doc_filename} - -文档内容： -{truncated_content} - -请以JSON格式返回，格式如下（只返回JSON，不要包含其他内容）： -{{"entities":[{{"name":"实体名","type":"entity/attribute/concept/event","description":"简介","attributes":{{"key":"value"}}}}],"relations":[{{"source":"源实体名","target":"目标实体名","relation":"关系类型","description":"关系描述"}}]}} - -要求： -1. 实体名称使用文档原文中的表述 -2. 实体类型只能从 entity（命名实体）、attribute（属性特征）、concept（核心概念）、event（事件）中选择 -3. 每个实体尽量给出简短描述 -4. 关系需要明确源实体和目标实体的名称 -5. 不要返回不存在的实体或关系""" - - try: - with httpx.Client(timeout=timeout) as client: - resp = client.post( - api_url, - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - }, - json={ - "model": model_name, - "messages": [ - {"role": "system", "content": "你是一个专业的知识图谱构建助手，擅长从文档中提取实体和关系。你只返回严格符合要求的JSON格式数据。"}, - {"role": "user", "content": prompt}, - ], - "max_tokens": 4096, - "temperature": 0.3, - }, - ) - - if resp.status_code != 200: - logger.error(f"LLM API调用失败 (HTTP {resp.status_code}): {resp.text[:500]}") - return [], [] - - data = resp.json() - content = data.get("choices", [{}])[0].get("message", {}).get("content", "") - - # 记录token使用量（可选） - usage = data.get("usage", {}) - logger.info( - f"LLM图谱构建 token使用: prompt={usage.get('prompt_tokens', 0)}, " - f"completion={usage.get('completion_tokens', 0)}, " - f"total={usage.get('total_tokens', 0)}" - ) - - if not content: - logger.error("LLM返回空内容") - return [], [] - - # 解析LLM返回的JSON - entities, relations = _parse_llm_graph_response(content) - return entities, relations - - except httpx.TimeoutException: - logger.error(f"LLM API调用超时: {api_url}") - return [], [] - except httpx.ConnectError: - logger.error(f"LLM API连接失败: {api_url}") - return [], [] - except Exception as e: - logger.error(f"LLM API调用异常: {e}") - return [], [] - - -def _parse_llm_graph_response(content: str): - """解析LLM返回的图谱JSON，做容错处理""" - entities = [] - relations = [] - - # 尝试多种方式提取JSON - # 方式1: 直接解析 - try: - result = json.loads(content) - entities = result.get("entities", []) - relations = result.get("relations", []) - return entities, relations - except json.JSONDecodeError: - pass - - # 方式2: 提取```json ... ```代码块 - json_match = re.search(r'```(?:json)?\s*([\s\S]*?)```', content) - if json_match: - try: - result = json.loads(json_match.group(1)) - entities = result.get("entities", []) - relations = result.get("relations", []) - return entities, relations - except json.JSONDecodeError: - pass - - # 方式3: 提取最外层的{...} - brace_match = re.search(r'\{[\s\S]*\}', content) - if brace_match: - try: - result = json.loads(brace_match.group(0)) - entities = result.get("entities", []) - relations = result.get("relations", []) - return entities, relations - except json.JSONDecodeError: - pass - - logger.warning(f"无法解析LLM返回的JSON: {content[:500]}") - return [], [] - - -def _persist_graph_data(db: Session, base_id: str, doc_id: str, entities: list, relations: list): - """将LLM提取的实体和关系持久化到数据库，返回(节点数, 边数)""" - # 先创建节点，记录name到node_id的映射 - name_to_node_id = {} - node_count = 0 - - for entity in entities: - if not isinstance(entity, dict): - continue - name = entity.get("name", "").strip() - if not name: - continue - entity_type = entity.get("type", "entity") - if entity_type not in ("entity", "attribute", "concept", "event"): - entity_type = "entity" - description = entity.get("description", "") - attributes = entity.get("attributes", {}) - if not isinstance(attributes, dict): - attributes = {} - - node_id = gen_uuid() - node = KnowledgeGraphNode( - id=node_id, - knowledge_base_id=base_id, - source_doc_id=doc_id, - label=name[:500], - type=entity_type, - properties=attributes, - description=description[:65535] if description else None, - created_at=datetime.now(), - ) - db.add(node) - name_to_node_id[name] = node_id - node_count += 1 - - # 必须flush才能获取节点ID用于外键关联 - db.flush() - - # 创建边 - edge_count = 0 - for rel in relations: - if not isinstance(rel, dict): - continue - source_name = rel.get("source", "").strip() - target_name = rel.get("target", "").strip() - if not source_name or not target_name: - continue - source_node_id = name_to_node_id.get(source_name) - target_node_id = name_to_node_id.get(target_name) - if not source_node_id or not target_node_id: - # 如果引用了LLM返回中不存在的实体名，跳过 - logger.debug(f"关系引用了未注册实体: {source_name} -> {target_name}") - continue - - edge = KnowledgeGraphEdge( - id=gen_uuid(), - knowledge_base_id=base_id, - source_node_id=source_node_id, - target_node_id=target_node_id, - relation=rel.get("relation", "关联")[:200], - weight=float(rel.get("weight", 1.0)) if rel.get("weight") else 1.0, - description=rel.get("description", "")[:65535] if rel.get("description") else None, - created_at=datetime.now(), - ) - db.add(edge) - edge_count += 1 - - return node_count, edge_count - - -def _build_graph_response(db: Session, base_id: str, total_nodes: int, total_edges: int, doc_count: int): - """从数据库读取图谱数据并构建响应""" - nodes = db.query(KnowledgeGraphNode).filter( - KnowledgeGraphNode.knowledge_base_id == base_id - ).order_by(KnowledgeGraphNode.created_at.asc()).all() - - edges = db.query(KnowledgeGraphEdge).filter( - KnowledgeGraphEdge.knowledge_base_id == base_id - ).order_by(KnowledgeGraphEdge.created_at.asc()).all() - - graph_nodes = [] - for node in nodes: - graph_nodes.append({ - "id": node.id, - "label": node.label, - "type": node.type, - "size": 10 + (len(node.description) % 10 if node.description else 5), - "group": node.type, - "description": node.description, - "properties": node.properties or {}, - "source_doc_id": node.source_doc_id, - }) - - graph_edges = [] - for edge in edges: - graph_edges.append({ - "id": edge.id, - "source": edge.source_node_id, - "target": edge.target_node_id, - "label": edge.relation, - "weight": edge.weight, - "description": edge.description, - }) - - # 统计各类型节点数量 - entity_types = {} - for node in nodes: - t = node.type - entity_types[t] = entity_types.get(t, 0) + 1 - - return { - "nodes": graph_nodes, - "edges": graph_edges, - "stats": { - "total_nodes": total_nodes, - "total_edges": total_edges, - "document_count": doc_count, - "entity_types": entity_types, - }, - } - - -@router.get("/bases/{base_id}/graph") -def get_knowledge_graph( - base_id: str, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """获取知识图谱数据：从持久化表中读取已构建的图谱""" - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 从数据库表读取图谱节点 - nodes = db.query(KnowledgeGraphNode).filter( - KnowledgeGraphNode.knowledge_base_id == base_id - ).order_by(KnowledgeGraphNode.created_at.asc()).all() - - if not nodes: - # 清理旧残留数据：如果 knowledge_graph_count > 0 但图谱表中无节点，清零旧值 - if (kb.knowledge_graph_count or 0) > 0: - kb.knowledge_graph_count = 0 - db.commit() - - # 检查是否有文档但尚未构建图谱 - doc_count = db.query(KnowledgeDocument).filter( - and_( - KnowledgeDocument.knowledge_base_id == base_id, - KnowledgeDocument.status == "completed", - ) - ).count() - - return { - "code": 0, - "data": { - "nodes": [], - "edges": [], - "stats": {"total_nodes": 0, "total_edges": 0, "document_count": doc_count}, - }, - "message": "暂无可用于构建图谱的文档，请先构建知识图谱", - } - - # 从数据库表读取图谱边 - edges = db.query(KnowledgeGraphEdge).filter( - KnowledgeGraphEdge.knowledge_base_id == base_id - ).order_by(KnowledgeGraphEdge.created_at.asc()).all() - - graph_nodes = [] - for node in nodes: - graph_nodes.append({ - "id": node.id, - "label": node.label, - "type": node.type, - "size": 10 + (len(node.description) % 10 if node.description else 5), - "group": node.type, - "description": node.description, - "properties": node.properties or {}, - "source_doc_id": node.source_doc_id, - }) - - graph_edges = [] - for edge in edges: - graph_edges.append({ - "id": edge.id, - "source": edge.source_node_id, - "target": edge.target_node_id, - "label": edge.relation, - "weight": edge.weight, - "description": edge.description, - }) - - # 统计各类型节点数量 - entity_types = {} - for node in nodes: - t = node.type - entity_types[t] = entity_types.get(t, 0) + 1 - - graph_data = { - "nodes": graph_nodes, - "edges": graph_edges, - "stats": { - "total_nodes": len(graph_nodes), - "total_edges": len(graph_edges), - "document_count": len(set(n.source_doc_id for n in nodes if n.source_doc_id)), - "entity_types": entity_types, - }, - } - - return {"code": 0, "data": graph_data, "message": "ok"} - -@router.get("/bases/{base_id}") -def get_knowledge_base( - base_id: str, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - return { - "code": 0, - "data": { - "id": kb.id, "name": kb.name, "description": kb.description, - "document_count": kb.document_count or 0, - "created_at": str(kb.created_at), "updated_at": str(kb.updated_at), - }, - "message": "ok", - } - - -@router.put("/bases/{base_id}") -def update_knowledge_base( - base_id: str, - req: UpdateKnowledgeBaseRequest, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - if req.name is not None: - kb.name = req.name - if req.description is not None: - kb.description = req.description - if req.icon is not None: - pass # icon not in model - if req.status is not None: - pass # status not in model - kb.updated_at = datetime.now() - - db.commit() - db.refresh(kb) - - return {"code": 0, "data": { - "id": kb.id, "name": kb.name, "description": kb.description, - }, "message": "更新成功"} - - -@router.delete("/bases/{base_id}") -def delete_knowledge_base( - base_id: str, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 删除知识库下的所有文档 - db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == base_id).delete() - db.delete(kb) - db.commit() - - return {"code": 0, "message": "知识库已删除"} - - -@router.get("/bases/{base_id}/documents") -def get_documents( - base_id: str, - status: Optional[str] = Query(None), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - # 先验证知识库归属 - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - query = db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == base_id) - if status: - query = query.filter(KnowledgeDocument.status == status) - - documents = query.order_by(desc(KnowledgeDocument.created_at)).all() - - result = [] - for doc in documents: - result.append({ - "id": doc.id, - "base_id": doc.knowledge_base_id, - "filename": doc.filename, - "file_size": doc.file_size, - "file_type": doc.file_type, - "status": doc.status, - "chunk_count": doc.chunk_count or 0, - "created_at": str(doc.created_at) if doc.created_at else None, - "updated_at": str(doc.updated_at) if doc.updated_at else None, - }) - - return {"code": 0, "data": result, "message": "ok"} - - -@router.post("/bases/{base_id}/documents") -async def _index_kb_doc_to_pgvector(doc_id: str, kb_id: str, content: str): - """将文档内容异步索引到 pgvector，供语义向量检索使用（best-effort）。""" - try: - from app.pgvector_search import index_document - await index_document(doc_id, kb_id, content, metadata={"source": "kb_upload"}) - except Exception as e: - logger.warning(f"[kb] 向量索引失败 doc={doc_id}: {e}") - - -async def upload_document( - base_id: str, - file: Optional[UploadFile] = File(None), - filename: str = Form(...), - file_type: Optional[str] = Form(None), - file_size: Optional[int] = Form(None), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 确定文件扩展名和存储路径 - file_ext = (file_type or "txt").lower() - if file_ext.startswith("."): - file_ext = file_ext[1:] - - # 创建知识库存储目录 - storage_dir = os.path.join(KNOWLEDGE_STORAGE_DIR, base_id) - os.makedirs(storage_dir, exist_ok=True) - - # 生成唯一文件名并存储 - doc_id = gen_uuid() - stored_filename = f"{doc_id}.{file_ext}" - stored_path = os.path.join(storage_dir, stored_filename) - - # 文件内容解析 - actual_size = file_size or 0 - content = "" - uploaded_content = b"" - - # 如果有真实文件上传，先保存并读取内容 - if file and file.filename: - try: - uploaded_content = await file.read() - actual_size = len(uploaded_content) - # 先写入原始文件 - with open(stored_path, "wb") as f: - f.write(uploaded_content) - # 对于文本类格式，尝试直接解析 - text_exts = {"txt", "md", "csv", "json", "yaml", "yml", "xml", "py", "js", "ts", "html", "css", "log"} - if file_ext in text_exts: - try: - content = uploaded_content.decode("utf-8") - except UnicodeDecodeError: - try: - content = uploaded_content.decode("gbk") - except UnicodeDecodeError: - content = f"[二进制文件: {filename}, 大小: {actual_size} bytes]" - else: - # 非文本格式：记录文件但内容用摘要代替 - content = _generate_document_content(filename, file_ext, actual_size) - logger.info(f"非文本文件 {filename}({file_ext}) 已保存到 {stored_path}, 使用模拟内容用于检索") - except Exception as e: - logger.error(f"文件读取失败: {e}") - content = _generate_document_content(filename, file_ext, actual_size) - else: - # 无文件上传（兼容旧 JSON 模式）：生成模拟内容 - actual_size = file_size or len(filename.encode('utf-8')) * 100 - content = _generate_document_content(filename, file_ext, actual_size) - # 写入模拟内容 - try: - with open(stored_path, "w", encoding="utf-8") as f: - f.write(content) - except Exception as e: - logger.error(f"知识库文档写入失败: {stored_path}, error={e}") - raise HTTPException(status_code=500, detail=f"文件存储失败: {str(e)}") - - # 确保文件已写入（防止重复写入） - if uploaded_content and file_ext not in {"txt", "md", "csv", "json", "yaml", "yml", "xml", "py", "js", "ts", "html", "css", "log"}: - pass # 已在上面 wb 写入 - elif not os.path.exists(stored_path): - try: - with open(stored_path, "w", encoding="utf-8") as f: - f.write(content) - except Exception as e: - logger.error(f"知识库文档写入失败: {stored_path}, error={e}") - raise HTTPException(status_code=500, detail=f"文件存储失败: {str(e)}") - - logger.info(f"知识库文档已处理: {stored_path}, ext={file_ext}, size={actual_size}, content_len={len(content)}") - - # 文本分块 - chunks = _split_content_into_chunks(content, chunk_size=500) - - doc = KnowledgeDocument( - id=doc_id, - knowledge_base_id=base_id, - filename=filename, - file_size=actual_size, - file_type=file_ext, - status="completed", - chunk_count=len(chunks), - created_at=datetime.now(), - updated_at=datetime.now(), - ) - db.add(doc) - - # 更新知识库统计 - kb.document_count = (kb.document_count or 0) + 1 - kb.updated_at = datetime.now() - - # 同步创建Asset记录 - _sync_asset_for_knowledge_doc(db, current_user.id, doc_id, filename, file_ext, actual_size) - - db.commit() - db.refresh(doc) - - # T1: 异步索引到 pgvector 语义库（best-effort，不阻塞上传响应） - try: - asyncio.create_task(_index_kb_doc_to_pgvector(doc.id, base_id, content)) - except Exception: - pass - - LogService.log_operation( - user_id=current_user.id, username=current_user.username, action="upload", - target="knowledge_document", target_id=doc.id, - detail=f"上传文档 {doc.filename} 到知识库 {kb.name}, 存储路径: {stored_path}, 分块数: {len(chunks)}", - ) - - return { - "code": 0, - "data": { - "id": doc.id, "base_id": doc.knowledge_base_id, "filename": doc.filename, - "file_size": doc.file_size, "file_type": doc.file_type, - "status": "completed", "chunk_count": doc.chunk_count, - "created_at": str(doc.created_at), - "stored_path": stored_path, - }, - "message": "上传成功", - } - - -def _generate_document_content(filename: str, file_ext: str, file_size: int) -> str: - """根据文件类型生成真实的模拟文档内容（文本解析后的结果）""" - name_base = filename.rsplit(".", 1)[0] - paragraphs = [] - - if file_ext in ("txt", "md"): - paragraphs = [ - f"# {name_base}", - f"", - f"## 概述", - f"本文档记录了关于 {name_base} 的详细内容。该文档由用户上传至知识库，" - f"经过文本解析引擎处理后，提取了以下关键信息。", - f"", - f"## 核心内容", - f"1. 主题分析：{name_base} 涉及的核心领域包括数据管理、知识工程和智能分析。", - f"2. 关键指标：通过对文档内容的深度解析，识别出 {max(3, file_size // 10000)} 个关键概念节点。", - f"3. 关联关系：文档中各主题之间存在 {max(2, file_size // 20000)} 条语义关联。", - f"", - f"## 详细分析", - f"在 {name_base} 的深入研究中，我们发现以下几点值得关注：", - f"- 数据结构层面的优化空间约为 15-20%", - f"- 知识图谱构建的准确率达到 87.3%", - f"- 检索效率相比传统方法提升了 2.3 倍", - f"", - f"## 结论与建议", - f"基于以上分析，建议在以下几个方面进行改进：", - f"1. 持续优化知识提取的准确度", - f"2. 扩展实体识别覆盖范围", - f"3. 增强跨文档关联分析能力", - f"", - f"---", - f"*本文档由 AIFlow 知识库系统自动解析生成，上载时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*", - ] - elif file_ext == "pdf": - paragraphs = [ - f"PDF文档: {name_base}", - f"", - f"【文档信息】", - f"标题：{name_base}", - f"页数估计：{max(1, file_size // 50000)} 页", - f"创建者：AIFlow 用户", - f"", - f"【提取的文本内容】", - f"第一章 引言", - f"本文档旨在阐述 {name_base} 相关的核心概念和实践方法。", - f"随着人工智能技术的不断发展，知识管理系统在企业数字化转型中扮演着越来越重要的角色。", - f"", - f"第二章 方法论", - f"本研究采用混合方法，结合定量分析与定性评估，对 {name_base} 进行全面探讨。", - f"数据来源包括：内部数据库查询结果、用户行为日志、以及与领域专家的深度访谈。", - f"", - f"第三章 结果分析", - f"通过对收集数据的系统分析，得出以下关键发现：", - f"1. 核心指标X的均值为 78.5，标准差为 12.3", - f"2. 关键因素Y与Z之间存在显著相关性（r=0.72, p<0.001）", - f"3. 85% 的受访者认为 {name_base} 是未来发展的关键方向", - f"", - f"第四章 结论", - f"综合以上分析，{name_base} 具有重要的研究价值和实践意义。", - ] - elif file_ext in ("docx", "doc"): - paragraphs = [ - f"Word文档: {name_base}", - f"", - f"{name_base} - 正式文档", - f"", - f"【摘要】", - f"本报告围绕 {name_base} 展开全面分析，涵盖市场趋势、技术演进和应用场景三个维度。", - f"通过系统性的研究和数据采集，为相关决策提供参考依据。", - f"", - f"【正文】", - f"一、背景介绍", - f"在当前的数字化转型浪潮中，{name_base} 已经成为组织提升竞争力的关键要素。", - f"根据Gartner的报告，到2026年将有超过60%的企业采用AI驱动的知识管理方案。", - f"", - f"二、技术架构", - f"系统采用微服务架构设计，包括数据采集层、处理引擎层、存储层和应用层四个核心模块。", - f"其中处理引擎层集成了NLP自然语言处理、实体识别和关系抽取等多种AI能力。", - f"", - f"三、实施计划", - f"第一阶段（1-2月）：完成基础架构搭建和数据迁移", - f"第二阶段（3-4月）：部署AI模型并开始增量学习", - f"第三阶段（5-6月）：全面上线并进行效果评估", - ] - elif file_ext in ("xlsx", "xls"): - paragraphs = [ - f"Excel表格: {name_base}", - f"", - f"【表格数据概览】", - f"| 指标类别 | Q1 | Q2 | Q3 | Q4 | 年度总计 |", - f"|---------|-----|-----|-----|-----|---------|", - f"| 用户增长 | 12% | 18% | 22% | 28% | 80% |", - f"| 营收增幅 | 8% | 14% | 19% | 25% | 66% |", - f"| 客户满意度 | 85 | 88 | 91 | 93 | 89.3 |", - f"| 市场份额 | 15% | 17% | 20% | 23% | -- |", - f"", - f"【关键指标分析】", - f"1. 用户增长率呈逐季上升趋势，Q4达到峰值28%", - f"2. 营收增长与用户增长高度正相关", - f"3. 客户满意度持续改善，Q4达到93分", - f"4. 市场份额稳步扩大，年度总增长约8个百分点", - f"", - f"【数据来源】内部经营分析系统，数据截止日期：{datetime.now().strftime('%Y-%m-%d')}", - ] - elif file_ext in ("pptx", "ppt"): - paragraphs = [ - f"PPT演示文稿: {name_base}", - f"", - f"【幻灯片结构】", - f"第1页：封面 - {name_base}", - f"第2页：目录与议程", - f"第3页：背景与现状分析", - f"第4页：核心挑战与机遇", - f"第5页：解决方案概述", - f"第6-8页：详细方案展开", - f"第9页：实施路线图", - f"第10页：总结与展望", - f"", - f"【关键要点提取】", - f"- 战略目标：实现 {name_base} 的全面数字化转型", - f"- 核心路径：AI赋能 + 数据驱动 + 敏捷迭代", - f"- 预期成果：效率提升30%、成本降低20%、用户满意度提升15%", - f"- 投入预算：约500万元（分三期投入）", - ] - else: - paragraphs = [ - f"文档: {name_base}", - f"", - f"文件类型：{file_ext.upper()}", - f"文件大小：{file_size} bytes", - f"上传时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - f"", - f"【内容摘要】", - f"本文档类型为 {file_ext.upper()} 格式，包含与 {name_base} 相关的内容。", - f"系统已完成基本的文本解析和索引建立，支持关键词检索和语义匹配。", - ] - - return "\n".join(paragraphs) - - -def _split_content_into_chunks(content: str, chunk_size: int = 500) -> list: - """将文档内容分割为文本块（模拟embedding的chunking过程）""" - chunks = [] - paragraphs = content.split("\n\n") - current_chunk = "" - - for para in paragraphs: - if len(current_chunk) + len(para) + 2 <= chunk_size: - current_chunk = (current_chunk + "\n\n" + para).strip() - else: - if current_chunk: - chunks.append(current_chunk) - if len(para) > chunk_size: - # 长段落进一步切分 - for i in range(0, len(para), chunk_size): - chunks.append(para[i:i+chunk_size]) - current_chunk = "" - else: - current_chunk = para - - if current_chunk: - chunks.append(current_chunk) - - return chunks if chunks else [content[:chunk_size]] - - -def _sync_asset_for_knowledge_doc(db: Session, user_id: str, doc_id: str, filename: str, file_type: str, file_size: int): - """将知识库文档同步为资产记录（问题9：资产关联知识库文档用量）""" - try: - # 文件类型归一化为前端子分类标签（docx/doc->word 等），确保出现在「文件 > Word」子栏 - _ft_label = { - "pdf": "pdf", "docx": "word", "doc": "word", "xlsx": "excel", "xls": "excel", - "pptx": "ppt", "ppt": "ppt", "csv": "txt", "md": "md", "txt": "txt", - }.get((file_type or "").lower(), "other") - asset = Asset( - id=gen_uuid(), - user_id=user_id, - name=filename, - category="file", - description=f"知识库文档 (ID: {doc_id})", - file_type=_ft_label, - size=file_size, - status="active", - knowledge_doc_id=doc_id, - created_at=datetime.now(), - updated_at=datetime.now(), - ) - db.add(asset) - logger.info(f"已同步知识库文档到资产: {filename} -> asset_id={asset.id}") - except Exception as e: - logger.warning(f"同步知识库文档到资产失败: {e}") - - -@router.delete("/bases/{base_id}/documents/{doc_id}") -def delete_document( - base_id: str, - doc_id: str, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - doc = db.query(KnowledgeDocument).filter( - and_(KnowledgeDocument.id == doc_id, KnowledgeDocument.knowledge_base_id == base_id) - ).first() - if not doc: - raise HTTPException(status_code=404, detail="文档不存在") - - # 更新知识库统计 - kb.document_count = max(0, (kb.document_count or 0) - 1) - pass # total_size removed - kb.updated_at = datetime.now() - - # 同步删除知识库文档对应的资产记录（双向同步） - try: - linked_asset = db.query(Asset).filter(Asset.knowledge_doc_id == doc_id).first() - if linked_asset: - linked_asset.status = "deleted" - linked_asset.updated_at = datetime.now() - except Exception as _e: - logger.warning(f"同步删除关联资产失败: {_e}") - - db.delete(doc) - db.commit() - - return {"code": 0, "message": "文档已删除"} - - -@router.get("/directories") -def list_directories( - base_id: Optional[str] = Query(None), - parent_id: Optional[str] = Query(None), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - query = db.query(KnowledgeDirectory).filter( - KnowledgeDirectory.user_id == current_user.id - ) - - if base_id: - query = query.filter(KnowledgeDirectory.knowledge_base_id == base_id) - if parent_id is not None: - query = query.filter(KnowledgeDirectory.parent_id == parent_id) - else: - # 默认查根目录 - query = query.filter(KnowledgeDirectory.parent_id.is_(None)) - - directories = query.order_by(asc(KnowledgeDirectory.name)).all() - - result = [] - for d in directories: - result.append({ - "id": d.id, - "name": d.name, - "parent_id": d.parent_id, - "knowledge_base_id": d.knowledge_base_id, - "created_at": str(d.created_at) if d.created_at else None, - }) - - return {"code": 0, "data": result, "message": "ok"} - - -@router.post("/directories") -def create_directory( - req: CreateDirectoryRequest, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - directory = KnowledgeDirectory( - id=gen_uuid(), - user_id=current_user.id, - name=req.name, - parent_id=req.parent_id, - knowledge_base_id=req.knowledge_base_id, - created_at=datetime.now(), - ) - db.add(directory) - db.commit() - db.refresh(directory) - - return { - "code": 0, - "data": { - "id": directory.id, "name": directory.name, - "parent_id": directory.parent_id, "knowledge_base_id": directory.knowledge_base_id, - "created_at": str(directory.created_at), - }, - "message": "创建成功", - } - - -# ============================================================ -# 知识库问答溯源 API -# ============================================================ - -@router.post("/qa") -def knowledge_qa( - req: KnowledgeQARequest, - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """知识库问答：在用户的知识库中检索相关文档并生成回答，返回原文引用和来源路径""" - # 验证知识库归属 - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == req.base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - # 检索知识库中的文档 - documents = db.query(KnowledgeDocument).filter( - and_( - KnowledgeDocument.knowledge_base_id == req.base_id, - KnowledgeDocument.status == "completed", - ) - ).all() - - if not documents: - return { - "code": 0, - "data": { - "question": req.question, - "answer": "该知识库中暂无已完成的文档。请先上传并处理文档后再进行问答。", - "sources": [], - "knowledge_base_name": kb.name, - }, - "message": "ok", - } - - # 构建引用来源列表（基于文档关键词匹配模拟检索） - import re - sources = [] - relevant_docs = [] - question_lower = req.question.lower() - - for doc in documents: - filename_lower = doc.filename.lower() - # 简单的关键词匹配检索（实际生产环境应使用向量检索） - keywords = re.split(r'[\s\-\_\.\,\;\:\!\?\(\)\[\]\{\}]+', question_lower) - filename_words = re.split(r'[\s\-\_\.\,\;\:\!\?\(\)\[\]\{\}]+', filename_lower) - match_count = sum(1 for kw in keywords if kw and any(kw in fw for fw in filename_words)) - - if match_count > 0 or len(documents) <= 3: - relevant_docs.append(doc) - # 为每个匹配文档生成模拟原文引用 - sample_content = generate_document_excerpt(doc, req.question) - sources.append({ - "document_id": doc.id, - "document_name": doc.filename, - "file_type": doc.file_type or "", - "file_size": doc.file_size or 0, - "source_path": f"/knowledge/{kb.id}/documents/{doc.id}", - "excerpt": sample_content, - "relevance": min(0.99, max(0.3, match_count * 0.2 + 0.3)) if match_count > 0 else 0.3, - }) - - # 生成模拟回答 - kb_name = kb.name - answer_parts = [f"根据知识库「{kb_name}」中相关文档的内容，针对您的问题回复如下：\n"] - - if sources: - answer_parts.append(f"共检索到 {len(sources)} 篇相关文档，以下为引用分析：\n") - for i, src in enumerate(sources[:5], 1): - answer_parts.append(f"\n【引用 {i}】来源：《{src['document_name']}》") - answer_parts.append(f"原文摘录：\"{src['excerpt']}\"") - answer_parts.append(f"文档类型：{src['file_type'].upper()} | 文档大小：{format_file_size(src['file_size'])}") - answer_parts.append(f"\n---\n综合以上文档内容，我们建议您参考上述引用来源获取详细信息。") - else: - answer_parts.append("未找到与问题高度相关的文档内容，建议上传更多相关文档后重试。") - - final_answer = "\n".join(answer_parts) - - return { - "code": 0, - "data": { - "question": req.question, - "answer": final_answer, - "sources": sources, - "knowledge_base_name": kb_name, - "total_documents": len(documents), - }, - "message": "ok", - } - - -@router.get("/bases/{base_id}/search") -def search_knowledge( - base_id: str, - keyword: str = Query(min_length=1), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """知识库文档检索：按关键词搜索文档并返回引用片段""" - kb = db.query(KnowledgeBase).filter( - and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) - ).first() - if not kb: - raise HTTPException(status_code=404, detail="知识库不存在") - - documents = db.query(KnowledgeDocument).filter( - and_( - KnowledgeDocument.knowledge_base_id == base_id, - KnowledgeDocument.status == "completed", - KnowledgeDocument.filename.ilike(f"%{keyword}%"), - ) - ).all() - - result = [] - for doc in documents: - result.append({ - "document_id": doc.id, - "document_name": doc.filename, - "file_type": doc.file_type or "", - "file_size": doc.file_size or 0, - "source_path": f"/knowledge/{kb.id}/documents/{doc.id}", - "excerpt": generate_document_excerpt(doc, keyword), - "created_at": str(doc.created_at) if doc.created_at else None, - }) - - return { - "code": 0, - "data": { - "keyword": keyword, - "knowledge_base_name": kb.name, - "results": result, - "total": len(result), - }, - "message": "ok", - } - - -@router.get("/tree-search") -async def tree_search_endpoint( - query: str = Query(min_length=1, description="检索查询"), - kb_id: Optional[str] = Query(default=None, description="限定知识库ID"), - scope: Optional[str] = Query( - default=None, - description="作用域: doc: | dir: | base:", - ), - top_k: int = Query(default=5, ge=1, le=20), - db: Session = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """TreeSearch 结构化文档两级检索。 - - Level 1: 返回用户可见的知识库 -> 目录 -> 文档结构树(tree)。 - Level 2: 在作用域内做分片段(chunk)级精准检索，命中带完整面包屑与片段摘要。 - """ - from app.tree_search import build_kb_tree, tree_search - - # Level 1: 结构树 - tree = build_kb_tree(db, user_id=current_user.id, kb_id=kb_id) - - # Level 2: 分片段检索 - scope_doc = scope_dir = scope_base = None - if scope: - if scope.startswith("doc:"): - scope_doc = scope[len("doc:"):] - elif scope.startswith("dir:"): - scope_dir = scope[len("dir:"):] - elif scope.startswith("base:"): - scope_base = scope[len("base:"):] - - hits = await tree_search( - db, query, - user_id=current_user.id, - kb_ids=[kb_id] if kb_id else None, - scope_base=scope_base, scope_dir=scope_dir, scope_doc=scope_doc, - top_k=top_k, mode="hybrid", - ) - - return { - "code": 0, - "data": { - "query": query, - "tree": tree, - "hits": hits, - "total": len(hits), - }, - "message": "ok", - } - - -def generate_document_excerpt(doc, query: str) -> str: - """根据文档类型生成模拟的原文摘录""" - excerpts = { - "pdf": f"本文档详细阐述了与「{query}」相关的核心内容。根据第3章第2节的分析，关键指标呈现明显的增长趋势，其中核心参数值在统计周期内提升了约27.3%。进一步的研究表明，该趋势与市场整体发展方向高度一致...", - "docx": f"关于「{query}」的讨论，文档中明确指出：\"从数据层面来看，该领域在过去四个季度中保持了平均15%的复合增长率。这一结论基于对超过2000个样本的综合分析，置信度达到95%以上。建议决策层关注以下三个关键方向...\"", - "xlsx": f"数据表格「{query}」相关统计：\n| 指标 | Q1 | Q2 | Q3 | Q4 |\n|------|----|----|----|----|\n| 用户增长 | 12% | 18% | 22% | 28% |\n| 营收增幅 | 8% | 14% | 19% | 25% |\n数据来源：内部经营分析系统", - "ppt": f"演示文稿第15页「{query}」关键要点：\n• 战略目标：实现年度增长30%\n• 核心路径：优化产品矩阵，拓展海外市场\n• 资源配置：预算投入约500万元\n• 时间节点：Q2完成试点，Q4全面推广\n详细方案见第15-22页", - "md": f"# {query}\n\n## 核心观点\n\n文档明确指出：*\"在当前的行业格局下，企业需要通过技术创新和模式变革来建立竞争壁垒。具体来说，可以从以下三个维度切入：数据驱动决策、自动化流程优化、以及个性化用户体验。\"*\n\n## 关键数据\n\n- 市场规模：预计2026年达到200亿\n- 年复合增长率：35%\n- 头部企业市占率：约40%\n\n> 详细分析参见完整文档。", - "txt": f"文档内容摘录：\n\n\"...针对「{query}」这一课题，我们进行了深入的调研和分析。研究发现，超过78%的受访者认为该领域的创新速度正在加快。同时，技术成熟度曲线显示，相关技术有望在未来12-18个月内进入主流应用阶段。建议前期投入研发资源约50人月，预期投资回报周期为18个月...\"", - } - file_type = (doc.file_type or "txt").lower() - return excerpts.get(file_type, excerpts["txt"]) - - -def format_file_size(size_bytes: int) -> str: - """格式化文件大小""" - if size_bytes >= 1024 * 1024: - return f"{size_bytes / (1024 * 1024):.1f} MB" - elif size_bytes >= 1024: - return f"{size_bytes / 1024:.1f} KB" - return f"{size_bytes} B" - - -# ============================================================ -# 知识图谱构建与查询 API -# ============================================================ - -class BuildGraphRequest(BaseModel): - base_id: str - +import asyncio +from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File, Form +from sqlalchemy.orm import Session +from sqlalchemy import and_, desc, asc +from pydantic import BaseModel, Field +from typing import Optional, List +import uuid +import os +import json +import re +import logging +import httpx +from datetime import datetime + +from ..database import get_db +from ..models import ( + User, KnowledgeBase, KnowledgeDocument, KnowledgeDirectory, Asset, + KnowledgeGraphNode, KnowledgeGraphEdge, KnowledgeGraphBuildLog, + AIModel, ModelProvider, + gen_uuid +) +from ..auth import get_current_user +from ..middleware.log_service import LogService + +router = APIRouter() +logger = logging.getLogger("aiflow") + +# 知识库文件存储根目录 +KNOWLEDGE_STORAGE_DIR = os.environ.get("KNOWLEDGE_STORAGE_DIR", "/data/share/project/data/knowledge") + + +class CreateKnowledgeBaseRequest(BaseModel): + name: str = Field(min_length=1, max_length=200) + description: Optional[str] = None + icon: Optional[str] = None + + +class UpdateKnowledgeBaseRequest(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + icon: Optional[str] = None + status: Optional[str] = None + + +class CreateDocumentRequest(BaseModel): + filename: str + file_size: Optional[int] = None + file_type: Optional[str] = None + + +class CreateDirectoryRequest(BaseModel): + name: str = Field(min_length=1, max_length=200) + parent_id: Optional[str] = None + knowledge_base_id: Optional[str] = None + + +class KnowledgeQARequest(BaseModel): + base_id: str + question: str = Field(min_length=1, max_length=5000) + + +@router.get("/bases") +def list_knowledge_bases( + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + bases = ( + db.query(KnowledgeBase) + .filter(KnowledgeBase.user_id == current_user.id) + .order_by(desc(KnowledgeBase.updated_at)) + .all() + ) + + # 如果用户没有知识库，自动创建两个默认知识库（每个用户只创建一次） + if not bases: + # 检查是否已存在名为"默认知识库"或"知识图谱库"的知识库（兜底检查） + existing_default = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.user_id == current_user.id, KnowledgeBase.name == "默认知识库") + ).first() + existing_graph = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.user_id == current_user.id, KnowledgeBase.name == "知识图谱库") + ).first() + + if not existing_default: + default_kb = KnowledgeBase( + id=gen_uuid(), + user_id=current_user.id, + name="默认知识库", + description="联网采集、直接上传的内容默认存储", + directory_path="/", + document_count=0, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + db.add(default_kb) + LogService.log_operation( + user_id=current_user.id, username=current_user.username, action="auto_create", + target="knowledge_base", target_id=default_kb.id, + detail="自动创建默认知识库", + ) + + if not existing_graph: + graph_kb = KnowledgeBase( + id=gen_uuid(), + user_id=current_user.id, + name="知识图谱库", + description="对话转知识图谱、构建知识图谱的默认存储", + directory_path="/", + document_count=0, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + db.add(graph_kb) + LogService.log_operation( + user_id=current_user.id, username=current_user.username, action="auto_create", + target="knowledge_base", target_id=graph_kb.id, + detail="自动创建知识图谱库", + ) + + db.commit() + + # 重新查询 + bases = ( + db.query(KnowledgeBase) + .filter(KnowledgeBase.user_id == current_user.id) + .order_by(desc(KnowledgeBase.updated_at)) + .all() + ) + + # 批量查询所有知识库的真实图谱节点数（从 knowledge_graph_nodes 表统计） + kb_ids = [kb.id for kb in bases] + graph_node_counts = {} + if kb_ids: + from sqlalchemy import func as sa_func + node_stats = db.query( + KnowledgeGraphNode.knowledge_base_id, + sa_func.count(KnowledgeGraphNode.id) + ).filter( + KnowledgeGraphNode.knowledge_base_id.in_(kb_ids) + ).group_by(KnowledgeGraphNode.knowledge_base_id).all() + graph_node_counts = {row[0]: row[1] for row in node_stats} + + result = [] + for kb in bases: + # 使用真实图谱数据，如果图谱表中有数据则用它，否则用旧字段（可能残留旧值） + real_node_count = graph_node_counts.get(kb.id) + if real_node_count is not None and real_node_count > 0: + display_graph_count = real_node_count + else: + display_graph_count = 0 # 清零旧残留值 + + result.append({ + "id": kb.id, + "name": kb.name, + "description": kb.description, + "document_count": kb.document_count or 0, + "knowledge_graph_count": display_graph_count, + "created_at": str(kb.created_at) if kb.created_at else None, + "updated_at": str(kb.updated_at) if kb.updated_at else None, + }) + + # 计算存储空间信息 + # 存储已用：所有知识库文档的真实文件大小总和 + if bases: + from sqlalchemy import func as sa_func + total_size_result = db.query(sa_func.sum(KnowledgeDocument.file_size)).filter( + KnowledgeDocument.knowledge_base_id.in_([kb.id for kb in bases]) + ).scalar() + storage_used = total_size_result or 0 + else: + storage_used = 0 + + # 根据用户会员等级计算存储限额 + membership_level = current_user.membership_level or 0 + storage_limits = {0: 1 * 1024 * 1024 * 1024, 2: 10 * 1024 * 1024 * 1024, 3: 100 * 1024 * 1024 * 1024} + storage_limit = storage_limits.get(membership_level, 1 * 1024 * 1024 * 1024) + + return { + "code": 0, + "data": { + "bases": result, + "storage_used": storage_used, + "storage_limit": storage_limit, + "membership_level": membership_level, + }, + "message": "ok", + } + + +@router.post("/bases") +def create_knowledge_base( + req: CreateKnowledgeBaseRequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = KnowledgeBase( + id=gen_uuid(), + user_id=current_user.id, + name=req.name, + description=req.description, + directory_path="/", + document_count=0, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + db.add(kb) + db.commit() + db.refresh(kb) + + LogService.log_operation( + user_id=current_user.id, username=current_user.username, action="create", + target="knowledge_base", target_id=kb.id, + detail=f"创建知识库 {kb.name}", + ) + + return { + "code": 0, + "data": { + "id": kb.id, "name": kb.name, "description": kb.description, + "created_at": str(kb.created_at), + }, + "message": "创建成功", + } + + +@router.post("/bases/{base_id}/build-graph") +def build_knowledge_graph( + base_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """构建知识图谱：调用LLM分析文档内容提取实体和关系，持久化到数据库""" + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 获取知识库中的已完成文档 + documents = db.query(KnowledgeDocument).filter( + and_( + KnowledgeDocument.knowledge_base_id == base_id, + KnowledgeDocument.status == "completed", + ) + ).all() + + if not documents: + raise HTTPException(status_code=400, detail="知识库中没有已完成的文档，请先上传文档并等待处理完成") + + # 获取LLM配置（获取默认启用的模型及其provider信息） + llm_config = _get_llm_config(db) + if not llm_config: + raise HTTPException(status_code=500, detail="没有可用的AI模型，请先在后台管理中配置模型") + + api_url, api_key, model_name, timeout = llm_config + + # 先清理旧图谱数据 + db.query(KnowledgeGraphEdge).filter( + KnowledgeGraphEdge.knowledge_base_id == base_id + ).delete() + db.query(KnowledgeGraphNode).filter( + KnowledgeGraphNode.knowledge_base_id == base_id + ).delete() + db.query(KnowledgeGraphBuildLog).filter( + KnowledgeGraphBuildLog.knowledge_base_id == base_id + ).delete() + db.flush() + + total_nodes = 0 + total_edges = 0 + + for doc in documents: + # 读取文档文件内容 + file_content = _read_document_content(doc) + + if not file_content or len(file_content.strip()) < 10: + logger.warning(f"文档 {doc.filename} 内容为空或过短，跳过图谱构建") + # 记录构建日志 + build_log = KnowledgeGraphBuildLog( + id=gen_uuid(), + knowledge_base_id=base_id, + document_id=doc.id, + node_count=0, + edge_count=0, + status="completed", + created_at=datetime.now(), + ) + db.add(build_log) + continue + + # 调用LLM提取实体和关系 + entities, relations = _extract_entities_with_llm( + api_url, api_key, model_name, timeout, doc.filename, file_content + ) + + # 将提取结果写入数据库 + doc_node_count, doc_edge_count = _persist_graph_data( + db, base_id, doc.id, entities, relations + ) + total_nodes += doc_node_count + total_edges += doc_edge_count + + # 记录构建日志 + build_log = KnowledgeGraphBuildLog( + id=gen_uuid(), + knowledge_base_id=base_id, + document_id=doc.id, + node_count=doc_node_count, + edge_count=doc_edge_count, + status="completed", + created_at=datetime.now(), + ) + db.add(build_log) + + db.commit() + + # 更新知识库的图谱计数 + kb.knowledge_graph_count = total_nodes + kb.updated_at = datetime.now() + db.commit() + + LogService.log_operation( + user_id=current_user.id, username=current_user.username, action="build_graph", + target="knowledge_base", target_id=base_id, + detail=f"为知识库 {kb.name} 构建知识图谱，共 {total_nodes} 节点、{total_edges} 边", + ) + + # 构建返回数据 + graph_data = _build_graph_response(db, base_id, total_nodes, total_edges, len(documents)) + + return { + "code": 0, + "data": graph_data, + "message": f"知识图谱构建完成，共 {total_nodes} 个节点、{total_edges} 条边", + } + + +def _get_llm_config(db: Session): + """获取LLM配置：优先默认模型，否则使用第一个启用的模型""" + logger.info("[图谱构建] 开始查找LLM配置...") + ai_model = db.query(AIModel).filter( + AIModel.is_default == True, + AIModel.is_enabled == True, + ).first() + + if not ai_model: + ai_model = db.query(AIModel).filter( + AIModel.is_enabled == True, + AIModel.endpoint.isnot(None), + ).first() + logger.info(f"[图谱构建] 无默认模型，尝试使用第一个启用模型: {ai_model.name if ai_model else 'None'}") + + if not ai_model: + logger.error("[图谱构建] 没有可用的AI模型！") + return None + + api_key = ai_model.api_key + api_url = ai_model.endpoint + + # 如果模型本身没有独立的api_key/endpoint，从provider获取 + if (not api_key or not api_url) and ai_model.provider_id: + provider = db.query(ModelProvider).filter(ModelProvider.id == ai_model.provider_id).first() + if provider: + if not api_key: + api_key = provider.api_key + if not api_url and provider.api_base_url: + api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" + + if not api_url or not api_key: + logger.error(f"[图谱构建] LLM配置不完整: api_url={api_url}, api_key={'***' if api_key else 'None'}") + return None + + logger.info(f"[图谱构建] 找到LLM配置: model={ai_model.name}, url={api_url[:50]}..., timeout={ai_model.timeout or 60}") + return (api_url, api_key, ai_model.name, ai_model.timeout or 60) + + +def _read_document_content(doc: KnowledgeDocument) -> str: + """从磁盘读取文档的实际内容""" + storage_dir = os.environ.get("KNOWLEDGE_STORAGE_DIR", "/data/share/project/data/knowledge") + file_path = os.path.join(storage_dir, doc.knowledge_base_id, f"{doc.id}.{doc.file_type}") + + logger.info(f"[图谱构建] 读取文档: {doc.filename}, id={doc.id}, type={doc.file_type}, path={file_path}, status={doc.status}") + + if not os.path.exists(file_path): + # 尝试其他可能的路径 + alt_path = os.path.join(storage_dir, doc.knowledge_base_id, f"{doc.id}.{doc.file_type.lower()}") + if os.path.exists(alt_path): + file_path = alt_path + logger.info(f"[图谱构建] 使用备用路径: {file_path}") + else: + # 尝试直接读file_path字段 + if doc.file_path and os.path.exists(doc.file_path): + file_path = doc.file_path + logger.info(f"[图谱构建] 使用数据库file_path: {file_path}") + else: + logger.warning(f"[图谱构建] 文档文件不存在: {file_path} (备用: {alt_path}, 数据库file_path: {doc.file_path})") + return "" + + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + return content + except Exception as e: + logger.error(f"读取文档文件失败: {file_path}, error={e}") + return "" + + +def _extract_entities_with_llm( + api_url: str, api_key: str, model_name: str, timeout: int, + doc_filename: str, file_content: str +): + """调用LLM分析文档内容，提取实体和关系""" + # 限制内容长度（防止token溢出） + max_content_len = 8000 + if len(file_content) > max_content_len: + truncated_content = file_content[:max_content_len] + "\n\n...(内容已截断)" + else: + truncated_content = file_content + + prompt = f"""分析以下文档内容，识别所有命名实体（人物、组织、地点、产品、技术术语等）和它们之间的关系。 + +文档标题：{doc_filename} + +文档内容： +{truncated_content} + +请以JSON格式返回，格式如下（只返回JSON，不要包含其他内容）： +{{"entities":[{{"name":"实体名","type":"entity/attribute/concept/event","description":"简介","attributes":{{"key":"value"}}}}],"relations":[{{"source":"源实体名","target":"目标实体名","relation":"关系类型","description":"关系描述"}}]}} + +要求： +1. 实体名称使用文档原文中的表述 +2. 实体类型只能从 entity（命名实体）、attribute（属性特征）、concept（核心概念）、event（事件）中选择 +3. 每个实体尽量给出简短描述 +4. 关系需要明确源实体和目标实体的名称 +5. 不要返回不存在的实体或关系""" + + try: + with httpx.Client(timeout=timeout) as client: + resp = client.post( + api_url, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": model_name, + "messages": [ + {"role": "system", "content": "你是一个专业的知识图谱构建助手，擅长从文档中提取实体和关系。你只返回严格符合要求的JSON格式数据。"}, + {"role": "user", "content": prompt}, + ], + "max_tokens": 4096, + "temperature": 0.3, + }, + ) + + if resp.status_code != 200: + logger.error(f"LLM API调用失败 (HTTP {resp.status_code}): {resp.text[:500]}") + return [], [] + + data = resp.json() + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + + # 记录token使用量（可选） + usage = data.get("usage", {}) + logger.info( + f"LLM图谱构建 token使用: prompt={usage.get('prompt_tokens', 0)}, " + f"completion={usage.get('completion_tokens', 0)}, " + f"total={usage.get('total_tokens', 0)}" + ) + + if not content: + logger.error("LLM返回空内容") + return [], [] + + # 解析LLM返回的JSON + entities, relations = _parse_llm_graph_response(content) + return entities, relations + + except httpx.TimeoutException: + logger.error(f"LLM API调用超时: {api_url}") + return [], [] + except httpx.ConnectError: + logger.error(f"LLM API连接失败: {api_url}") + return [], [] + except Exception as e: + logger.error(f"LLM API调用异常: {e}") + return [], [] + + +def _parse_llm_graph_response(content: str): + """解析LLM返回的图谱JSON，做容错处理""" + entities = [] + relations = [] + + # 尝试多种方式提取JSON + # 方式1: 直接解析 + try: + result = json.loads(content) + entities = result.get("entities", []) + relations = result.get("relations", []) + return entities, relations + except json.JSONDecodeError: + pass + + # 方式2: 提取```json ... ```代码块 + json_match = re.search(r'```(?:json)?\s*([\s\S]*?)```', content) + if json_match: + try: + result = json.loads(json_match.group(1)) + entities = result.get("entities", []) + relations = result.get("relations", []) + return entities, relations + except json.JSONDecodeError: + pass + + # 方式3: 提取最外层的{...} + brace_match = re.search(r'\{[\s\S]*\}', content) + if brace_match: + try: + result = json.loads(brace_match.group(0)) + entities = result.get("entities", []) + relations = result.get("relations", []) + return entities, relations + except json.JSONDecodeError: + pass + + logger.warning(f"无法解析LLM返回的JSON: {content[:500]}") + return [], [] + + +def _persist_graph_data(db: Session, base_id: str, doc_id: str, entities: list, relations: list): + """将LLM提取的实体和关系持久化到数据库，返回(节点数, 边数)""" + # 先创建节点，记录name到node_id的映射 + name_to_node_id = {} + node_count = 0 + + for entity in entities: + if not isinstance(entity, dict): + continue + name = entity.get("name", "").strip() + if not name: + continue + entity_type = entity.get("type", "entity") + if entity_type not in ("entity", "attribute", "concept", "event"): + entity_type = "entity" + description = entity.get("description", "") + attributes = entity.get("attributes", {}) + if not isinstance(attributes, dict): + attributes = {} + + node_id = gen_uuid() + node = KnowledgeGraphNode( + id=node_id, + knowledge_base_id=base_id, + source_doc_id=doc_id, + label=name[:500], + type=entity_type, + properties=attributes, + description=description[:65535] if description else None, + created_at=datetime.now(), + ) + db.add(node) + name_to_node_id[name] = node_id + node_count += 1 + + # 必须flush才能获取节点ID用于外键关联 + db.flush() + + # 创建边 + edge_count = 0 + for rel in relations: + if not isinstance(rel, dict): + continue + source_name = rel.get("source", "").strip() + target_name = rel.get("target", "").strip() + if not source_name or not target_name: + continue + source_node_id = name_to_node_id.get(source_name) + target_node_id = name_to_node_id.get(target_name) + if not source_node_id or not target_node_id: + # 如果引用了LLM返回中不存在的实体名，跳过 + logger.debug(f"关系引用了未注册实体: {source_name} -> {target_name}") + continue + + edge = KnowledgeGraphEdge( + id=gen_uuid(), + knowledge_base_id=base_id, + source_node_id=source_node_id, + target_node_id=target_node_id, + relation=rel.get("relation", "关联")[:200], + weight=float(rel.get("weight", 1.0)) if rel.get("weight") else 1.0, + description=rel.get("description", "")[:65535] if rel.get("description") else None, + created_at=datetime.now(), + ) + db.add(edge) + edge_count += 1 + + return node_count, edge_count + + +def _build_graph_response(db: Session, base_id: str, total_nodes: int, total_edges: int, doc_count: int): + """从数据库读取图谱数据并构建响应""" + nodes = db.query(KnowledgeGraphNode).filter( + KnowledgeGraphNode.knowledge_base_id == base_id + ).order_by(KnowledgeGraphNode.created_at.asc()).all() + + edges = db.query(KnowledgeGraphEdge).filter( + KnowledgeGraphEdge.knowledge_base_id == base_id + ).order_by(KnowledgeGraphEdge.created_at.asc()).all() + + graph_nodes = [] + for node in nodes: + graph_nodes.append({ + "id": node.id, + "label": node.label, + "type": node.type, + "size": 10 + (len(node.description) % 10 if node.description else 5), + "group": node.type, + "description": node.description, + "properties": node.properties or {}, + "source_doc_id": node.source_doc_id, + }) + + graph_edges = [] + for edge in edges: + graph_edges.append({ + "id": edge.id, + "source": edge.source_node_id, + "target": edge.target_node_id, + "label": edge.relation, + "weight": edge.weight, + "description": edge.description, + }) + + # 统计各类型节点数量 + entity_types = {} + for node in nodes: + t = node.type + entity_types[t] = entity_types.get(t, 0) + 1 + + return { + "nodes": graph_nodes, + "edges": graph_edges, + "stats": { + "total_nodes": total_nodes, + "total_edges": total_edges, + "document_count": doc_count, + "entity_types": entity_types, + }, + } + + +@router.get("/bases/{base_id}/graph") +def get_knowledge_graph( + base_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """获取知识图谱数据：从持久化表中读取已构建的图谱""" + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 从数据库表读取图谱节点 + nodes = db.query(KnowledgeGraphNode).filter( + KnowledgeGraphNode.knowledge_base_id == base_id + ).order_by(KnowledgeGraphNode.created_at.asc()).all() + + if not nodes: + # 清理旧残留数据：如果 knowledge_graph_count > 0 但图谱表中无节点，清零旧值 + if (kb.knowledge_graph_count or 0) > 0: + kb.knowledge_graph_count = 0 + db.commit() + + # 检查是否有文档但尚未构建图谱 + doc_count = db.query(KnowledgeDocument).filter( + and_( + KnowledgeDocument.knowledge_base_id == base_id, + KnowledgeDocument.status == "completed", + ) + ).count() + + return { + "code": 0, + "data": { + "nodes": [], + "edges": [], + "stats": {"total_nodes": 0, "total_edges": 0, "document_count": doc_count}, + }, + "message": "暂无可用于构建图谱的文档，请先构建知识图谱", + } + + # 从数据库表读取图谱边 + edges = db.query(KnowledgeGraphEdge).filter( + KnowledgeGraphEdge.knowledge_base_id == base_id + ).order_by(KnowledgeGraphEdge.created_at.asc()).all() + + graph_nodes = [] + for node in nodes: + graph_nodes.append({ + "id": node.id, + "label": node.label, + "type": node.type, + "size": 10 + (len(node.description) % 10 if node.description else 5), + "group": node.type, + "description": node.description, + "properties": node.properties or {}, + "source_doc_id": node.source_doc_id, + }) + + graph_edges = [] + for edge in edges: + graph_edges.append({ + "id": edge.id, + "source": edge.source_node_id, + "target": edge.target_node_id, + "label": edge.relation, + "weight": edge.weight, + "description": edge.description, + }) + + # 统计各类型节点数量 + entity_types = {} + for node in nodes: + t = node.type + entity_types[t] = entity_types.get(t, 0) + 1 + + graph_data = { + "nodes": graph_nodes, + "edges": graph_edges, + "stats": { + "total_nodes": len(graph_nodes), + "total_edges": len(graph_edges), + "document_count": len(set(n.source_doc_id for n in nodes if n.source_doc_id)), + "entity_types": entity_types, + }, + } + + return {"code": 0, "data": graph_data, "message": "ok"} + +@router.get("/bases/{base_id}") +def get_knowledge_base( + base_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + return { + "code": 0, + "data": { + "id": kb.id, "name": kb.name, "description": kb.description, + "document_count": kb.document_count or 0, + "created_at": str(kb.created_at), "updated_at": str(kb.updated_at), + }, + "message": "ok", + } + + +@router.put("/bases/{base_id}") +def update_knowledge_base( + base_id: str, + req: UpdateKnowledgeBaseRequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + if req.name is not None: + kb.name = req.name + if req.description is not None: + kb.description = req.description + if req.icon is not None: + pass # icon not in model + if req.status is not None: + pass # status not in model + kb.updated_at = datetime.now() + + db.commit() + db.refresh(kb) + + return {"code": 0, "data": { + "id": kb.id, "name": kb.name, "description": kb.description, + }, "message": "更新成功"} + + +@router.delete("/bases/{base_id}") +def delete_knowledge_base( + base_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 删除知识库下的所有文档 + db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == base_id).delete() + db.delete(kb) + db.commit() + + return {"code": 0, "message": "知识库已删除"} + + +@router.get("/bases/{base_id}/documents") +def get_documents( + base_id: str, + status: Optional[str] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + # 先验证知识库归属 + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + query = db.query(KnowledgeDocument).filter(KnowledgeDocument.knowledge_base_id == base_id) + if status: + query = query.filter(KnowledgeDocument.status == status) + + documents = query.order_by(desc(KnowledgeDocument.created_at)).all() + + result = [] + for doc in documents: + result.append({ + "id": doc.id, + "base_id": doc.knowledge_base_id, + "filename": doc.filename, + "file_size": doc.file_size, + "file_type": doc.file_type, + "status": doc.status, + "chunk_count": doc.chunk_count or 0, + "created_at": str(doc.created_at) if doc.created_at else None, + "updated_at": str(doc.updated_at) if doc.updated_at else None, + }) + + return {"code": 0, "data": result, "message": "ok"} + + +@router.post("/bases/{base_id}/documents") +async def _index_kb_doc_to_pgvector(doc_id: str, kb_id: str, content: str): + """将文档内容异步索引到 pgvector，供语义向量检索使用（best-effort）。""" + try: + from app.pgvector_search import index_document + await index_document(doc_id, kb_id, content, metadata={"source": "kb_upload"}) + except Exception as e: + logger.warning(f"[kb] 向量索引失败 doc={doc_id}: {e}") + + +async def upload_document( + base_id: str, + file: Optional[UploadFile] = File(None), + filename: str = Form(...), + file_type: Optional[str] = Form(None), + file_size: Optional[int] = Form(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 确定文件扩展名和存储路径 + file_ext = (file_type or "txt").lower() + if file_ext.startswith("."): + file_ext = file_ext[1:] + + # 创建知识库存储目录 + storage_dir = os.path.join(KNOWLEDGE_STORAGE_DIR, base_id) + os.makedirs(storage_dir, exist_ok=True) + + # 生成唯一文件名并存储 + doc_id = gen_uuid() + stored_filename = f"{doc_id}.{file_ext}" + stored_path = os.path.join(storage_dir, stored_filename) + + # 文件内容解析 + actual_size = file_size or 0 + content = "" + uploaded_content = b"" + + # 如果有真实文件上传，先保存并读取内容 + if file and file.filename: + try: + uploaded_content = await file.read() + actual_size = len(uploaded_content) + # 先写入原始文件 + with open(stored_path, "wb") as f: + f.write(uploaded_content) + # 对于文本类格式，尝试直接解析 + text_exts = {"txt", "md", "csv", "json", "yaml", "yml", "xml", "py", "js", "ts", "html", "css", "log"} + if file_ext in text_exts: + try: + content = uploaded_content.decode("utf-8") + except UnicodeDecodeError: + try: + content = uploaded_content.decode("gbk") + except UnicodeDecodeError: + content = f"[二进制文件: {filename}, 大小: {actual_size} bytes]" + else: + # 非文本格式：记录文件但内容用摘要代替 + content = _generate_document_content(filename, file_ext, actual_size) + logger.info(f"非文本文件 {filename}({file_ext}) 已保存到 {stored_path}, 使用模拟内容用于检索") + except Exception as e: + logger.error(f"文件读取失败: {e}") + content = _generate_document_content(filename, file_ext, actual_size) + else: + # 无文件上传（兼容旧 JSON 模式）：生成模拟内容 + actual_size = file_size or len(filename.encode('utf-8')) * 100 + content = _generate_document_content(filename, file_ext, actual_size) + # 写入模拟内容 + try: + with open(stored_path, "w", encoding="utf-8") as f: + f.write(content) + except Exception as e: + logger.error(f"知识库文档写入失败: {stored_path}, error={e}") + raise HTTPException(status_code=500, detail=f"文件存储失败: {str(e)}") + + # 确保文件已写入（防止重复写入） + if uploaded_content and file_ext not in {"txt", "md", "csv", "json", "yaml", "yml", "xml", "py", "js", "ts", "html", "css", "log"}: + pass # 已在上面 wb 写入 + elif not os.path.exists(stored_path): + try: + with open(stored_path, "w", encoding="utf-8") as f: + f.write(content) + except Exception as e: + logger.error(f"知识库文档写入失败: {stored_path}, error={e}") + raise HTTPException(status_code=500, detail=f"文件存储失败: {str(e)}") + + logger.info(f"知识库文档已处理: {stored_path}, ext={file_ext}, size={actual_size}, content_len={len(content)}") + + # 文本分块 + chunks = _split_content_into_chunks(content, chunk_size=500) + + doc = KnowledgeDocument( + id=doc_id, + knowledge_base_id=base_id, + filename=filename, + file_size=actual_size, + file_type=file_ext, + status="completed", + chunk_count=len(chunks), + created_at=datetime.now(), + updated_at=datetime.now(), + ) + db.add(doc) + + # 更新知识库统计 + kb.document_count = (kb.document_count or 0) + 1 + kb.updated_at = datetime.now() + + # 同步创建Asset记录 + _sync_asset_for_knowledge_doc(db, current_user.id, doc_id, filename, file_ext, actual_size) + + db.commit() + db.refresh(doc) + + # T1: 异步索引到 pgvector 语义库（best-effort，不阻塞上传响应） + try: + asyncio.create_task(_index_kb_doc_to_pgvector(doc.id, base_id, content)) + except Exception: + pass + + LogService.log_operation( + user_id=current_user.id, username=current_user.username, action="upload", + target="knowledge_document", target_id=doc.id, + detail=f"上传文档 {doc.filename} 到知识库 {kb.name}, 存储路径: {stored_path}, 分块数: {len(chunks)}", + ) + + return { + "code": 0, + "data": { + "id": doc.id, "base_id": doc.knowledge_base_id, "filename": doc.filename, + "file_size": doc.file_size, "file_type": doc.file_type, + "status": "completed", "chunk_count": doc.chunk_count, + "created_at": str(doc.created_at), + "stored_path": stored_path, + }, + "message": "上传成功", + } + + +def _generate_document_content(filename: str, file_ext: str, file_size: int) -> str: + """根据文件类型生成真实的模拟文档内容（文本解析后的结果）""" + name_base = filename.rsplit(".", 1)[0] + paragraphs = [] + + if file_ext in ("txt", "md"): + paragraphs = [ + f"# {name_base}", + f"", + f"## 概述", + f"本文档记录了关于 {name_base} 的详细内容。该文档由用户上传至知识库，" + f"经过文本解析引擎处理后，提取了以下关键信息。", + f"", + f"## 核心内容", + f"1. 主题分析：{name_base} 涉及的核心领域包括数据管理、知识工程和智能分析。", + f"2. 关键指标：通过对文档内容的深度解析，识别出 {max(3, file_size // 10000)} 个关键概念节点。", + f"3. 关联关系：文档中各主题之间存在 {max(2, file_size // 20000)} 条语义关联。", + f"", + f"## 详细分析", + f"在 {name_base} 的深入研究中，我们发现以下几点值得关注：", + f"- 数据结构层面的优化空间约为 15-20%", + f"- 知识图谱构建的准确率达到 87.3%", + f"- 检索效率相比传统方法提升了 2.3 倍", + f"", + f"## 结论与建议", + f"基于以上分析，建议在以下几个方面进行改进：", + f"1. 持续优化知识提取的准确度", + f"2. 扩展实体识别覆盖范围", + f"3. 增强跨文档关联分析能力", + f"", + f"---", + f"*本文档由 AIFlow 知识库系统自动解析生成，上载时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*", + ] + elif file_ext == "pdf": + paragraphs = [ + f"PDF文档: {name_base}", + f"", + f"【文档信息】", + f"标题：{name_base}", + f"页数估计：{max(1, file_size // 50000)} 页", + f"创建者：AIFlow 用户", + f"", + f"【提取的文本内容】", + f"第一章 引言", + f"本文档旨在阐述 {name_base} 相关的核心概念和实践方法。", + f"随着人工智能技术的不断发展，知识管理系统在企业数字化转型中扮演着越来越重要的角色。", + f"", + f"第二章 方法论", + f"本研究采用混合方法，结合定量分析与定性评估，对 {name_base} 进行全面探讨。", + f"数据来源包括：内部数据库查询结果、用户行为日志、以及与领域专家的深度访谈。", + f"", + f"第三章 结果分析", + f"通过对收集数据的系统分析，得出以下关键发现：", + f"1. 核心指标X的均值为 78.5，标准差为 12.3", + f"2. 关键因素Y与Z之间存在显著相关性（r=0.72, p<0.001）", + f"3. 85% 的受访者认为 {name_base} 是未来发展的关键方向", + f"", + f"第四章 结论", + f"综合以上分析，{name_base} 具有重要的研究价值和实践意义。", + ] + elif file_ext in ("docx", "doc"): + paragraphs = [ + f"Word文档: {name_base}", + f"", + f"{name_base} - 正式文档", + f"", + f"【摘要】", + f"本报告围绕 {name_base} 展开全面分析，涵盖市场趋势、技术演进和应用场景三个维度。", + f"通过系统性的研究和数据采集，为相关决策提供参考依据。", + f"", + f"【正文】", + f"一、背景介绍", + f"在当前的数字化转型浪潮中，{name_base} 已经成为组织提升竞争力的关键要素。", + f"根据Gartner的报告，到2026年将有超过60%的企业采用AI驱动的知识管理方案。", + f"", + f"二、技术架构", + f"系统采用微服务架构设计，包括数据采集层、处理引擎层、存储层和应用层四个核心模块。", + f"其中处理引擎层集成了NLP自然语言处理、实体识别和关系抽取等多种AI能力。", + f"", + f"三、实施计划", + f"第一阶段（1-2月）：完成基础架构搭建和数据迁移", + f"第二阶段（3-4月）：部署AI模型并开始增量学习", + f"第三阶段（5-6月）：全面上线并进行效果评估", + ] + elif file_ext in ("xlsx", "xls"): + paragraphs = [ + f"Excel表格: {name_base}", + f"", + f"【表格数据概览】", + f"| 指标类别 | Q1 | Q2 | Q3 | Q4 | 年度总计 |", + f"|---------|-----|-----|-----|-----|---------|", + f"| 用户增长 | 12% | 18% | 22% | 28% | 80% |", + f"| 营收增幅 | 8% | 14% | 19% | 25% | 66% |", + f"| 客户满意度 | 85 | 88 | 91 | 93 | 89.3 |", + f"| 市场份额 | 15% | 17% | 20% | 23% | -- |", + f"", + f"【关键指标分析】", + f"1. 用户增长率呈逐季上升趋势，Q4达到峰值28%", + f"2. 营收增长与用户增长高度正相关", + f"3. 客户满意度持续改善，Q4达到93分", + f"4. 市场份额稳步扩大，年度总增长约8个百分点", + f"", + f"【数据来源】内部经营分析系统，数据截止日期：{datetime.now().strftime('%Y-%m-%d')}", + ] + elif file_ext in ("pptx", "ppt"): + paragraphs = [ + f"PPT演示文稿: {name_base}", + f"", + f"【幻灯片结构】", + f"第1页：封面 - {name_base}", + f"第2页：目录与议程", + f"第3页：背景与现状分析", + f"第4页：核心挑战与机遇", + f"第5页：解决方案概述", + f"第6-8页：详细方案展开", + f"第9页：实施路线图", + f"第10页：总结与展望", + f"", + f"【关键要点提取】", + f"- 战略目标：实现 {name_base} 的全面数字化转型", + f"- 核心路径：AI赋能 + 数据驱动 + 敏捷迭代", + f"- 预期成果：效率提升30%、成本降低20%、用户满意度提升15%", + f"- 投入预算：约500万元（分三期投入）", + ] + else: + paragraphs = [ + f"文档: {name_base}", + f"", + f"文件类型：{file_ext.upper()}", + f"文件大小：{file_size} bytes", + f"上传时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + f"", + f"【内容摘要】", + f"本文档类型为 {file_ext.upper()} 格式，包含与 {name_base} 相关的内容。", + f"系统已完成基本的文本解析和索引建立，支持关键词检索和语义匹配。", + ] + + return "\n".join(paragraphs) + + +def _split_content_into_chunks(content: str, chunk_size: int = 500) -> list: + """将文档内容分割为文本块（模拟embedding的chunking过程）""" + chunks = [] + paragraphs = content.split("\n\n") + current_chunk = "" + + for para in paragraphs: + if len(current_chunk) + len(para) + 2 <= chunk_size: + current_chunk = (current_chunk + "\n\n" + para).strip() + else: + if current_chunk: + chunks.append(current_chunk) + if len(para) > chunk_size: + # 长段落进一步切分 + for i in range(0, len(para), chunk_size): + chunks.append(para[i:i+chunk_size]) + current_chunk = "" + else: + current_chunk = para + + if current_chunk: + chunks.append(current_chunk) + + return chunks if chunks else [content[:chunk_size]] + + +def _sync_asset_for_knowledge_doc(db: Session, user_id: str, doc_id: str, filename: str, file_type: str, file_size: int): + """将知识库文档同步为资产记录（问题9：资产关联知识库文档用量）""" + try: + # 文件类型归一化为前端子分类标签（docx/doc->word 等），确保出现在「文件 > Word」子栏 + _ft_label = { + "pdf": "pdf", "docx": "word", "doc": "word", "xlsx": "excel", "xls": "excel", + "pptx": "ppt", "ppt": "ppt", "csv": "txt", "md": "md", "txt": "txt", + }.get((file_type or "").lower(), "other") + asset = Asset( + id=gen_uuid(), + user_id=user_id, + name=filename, + category="file", + description=f"知识库文档 (ID: {doc_id})", + file_type=_ft_label, + size=file_size, + status="active", + knowledge_doc_id=doc_id, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + db.add(asset) + logger.info(f"已同步知识库文档到资产: {filename} -> asset_id={asset.id}") + except Exception as e: + logger.warning(f"同步知识库文档到资产失败: {e}") + + +@router.delete("/bases/{base_id}/documents/{doc_id}") +def delete_document( + base_id: str, + doc_id: str, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + doc = db.query(KnowledgeDocument).filter( + and_(KnowledgeDocument.id == doc_id, KnowledgeDocument.knowledge_base_id == base_id) + ).first() + if not doc: + raise HTTPException(status_code=404, detail="文档不存在") + + # 更新知识库统计 + kb.document_count = max(0, (kb.document_count or 0) - 1) + pass # total_size removed + kb.updated_at = datetime.now() + + # 同步删除知识库文档对应的资产记录（双向同步） + try: + linked_asset = db.query(Asset).filter(Asset.knowledge_doc_id == doc_id).first() + if linked_asset: + linked_asset.status = "deleted" + linked_asset.updated_at = datetime.now() + except Exception as _e: + logger.warning(f"同步删除关联资产失败: {_e}") + + db.delete(doc) + db.commit() + + return {"code": 0, "message": "文档已删除"} + + +@router.get("/directories") +def list_directories( + base_id: Optional[str] = Query(None), + parent_id: Optional[str] = Query(None), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + query = db.query(KnowledgeDirectory).filter( + KnowledgeDirectory.user_id == current_user.id + ) + + if base_id: + query = query.filter(KnowledgeDirectory.knowledge_base_id == base_id) + if parent_id is not None: + query = query.filter(KnowledgeDirectory.parent_id == parent_id) + else: + # 默认查根目录 + query = query.filter(KnowledgeDirectory.parent_id.is_(None)) + + directories = query.order_by(asc(KnowledgeDirectory.name)).all() + + result = [] + for d in directories: + result.append({ + "id": d.id, + "name": d.name, + "parent_id": d.parent_id, + "knowledge_base_id": d.knowledge_base_id, + "created_at": str(d.created_at) if d.created_at else None, + }) + + return {"code": 0, "data": result, "message": "ok"} + + +@router.post("/directories") +def create_directory( + req: CreateDirectoryRequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + directory = KnowledgeDirectory( + id=gen_uuid(), + user_id=current_user.id, + name=req.name, + parent_id=req.parent_id, + knowledge_base_id=req.knowledge_base_id, + created_at=datetime.now(), + ) + db.add(directory) + db.commit() + db.refresh(directory) + + return { + "code": 0, + "data": { + "id": directory.id, "name": directory.name, + "parent_id": directory.parent_id, "knowledge_base_id": directory.knowledge_base_id, + "created_at": str(directory.created_at), + }, + "message": "创建成功", + } + + +# ============================================================ +# 知识库问答溯源 API +# ============================================================ + +@router.post("/qa") +def knowledge_qa( + req: KnowledgeQARequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """知识库问答：在用户的知识库中检索相关文档并生成回答，返回原文引用和来源路径""" + # 验证知识库归属 + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == req.base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + # 检索知识库中的文档 + documents = db.query(KnowledgeDocument).filter( + and_( + KnowledgeDocument.knowledge_base_id == req.base_id, + KnowledgeDocument.status == "completed", + ) + ).all() + + if not documents: + return { + "code": 0, + "data": { + "question": req.question, + "answer": "该知识库中暂无已完成的文档。请先上传并处理文档后再进行问答。", + "sources": [], + "knowledge_base_name": kb.name, + }, + "message": "ok", + } + + # 构建引用来源列表（基于文档关键词匹配模拟检索） + import re + sources = [] + relevant_docs = [] + question_lower = req.question.lower() + + for doc in documents: + filename_lower = doc.filename.lower() + # 简单的关键词匹配检索（实际生产环境应使用向量检索） + keywords = re.split(r'[\s\-\_\.\,\;\:\!\?\(\)\[\]\{\}]+', question_lower) + filename_words = re.split(r'[\s\-\_\.\,\;\:\!\?\(\)\[\]\{\}]+', filename_lower) + match_count = sum(1 for kw in keywords if kw and any(kw in fw for fw in filename_words)) + + if match_count > 0 or len(documents) <= 3: + relevant_docs.append(doc) + # 为每个匹配文档生成模拟原文引用 + sample_content = generate_document_excerpt(doc, req.question) + sources.append({ + "document_id": doc.id, + "document_name": doc.filename, + "file_type": doc.file_type or "", + "file_size": doc.file_size or 0, + "source_path": f"/knowledge/{kb.id}/documents/{doc.id}", + "excerpt": sample_content, + "relevance": min(0.99, max(0.3, match_count * 0.2 + 0.3)) if match_count > 0 else 0.3, + }) + + # 生成模拟回答 + kb_name = kb.name + answer_parts = [f"根据知识库「{kb_name}」中相关文档的内容，针对您的问题回复如下：\n"] + + if sources: + answer_parts.append(f"共检索到 {len(sources)} 篇相关文档，以下为引用分析：\n") + for i, src in enumerate(sources[:5], 1): + answer_parts.append(f"\n【引用 {i}】来源：《{src['document_name']}》") + answer_parts.append(f"原文摘录：\"{src['excerpt']}\"") + answer_parts.append(f"文档类型：{src['file_type'].upper()} | 文档大小：{format_file_size(src['file_size'])}") + answer_parts.append(f"\n---\n综合以上文档内容，我们建议您参考上述引用来源获取详细信息。") + else: + answer_parts.append("未找到与问题高度相关的文档内容，建议上传更多相关文档后重试。") + + final_answer = "\n".join(answer_parts) + + return { + "code": 0, + "data": { + "question": req.question, + "answer": final_answer, + "sources": sources, + "knowledge_base_name": kb_name, + "total_documents": len(documents), + }, + "message": "ok", + } + + +@router.get("/bases/{base_id}/search") +def search_knowledge( + base_id: str, + keyword: str = Query(min_length=1), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """知识库文档检索：按关键词搜索文档并返回引用片段""" + kb = db.query(KnowledgeBase).filter( + and_(KnowledgeBase.id == base_id, KnowledgeBase.user_id == current_user.id) + ).first() + if not kb: + raise HTTPException(status_code=404, detail="知识库不存在") + + documents = db.query(KnowledgeDocument).filter( + and_( + KnowledgeDocument.knowledge_base_id == base_id, + KnowledgeDocument.status == "completed", + KnowledgeDocument.filename.ilike(f"%{keyword}%"), + ) + ).all() + + result = [] + for doc in documents: + result.append({ + "document_id": doc.id, + "document_name": doc.filename, + "file_type": doc.file_type or "", + "file_size": doc.file_size or 0, + "source_path": f"/knowledge/{kb.id}/documents/{doc.id}", + "excerpt": generate_document_excerpt(doc, keyword), + "created_at": str(doc.created_at) if doc.created_at else None, + }) + + return { + "code": 0, + "data": { + "keyword": keyword, + "knowledge_base_name": kb.name, + "results": result, + "total": len(result), + }, + "message": "ok", + } + + +@router.get("/tree-search") +async def tree_search_endpoint( + query: str = Query(min_length=1, description="检索查询"), + kb_id: Optional[str] = Query(default=None, description="限定知识库ID"), + scope: Optional[str] = Query( + default=None, + description="作用域: doc: | dir: | base:", + ), + top_k: int = Query(default=5, ge=1, le=20), + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """TreeSearch 结构化文档两级检索。 + + Level 1: 返回用户可见的知识库 -> 目录 -> 文档结构树(tree)。 + Level 2: 在作用域内做分片段(chunk)级精准检索，命中带完整面包屑与片段摘要。 + """ + from app.tree_search import build_kb_tree, tree_search + + # Level 1: 结构树 + tree = build_kb_tree(db, user_id=current_user.id, kb_id=kb_id) + + # Level 2: 分片段检索 + scope_doc = scope_dir = scope_base = None + if scope: + if scope.startswith("doc:"): + scope_doc = scope[len("doc:"):] + elif scope.startswith("dir:"): + scope_dir = scope[len("dir:"):] + elif scope.startswith("base:"): + scope_base = scope[len("base:"):] + + hits = await tree_search( + db, query, + user_id=current_user.id, + kb_ids=[kb_id] if kb_id else None, + scope_base=scope_base, scope_dir=scope_dir, scope_doc=scope_doc, + top_k=top_k, mode="hybrid", + ) + + return { + "code": 0, + "data": { + "query": query, + "tree": tree, + "hits": hits, + "total": len(hits), + }, + "message": "ok", + } + + +class WebSearchRequest(BaseModel): + query: str = Field(min_length=1, max_length=500) + top_k: int = 5 + + +@router.post("/web-search", summary="AnySearch 联网检索") +async def web_search_endpoint( + req: WebSearchRequest, + db: Session = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """联网检索（AnySearch）：返回实时网页结果，供前端 / 智能体引用。""" + try: + from sqlalchemy import text + import json as _json + from ..anysearch_client import anysearch_search + row = db.execute(text( + "SELECT api_key, api_url, config FROM external_kb_services " + "WHERE service_type='anysearch' AND is_enabled=1 LIMIT 1" + )).fetchone() + if not row: + return {"code": 0, "data": {"results": [], "enabled": False, "message": "未配置 AnySearch 联网检索服务"}} + api_key = row[0] or "" + api_url = row[1] or "https://api.anysearch.com/v1/search" + cfg = row[2] + if isinstance(cfg, str): + try: + cfg = _json.loads(cfg) + except Exception: + cfg = {} + cfg = cfg or {} + zone = cfg.get("zone", "cn") + freshness = cfg.get("freshness") + hits = await anysearch_search(api_key, api_url, req.query, top_k=req.top_k, zone=zone, freshness=freshness) + return {"code": 0, "data": {"results": hits, "enabled": True}} + except Exception as e: + logger.warning(f"web-search endpoint failed: {e}") + return {"code": 1, "message": str(e), "data": {"results": []}} + + +def generate_document_excerpt(doc, query: str) -> str: + """根据文档类型生成模拟的原文摘录""" + excerpts = { + "pdf": f"本文档详细阐述了与「{query}」相关的核心内容。根据第3章第2节的分析，关键指标呈现明显的增长趋势，其中核心参数值在统计周期内提升了约27.3%。进一步的研究表明，该趋势与市场整体发展方向高度一致...", + "docx": f"关于「{query}」的讨论，文档中明确指出：\"从数据层面来看，该领域在过去四个季度中保持了平均15%的复合增长率。这一结论基于对超过2000个样本的综合分析，置信度达到95%以上。建议决策层关注以下三个关键方向...\"", + "xlsx": f"数据表格「{query}」相关统计：\n| 指标 | Q1 | Q2 | Q3 | Q4 |\n|------|----|----|----|----|\n| 用户增长 | 12% | 18% | 22% | 28% |\n| 营收增幅 | 8% | 14% | 19% | 25% |\n数据来源：内部经营分析系统", + "ppt": f"演示文稿第15页「{query}」关键要点：\n• 战略目标：实现年度增长30%\n• 核心路径：优化产品矩阵，拓展海外市场\n• 资源配置：预算投入约500万元\n• 时间节点：Q2完成试点，Q4全面推广\n详细方案见第15-22页", + "md": f"# {query}\n\n## 核心观点\n\n文档明确指出：*\"在当前的行业格局下，企业需要通过技术创新和模式变革来建立竞争壁垒。具体来说，可以从以下三个维度切入：数据驱动决策、自动化流程优化、以及个性化用户体验。\"*\n\n## 关键数据\n\n- 市场规模：预计2026年达到200亿\n- 年复合增长率：35%\n- 头部企业市占率：约40%\n\n> 详细分析参见完整文档。", + "txt": f"文档内容摘录：\n\n\"...针对「{query}」这一课题，我们进行了深入的调研和分析。研究发现，超过78%的受访者认为该领域的创新速度正在加快。同时，技术成熟度曲线显示，相关技术有望在未来12-18个月内进入主流应用阶段。建议前期投入研发资源约50人月，预期投资回报周期为18个月...\"", + } + file_type = (doc.file_type or "txt").lower() + return excerpts.get(file_type, excerpts["txt"]) + + +def format_file_size(size_bytes: int) -> str: + """格式化文件大小""" + if size_bytes >= 1024 * 1024: + return f"{size_bytes / (1024 * 1024):.1f} MB" + elif size_bytes >= 1024: + return f"{size_bytes / 1024:.1f} KB" + return f"{size_bytes} B" + + +# ============================================================ +# 知识图谱构建与查询 API +# ============================================================ + +class BuildGraphRequest(BaseModel): + base_id: str + diff --git a/backend/python/app/chat_context.py b/backend/python/app/chat_context.py new file mode 100644 index 0000000..bb4173b --- /dev/null +++ b/backend/python/app/chat_context.py @@ -0,0 +1,2948 @@ +# Auto-extracted from chat.py during Phase 1 refactor (Strangler Fig). + +# Source of truth: scripts/phase1_extract.py. Do not hand-merge back. + + + +from fastapi import APIRouter, Depends, HTTPException, Query, Response + +from sqlalchemy.orm import Session + +from sqlalchemy import and_, desc, asc, func + +from pydantic import BaseModel, Field + +from typing import Optional + +import uuid + +from datetime import datetime, timezone, timedelta + +import re + +import json + +import asyncio + +import math + +import httpx + +import os + +import logging + +from ..database import get_db, SessionLocal + +from ..models import ( + + + + + + + + User, Conversation, Message, AIModel, ModelProvider, TokenUsageLog, ConversationSummary, + + + + + + + + KnowledgeBase, KnowledgeDocument, KnowledgeDocumentChunk, EvolutionSkill, + + + + + + + + EvolutionTrajectory, MarketApp, AppPurchase, AppInstallation, + + + + + + + + MCPServer, + + + + + + + + gen_uuid + + + + + + + +) + +from ..auth import get_current_user + +from ..platform_knowledge import get_platform_knowledge + +from ..mcp_client import list_mcp_tools, call_mcp_tool + +from ..middleware.log_service import LogService + +from ..capability_registry import resolve_auto_model + +from app.api.chat_skill import (_bm25_search) + +from app.kb_router import kb_router_search + +from app.api.chat_utils import (logger) + + + +async def _retrieve_kb_context(db: Session, kb_ids: list, query: str, max_tokens: int = 1000) -> str: + + + + + + + + """ + + + + + + + + Stage C: 增强版知识库检索 — 使用BM25排序替代朴素拼接 + + + + + + + + 1. 获取每个KB的文档 + + + + + + + + 2. 构建文档列表(content+title+metadata) + + + + + + + + 3. BM25检索最相关的段落 + + + + + + + + 4. 拼接为上下文注入system prompt + + + + + + + + 同时叠加 Dify 外部知识库(RAG)检索结果。 + + + + + + + + """ + + + + + + + + local_context = "" + + + + + + + + + + + + + + + + if kb_ids: + + + + + + + + kbs = db.query(KnowledgeBase).filter(KnowledgeBase.id.in_(kb_ids)).all() + + + + + + + + if kbs: + + + + + + + + # 收集所有文档段落 + + + + + + + + # Stage C.1: 优先使用 KnowledgeDocumentChunk 表的预分段数据 + + + + + + + + all_segments = [] + + + + + + + + for kb in kbs: + + + + + + + + # 先查chunk表 + + + + + + + + chunks = db.query(KnowledgeDocumentChunk).filter( + + + + + + + + KnowledgeDocumentChunk.knowledge_base_id == kb.id + + + + + + + + ).order_by(KnowledgeDocumentChunk.chunk_index).limit(30).all() + + + + + + + + + + + + + + + + if chunks: + + + + + + + + # 使用预分段数据 + + + + + + + + for chunk in chunks: + + + + + + + + if chunk.content and len(chunk.content.strip()) >= 20: + + + + + + + + all_segments.append({ + + + + + + + + "content": "[{}/{}/段{}]: {}".format( + + + + + + + + kb.name, chunk.document_id[:8], chunk.chunk_index, chunk.content[:500] + + + + + + + + ), + + + + + + + + "raw_content": chunk.content, + + + + + + + + "kb_name": kb.name, + + + + + + + + }) + + + + + + + + else: + + + + + + + + # 退化: 使用原始文档分段 + + + + + + + + docs = db.query(KnowledgeDocument).filter( + + + + + + + + KnowledgeDocument.knowledge_base_id == kb.id, + + + + + + + + KnowledgeDocument.status == "completed" + + + + + + + + ).order_by(KnowledgeDocument.updated_at.desc()).limit(20).all() + + + + + + + + + + + + + + + + for doc in docs: + + + + + + + + content = doc.content or "" + + + + + + + + if not content: + + + + + + + + continue + + + + + + + + + + + + + + + + # 如果文档较长, 分段处理 (每段~800字) + + + + + + + + if len(content) > 800: + + + + + + + + paragraphs = re.split(r'\n\n+', content) + + + + + + + + for pi, para in enumerate(paragraphs): + + + + + + + + if len(para.strip()) >= 20: + + + + + + + + all_segments.append({ + + + + + + + + "content": "[{}/{}/段{}]: {}".format( + + + + + + + + kb.name, doc.name, pi+1, para.strip()[:500] + + + + + + + + ), + + + + + + + + "raw_content": para.strip(), + + + + + + + + "kb_name": kb.name, + + + + + + + + "doc_name": doc.name, + + + + + + + + }) + + + + + + + + else: + + + + + + + + all_segments.append({ + + + + + + + + "content": "[{}/{}]: {}".format(kb.name, doc.name, content[:500]), + + + + + + + + "raw_content": content, + + + + + + + + "kb_name": kb.name, + + + + + + + + "doc_name": doc.name, + + + + + + + + }) + + + + + + + + + + + + + + + + if not all_segments: + + + + + + + + # 退化: 使用原始朴素拼接 + + + + + + + + kb_names = ", ".join([k.name for k in kbs]) + + + + + + + + local_context = f"参考知识库: {kb_names}" + + + + + + + + else: + + + + + + + + # BM25检索最相关的段落 + + + + + + + + # 混合检索：BM25(关键词) + 向量语义 + RRF 融合；失败回退 BM25 + bm25_segs = _bm25_search(query, all_segments, top_k=8) + bm25_for_router = [ + {"content": seg.get("raw_content", seg.get("content", "")), + "metadata": {"label": seg.get("content", "")}} + for seg in bm25_segs + ] + merged = None + try: + merged = await kb_router_search( + query, kb_ids=kb_ids, top_k=8, + bm25_results=bm25_for_router, db_session=db, + use_vector=True, use_external=False, + ) + except Exception as e: + logger.warning(f"[kb] 混合检索失败, 回退 BM25: {e}") + if merged: + ctx_parts = [] + used = 0 + for seg in merged: + label = (seg.get("metadata") or {}).get("label") + c = label if label else seg.get("content", "") + t = _estimate_tokens(c) + if ctx_parts and used + t > max_tokens: + break + ctx_parts.append(c) + used += t + local_context = "\n".join(ctx_parts) + else: + ctx_parts = [seg["content"] for seg in bm25_segs] + joined = "\n".join(ctx_parts) + if _estimate_tokens(joined) > max_tokens: + trunc, u = [], 0 + for c in ctx_parts: + if u + _estimate_tokens(c) > max_tokens: + break + trunc.append(c) + u += _estimate_tokens(c) + joined = "\n".join(trunc) + "\n...(内容截断)" + local_context = joined + + + + + + + + + + + + + + + + # 叠加 Dify 外部知识库(RAG)检索 + + + + + + + + dify_context = "" + + + + + + + + try: + + + + + + + + from app.dify_kb import get_dify_kb_context + + + + + + + + dify_context = get_dify_kb_context(db, query, top_k=5) + + + + + + + + except Exception as e: + + + + + + + + logger.warning(f"Dify KB context retrieval failed: {e}") + + + + + + + + + + + + + + + + # 联网检索 (AnySearch) — 实时网页结果注入 + web_context = "" + try: + web_context = await _retrieve_web_context(db, query, top_k=3) + except Exception as e: + logger.warning(f"web search context failed: {e}") + + parts = [] + + + + + + + + if local_context: + + + + + + + + parts.append(local_context) + + + + + + + + if dify_context: + + + + + + + + parts.append("【Dify 知识库检索结果】\n" + dify_context) + + + + + + + + if web_context: + parts.append("【联网检索结果】\n" + web_context) + + return "\n\n".join(parts) + + + +async def _retrieve_web_context(db: Session, query: str, top_k: int = 3) -> str: + """联网检索上下文：调用 AnySearch 获取实时网页结果，格式化为引用列表。 + + 仅在存在已启用的 AnySearch 服务且全局开关开启时生效。失败静默返回空串。 + """ + try: + # 全局开关（默认开启）：ANYSEARCH_IN_CHAT=0 可关闭 + if os.environ.get("ANYSEARCH_IN_CHAT", "1") != "1": + return "" + from app.kb_router import kb_router_search + hits = await kb_router_search( + query, top_k=top_k, db_session=db, + use_web=True, use_vector=False, use_external=False, use_tree=False, + ) + if not hits: + return "" + lines = [] + for h in hits: + meta = h.get("metadata", {}) or {} + title = meta.get("title", "") or "(无标题)" + url = meta.get("url", "") + snippet = (h.get("content", "") or "").strip() + if not snippet: + continue + line = f"- 【{title}】{url}\n {snippet}" + lines.append(line) + return "\n".join(lines) + except Exception as e: + logger.warning(f"web search context failed: {e}") + return "" + + +async def _ingest_answer_to_dify(conv_id: str, msg_id: str, content: str): + + + + + + + + """后台任务：将助手回答入库到 Dify 知识库（量化后可供 RAG 检索复用）。""" + + + + + + + + try: + + + + + + + + from app.dify_kb import aingest_text + + + + + + + + from app.database import SessionLocal + + + + + + + + from sqlalchemy import text as _t + + + + + + + + db = SessionLocal() + + + + + + + + try: + + + + + + + + row = db.execute(_t( + + + + + + + + "SELECT api_url, api_key, config FROM external_kb_services " + + + + + + + + "WHERE service_type='dify' AND is_enabled=1 LIMIT 1" + + + + + + + + )).fetchone() + + + + + + + + finally: + + + + + + + + db.close() + + + + + + + + if not row: + + + + + + + + return + + + + + + + + api_url, api_key, config_raw = row + + + + + + + + dataset_name = None + + + + + + + + if config_raw: + + + + + + + + try: + + + + + + + + import json + + + + + + + + cfg = config_raw if isinstance(config_raw, dict) else json.loads(config_raw) + + + + + + + + dataset_name = cfg.get("dataset_name") + + + + + + + + except Exception: + + + + + + + + pass + + + + + + + + await aingest_text(api_url, api_key, content, name=f"chat_{conv_id}_{msg_id}", dataset_name=dataset_name) + + + + + + + + except Exception as e: + + + + + + + + logger.warning(f"后台入库 Dify 失败: {e}") + + + +def _build_system_prompt( + + + + + + + + skills: list, + + + + + + + + agents: list, + + + + + + + + kb_context: str, + + + + + + + + user_content: str, + + + + + + + + mcp: list = None, + + + + mcp_functions: list = None, + + + + auto_discovered: bool = False, + + all_mcp: list = None, + + + +) -> tuple: + + + + + + + + """ + + + + + + + + 动态组装system prompt + function definitions。 + + + + + + + + 返回: (system_prompt_str, function_definitions_list) + + + + + + + + """ + + + + + + + + system_parts = ["你是AIFlow平台的AI助手，具备专业的知识储备和工具调用能力。"] + + + + + + + + + + + + + + + + # 平台知识库注入（始终开启）：确保AI优先使用平台已有能力 + + + + + + + + try: + + + + + + + + pk = get_platform_knowledge() + + + + + + + + if pk: + + + + + + + + system_parts.append(f"【平台知识库——优先使用以下知识回答用户问题】\n{pk}") + + + + + + + + except Exception: + + + + + + + + pass + + + + + + + + + + + + + + + + # 回复格式要求：强制使用Markdown + + + + + + + + system_parts.append("回复格式要求：使用 Markdown 格式，层次分明即可。对于简短问题，用 1-2 段自然文字直接回答，不必强行分点、加标题或表格；仅在确实需要分步骤或列举时才使用列表，代码/配置用 ``` 代码块包裹。避免冗余空行，保持紧凑、美观、易读。") + + + + + + + + + + + + + + + + function_definitions = [] + + + + + + + + + + + + + + + + # Stage B.1: Agent角色注入（Issue 2：让用户询问 agent 能力时，AI 能“看到”该 agent 资料） + + + + + + + + if agents: + + + + + + + + for agent in agents: + + + + + + + + name = agent.get("name", "") + + + + + + + + role = agent.get("role_prompt", "") + + + + + + + + instructions = agent.get("instructions", "") + + + + + + + + desc = agent.get("description", "") + + + + + + + + block = [] + + + + + + + + if name: + + + + + + + + block.append( + + + + + + + + f"用户已选择智能体「{name}」。当用户询问该智能体的功能、能力、用法或相关问题时，" + + + + + + + + f"请基于以下资料准确作答，不要说“不知道这个智能体”。" + + + + + + + + ) + + + + + + + + if role: + + + + + + + + block.append(f"该智能体的角色设定: {role}") + + + + + + + + if desc: + + + + + + + + block.append(f"该智能体的简介: {desc}") + + + + + + + + if instructions: + + + + + + + + block.append(f"该智能体的行为指引: {instructions}") + + + + + + + + if block: + + + + + + + + system_parts.append("\n".join(block)) + + + + + + + + + + + + + + + + # Stage A.2: Skill工具注入 + + + + + + + + if skills: + + + + + + + + skill_descs = ", ".join([f"{s['name']}({s.get('category', 'general')})" for s in skills]) + + + + + + + + system_parts.append(f"可用技能工具: {skill_descs}") + + + + + + + + if auto_discovered: + + + + + + + + # Issue 8：每轮默认先寻找本账号已安装且平台已部署的技能，多个适用时 AI 自行判断选 1 个最优 + + + + + + + + system_parts.append( + + + + + + + + "重要规则: 以上技能为本账号已安装且平台已部署的技能。每轮对话请先判断用户请求是否可由其中某个技能处理；" + + + + + + + + "若多个技能都适用，请自行判断并选择最贴切的 1 个优先调用，不要一次性全部调用。" + + + + + + + + "调用对应工具后将获得执行结果，基于结果生成最终回复。若这些技能均不适用，则直接基于你的知识回答。" + + + + + + + + ) + + + + + + + + else: + + + + + + + + system_parts.append( + + "重要规则：\n" + + "1) 当用户请求需要由该技能【执行】某项任务时，优先通过 function calling 调用对应工具；调用后基于工具返回结果生成回复。\n" + + "2) 当用户询问所选技能【本身的介绍、功能、用途、用法、特点】（例如“介绍一下这个技能”“它是什么”“怎么用”“能做什么”）时，" + + "必须基于下方【已选技能资料】中的名称/描述/指令直接作答，不要泛泛介绍平台整体，也不要说“不知道这个技能”。\n" + + "3) 若用户问题既不执行也不询问该技能本身，则按常规方式回答。" + + ) + + + + + + + + + + + + + + + + # 为每个skill构建function definition + + + + + + + + for s in skills: + + + + + + + + # 从skill_content提取指令注入system prompt + + + + + + + + instructions = s.get("instructions", "") + + + + + + + + name = s.get("name", "") + + if not auto_discovered: + + # 显式选中技能：注入可引用的技能资料，便于“介绍/说明”类问题据此作答 + + ref_parts = [f"【已选技能资料】名称：{name}"] + + desc = s.get("description", "") + + if desc: + + ref_parts.append(f"简介：{desc}") + + if instructions and len(instructions) < 1200: + + ref_parts.append(f"使用指令：{instructions}") + + system_parts.append("\n".join(ref_parts)) + + elif instructions and len(instructions) < 800: + + system_parts.append(f"技能「{name}」的详细指令: {instructions}") + + + + + + + + + + + + + + + + # 构建参数schema + + + + + + + + params = s.get("parameters", {}) + + + + + + + + if isinstance(params, dict) and len(params) > 0: + + + + + + + + properties = {} + + + + + + + + required = [] + + + + + + + + for pname, pdef in params.items(): + + + + + + + + if isinstance(pdef, dict): + + + + + + + + properties[pname] = { + + + + + + + + "type": pdef.get("type", "string"), + + + + + + + + "description": pdef.get("description", f"传递给{s['name']}的{pname}参数"), + + + + + + + + } + + + + + + + + if pdef.get("required", True): + + + + + + + + required.append(pname) + + + + + + + + else: + + + + + + + + properties[pname] = {"type": "string", "description": f"传递给{s['name']}的{pname}参数"} + + + + + + + + required.append(pname) + + + + + + + + + + + + + + + + if not properties: + + + + + + + + properties = {"input": {"type": "string", "description": f"需要传递给{s['name']}的完整用户输入"}} + + + + + + + + required = ["input"] + + + + + + + + else: + + + + + + + + properties = {"input": {"type": "string", "description": f"需要传递给{s['name']}的完整用户输入"}} + + + + + + + + required = ["input"] + + + + + + + + + + + + + + + + desc = s.get("description", "") + + + + + + + + category = s.get("category", "general") + + + + + + + + func_desc = f"调用技能「{s['name']}」处理用户请求。使用场景: {desc}。当用户请求涉及{category}类任务时优先调用此工具。" + + + + + + + + + + + + + + + + function_definitions.append({ + + + + + + + + "name": s["func_name"], + + + + + + + + "description": func_desc, + + + + + + + + "parameters": { + + + + + + + + "type": "object", + + + + + + + + "properties": properties, + + + + + + + + "required": required, + + + + + + + + }, + + + + + + + + }) + + + + + + + + + + + + + + + + # KB context injection + + + + + + + + if kb_context: + + + + + + + + system_parts.append(f"参考以下知识库内容回答用户问题:\n{kb_context}") + + + + + + + + + + + + + + + + # 平台全量 MCP 清单（Issue：让用户询问"平台有哪些 MCP"时 AI 能准确列举，而非凭空编造） + + if all_mcp: + + _mcp_inventory = "\n".join([f"- {m.get('name','')}: {m.get('description','') or '（未填写描述）'}" for m in all_mcp]) + + system_parts.append( + + f"【平台已安装的 MCP 应用清单（共 {len(all_mcp)} 个，均为平台真实接入并启用）】\n{_mcp_inventory}\n" + + "当用户询问“平台有哪些 MCP / 接入了哪些 MCP / 有哪些工具服务”时，请严格依据以上清单准确列举名称与功能，" + + "不要编造或遗漏，也不要把未列出的服务说成已接入。" + + ) + + + + # MCP 应用注入（Issue 3：让用户询问 MCP 能力时，AI 能“看到”该 MCP 的资料与地址） + + + + + + + + if mcp: + + + + + + + + mcp_lines = [] + + + + + + + + for m in mcp: + + + + + + + + name = m.get("name", "") + + + + + + + + desc = m.get("description", "") + + + + + + + + transport = m.get("transport", "stdio") + + + + + + + + endpoint = m.get("endpoint", "") + + + + + + + + line = f"用户已选择 MCP 应用「{name}」（传输方式: {transport}）" + + + + + + + + if desc: + + + + + + + + line += f"，功能简介: {desc}" + + + + + + + + if endpoint: + + + + + + + + line += f"，服务地址: {endpoint}" + + + + + + + + mcp_lines.append(line) + + + + + + + + system_parts.append("\n".join(mcp_lines)) + + + + + + + + system_parts.append( + + + + + + + + "用户已选择上述 MCP 应用。当请求需要借助外部工具/系统（如查询数据、调用 API、操作第三方服务）时，" + + + + + + + + "请优先参考这些 MCP 应用的能力来回答；若平台已接入对应工具，优先通过工具调用完成；" + + + + + + + + "不要说“不了解该 MCP”。" + + + + + + + + ) + + + + # 高优先级覆盖规则：显式选中工具时，回答必须基于该工具资料 + + if (skills and not auto_discovered) or (mcp): + + system_parts.append( + + "【最高优先级覆盖规则】\n" + + "用户在本轮已明确选择了工具（见上文【已选技能资料】或【用户已选择 MCP 应用】）。\n" + + "如果用户的问题是在询问这些已选工具/技能/应用本身的介绍、功能、能做什么、怎么用、特点或用法，" + + "你必须只基于上文提供的该工具具体资料作答；\n" + + "不要引用《平台知识库》中关于技能市场、MCP 协议或平台整体能力的通用描述；" + + "不要列举其它未选中的技能或 MCP 作为示例；也不要说“不知道”或“无法回答”。\n" + + "若用户问题与已选工具本身无关，则按常规方式回答。" + + ) + + + + + + + + + + + + + + + + + + system_prompt = "\n".join(system_parts) + + + + + + + + # MCP function definitions 注入（让 LLM 可通过 function calling 真实调用 MCP 工具） + + + + if mcp_functions: + + + + function_definitions.extend(mcp_functions) + + + + + + + + return system_prompt, function_definitions + + + +def _get_conversation_context(db: Session, conv_id: str) -> dict: + + + + + + + + """ + + + + + + + + Stage B.2: 获取对话的持久化上下文 + + + + + + + + 对话级别的agent/skill/kb选择保存在Conversation字段中。 + + + + + + + + """ + + + + + + + + conv = db.query(Conversation).filter(Conversation.id == conv_id).first() + + + + + + + + if not conv: + + + + + + + + return {} + + + + + + + + + + + + + + + + # Conversation目前没有存储agent/skill/kb的JSON字段 + + + + + + + + # 我们从第一条消息中获取(如果存在), 或返回空 + + + + + + + + # 后续可通过数据库迁移添加Conversation.context字段 + + + + + + + + + + + + + + + + # 查询该对话中最早一条含有skill_ids/agent_ids/knowledge_ids的消息 + + + + + + + + first_tool_msg = db.query(Message).filter( + + + + + + + + Message.conversation_id == conv_id, + + + + + + + + Message.role == "user", + + + + + + + + ).order_by(asc(Message.created_at)).first() + + + + + + + + + + + + + + + + context = {} + + + + + + + + if first_tool_msg: + + + + + + + + if first_tool_msg.skill_ids: + + + + + + + + context["skill_ids"] = first_tool_msg.skill_ids + + + + + + + + if first_tool_msg.agent_ids: + + + + + + + + context["agent_ids"] = first_tool_msg.agent_ids + + + + + + + + if first_tool_msg.knowledge_ids: + + + + + + + + context["knowledge_base_ids"] = first_tool_msg.knowledge_ids + + + + + + + + if getattr(first_tool_msg, "mcp_ids", None): + + + + + + + + context["mcp_ids"] = first_tool_msg.mcp_ids + + + + + + + + + + + + + + + + return context + + + + + + + +# ===== Phase 2 上下文引擎升级：分层记忆 + 压缩 + 隔离 ===== + +logger = logging.getLogger(__name__) + + + + + +_tiktoken_enc = None + +def _estimate_tokens(text): + + """精确 token 估算：优先 tiktoken(cl100k_base)，失败回退 2 字符/token。""" + + if not text: + + return 0 + + global _tiktoken_enc + + if _tiktoken_enc is None: + + try: + + import tiktoken + + _tiktoken_enc = tiktoken.get_encoding("cl100k_base") + + except Exception: + + _tiktoken_enc = False + + if _tiktoken_enc: + + try: + + return len(_tiktoken_enc.encode(text)) + + except Exception: + + pass + + return max(1, len(text) // 2) + + + + + +def _build_chat_history(db, conv_id, max_tokens=6000, max_messages=40, + exclude_latest_user=True, include_summary=True): + """会话层上下文引擎：预算感知的历史组装 + 压缩摘要/结构化长期记忆前置。 + + - 若 conversation_summaries 存在结构化长期记忆(structured_memory)，以紧凑 + 结构化区块前置（Semantica 式：实体/事实/偏好/决策/待办），比扁平摘要更省 + token 且更便于模型检索关键信息。 + - 否则回退到扁平 summary_text 前置。 + - 按 token 估算从新到旧累加，超预算即截断（隔离：不溢出上下文窗口）。 + - exclude_latest_user=True 时丢弃最新一条 user 消息（当前轮次另行发送）。 + """ + ensure_structured_column() + summary_block = None + if include_summary: + try: + summ = db.query(ConversationSummary).filter( + ConversationSummary.conversation_id == conv_id + ).first() + if summ and summ.structured_memory and isinstance(summ.structured_memory, dict): + block = _format_structured_memory(summ.structured_memory) + if block: + summary_block = { + "role": "system", + "content": "【长期记忆·结构化】请据此理解用户此前的偏好、已确认决策与待办：\n" + block, + } + elif summ and summ.summary_text: + summary_block = { + "role": "system", + "content": "【对话历史摘要，请据此理解用户此前的意图与已确认结论】\n" + summ.summary_text, + } + except Exception: + pass + + rows = db.query(Message).filter( + Message.conversation_id == conv_id, + Message.role.in_(["user", "assistant"]), + ).order_by(desc(Message.created_at)).limit(max_messages).all() + if exclude_latest_user: + for i, m in enumerate(rows): + if m.role == "user": + rows = rows[:i] + rows[i + 1:] + break + + kept = [] + used = 0 + for m in rows: # 新->旧 + t = _estimate_tokens(m.content or "") + if kept and used + t > max_tokens: + break + kept.append({"role": m.role, "content": m.content}) + used += t + + kept.reverse() # 时间正序 + + out = [] + if summary_block: + out.append(summary_block) + out.extend(kept) + return out + + +def _format_structured_memory(mem: dict) -> str: + """将结构化记忆格式化为紧凑可读区块。""" + labels = { + "entities": "实体", + "facts": "事实", + "preferences": "偏好", + "decisions": "决策", + "open_tasks": "待办", + } + lines = [] + for key, label in labels.items(): + items = mem.get(key) + if not items: + continue + if isinstance(items, list): + for it in items: + if it: + lines.append(f"· {label}: {it}") + elif isinstance(items, str): + lines.append(f"· {label}: {items}") + return "\n".join(lines) +async def _summarize_via_llm(prompt): + + """调用默认文本模型生成对话摘要；失败返回 None（压缩为可选项，不阻塞）。""" + + try: + + from ..capability_registry import resolve_auto_model + + from app.api.chat_utils import _proxy_for_url + + sdb = SessionLocal() + + try: + + ai_model = resolve_auto_model("text", sdb) + + if not ai_model: + + return None + + provider = sdb.query(ModelProvider).filter( + + ModelProvider.id == ai_model.provider_id + + ).first() + + api_key = ai_model.api_key or (provider.api_key if provider else None) + + if not provider or not provider.api_base_url or not api_key: + + return None + + api_url = f"{provider.api_base_url.rstrip('/')}/v1/chat/completions" + + body = { + + "model": ai_model.name, + + "messages": [ + + {"role": "system", "content": "你是专业的对话摘要助手，输出简洁中文摘要。"}, + + {"role": "user", "content": prompt}, + + ], + + "max_tokens": 512, + + "temperature": 0.2, + + } + + async with httpx.AsyncClient(proxy=_proxy_for_url(api_url), trust_env=False, timeout=60) as client: + + resp = await client.post( + + api_url, + + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + + json=body, + + ) + + if resp.status_code != 200: + + return None + + data = resp.json() + + return data.get("choices", [{}])[0].get("message", {}).get("content", "").strip() + + finally: + + sdb.close() + + except Exception as e: + + logger.warning(f"[context] summarize llm failed: {e}") + + return None + + + + + +_STRUCT_COL_READY = False + + +def ensure_structured_column(): + """幂等地为 conversation_summaries 增加 structured_memory 列（MariaDB 安全）。""" + global _STRUCT_COL_READY + if _STRUCT_COL_READY: + return True + try: + from app.database import engine + from sqlalchemy import text + with engine.connect() as conn: + exists = conn.execute(text( + "SELECT COUNT(*) FROM information_schema.columns " + "WHERE table_schema = DATABASE() AND table_name='conversation_summaries' " + "AND column_name='structured_memory'" + )).scalar() + if not exists: + conn.execute(text( + "ALTER TABLE conversation_summaries ADD COLUMN structured_memory JSON DEFAULT NULL" + )) + conn.commit() + _STRUCT_COL_READY = True + return True + except Exception as e: + logger.warning(f"[context] ensure structured column failed: {e}") + return False + + +async def _recursive_compress(text, depth=0, max_depth=3, budget=700): + """递归再压缩：若摘要仍超过 token 预算，则进一步浓缩，直至收敛或达最大深度。""" + if not text: + return text + if _estimate_tokens(text) <= budget or depth >= max_depth: + return text + s = await _summarize_via_llm("请将以下摘要进一步浓缩，保留全部关键信息，篇幅更短：\n" + text) + if not s: + return text + return await _recursive_compress(s, depth + 1, max_depth, budget) + + +async def _extract_structured_memory(text): + """Semantica 式结构化长期记忆抽取：将对话内容抽取为类型化结构。 + + 模型对抽取类提示偶发返回空，故做最多 3 次重试；并兼容有标签与无标签两种输出。 + """ + if not text or len(text.strip()) < 10: + return None + prompt = ( + "提取要点，按 实体/事实/偏好/决策/待办 五类，用分号分隔输出：" + text[:6000] + ) + raw = None + for _ in range(3): + r = await _summarize_via_llm(prompt) + if r and r.strip(): + raw = r.strip() + break + if not raw: + return None + # JSON 回退 + if "{" in raw: + try: + s = raw.find("{") + e = raw.rfind("}") + data = json.loads(raw[s:e + 1]) + if isinstance(data, dict): + return _normalize_structured(data) + except Exception: + pass + norm = raw.replace("：", ":").replace("；", ";").replace("，", ",").replace("。", "") + result = _parse_structured_segments(norm) + if result: + return result + # 无标签回退：按分号切分，按顺序映射到五类 + segs = [s.strip() for s in norm.replace(chr(10), ";").split(";") if s.strip()] + if segs: + cats = ["entities", "facts", "preferences", "decisions", "open_tasks"] + fb = {} + for i, s in enumerate(segs[:5]): + fb[cats[i]] = [s] + return fb + return None + + +def _parse_structured_segments(norm): + """解析『类别: 内容』格式（兼容全角冒号/分号与单行多类别）。""" + result = {} + for seg in norm.replace(chr(10), ";").split(";"): + if ":" not in seg: + continue + key, _, val = seg.partition(":") + key = key.strip() + val = val.strip().strip(",") + if not val or val in ("无", "none", "None"): + continue + mapped = { + "实体": "entities", "事实": "facts", "偏好": "preferences", + "决策": "decisions", "待办": "open_tasks", + }.get(key) + if mapped: + items = [x.strip() for x in val.split(",") if x.strip()] + if items: + result[mapped] = items + return result + + +def _normalize_structured(data): + """将模型的 JSON 输出规范化为结构化记忆字典。""" + keys = ["entities", "facts", "preferences", "decisions", "open_tasks"] + out = {} + for k in keys: + v = data.get(k) + if isinstance(v, list): + out[k] = [str(x) for x in v if x] + elif isinstance(v, str) and v: + out[k] = [v] + return out + + +def _merge_structured_memory(old, new): + """合并新旧结构化记忆，按字符串去重并控制规模。""" + keys = ["entities", "facts", "preferences", "decisions", "open_tasks"] + merged = {} + for k in keys: + a = old.get(k) if isinstance(old, dict) else None + b = new.get(k) if isinstance(new, dict) else None + items = [] + for x in (a or []) + (b or []): + if x and x not in items: + items.append(x) + merged[k] = items[:50] + return merged + + +async def _maybe_compress_history(conv_id): + """后台异步压缩：将受保护窗口外的旧消息递归再压缩，并抽取结构化长期记忆。 + + 失败静默回滚，绝不阻塞用户本次响应（fire-and-forget）。 + """ + ensure_structured_column() + sdb = None + try: + sdb = SessionLocal() + all_msgs = sdb.query(Message).filter( + Message.conversation_id == conv_id, + Message.role.in_(["user", "assistant"]), + ).order_by(asc(Message.created_at)).all() + PROTECTED = 20 + if len(all_msgs) <= PROTECTED: + return + compressible = all_msgs[: len(all_msgs) - PROTECTED] + if len(compressible) < 8: + return + + # 1) 分块摘要：覆盖全部可压缩消息（而非仅前 30 条） + CHUNK = 30 + chunk_summaries = [] + for i in range(0, len(compressible), CHUNK): + piece = compressible[i:i + CHUNK] + text = "\n".join(f"{m.role}: {m.content}" for m in piece) + s = await _summarize_via_llm("请将以下对话片段浓缩为简明要点（保留关键事实、诉求与结论）：\n" + text) + if s: + chunk_summaries.append(s) + + existing = sdb.query(ConversationSummary).filter( + ConversationSummary.conversation_id == conv_id + ).first() + prior = existing.summary_text if existing else "" + + combined = "\n".join(chunk_summaries) + if prior: + combined = prior + "\n" + combined + + # 2) 递归再压缩：合并摘要超预算时逐层再压缩直到收敛 + final_summary = await _recursive_compress(combined) + + # 3) 结构化长期记忆抽取（Semantica 式），并与既有记忆合并 + structured = await _extract_structured_memory(combined) + if structured is None: + structured = {} + if existing and existing.structured_memory and isinstance(existing.structured_memory, dict): + structured = _merge_structured_memory(existing.structured_memory, structured) + + now = datetime.now() + if existing: + existing.summary_text = final_summary + existing.structured_memory = structured + existing.token_count = _estimate_tokens(final_summary) + existing.updated_at = now + else: + sdb.add(ConversationSummary( + conversation_id=conv_id, + summary_text=final_summary, + structured_memory=structured, + token_count=_estimate_tokens(final_summary), + updated_at=now, + )) + sdb.commit() + except Exception as e: + logger.warning(f"[context] compress history failed conv={conv_id}: {e}") + if sdb is not None: + try: + sdb.rollback() + except Exception: + pass + finally: + if sdb is not None: + try: + sdb.close() + except Exception: + pass \ No newline at end of file diff --git a/backend/python/app/kb_router.py b/backend/python/app/kb_router.py index 5632bdc..8425bb0 100644 --- a/backend/python/app/kb_router.py +++ b/backend/python/app/kb_router.py @@ -1,204 +1,262 @@ -""" -知识库路由层 -============ -将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。 -根据知识库配置自动选择最优检索策略。 - -使用方式: - from app.kb_router import kb_router_search - results = await kb_router_search(query, kb_id, top_k=5) -""" - -import logging -from typing import List, Dict, Optional - -logger = logging.getLogger("aiflow.kb_router") - - -async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]: - """获取已配置的外部KB服务（仅 dify）""" - from sqlalchemy import text - try: - query = "SELECT * FROM external_kb_services" - if enabled_only: - query += " WHERE is_enabled = 1" - query += " ORDER BY created_at DESC" - result = db_session.execute(text(query)) - columns = result.keys() - return [dict(zip(columns, row)) for row in result.fetchall()] - except Exception as e: - logger.error(f"Failed to get external services: {e}") - return [] - - -async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]: - """检索 Dify 知识库（走 app.dify_kb 客户端）""" - try: - from app.dify_kb import dify_retrieve - api_url = service.get("api_url", "") - api_key = service.get("api_key", "") - dataset_name = None - cfg = service.get("config") - if isinstance(cfg, str): - try: - import json - cfg = json.loads(cfg) - except Exception: - cfg = None - if isinstance(cfg, dict): - dataset_name = cfg.get("dataset_name") - hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name) - return hits - except Exception as e: - logger.error(f"Dify search error: {e}") - return [] - - -async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]: - """路由到对应的外部KB服务（当前仅支持 dify）""" - service_type = service.get("service_type", "") - if service_type == "dify": - return await _search_dify(service, query, top_k) - else: - logger.warning(f"Unknown external KB type: {service_type}") - return [] - - -async def kb_router_search( - query: str, - kb_id: Optional[str] = None, - kb_ids: Optional[List[str]] = None, - top_k: int = 5, - bm25_results: Optional[List[Dict]] = None, - db_session=None, - use_vector: bool = True, - use_external: bool = True, - use_tree: bool = True, -) -> List[Dict]: - """ - 知识库路由层 — 混合检索 - - 检索策略: - 1. 本地 BM25 检索 (MySQL, 已有) - 2. 向量语义检索 (PostgreSQL+pgvector, 新增) - 3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中) - 4. 外部KB检索 (Dify, 按配置) - - 融合方式: Reciprocal Rank Fusion (RRF) - - Args: - query: 查询文本 - kb_id: 知识库ID - top_k: 返回结果数 - bm25_results: BM25检索结果 - db_session: 数据库会话(用于获取外部KB配置与分片段检索) - use_vector: 是否使用向量检索 - use_external: 是否使用外部KB检索 - use_tree: 是否使用 TreeSearch 结构化分片段检索 - - Returns: - 融合后的检索结果列表 - """ - all_results = [] - - # 1. BM25 结果 - if bm25_results: - for rank, item in enumerate(bm25_results): - all_results.append({ - "content": item.get("content", item.get("text", "")), - "score": item.get("score", 1.0 / (60 + rank + 1)), - "source": "local_bm25", - "rank": rank + 1, - "metadata": item.get("metadata", {}), - }) - - # 2. 向量检索 - if use_vector: - try: - from app.pgvector_search import vector_search - kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else [None]) - vec_results = [] - for _kid in kb_targets: - try: - vec_results.extend(await vector_search(query, _kid, top_k)) - except Exception as e: - logger.warning(f"Vector search failed for kb={_kid}: {e}") - for rank, item in enumerate(vec_results): - all_results.append({ - "content": item.get("content", ""), - "score": item.get("score", 0.5), - "source": "vector_pg", - "rank": rank + 1, - "metadata": item.get("metadata", {}), - }) - except ImportError: - logger.debug("pgvector_search not available, skipping vector search") - except Exception as e: - logger.warning(f"Vector search failed: {e}") - - # 2.5 结构化分片段检索 (TreeSearch, Level 2) - if use_tree and db_session is not None: - try: - from app.tree_search import tree_search - kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else None) - tree_hits = await tree_search( - db_session, query, kb_ids=kb_targets, top_k=top_k, mode="hybrid", - ) - for rank, item in enumerate(tree_hits): - all_results.append({ - "content": item.get("content", ""), - "score": item.get("score", 0.5), - "source": "tree_chunk", - "rank": rank + 1, - "metadata": item.get("metadata", {}), - }) - except Exception as e: - logger.warning(f"TreeSearch failed: {e}") - - # 3. 外部KB检索 (Dify) - if use_external and db_session: - services = await _get_external_services(db_session) - for service in services: - if service.get("status") == "connected": - ext_results = await _search_external(service, query, top_k) - for rank, item in enumerate(ext_results): - all_results.append({ - "content": item.get("content", ""), - "score": item.get("score", 0.5), - "source": item.get("source", "external_dify"), - "rank": rank + 1, - "metadata": item.get("metadata", {}), - }) - - if not all_results: - return [] - - # RRF 融合 - rrf_k = 60 - score_map = {} - for item in all_results: - content_key = item["content"][:200] # 用内容前200字符作为去重键 - rrf_score = 1.0 / (rrf_k + item.get("rank", 1)) - - if content_key in score_map: - score_map[content_key]["score"] += rrf_score - score_map[content_key]["sources"].append(item["source"]) - else: - score_map[content_key] = { - "content": item["content"], - "score": rrf_score, - "sources": [item["source"]], - "metadata": item.get("metadata", {}), - } - - # 按融合分数排序 - merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True) - - # 标准化分数到 0-1 - if merged: - max_score = merged[0]["score"] - for item in merged: - item["score"] = item["score"] / max_score if max_score > 0 else 0 - item["sources"] = list(set(item["sources"])) - - return merged[:top_k] +""" +知识库路由层 +============ +将检索请求智能路由到本地BM25、向量检索、或外部KB服务(Dify)。 +根据知识库配置自动选择最优检索策略。 + +使用方式: + from app.kb_router import kb_router_search + results = await kb_router_search(query, kb_id, top_k=5) +""" + +import logging +from typing import List, Dict, Optional + +logger = logging.getLogger("aiflow.kb_router") + + +async def _get_external_services(db_session, enabled_only: bool = True) -> List[Dict]: + """获取已配置的外部KB服务（仅 dify）""" + from sqlalchemy import text + try: + query = "SELECT * FROM external_kb_services" + if enabled_only: + query += " WHERE is_enabled = 1" + query += " ORDER BY created_at DESC" + result = db_session.execute(text(query)) + columns = result.keys() + return [dict(zip(columns, row)) for row in result.fetchall()] + except Exception as e: + logger.error(f"Failed to get external services: {e}") + return [] + + +async def _search_dify(service: Dict, query: str, top_k: int = 5) -> List[Dict]: + """检索 Dify 知识库（走 app.dify_kb 客户端）""" + try: + from app.dify_kb import dify_retrieve + api_url = service.get("api_url", "") + api_key = service.get("api_key", "") + dataset_name = None + cfg = service.get("config") + if isinstance(cfg, str): + try: + import json + cfg = json.loads(cfg) + except Exception: + cfg = None + if isinstance(cfg, dict): + dataset_name = cfg.get("dataset_name") + hits = dify_retrieve(api_url, api_key, query, top_k=top_k, dataset_name=dataset_name) + return hits + except Exception as e: + logger.error(f"Dify search error: {e}") + return [] + + +async def _search_external(service: Dict, query: str, top_k: int = 5) -> List[Dict]: + """路由到对应的外部KB服务（当前仅支持 dify）""" + service_type = service.get("service_type", "") + if service_type == "dify": + return await _search_dify(service, query, top_k) + else: + logger.warning(f"Unknown external KB type: {service_type}") + return [] + +async def _get_web_services(db_session, enabled_only: bool = True) -> List[Dict]: + """获取已启用的 AnySearch 联网检索服务配置。""" + from sqlalchemy import text + try: + query = "SELECT * FROM external_kb_services WHERE service_type='anysearch'" + if enabled_only: + query += " AND is_enabled = 1" + query += " ORDER BY created_at DESC" + result = db_session.execute(text(query)) + columns = result.keys() + return [dict(zip(columns, row)) for row in result.fetchall()] + except Exception as e: + logger.error(f"Failed to get AnySearch services: {e}") + return [] + + +async def _search_anysearch(service: Dict, query: str, top_k: int = 5) -> List[Dict]: + """调用 AnySearch 联网检索（走 app.anysearch_client）。""" + try: + from app.anysearch_client import anysearch_search + api_key = service.get("api_key", "") + api_url = service.get("api_url") or "https://api.anysearch.com/v1/search" + cfg = service.get("config") + if isinstance(cfg, str): + try: + import json + cfg = json.loads(cfg) + except Exception: + cfg = None + cfg = cfg or {} + zone = cfg.get("zone", "cn") + freshness = cfg.get("freshness") + k = int(cfg.get("top_k", top_k)) + return await anysearch_search(api_key, api_url, query, top_k=k, zone=zone, freshness=freshness) + except Exception as e: + logger.error(f"AnySearch search error: {e}") + return [] + + +async def kb_router_search( + query: str, + kb_id: Optional[str] = None, + kb_ids: Optional[List[str]] = None, + top_k: int = 5, + bm25_results: Optional[List[Dict]] = None, + db_session=None, + use_vector: bool = True, + use_external: bool = True, + use_tree: bool = True, + use_web: bool = False, +) -> List[Dict]: + """ + 知识库路由层 — 混合检索 + + 检索策略: + 1. 本地 BM25 检索 (MySQL, 已有) + 2. 向量语义检索 (PostgreSQL+pgvector, 新增) + 3. 结构化分片段检索 (TreeSearch, 新增: 知识库目录树 + chunk 级精准命中) + 4. 外部KB检索 (Dify, 按配置) + 5. 联网检索 (AnySearch, 可选 use_web) + + 融合方式: Reciprocal Rank Fusion (RRF) + + Args: + query: 查询文本 + kb_id: 知识库ID + top_k: 返回结果数 + bm25_results: BM25检索结果 + db_session: 数据库会话(用于获取外部KB配置与分片段检索) + use_vector: 是否使用向量检索 + use_external: 是否使用外部KB检索 + use_tree: 是否使用 TreeSearch 结构化分片段检索 + use_web: 是否使用 AnySearch 联网检索 + + Returns: + 融合后的检索结果列表 + """ + all_results = [] + + # 1. BM25 结果 + if bm25_results: + for rank, item in enumerate(bm25_results): + all_results.append({ + "content": item.get("content", item.get("text", "")), + "score": item.get("score", 1.0 / (60 + rank + 1)), + "source": "local_bm25", + "rank": rank + 1, + "metadata": item.get("metadata", {}), + }) + + # 2. 向量检索 + if use_vector: + try: + from app.pgvector_search import vector_search + kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else [None]) + vec_results = [] + for _kid in kb_targets: + try: + vec_results.extend(await vector_search(query, _kid, top_k)) + except Exception as e: + logger.warning(f"Vector search failed for kb={_kid}: {e}") + for rank, item in enumerate(vec_results): + all_results.append({ + "content": item.get("content", ""), + "score": item.get("score", 0.5), + "source": "vector_pg", + "rank": rank + 1, + "metadata": item.get("metadata", {}), + }) + except ImportError: + logger.debug("pgvector_search not available, skipping vector search") + except Exception as e: + logger.warning(f"Vector search failed: {e}") + + # 2.5 结构化分片段检索 (TreeSearch, Level 2) + if use_tree and db_session is not None: + try: + from app.tree_search import tree_search + kb_targets = kb_ids if kb_ids else ([kb_id] if kb_id else None) + tree_hits = await tree_search( + db_session, query, kb_ids=kb_targets, top_k=top_k, mode="hybrid", + ) + for rank, item in enumerate(tree_hits): + all_results.append({ + "content": item.get("content", ""), + "score": item.get("score", 0.5), + "source": "tree_chunk", + "rank": rank + 1, + "metadata": item.get("metadata", {}), + }) + except Exception as e: + logger.warning(f"TreeSearch failed: {e}") + + # 3. 外部KB检索 (Dify) + if use_external and db_session: + services = await _get_external_services(db_session) + for service in services: + if service.get("status") == "connected": + ext_results = await _search_external(service, query, top_k) + for rank, item in enumerate(ext_results): + all_results.append({ + "content": item.get("content", ""), + "score": item.get("score", 0.5), + "source": item.get("source", "external_dify"), + "rank": rank + 1, + "metadata": item.get("metadata", {}), + }) + + # 4. 联网检索 (AnySearch, 可选) + if use_web and db_session is not None: + try: + web_services = await _get_web_services(db_session) + for svc in web_services: + web_hits = await _search_anysearch(svc, query, top_k) + for rank, item in enumerate(web_hits): + all_results.append({ + "content": item.get("content", ""), + "score": item.get("score", 0.6), + "source": item.get("source", "web_anysearch"), + "rank": rank + 1, + "metadata": item.get("metadata", {}), + }) + except Exception as e: + logger.warning(f"AnySearch web search failed: {e}") + + if not all_results: + return [] + + # RRF 融合 + rrf_k = 60 + score_map = {} + for item in all_results: + content_key = item["content"][:200] # 用内容前200字符作为去重键 + rrf_score = 1.0 / (rrf_k + item.get("rank", 1)) + + if content_key in score_map: + score_map[content_key]["score"] += rrf_score + score_map[content_key]["sources"].append(item["source"]) + else: + score_map[content_key] = { + "content": item["content"], + "score": rrf_score, + "sources": [item["source"]], + "metadata": item.get("metadata", {}), + } + + # 按融合分数排序 + merged = sorted(score_map.values(), key=lambda x: x["score"], reverse=True) + + # 标准化分数到 0-1 + if merged: + max_score = merged[0]["score"] + for item in merged: + item["score"] = item["score"] / max_score if max_score > 0 else 0 + item["sources"] = list(set(item["sources"])) + + return merged[:top_k]